import asyncio import json from contextlib import asynccontextmanager from copy import deepcopy from datetime import datetime, timezone from types import SimpleNamespace from unittest.mock import AsyncMock from uuid import uuid4 import pytest from fastapi import HTTPException from app.domain.schemas.periodic_task import PeriodicTaskExecution from app.services.periodic_diagnosis import diagnosis_execution, execution_record from app.services import periodic_tasks def document(): return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()), "monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest", "devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"}, {"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}], "config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"], "rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]}, "latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}} def monitoring(): return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00", "period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00", "status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None} def test_water_risk_is_independent_of_incomplete_pressure_model(): source, period = document(), monitoring() period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[ {"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}]) result = diagnosis_execution(source, period) assert result["conclusion"] == "suspected_pollution" assert result["investigation_task_id"] == period["task_id"] assert result["risk_signals"][0]["metric_name"] == "余氯" assert result["data_gaps"] == ["压力基线不足"] assert all(step["status"] == "completed" for step in result["steps"]) def detected_water_event(): from datetime import timedelta from app.domain.schemas.scada_replay import ScadaReplayCreate from app.services.scada_replay_processing import detect_frame, prepare_cleaning source = document() at = datetime.fromisoformat(monitoring()["period_end"]) config = ScadaReplayCreate( request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15), device_ids=["Q1"], rules=[{ "rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "lower": 0.2, "unit": "mg/L", "metric_name": "余氯", }], ) source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules] source, events = detect_frame(source, prepare_cleaning([ {"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"} ], config), config, at) event = next(event for event in events if event["rule_id"] == "water") task_id = str(uuid4()) event["task_id"] = task_id source["rule_states"]["water"]["event"]["task_id"] = task_id return source, event def test_periodic_report_includes_water_event_excluded_from_burst_monitoring(): from app.services.scada_normal_profile import diagnostic_signal source, event = detected_water_event() period = monitoring() assert not diagnostic_signal(source, event) original = deepcopy(period) result = diagnosis_execution(source, period) assert result["conclusion"] == "suspected_pollution" assert result["investigation_task_id"] == event["task_id"] water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution") assert water["device_ids"] == ["Q1"] assert water["task_id"] == event["task_id"] assert period == original def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence(): source, event = detected_water_event() source["rule_states"]["water"]["event"] = None event.update(status="resolved", resolved_at=monitoring()["period_end"]) period = monitoring() hydraulic_task_id = str(uuid4()) hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()), "device_ids": ["P1"], "detail": "压力变化需要复核"} period.update(status="attention", task_id=hydraulic_task_id, weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal]) original = deepcopy(period) result = diagnosis_execution(source, period, [event, deepcopy(event)]) assert result["conclusion"] == "suspected_pollution" assert result["investigation_task_id"] == hydraulic_task_id water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"] assert len(water) == 1 assert water[0]["task_id"] == event["task_id"] assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"] for signal in result["risk_signals"]) assert period == original assert result["result"]["monitoring"] == original def test_periodic_water_projection_excludes_telemetry_quality_events(): source, event = detected_water_event() event["category"] = "data_quality" source["rule_states"]["water"]["event"] = deepcopy(event) result = diagnosis_execution(source, monitoring(), [event]) assert result["conclusion"] == "normal" assert result["risk_signals"] == [] assert result["investigation_task_id"] is None @pytest.mark.parametrize("future", [ {"occurred_at": "2026-07-16T08:15:00+08:00"}, {"batch_sequence": 3}, ]) def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future): source, event = detected_water_event() event.update(future) source["rule_states"]["water"]["event"] = event result = diagnosis_execution(source, monitoring(), [event]) assert result["conclusion"] == "normal" assert result["risk_signals"] == [] assert result["investigation_task_id"] is None def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal(): source, period = document(), monitoring() source["latest"]["P1"]["quality"] = "invalid" first = diagnosis_execution(source, period) assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"] assert first["conclusion"] == "attention" assert first["result"]["invalid_device_ids"] == ["P1"] source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00" assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"] other = dict(source, replay_id=str(uuid4())) assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"] class Repository: def __init__(self): self.document = document() self.document["last_monitoring"] = monitoring() self.project_id = self.document["project_id"] self.connection = object() self.tasks = {} self.receipts = {} self.state = None self.events = [] @asynccontextmanager async def control(self): yield async def get(self, _): return deepcopy(self.document) async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)} async def periodic_events(self, *_): return deepcopy(self.events) async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution) async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution) async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id))) async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id) def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch): repo = Repository() enqueue = AsyncMock(return_value={"job_id": uuid4()}) monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue)) context = SimpleNamespace(project_id=repo.project_id) monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) async def run(): await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"]) await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"]) asyncio.run(run()) assert len(repo.tasks) == 2 assert enqueue.await_count == 1 payload = enqueue.call_args.kwargs assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id} assert payload["owner_user_id"] == repo.document["created_by"] assert repo.state["status"] == "pending" for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task) def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch): source, event = detected_water_event() source["rule_states"]["water"]["event"] = None source["last_monitoring"] = monitoring() repo = Repository() repo.document = source repo.project_id = source["project_id"] repo.events = [event] read_events = AsyncMock(wraps=repo.periodic_events) repo.periodic_events = read_events monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( enqueue=AsyncMock(return_value={"job_id": uuid4()}))) monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"])) read_events.assert_awaited_once() diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis") assert diagnosis["conclusion"] == "suspected_pollution" assert diagnosis["investigation_task_id"] == event["task_id"] def test_worker_interruption_is_reported_as_retryable_failure(): record = execution_record(document(), monitoring(), "network-simulation") result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}}) assert result["status"] == "failed" assert "WORKER_INTERRUPTED" in result["summary"] assert record["status"] == "pending" @pytest.mark.parametrize("job", [None, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}}, {"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}]) def test_failed_calculation_marks_its_execution_step_and_skips_dependent_steps(job): record = execution_record(document(), monitoring(), "network-simulation") record["steps"][0]["status"] = "completed" original = deepcopy(record) result = periodic_tasks.project_job(record, job) assert result["status"] == "failed" assert result["steps"][0]["status"] == "completed" assert result["steps"][1]["status"] == "failed" assert result["steps"][1]["detail"] == result["summary"] assert all(step["status"] == "skipped" for step in result["steps"][2:]) assert record == original def test_worker_failure_persists_failed_and_skipped_steps_in_the_business_report(monkeypatch): repo = Repository() record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation") record["steps"][0]["status"] = "completed" record["calculation_job_id"] = str(uuid4()) repo.tasks[record["execution_id"]] = record monkeypatch.setattr(periodic_tasks, "ScadaReplayRepository", lambda *_: repo) row = {"project_id": repo.project_id, "job_id": record["calculation_job_id"], "parameters": { "replay_id": record["replay_id"], "execution_id": record["execution_id"], "batch_sequence": record["batch_sequence"]}} asyncio.run(periodic_tasks.persist_periodic_terminal(repo.connection, row, {"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}})) saved = repo.tasks[record["execution_id"]] assert saved["status"] == repo.state["status"] == "failed" assert [step["status"] for step in saved["steps"]] == ["completed", "failed", "skipped", "skipped", "skipped"] assert saved["execution_id"] == record["execution_id"] assert saved["batch_sequence"] == record["batch_sequence"] @pytest.mark.parametrize("action", ["step", "monitor"]) def test_failed_period_must_be_retried_before_advancing_but_can_be_stopped(monkeypatch, action): from tests.unit.test_scada_replay_service import setup from app.domain.schemas.scada_replay import ScadaReplayCommand repo, service, _, _, create = setup(monkeypatch) monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value)) async def run(): replay = await service.create(create, monitoring_interval_seconds=900) repo.document["config"]["scheduled_tasks_enabled"] = True repo.document["last_scheduled_simulation"] = {"status": "failed", "error": "EXECUTION_TIMEOUT"} original = deepcopy(repo.document) command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action=action, **({"at_time": replay.start_time} if action == "monitor" else {})) with pytest.raises(HTTPException) as error: await service.command(replay.replay_id, command) assert error.value.status_code == 409 assert "重试" in error.value.detail assert repo.document == original stopped = await service.command(replay.replay_id, ScadaReplayCommand( request_id=uuid4(), expected_revision=replay.revision, action="stop")) assert stopped.status == "stopped" asyncio.run(run()) def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch): repo = Repository() monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded()))) monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"])) assert len(repo.tasks) == 2 assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed" assert repo.state["status"] == "failed" assert repo.state["calculation_job_id"] is None assert "尚未启动" in repo.state["summary"] def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch): repo = Repository() record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation") record.update(calculation_job_id=str(uuid4()), status="failed") repo.tasks[record["execution_id"]] = deepcopy(record) enqueue = AsyncMock(return_value={"job_id": uuid4()}) monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"}))) monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) request = uuid4() async def run(): for _ in range(2): await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"], record["execution_id"], request) asyncio.run(run()) retried = repo.tasks[record["execution_id"]] assert retried["attempt"] == 2 assert retried["period_end"] == record["period_end"] assert retried["batch_sequence"] == record["batch_sequence"] assert enqueue.await_count == 1 def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch): monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"}) with pytest.raises(HTTPException) as error: asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {})) assert error.value.status_code == 403 @pytest.mark.parametrize("calculation_status", ["pending", "failed"]) def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch, calculation_status): from tests.unit.test_scada_replay_service import setup from app.domain.schemas.scada_replay import ScadaReplayCommand repo, service, _, _, create = setup(monkeypatch) monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value)) async def run(): replay = await service.create(create, monitoring_interval_seconds=900) command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step") accepted = await service.command(replay.replay_id, command) repo.document["config"]["scheduled_tasks_enabled"] = True repo.document["last_scheduled_simulation"] = {"status": calculation_status} # A transport retry returns the original accepted period, even while its # calculation is queued; a different step must wait. repeated = await service.command(replay.replay_id, command) assert repeated.revision == accepted.revision with pytest.raises(HTTPException) as error: await service.command(replay.replay_id, ScadaReplayCommand( request_id=uuid4(), expected_revision=accepted.revision, action="step")) assert error.value.status_code == 409 assert "水力计算" in error.value.detail asyncio.run(run()) def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path): from app.services import calculation_worker from app.mcp.models import QueryResult @asynccontextmanager async def scope(_): yield SimpleNamespace(project="authorized-project") monkeypatch.setattr(calculation_worker, "execution_scope", scope) internal = AsyncMock(return_value=QueryResult( source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test", queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"})) monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal) ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding")) monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary) source, target = tmp_path / "job.json", tmp_path / "result.json" source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL})) asyncio.run(calculation_worker.execute_child(str(source), str(target))) assert json.loads(target.read_text())["status"] == "succeeded" internal.assert_awaited_once() ordinary.assert_not_awaited()