feat(replay): execute periodic diagnosis and hydraulic checks

This commit is contained in:
2026-11-16 15:13:22 +08:00
parent 4c441fcbf2
commit 389892cb3f
22 changed files with 1934 additions and 12 deletions
+306
View File
@@ -0,0 +1,306 @@
import asyncio
import json
from contextlib import asynccontextmanager
from copy import deepcopy
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock
from uuid import uuid4
import pytest
from fastapi import HTTPException
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
from app.services import periodic_tasks
def document():
return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()),
"monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest",
"devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"},
{"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}],
"config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"],
"rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]},
"latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}}
def monitoring():
return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00",
"period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00",
"status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None}
def test_water_risk_is_independent_of_incomplete_pressure_model():
source, period = document(), monitoring()
period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[
{"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}])
result = diagnosis_execution(source, period)
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == period["task_id"]
assert result["risk_signals"][0]["metric_name"] == "余氯"
assert result["data_gaps"] == ["压力基线不足"]
assert all(step["status"] == "completed" for step in result["steps"])
def detected_water_event():
from datetime import timedelta
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
source = document()
at = datetime.fromisoformat(monitoring()["period_end"])
config = ScadaReplayCreate(
request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15),
device_ids=["Q1"], rules=[{
"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"],
"lower": 0.2, "unit": "mg/L", "metric_name": "余氯",
}],
)
source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules]
source, events = detect_frame(source, prepare_cleaning([
{"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"}
], config), config, at)
event = next(event for event in events if event["rule_id"] == "water")
task_id = str(uuid4())
event["task_id"] = task_id
source["rule_states"]["water"]["event"]["task_id"] = task_id
return source, event
def test_periodic_report_includes_water_event_excluded_from_burst_monitoring():
from app.services.scada_normal_profile import diagnostic_signal
source, event = detected_water_event()
period = monitoring()
assert not diagnostic_signal(source, event)
original = deepcopy(period)
result = diagnosis_execution(source, period)
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == event["task_id"]
water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution")
assert water["device_ids"] == ["Q1"]
assert water["task_id"] == event["task_id"]
assert period == original
def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence():
source, event = detected_water_event()
source["rule_states"]["water"]["event"] = None
event.update(status="resolved", resolved_at=monitoring()["period_end"])
period = monitoring()
hydraulic_task_id = str(uuid4())
hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()),
"device_ids": ["P1"], "detail": "压力变化需要复核"}
period.update(status="attention", task_id=hydraulic_task_id,
weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal])
original = deepcopy(period)
result = diagnosis_execution(source, period, [event, deepcopy(event)])
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == hydraulic_task_id
water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"]
assert len(water) == 1
assert water[0]["task_id"] == event["task_id"]
assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"]
for signal in result["risk_signals"])
assert period == original
assert result["result"]["monitoring"] == original
def test_periodic_water_projection_excludes_telemetry_quality_events():
source, event = detected_water_event()
event["category"] = "data_quality"
source["rule_states"]["water"]["event"] = deepcopy(event)
result = diagnosis_execution(source, monitoring(), [event])
assert result["conclusion"] == "normal"
assert result["risk_signals"] == []
assert result["investigation_task_id"] is None
@pytest.mark.parametrize("future", [
{"occurred_at": "2026-07-16T08:15:00+08:00"},
{"batch_sequence": 3},
])
def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future):
source, event = detected_water_event()
event.update(future)
source["rule_states"]["water"]["event"] = event
result = diagnosis_execution(source, monitoring(), [event])
assert result["conclusion"] == "normal"
assert result["risk_signals"] == []
assert result["investigation_task_id"] is None
def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal():
source, period = document(), monitoring()
source["latest"]["P1"]["quality"] = "invalid"
first = diagnosis_execution(source, period)
assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"]
assert first["conclusion"] == "attention"
assert first["result"]["invalid_device_ids"] == ["P1"]
source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00"
assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"]
other = dict(source, replay_id=str(uuid4()))
assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"]
class Repository:
def __init__(self):
self.document = document()
self.document["last_monitoring"] = monitoring()
self.project_id = self.document["project_id"]
self.connection = object()
self.tasks = {}
self.receipts = {}
self.state = None
self.events = []
@asynccontextmanager
async def control(self): yield
async def get(self, _): return deepcopy(self.document)
async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)}
async def periodic_events(self, *_): return deepcopy(self.events)
async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution)
async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution)
async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id)))
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id)
def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch):
repo = Repository()
enqueue = AsyncMock(return_value={"job_id": uuid4()})
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue))
context = SimpleNamespace(project_id=repo.project_id)
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
async def run():
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
asyncio.run(run())
assert len(repo.tasks) == 2
assert enqueue.await_count == 1
payload = enqueue.call_args.kwargs
assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id}
assert payload["owner_user_id"] == repo.document["created_by"]
assert repo.state["status"] == "pending"
for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task)
def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch):
source, event = detected_water_event()
source["rule_states"]["water"]["event"] = None
source["last_monitoring"] = monitoring()
repo = Repository()
repo.document = source
repo.project_id = source["project_id"]
repo.events = [event]
read_events = AsyncMock(wraps=repo.periodic_events)
repo.periodic_events = read_events
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=AsyncMock(return_value={"job_id": uuid4()})))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"]))
read_events.assert_awaited_once()
diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")
assert diagnosis["conclusion"] == "suspected_pollution"
assert diagnosis["investigation_task_id"] == event["task_id"]
def test_worker_interruption_is_reported_as_retryable_failure():
record = execution_record(document(), monitoring(), "network-simulation")
result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}})
assert result["status"] == "failed"
assert "WORKER_INTERRUPTED" in result["summary"]
assert record["status"] == "pending"
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
repo = Repository()
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded())))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"]))
assert len(repo.tasks) == 2
assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed"
assert repo.state["status"] == "failed"
assert repo.state["calculation_job_id"] is None
assert "尚未启动" in repo.state["summary"]
def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch):
repo = Repository()
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
record.update(calculation_job_id=str(uuid4()), status="failed")
repo.tasks[record["execution_id"]] = deepcopy(record)
enqueue = AsyncMock(return_value={"job_id": uuid4()})
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"})))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
request = uuid4()
async def run():
for _ in range(2):
await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id),
repo.document["replay_id"], record["execution_id"], request)
asyncio.run(run())
retried = repo.tasks[record["execution_id"]]
assert retried["attempt"] == 2
assert retried["period_end"] == record["period_end"]
assert retried["batch_sequence"] == record["batch_sequence"]
assert enqueue.await_count == 1
def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"})
with pytest.raises(HTTPException) as error:
asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {}))
assert error.value.status_code == 403
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
from tests.unit.test_scada_replay_service import setup
from app.domain.schemas.scada_replay import ScadaReplayCommand
repo, service, _, _, create = setup(monkeypatch)
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
async def run():
replay = await service.create(create, monitoring_interval_seconds=900)
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
accepted = await service.command(replay.replay_id, command)
repo.document["config"]["scheduled_tasks_enabled"] = True
repo.document["last_scheduled_simulation"] = {"status": "pending"}
# A transport retry returns the original accepted period, even while its
# calculation is queued; a different step must wait.
repeated = await service.command(replay.replay_id, command)
assert repeated.revision == accepted.revision
with pytest.raises(HTTPException) as error:
await service.command(replay.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=accepted.revision, action="step"))
assert error.value.status_code == 409
assert "水力计算" in error.value.detail
asyncio.run(run())
def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path):
from app.services import calculation_worker
from app.mcp.models import QueryResult
@asynccontextmanager
async def scope(_): yield SimpleNamespace(project="authorized-project")
monkeypatch.setattr(calculation_worker, "execution_scope", scope)
internal = AsyncMock(return_value=QueryResult(
source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test",
queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"}))
monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal)
ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding"))
monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary)
source, target = tmp_path / "job.json", tmp_path / "result.json"
source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL}))
asyncio.run(calculation_worker.execute_child(str(source), str(target)))
assert json.loads(target.read_text())["status"] == "succeeded"
internal.assert_awaited_once()
ordinary.assert_not_awaited()