OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scada): add calibrated monitoring and incident recovery

Browse Source
This commit is contained in:
jiang
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_operational_tasks.py
+195
View File
@@ -576,6 +576,65 @@ async def test_presentation_cursor_and_event_are_read_only_and_idempotent():
await service.presentation_cursor(uuid4())
@pytest.mark.anyio
async def test_replay_investigation_can_publish_evidenced_composite_chart_but_cannot_propose_real_action():
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task["source"]["replay_context"] = {
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
}
repository = RunTaskRepository(task, _active_run(task, now))
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
evidence, _ = await service.record_data_snapshot(task["task_id"], TaskDataSnapshot(
source="query_scada_readings", source_ref="mcp:frozen-scada", queried_at=now,
origin="synthetic", data={"items": []}, truncated=False,
))
result = await service.publish_presentation_event(task["task_id"], TaskPresentationEventCommand(
request_id=uuid4(), run_id=repository.run["run_id"], source_session_id="session-1",
command={"action": "open_chart", "chart_id": "pressure", "evidence_refs": [evidence.evidence_id],
"chart": {"title": "压力", "chart_type": "line", "x_data": ["19:45", "20:00"],
"y_axis_name": "压力 m", "series": [
{"name": "当前", "type": "line", "data": [27., 26.]},
{"name": "前一天", "type": "scatter", "data": [None, 28.],
"sample_times": [None, "2026-10-09T20:00:00+08:00"]},
]}},
))
assert result.recorded
assert repository.events[-1]["metadata"]["command"]["chart"]["series"][1]["data"] == [None, 28.]
with pytest.raises(DomainError) as caught:
await service.propose_action(task["task_id"], ActionProposalCommand(
request_id=uuid4(), kind="create_work_order", reason="演示不得创建真实工单",
payload={"title": "现场核查", "description": "核查", "work_order_type": "general"},
))
assert caught.value.code == "REPLAY_ACTION_DISABLED"
@pytest.mark.anyio
@pytest.mark.parametrize("operation", ["draft", "approve", "submit"])
async def test_replay_cannot_edit_approve_or_submit_preexisting_real_actions(operation):
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task["source"]["replay_context"] = {
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
}
repository = RunTaskRepository(task, _active_run(task, now))
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
dispatcher = SimpleNamespace(dispatch=AsyncMock())
args = {"request_id": uuid4(), "expected_version": 1}
with pytest.raises(DomainError) as caught:
if operation == "draft":
await service.update_work_order_draft(task["task_id"], uuid4(), WorkOrderDraftUpdateCommand(
**args, payload={"title": "现场核查", "description": "核查", "work_order_type": "general"}))
elif operation == "approve":
await service.decide_action(task["task_id"], uuid4(), ActionDecisionCommand(**args, decision="approved"))
else:
await service.submit_action(task["task_id"], uuid4(), ActionExecuteCommand(**args), dispatcher, None)
assert caught.value.code == "REPLAY_ACTION_DISABLED"
dispatcher.dispatch.assert_not_awaited()
@pytest.mark.anyio
@pytest.mark.parametrize("payload", [
{"action": "locate_features", "layer": "scada", "feature_ids": ["D-1"]},
@@ -1025,6 +1084,93 @@ def _seed_concluded_report(repository, now):
})
@pytest.mark.anyio
async def test_burst_waits_for_completion_then_scada_evidence_then_agent_report(monkeypatch):
from copy import deepcopy
from app.domain.schemas.project_settings import ProjectSettings
from app.services.operational_recovery import reconcile_recoveries, track_work_order
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
incident = now - timedelta(hours=4)
returned = incident + timedelta(hours=1)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task.update(workflow="burst_report", trigger={"kind": "event", "occurred_at": incident.isoformat(), "detail": "爆管"})
repository = RunTaskRepository(task, _active_run(task, now))
repository.connection = object()
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
monkeypatch.setattr(service, "_resolve_investigation_alerts", AsyncMock())
_seed_concluded_report(repository, now)
action = await service.propose_action(task["task_id"], ActionProposalCommand(
request_id=uuid4(), kind="create_work_order", reason="隔离抢修与恢复供水",
payload={"title": "抢修恢复", "description": "回传完工时间和恢复供水情况", "work_order_type": "general"}))
child_id = str(uuid4())
repository.action = action.model_copy(update={"approval_status": "approved", "execution_status": "succeeded"}).model_dump(mode="json")
repository.action["receipt"] = {"adapter": "peripheral_sample", "receipt_id": child_id,
"submitted_at": now.isoformat(), "completed_at": now.isoformat(), "external_status": "open", "detail": "仅创建成功"}
track_work_order(repository.task, repository.action)
finished = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert finished.status == "waiting_human"
service._resolve_investigation_alerts.assert_not_awaited()
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=2,
status="completed", occurred_at=returned, detail="抢修完成，已恢复供水"))
assert repository.task["handling_context"]["phase"] == "waiting_scada"
assert datetime.fromisoformat(repository.task["handling_context"]["returned_at"]) == returned
settings = ProjectSettings(scada_monitoring_interval_seconds=900, scada_monitoring={"pressure_device_ids": ["P1"]})
repository.recovery_candidates = AsyncMock(side_effect=lambda *_: [deepcopy(repository.task)]
if repository.task["handling_context"]["phase"] == "waiting_scada" else [])
timescale = SimpleNamespace(transaction=repository.transaction)
rows = []
for sample in (1, 2):
at = returned + timedelta(minutes=15 * sample)
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
reader = AsyncMock(return_value=rows[:-1])
monkeypatch.setattr("app.infra.db.timescaledb.repositories.scada.ScadaRepository.get_scada_by_ids_time_range", reader)
assert await reconcile_recoveries(service, timescale, settings) == 0
assert repository.task["handling_context"]["phase"] == "waiting_scada"
reader.return_value = rows
assert await reconcile_recoveries(service, timescale, settings) == 1
assert await reconcile_recoveries(service, timescale, settings) == 0
recovery = repository.task["handling_context"]
evidence_id = recovery["verification_evidence_id"]
assert repository.evidence[evidence_id]["data_snapshot"]["data"]["status"] == "passed"
service._resolve_investigation_alerts.assert_not_awaited()
# Correcting the appointment's field return time invalidates the old proof
# and requires two actual observations after the new business time.
corrected_time = returned + timedelta(minutes=15)
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=3,
status="completed", occurred_at=corrected_time, detail="修订现场实际完工时间"))
assert repository.task["handling_context"]["phase"] == "waiting_scada"
assert "verification_evidence_id" not in repository.task["handling_context"]
assert await reconcile_recoveries(service, timescale, settings) == 0
at = returned + timedelta(minutes=45)
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
assert await reconcile_recoveries(service, timescale, settings) == 1
revised_id = repository.task["handling_context"]["verification_evidence_id"]
assert revised_id != evidence_id and evidence_id in repository.evidence
evidence_id = revised_id
repository.task.update(status="running", agent_session_id="session-1")
repository.run = _active_run(repository.task, now)
_seed_concluded_report(repository, now)
with pytest.raises(DomainError) as rejected:
await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert rejected.value.code == "RECOVERY_EVIDENCE_REQUIRED"
repository.reports[-1]["report"]["evidence_refs"] = [evidence_id]
result = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert result.status == "completed" and result.handling_context["phase"] == "completed"
service._resolve_investigation_alerts.assert_awaited_once_with(task["task_id"], evidence_id)
@pytest.mark.anyio
async def test_plan_only_keeps_lease_until_same_generation_report_closes_run():
now = datetime(2026, 9, 23, 8, tzinfo=timezone.utc)
@@ -2087,3 +2233,52 @@ async def test_missing_calculation_migration_fails_finalization_closed(monkeypat
await repo.pending_calculation_jobs(str(uuid4()), str(uuid4()), str(uuid4()))
assert error.value.code == 'DEPENDENCY_UNAVAILABLE'
assert error.value.status == 503
@pytest.mark.anyio
async def test_recovery_alert_query_scopes_active_alerts_to_project_and_task():
alerts = [{"alert_id": "first"}, {"alert_id": "second"}]
cursor = SimpleNamespace(fetchall=AsyncMock(return_value=[{"document": item} for item in alerts]))
connection = SimpleNamespace(execute=AsyncMock(return_value=cursor))
project_id, task_id = uuid4(), str(uuid4())
repository = OperationalTaskRepository(connection, project_id)
assert await repository.list_investigation_alerts(task_id) == alerts
statement, parameters = connection.execute.await_args.args
assert parameters == (str(project_id), f"operational-task:{task_id}")
assert "status IN ('active','acknowledged')" in statement
@pytest.mark.anyio
async def test_recovery_resolves_alerts_with_original_facts_and_stable_event_id(monkeypatch):
from uuid import NAMESPACE_URL, uuid5
from app.services.task_alerts import TaskAlertService
task_id, project_id = str(uuid4()), str(uuid4())
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
alert = {
"alert_id": "incident-1", "producer_id": "producer-1", "revision": 2,
"condition_id": f"operational-task:{task_id}", "status": "acknowledged",
"source": {"task_type": "scheduled_task", "task_id": task_id, "run_id": "run-1"},
"title": "压力异常", "description": "等待恢复", "severity": "warning",
"occurred_at": now.isoformat(), "updated_at": now.isoformat(),
}
connection = object()
repository = SimpleNamespace(connection=connection, project_id=project_id,
list_investigation_alerts=AsyncMock(return_value=[alert]))
calls = []
async def push(service, command):
calls.append((service, command))
monkeypatch.setattr(TaskAlertService, "push", push)
service = OperationalTaskService(repository, _context(), set())
for _ in range(2):
await service._resolve_investigation_alerts(task_id, "recovery:evidence-1")
assert len(calls) == 2
for publisher, command in calls:
assert publisher.repo.connection is connection and publisher.repo.project_id == project_id
assert publisher.internal_producer_id == alert["producer_id"]
assert command.event_type == "resolved"
assert command.event_id == uuid5(NAMESPACE_URL, f"recovery:{task_id}:recovery:evidence-1:incident-1")
assert command.source.model_dump(mode="json") == alert["source"]
assert command.title == alert["title"] and command.description == alert["description"]