OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scada): add calibrated monitoring and incident recovery

Browse Source
This commit is contained in:
jiang
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/operational_tasks.py
+54 -10
View File
@@ -72,6 +72,7 @@ from app.domain.schemas.operational_task import (
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
from app.services.content_digest import digest
from app.services.operational_presentation import resolve_overlay
from app.services.operational_recovery import accept_completion, track_work_order
LEASE_DURATION = timedelta(seconds=60)
@@ -1005,7 +1006,7 @@ class OperationalTaskService:
# Compatibility for pre-015 repository adapters only.
if command.report:
if task["task_type"] == "event_investigation":
task["status"] = self._investigation_status(actions.values()) if command.report.stage == "final" else "running"
task["status"] = self._investigation_status(actions.values(), task) if command.report.stage == "final" else "running"
else:
task["status"] = self._status_from_plan(current_plan.plan.nodes, actions.values()) if current_plan else "running"
if task["status"] in {"waiting_human", "completed", "failed", "cancelled"}:
@@ -1069,8 +1070,6 @@ class OperationalTaskService:
await self.repo.lock(f"run:{command.run_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告，不执行真实工单或设备动作", status=409)
run = await self._active_run(task)
require(run is not None and str(run["run_id"]) == str(command.run_id)
and run["generation"] == task["execution_generation"],
@@ -1133,6 +1132,7 @@ class OperationalTaskService:
await self.repo.lock(f"task:{task_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
self._validate_action_payload(command)
if task["task_type"] == "event_investigation":
report = await self.repo.get_latest_report(str(task_id))
@@ -1201,7 +1201,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.kind == "create_work_order", "INVALID_ACTION_KIND",
"Only work-order drafts can be edited", status=422)
@@ -1255,7 +1257,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
@@ -1300,7 +1304,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
@@ -1396,6 +1402,7 @@ class OperationalTaskService:
)
if outcome.status in {"succeeded", "failed", "unsupported"}:
task_document = await self._task(task_id)
track_work_order(task_document, _json(current))
await self._requeue_task(task_document, outcome_recorded_at)
await self.repo.update_task(
_task_json(OperationalTaskSummary.model_validate(task_document))
@@ -1469,6 +1476,7 @@ class OperationalTaskService:
)
if outcome.status in {"succeeded", "failed", "unsupported"}:
task_document = await self._task(task_id)
track_work_order(task_document, _json(current))
await self._requeue_task(task_document, outcome_recorded_at)
await self.repo.update_task(
_task_json(OperationalTaskSummary.model_validate(task_document))
@@ -1589,6 +1597,10 @@ class OperationalTaskService:
),
)
if task["task_type"] == "event_investigation":
# Recover older tasks whose successful dispatch predates tracking.
for item in await self.repo.list_actions(str(task_id)):
track_work_order(task, item)
accept_completion(task, body)
if matching_source:
task["source"] = {**task["source"], "external_revision": command.source_revision}
await self._requeue_task(task, now)
@@ -2032,7 +2044,15 @@ class OperationalTaskService:
for item in await self._list_access_requests(task_id)
]
if task["task_type"] == "event_investigation":
lifecycle = self._investigation_status(actions)
lifecycle = self._investigation_status(actions, task)
recovery = task.get("handling_context") or {}
if lifecycle == "completed" and recovery:
require(recovery.get("verification_evidence_id") in report.report.evidence_refs,
"RECOVERY_EVIDENCE_REQUIRED", "最终复核报告必须引用恢复监测证据", status=409)
require(report.report.outcome == "concluded", "RECOVERY_EVIDENCE_REQUIRED",
"恢复复核通过后才能结束事件", status=409)
recovery.update(phase="completed", title="恢复供水复核成功", completed_at=now.isoformat())
await self._resolve_investigation_alerts(str(task_id), recovery["verification_evidence_id"])
elif report.report.outcome == "insufficient_data":
terminal = all(node.status in {"completed", "skipped", "cancelled"}
for node in plan.plan.nodes)
@@ -2081,7 +2101,8 @@ class OperationalTaskService:
failure_detail or "工况任务执行失败",
now,
)
elif lifecycle == "waiting_human" and not pending_question:
elif (lifecycle == "waiting_human" and not pending_question
and (task.get("handling_context") or {}).get("phase") != "waiting_scada"):
pending_actions = any(
action.approval_status == "pending" for action in actions
)
@@ -2108,11 +2129,32 @@ class OperationalTaskService:
)
return run
async def _resolve_investigation_alerts(self, task_id: str, evidence_id: str) -> None:
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
if not hasattr(self.repo, "list_investigation_alerts"):
return
for alert in await self.repo.list_investigation_alerts(task_id):
command = TaskAlertEvent.model_validate({
**{key: value for key, value in alert.items() if key not in {"status", "updated_at", "revision"}},
"event_id": uuid5(NAMESPACE_URL, f"recovery:{task_id}:{evidence_id}:{alert['alert_id']}"),
"event_type": "resolved",
})
await TaskAlertService(TaskAlertRepository(self.repo.connection, self.repo.project_id), None,
internal_producer_id=command.producer_id).push(command)
async def _task(self, task_id: UUID) -> dict:
task = await self.repo.get_task(str(task_id))
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
return task
@staticmethod
def _require_real_action(task: dict) -> None:
require(not (task.get("source") or {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告，不执行真实工单或设备动作", status=409)
@staticmethod
def _require_actionable_task(task: dict) -> None:
source = task.get("source") or {}
@@ -2176,8 +2218,10 @@ class OperationalTaskService:
return "running"
@staticmethod
def _investigation_status(actions: Iterable[HumanApprovalAction]) -> str:
"""Only recorded business actions can keep an investigation waiting."""
def _investigation_status(actions: Iterable[HumanApprovalAction], task: dict | None = None) -> str:
"""Field completion and recovery evidence precede incident closure."""
if (task or {}).get("handling_context", {}).get("phase") in {"waiting_work_order", "waiting_scada"}:
return "waiting_human"
return "waiting_human" if any(
action.approval_status == "pending" or (
action.approval_status == "approved"