"""Work-order completion and causal SCADA recovery evidence for incident tasks.""" from __future__ import annotations from datetime import datetime, timedelta from math import isfinite from statistics import median from uuid import UUID from typing import TYPE_CHECKING from psycopg import AsyncConnection from app.domain.schemas.project_settings import ProjectSettings from app.services.content_digest import digest if TYPE_CHECKING: from app.services.operational_tasks import OperationalTaskService RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"} def track_work_order(task: dict, action: dict) -> None: if task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order": return receipt = action.get("receipt") or {} if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample": return context = task.setdefault("handling_context", {}) orders = context.setdefault("work_orders", {}) order_id = receipt["receipt_id"] if order_id not in orders: orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])} context.update(phase="waiting_work_order", title="等待现场工单返回") context.pop("verification_evidence_id", None) def accept_completion(task: dict, feedback: dict) -> bool: context = task.get("handling_context") or {} order_id = feedback["source_event_id"].removeprefix("peripheral_sample:") order = context.get("work_orders", {}).get(order_id) if order is None: return False order.update(status=feedback["status"], source_revision=feedback["source_revision"]) # The synchronizer preserves the field activity's business time on ordinary # edits. A revised field receipt can explicitly correct the verification start. if feedback["status"] == "completed": order["returned_at"] = feedback["occurred_at"] else: order.pop("returned_at", None) orders = list(context["work_orders"].values()) if all(item["status"] == "completed" for item in orders): returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat) previous = context.get("returned_at") if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at) or context.get("phase") == "waiting_work_order"): context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at) for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"): context.pop(key, None) else: context.update(phase="waiting_work_order", title="等待现场工单返回") context.pop("verification_evidence_id", None) return True def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime, incident_at: datetime, interval_seconds: int) -> dict: """Verify the first SCADA sample strictly after the field feedback time T.""" base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(), "required_samples": 1, "interval_seconds": interval_seconds, "selection_method": "first_sample_after_feedback", "baseline_method": "normal_same_phase_range_plus_3_robust_sigma"} if not device_ids: return {**base, "reason": "monitoring_configuration_missing"} if returned_at < incident_at: return {**base, "reason": "completion_precedes_incident"} selected = set(device_ids) baseline: dict[tuple[str, int], list[float]] = {} observations: dict[datetime, dict[str, float]] = {} origin = incident_at - timedelta(days=15) for row in rows: if row["device_id"] not in selected: continue at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"]) if at > returned_at: # Keep the nearest frame even when it contains null/invalid values; # never skip it in order to find a later normal-looking sample. observations.setdefault(at, {}) if row.get("quality") not in {None, "valid", "corrected"}: continue value = row.get("cleaned_value") if value is None: value = row.get("monitored_value") if value is None or not isfinite(float(value)): continue value = float(value) phase = int(at.timestamp()) % 86400 if origin <= at < incident_at: baseline.setdefault((row["device_id"], phase), []).append(value) elif at > returned_at: observations[at][row["device_id"]] = value if not observations: return {**base, "reason": "post_completion_reading_missing"} at = min(observations) values = observations[at] phase = int(at.timestamp()) % 86400 checks = [] for device in device_ids: reference = baseline.get((device, phase), []) if device not in values or len(reference) < 15: checks.append({"device_id": device, "status": "missing_data"}) continue center = median(reference) sigma = 1.4826 * median(abs(value - center) for value in reference) margin = max(3 * sigma, abs(center) * 1e-6, 1e-9) lower, upper = min(reference) - margin, max(reference) + margin checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper, "normal_day_count": len(reference), "status": "normal" if lower <= values[device] <= upper else "abnormal"}) sample = {"at": at.isoformat(), "devices": checks} if all(item["status"] == "normal" for item in checks): return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]} return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample} async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection, settings: ProjectSettings, *, limit: int = 20, created_from: datetime | None = None) -> int: """Poll only persisted recovery waits; publish evidence before waking the Agent.""" from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord from app.domain.schemas.operational_data import TaskDataSnapshot from app.infra.db.timescaledb.repositories.scada import ScadaRepository async with service.repo.transaction(): candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope) resumed = 0 for task in candidates: if task.get("execution_scope") != service.execution_scope: continue context = task["handling_context"] returned_at = datetime.fromisoformat(context["returned_at"]) incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"]) interval = settings.scada_monitoring_interval_seconds roles = settings.scada_monitoring devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else [] source = "scada_readings" through = service._now() if context.get("device_ids"): devices = context["device_ids"] interval = context.get("interval_seconds", interval) else: # Retain the points the Agent actually inspected, including water # quality or local incident sensors outside the project role lists. for evidence in await service.repo.list_evidence(task["task_id"]): saved = evidence.get("data_snapshot") or {} if saved.get("source") not in {"query_scada_readings", "scada_readings"}: continue data = saved.get("data") if isinstance(data, dict): data = data.get("data", data.get("items", [])) if isinstance(data, list): devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str)) devices = list(dict.fromkeys(devices)) async with timescale.transaction(): # Field completion authorizes a separate recovery observation even # when the incident was found in a frozen replay. Keep that replay's # query window and batch boundary unchanged. rows = await ScadaRepository.get_recovery_scada_readings( timescale, devices, incident_at, returned_at, through, baseline_limit=100_001, ) if through > returned_at and devices else [] result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else evaluate_recovery(rows, device_ids=devices, returned_at=returned_at, incident_at=incident_at, interval_seconds=interval or 0)) async with service.repo.transaction(): await service.repo.lock(f"task:{task['task_id']}") current = await service._task(UUID(task["task_id"])) # A new receipt, cancellation or another poll may have won the race. if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}: continue now = service._now() if devices: current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0) current["handling_context"]["last_checked_at"] = now.isoformat() current["handling_context"]["last_check"] = result if result["status"] == "passed": evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result}) evidence = StoredEvidenceRecord( evidence_id=evidence_id, kind="system_data", summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常，等待 Agent 完成复核报告", source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}", observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now, recorded_by=service._actor(), has_snapshot=True, data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now, origin="system_data", data=result, truncated=False), ) if not await service.repo.get_evidence(task["task_id"], evidence_id): await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json")) current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id) await service._requeue_task(current, now) resumed += 1 else: current.update(revision=current["revision"] + 1, updated_at=now) await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump( mode="json", exclude={"active_run", "latest_run"})) await service._task_event(current, now) return resumed