"""Work-order completion and causal SCADA recovery evidence for incident tasks.""" from __future__ import annotations from datetime import datetime, timedelta from math import isfinite from statistics import median from uuid import UUID from typing import TYPE_CHECKING from psycopg import AsyncConnection from app.domain.schemas.project_settings import ProjectSettings from app.services.content_digest import digest if TYPE_CHECKING: from app.services.operational_tasks import OperationalTaskService RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"} def track_work_order(task: dict, action: dict) -> None: if (task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order" or task.get("source", {}).get("replay_context")): return receipt = action.get("receipt") or {} if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample": return context = task.setdefault("handling_context", {}) orders = context.setdefault("work_orders", {}) order_id = receipt["receipt_id"] if order_id not in orders: orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])} context.update(phase="waiting_work_order", title="等待现场工单返回") context.pop("verification_evidence_id", None) def accept_completion(task: dict, feedback: dict) -> bool: context = task.get("handling_context") or {} order_id = feedback["source_event_id"].removeprefix("peripheral_sample:") order = context.get("work_orders", {}).get(order_id) if order is None: return False order.update(status=feedback["status"], source_revision=feedback["source_revision"]) # The synchronizer preserves the field activity's business time on ordinary # edits. A revised field receipt can explicitly correct the verification start. if feedback["status"] == "completed": order["returned_at"] = feedback["occurred_at"] else: order.pop("returned_at", None) orders = list(context["work_orders"].values()) if all(item["status"] == "completed" for item in orders): returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat) previous = context.get("returned_at") if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at) or context.get("phase") == "waiting_work_order"): context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at) for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"): context.pop(key, None) else: context.update(phase="waiting_work_order", title="等待现场工单返回") context.pop("verification_evidence_id", None) return True def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime, incident_at: datetime, interval_seconds: int) -> dict: """Require two fresh complete samples inside the pre-event normal envelope. Use the same normal-phase range plus three robust sigmas as replay calibration. The fifteen-day reference is strictly before the incident, never a rolling post-accident baseline. Only finite, usable observations can prove recovery. """ base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(), "required_samples": 2, "interval_seconds": interval_seconds, "baseline_method": "normal_same_phase_range_plus_3_robust_sigma"} if not device_ids or interval_seconds <= 0 or 86400 % interval_seconds: return {**base, "reason": "monitoring_configuration_missing"} if returned_at < incident_at: return {**base, "reason": "completion_precedes_incident"} selected = set(device_ids) baseline: dict[tuple[str, int], list[float]] = {} observations: dict[datetime, dict[str, float]] = {} origin = incident_at - timedelta(days=15) for row in rows: if row["device_id"] not in selected: continue at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"]) if row.get("quality") not in {None, "valid", "corrected"}: continue value = row.get("cleaned_value") if value is None: value = row.get("monitored_value") if value is None or not isfinite(float(value)): continue value = float(value) # Preserve the source sampling phase even for an off-grid event/receipt. phase = int(at.timestamp()) % 86400 if origin <= at < incident_at: baseline.setdefault((row["device_id"], phase), []).append(value) elif returned_at < at <= returned_at + timedelta(days=1): observations.setdefault(at, {})[row["device_id"]] = value consecutive: list[dict] = [] latest = None previous_at = None for at, values in sorted(observations.items()): phase = int(at.timestamp()) % 86400 checks = [] for device in device_ids: reference = baseline.get((device, phase), []) if device not in values or len(reference) < 15: checks.append({"device_id": device, "status": "missing_data"}) continue center = median(reference) sigma = 1.4826 * median(abs(value - center) for value in reference) margin = max(3 * sigma, abs(center) * 1e-6, 1e-9) lower, upper = min(reference) - margin, max(reference) + margin checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper, "normal_day_count": len(reference), "status": "normal" if lower <= values[device] <= upper else "abnormal"}) latest = {"at": at.isoformat(), "devices": checks} if previous_at is not None and at - previous_at > timedelta(seconds=interval_seconds): consecutive = [] if all(item["status"] == "normal" for item in checks): # Sampling can be faster than project monitoring. Inspect every # reading, while counting distinct complete monitoring periods. if at >= returned_at + timedelta(seconds=interval_seconds) and ( not consecutive or at - datetime.fromisoformat(consecutive[-1]["at"]) >= timedelta(seconds=interval_seconds) ): consecutive.append(latest) else: consecutive = [] previous_at = at if len(consecutive) >= 2: return {**base, "status": "passed", "verified_through": consecutive[-1]["at"], "samples": consecutive[-2:]} return {**base, "reason": "post_completion_data_not_normal_or_incomplete", "latest_sample": latest} async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection, settings: ProjectSettings, *, limit: int = 20, created_from: datetime | None = None) -> int: """Poll only persisted recovery waits; publish evidence before waking the Agent.""" from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord from app.domain.schemas.operational_data import TaskDataSnapshot from app.infra.db.timescaledb.repositories.scada import ScadaRepository async with service.repo.transaction(): candidates = await service.repo.recovery_candidates(limit, created_from) resumed = 0 for task in candidates: context = task["handling_context"] returned_at = datetime.fromisoformat(context["returned_at"]) incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"]) interval = settings.scada_monitoring_interval_seconds roles = settings.scada_monitoring devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else [] source = "scada_readings" through = min(service._now(), returned_at + timedelta(days=1)) if context.get("device_ids"): devices = context["device_ids"] interval = context.get("interval_seconds", interval) else: # Retain the points the Agent actually inspected, including water # quality or local incident sensors outside the project role lists. for evidence in await service.repo.list_evidence(task["task_id"]): saved = evidence.get("data_snapshot") or {} if saved.get("source") not in {"query_scada_readings", "scada_readings"}: continue data = saved.get("data") if isinstance(data, dict): data = data.get("data", data.get("items", [])) if isinstance(data, list): devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str)) devices = list(dict.fromkeys(devices)) async with timescale.transaction(): rows = await ScadaRepository.get_scada_by_ids_time_range( timescale, devices, incident_at - timedelta(days=15), through, row_limit=100_001, ) if through > returned_at and devices else [] result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else evaluate_recovery(rows, device_ids=devices, returned_at=returned_at, incident_at=incident_at, interval_seconds=interval or 0)) async with service.repo.transaction(): await service.repo.lock(f"task:{task['task_id']}") current = await service._task(UUID(task["task_id"])) # A new receipt, cancellation or another poll may have won the race. if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}: continue now = service._now() if devices and interval: current["handling_context"].update(device_ids=devices, interval_seconds=interval) current["handling_context"]["last_checked_at"] = now.isoformat() current["handling_context"]["last_check"] = result if result["status"] == "passed": evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result}) evidence = StoredEvidenceRecord( evidence_id=evidence_id, kind="system_data", summary="完工反馈后连续两个采样周期的 SCADA 恢复正常，等待 Agent 完成复核报告", source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}", observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now, recorded_by=service._actor(), has_snapshot=True, data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now, origin="system_data", data=result, truncated=False), ) if not await service.repo.get_evidence(task["task_id"], evidence_id): await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json")) current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id) await service._requeue_task(current, now) resumed += 1 else: current.update(revision=current["revision"] + 1, updated_at=now) await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump( mode="json", exclude={"active_run", "latest_run"})) await service._task_event(current, now) return resumed