202 lines
11 KiB
Python
202 lines
11 KiB
Python
"""Work-order completion and causal SCADA recovery evidence for incident tasks."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
from math import isfinite
|
|
from statistics import median
|
|
from uuid import UUID
|
|
from typing import TYPE_CHECKING
|
|
|
|
from psycopg import AsyncConnection
|
|
from app.domain.schemas.project_settings import ProjectSettings
|
|
|
|
from app.services.content_digest import digest
|
|
|
|
if TYPE_CHECKING:
|
|
from app.services.operational_tasks import OperationalTaskService
|
|
|
|
|
|
RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"}
|
|
|
|
|
|
def track_work_order(task: dict, action: dict) -> None:
|
|
if (task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order"
|
|
or task.get("source", {}).get("replay_context")):
|
|
return
|
|
receipt = action.get("receipt") or {}
|
|
if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample":
|
|
return
|
|
context = task.setdefault("handling_context", {})
|
|
orders = context.setdefault("work_orders", {})
|
|
order_id = receipt["receipt_id"]
|
|
if order_id not in orders:
|
|
orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])}
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
|
|
|
|
def accept_completion(task: dict, feedback: dict) -> bool:
|
|
context = task.get("handling_context") or {}
|
|
order_id = feedback["source_event_id"].removeprefix("peripheral_sample:")
|
|
order = context.get("work_orders", {}).get(order_id)
|
|
if order is None:
|
|
return False
|
|
order.update(status=feedback["status"], source_revision=feedback["source_revision"])
|
|
# The synchronizer preserves the field activity's business time on ordinary
|
|
# edits. A revised field receipt can explicitly correct the verification start.
|
|
if feedback["status"] == "completed":
|
|
order["returned_at"] = feedback["occurred_at"]
|
|
else:
|
|
order.pop("returned_at", None)
|
|
orders = list(context["work_orders"].values())
|
|
if all(item["status"] == "completed" for item in orders):
|
|
returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat)
|
|
previous = context.get("returned_at")
|
|
if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at)
|
|
or context.get("phase") == "waiting_work_order"):
|
|
context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at)
|
|
for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"):
|
|
context.pop(key, None)
|
|
else:
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
return True
|
|
|
|
|
|
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
|
|
incident_at: datetime, interval_seconds: int) -> dict:
|
|
"""Verify the first SCADA sample strictly after the field feedback time T."""
|
|
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
|
|
"required_samples": 1, "interval_seconds": interval_seconds,
|
|
"selection_method": "first_sample_after_feedback",
|
|
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
|
|
if not device_ids:
|
|
return {**base, "reason": "monitoring_configuration_missing"}
|
|
if returned_at < incident_at:
|
|
return {**base, "reason": "completion_precedes_incident"}
|
|
selected = set(device_ids)
|
|
baseline: dict[tuple[str, int], list[float]] = {}
|
|
observations: dict[datetime, dict[str, float]] = {}
|
|
origin = incident_at - timedelta(days=15)
|
|
for row in rows:
|
|
if row["device_id"] not in selected:
|
|
continue
|
|
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
|
|
if at > returned_at:
|
|
# Keep the nearest frame even when it contains null/invalid values;
|
|
# never skip it in order to find a later normal-looking sample.
|
|
observations.setdefault(at, {})
|
|
if row.get("quality") not in {None, "valid", "corrected"}:
|
|
continue
|
|
value = row.get("cleaned_value")
|
|
if value is None:
|
|
value = row.get("monitored_value")
|
|
if value is None or not isfinite(float(value)):
|
|
continue
|
|
value = float(value)
|
|
phase = int(at.timestamp()) % 86400
|
|
if origin <= at < incident_at:
|
|
baseline.setdefault((row["device_id"], phase), []).append(value)
|
|
elif at > returned_at:
|
|
observations[at][row["device_id"]] = value
|
|
if not observations:
|
|
return {**base, "reason": "post_completion_reading_missing"}
|
|
at = min(observations)
|
|
values = observations[at]
|
|
phase = int(at.timestamp()) % 86400
|
|
checks = []
|
|
for device in device_ids:
|
|
reference = baseline.get((device, phase), [])
|
|
if device not in values or len(reference) < 15:
|
|
checks.append({"device_id": device, "status": "missing_data"})
|
|
continue
|
|
center = median(reference)
|
|
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
|
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
|
lower, upper = min(reference) - margin, max(reference) + margin
|
|
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
|
"normal_day_count": len(reference),
|
|
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
|
sample = {"at": at.isoformat(), "devices": checks}
|
|
if all(item["status"] == "normal" for item in checks):
|
|
return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]}
|
|
return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample}
|
|
|
|
|
|
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
|
|
settings: ProjectSettings, *, limit: int = 20,
|
|
created_from: datetime | None = None) -> int:
|
|
"""Poll only persisted recovery waits; publish evidence before waking the Agent."""
|
|
from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord
|
|
from app.domain.schemas.operational_data import TaskDataSnapshot
|
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
|
|
|
async with service.repo.transaction():
|
|
candidates = await service.repo.recovery_candidates(limit, created_from)
|
|
resumed = 0
|
|
for task in candidates:
|
|
context = task["handling_context"]
|
|
returned_at = datetime.fromisoformat(context["returned_at"])
|
|
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
|
|
interval = settings.scada_monitoring_interval_seconds
|
|
roles = settings.scada_monitoring
|
|
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
|
|
source = "scada_readings"
|
|
through = service._now()
|
|
if context.get("device_ids"):
|
|
devices = context["device_ids"]
|
|
interval = context.get("interval_seconds", interval)
|
|
else:
|
|
# Retain the points the Agent actually inspected, including water
|
|
# quality or local incident sensors outside the project role lists.
|
|
for evidence in await service.repo.list_evidence(task["task_id"]):
|
|
saved = evidence.get("data_snapshot") or {}
|
|
if saved.get("source") not in {"query_scada_readings", "scada_readings"}:
|
|
continue
|
|
data = saved.get("data")
|
|
if isinstance(data, dict):
|
|
data = data.get("data", data.get("items", []))
|
|
if isinstance(data, list):
|
|
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
|
|
devices = list(dict.fromkeys(devices))
|
|
async with timescale.transaction():
|
|
rows = await ScadaRepository.get_recovery_scada_readings(
|
|
timescale, devices, incident_at, returned_at, through, baseline_limit=100_001,
|
|
) if through > returned_at and devices else []
|
|
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
|
|
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
|
|
incident_at=incident_at, interval_seconds=interval or 0))
|
|
async with service.repo.transaction():
|
|
await service.repo.lock(f"task:{task['task_id']}")
|
|
current = await service._task(UUID(task["task_id"]))
|
|
# A new receipt, cancellation or another poll may have won the race.
|
|
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
|
|
continue
|
|
now = service._now()
|
|
if devices:
|
|
current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0)
|
|
current["handling_context"]["last_checked_at"] = now.isoformat()
|
|
current["handling_context"]["last_check"] = result
|
|
if result["status"] == "passed":
|
|
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
|
|
evidence = StoredEvidenceRecord(
|
|
evidence_id=evidence_id, kind="system_data",
|
|
summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常，等待 Agent 完成复核报告",
|
|
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
|
|
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
|
|
recorded_by=service._actor(), has_snapshot=True,
|
|
data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now,
|
|
origin="system_data", data=result, truncated=False),
|
|
)
|
|
if not await service.repo.get_evidence(task["task_id"], evidence_id):
|
|
await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json"))
|
|
current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id)
|
|
await service._requeue_task(current, now)
|
|
resumed += 1
|
|
else:
|
|
current.update(revision=current["revision"] + 1, updated_at=now)
|
|
await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump(
|
|
mode="json", exclude={"active_run", "latest_run"}))
|
|
await service._task_event(current, now)
|
|
return resumed
|