jiang
33df25f22b
Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
206 lines
11 KiB
Python
206 lines
11 KiB
Python
"""Work-order completion and causal SCADA recovery evidence for incident tasks."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
from math import isfinite
|
|
from statistics import median
|
|
from uuid import UUID
|
|
from typing import TYPE_CHECKING
|
|
|
|
from psycopg import AsyncConnection
|
|
from app.domain.schemas.project_settings import ProjectSettings
|
|
|
|
from app.services.content_digest import digest
|
|
|
|
if TYPE_CHECKING:
|
|
from app.services.operational_tasks import OperationalTaskService
|
|
|
|
|
|
RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"}
|
|
|
|
|
|
def track_work_order(task: dict, action: dict) -> None:
|
|
if task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order":
|
|
return
|
|
receipt = action.get("receipt") or {}
|
|
if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample":
|
|
return
|
|
context = task.setdefault("handling_context", {})
|
|
orders = context.setdefault("work_orders", {})
|
|
order_id = receipt["receipt_id"]
|
|
if order_id not in orders:
|
|
orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])}
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
|
|
|
|
def accept_completion(task: dict, feedback: dict) -> bool:
|
|
context = task.get("handling_context") or {}
|
|
order_id = feedback["source_event_id"].removeprefix("peripheral_sample:")
|
|
order = context.get("work_orders", {}).get(order_id)
|
|
if order is None:
|
|
return False
|
|
order.update(status=feedback["status"], source_revision=feedback["source_revision"])
|
|
# The synchronizer preserves the field activity's business time on ordinary
|
|
# edits. A revised field receipt can explicitly correct the verification start.
|
|
if feedback["status"] == "completed":
|
|
order["returned_at"] = feedback["occurred_at"]
|
|
else:
|
|
order.pop("returned_at", None)
|
|
orders = list(context["work_orders"].values())
|
|
if all(item["status"] == "completed" for item in orders):
|
|
returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat)
|
|
previous = context.get("returned_at")
|
|
if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at)
|
|
or context.get("phase") == "waiting_work_order"):
|
|
context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at)
|
|
for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"):
|
|
context.pop(key, None)
|
|
else:
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
return True
|
|
|
|
|
|
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
|
|
incident_at: datetime, interval_seconds: int) -> dict:
|
|
"""Verify the first SCADA sample strictly after the field feedback time T."""
|
|
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
|
|
"required_samples": 1, "interval_seconds": interval_seconds,
|
|
"selection_method": "first_sample_after_feedback",
|
|
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
|
|
if not device_ids:
|
|
return {**base, "reason": "monitoring_configuration_missing"}
|
|
if returned_at < incident_at:
|
|
return {**base, "reason": "completion_precedes_incident"}
|
|
selected = set(device_ids)
|
|
baseline: dict[tuple[str, int], list[float]] = {}
|
|
observations: dict[datetime, dict[str, float]] = {}
|
|
origin = incident_at - timedelta(days=15)
|
|
for row in rows:
|
|
if row["device_id"] not in selected:
|
|
continue
|
|
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
|
|
if at > returned_at:
|
|
# Keep the nearest frame even when it contains null/invalid values;
|
|
# never skip it in order to find a later normal-looking sample.
|
|
observations.setdefault(at, {})
|
|
if row.get("quality") not in {None, "valid", "corrected"}:
|
|
continue
|
|
value = row.get("cleaned_value")
|
|
if value is None:
|
|
value = row.get("monitored_value")
|
|
if value is None or not isfinite(float(value)):
|
|
continue
|
|
value = float(value)
|
|
phase = int(at.timestamp()) % 86400
|
|
if origin <= at < incident_at:
|
|
baseline.setdefault((row["device_id"], phase), []).append(value)
|
|
elif at > returned_at:
|
|
observations[at][row["device_id"]] = value
|
|
if not observations:
|
|
return {**base, "reason": "post_completion_reading_missing"}
|
|
at = min(observations)
|
|
values = observations[at]
|
|
phase = int(at.timestamp()) % 86400
|
|
checks = []
|
|
for device in device_ids:
|
|
reference = baseline.get((device, phase), [])
|
|
if device not in values or len(reference) < 15:
|
|
checks.append({"device_id": device, "status": "missing_data"})
|
|
continue
|
|
center = median(reference)
|
|
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
|
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
|
lower, upper = min(reference) - margin, max(reference) + margin
|
|
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
|
"normal_day_count": len(reference),
|
|
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
|
sample = {"at": at.isoformat(), "devices": checks}
|
|
if all(item["status"] == "normal" for item in checks):
|
|
return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]}
|
|
return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample}
|
|
|
|
|
|
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
|
|
settings: ProjectSettings, *, limit: int = 20,
|
|
created_from: datetime | None = None) -> int:
|
|
"""Poll only persisted recovery waits; publish evidence before waking the Agent."""
|
|
from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord
|
|
from app.domain.schemas.operational_data import TaskDataSnapshot
|
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
|
|
|
async with service.repo.transaction():
|
|
candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope)
|
|
resumed = 0
|
|
for task in candidates:
|
|
if task.get("execution_scope") != service.execution_scope:
|
|
continue
|
|
context = task["handling_context"]
|
|
returned_at = datetime.fromisoformat(context["returned_at"])
|
|
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
|
|
interval = settings.scada_monitoring_interval_seconds
|
|
roles = settings.scada_monitoring
|
|
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
|
|
source = "scada_readings"
|
|
through = service._now()
|
|
if context.get("device_ids"):
|
|
devices = context["device_ids"]
|
|
interval = context.get("interval_seconds", interval)
|
|
else:
|
|
# Retain the points the Agent actually inspected, including water
|
|
# quality or local incident sensors outside the project role lists.
|
|
for evidence in await service.repo.list_evidence(task["task_id"]):
|
|
saved = evidence.get("data_snapshot") or {}
|
|
if saved.get("source") not in {"query_scada_readings", "scada_readings"}:
|
|
continue
|
|
data = saved.get("data")
|
|
if isinstance(data, dict):
|
|
data = data.get("data", data.get("items", []))
|
|
if isinstance(data, list):
|
|
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
|
|
devices = list(dict.fromkeys(devices))
|
|
async with timescale.transaction():
|
|
# Field completion authorizes a separate recovery observation even
|
|
# when the incident was found in a frozen replay. Keep that replay's
|
|
# query window and batch boundary unchanged.
|
|
rows = await ScadaRepository.get_recovery_scada_readings(
|
|
timescale, devices, incident_at, returned_at, through, baseline_limit=100_001,
|
|
) if through > returned_at and devices else []
|
|
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
|
|
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
|
|
incident_at=incident_at, interval_seconds=interval or 0))
|
|
async with service.repo.transaction():
|
|
await service.repo.lock(f"task:{task['task_id']}")
|
|
current = await service._task(UUID(task["task_id"]))
|
|
# A new receipt, cancellation or another poll may have won the race.
|
|
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
|
|
continue
|
|
now = service._now()
|
|
if devices:
|
|
current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0)
|
|
current["handling_context"]["last_checked_at"] = now.isoformat()
|
|
current["handling_context"]["last_check"] = result
|
|
if result["status"] == "passed":
|
|
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
|
|
evidence = StoredEvidenceRecord(
|
|
evidence_id=evidence_id, kind="system_data",
|
|
summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常，等待 Agent 完成复核报告",
|
|
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
|
|
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
|
|
recorded_by=service._actor(), has_snapshot=True,
|
|
data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now,
|
|
origin="system_data", data=result, truncated=False),
|
|
)
|
|
if not await service.repo.get_evidence(task["task_id"], evidence_id):
|
|
await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json"))
|
|
current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id)
|
|
await service._requeue_task(current, now)
|
|
resumed += 1
|
|
else:
|
|
current.update(revision=current["revision"] + 1, updated_at=now)
|
|
await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump(
|
|
mode="json", exclude={"active_run", "latest_run"}))
|
|
await service._task_event(current, now)
|
|
return resumed
|