jiang
58a907bfbe
Full-network browsing expanded persisted recovery gates to unrelated devices, while full fifteen-day reference reads exhausted the row budget. Rebuild legacy cached scopes from report devices and project monitoring roles, and query only the nearest post-feedback frame and its historical phase. Regression tests fail before the fix for both new and cached scopes and verify automatic Agent requeue after recovery.
239 lines
13 KiB
Python
239 lines
13 KiB
Python
"""Work-order completion and causal SCADA recovery evidence for incident tasks."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
from math import isfinite
|
|
from statistics import median
|
|
from uuid import UUID
|
|
from typing import TYPE_CHECKING
|
|
|
|
from psycopg import AsyncConnection
|
|
from app.domain.schemas.project_settings import ProjectSettings
|
|
|
|
from app.services.content_digest import digest
|
|
|
|
if TYPE_CHECKING:
|
|
from app.services.operational_tasks import OperationalTaskService
|
|
|
|
|
|
RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly", "water_pollution"}
|
|
|
|
|
|
def track_work_order(task: dict, action: dict) -> bool:
|
|
if task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order":
|
|
return False
|
|
receipt = action.get("receipt") or {}
|
|
if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample":
|
|
return False
|
|
context = task.setdefault("handling_context", {})
|
|
orders = context.setdefault("work_orders", {})
|
|
order_id = receipt["receipt_id"]
|
|
if order_id not in orders:
|
|
proposed_at = action.get("proposed_at")
|
|
for prior in orders.values():
|
|
terminal_at = prior.get("terminal_recorded_at")
|
|
# Only a new decision after the failed field attempt replaces it.
|
|
# A delayed receipt for an older independent dispatch does not.
|
|
if (not prior.get("superseded_by") and prior["status"] in {"failed", "cancelled"}
|
|
and proposed_at and terminal_at
|
|
and datetime.fromisoformat(proposed_at) > datetime.fromisoformat(terminal_at)):
|
|
prior["superseded_by"] = order_id
|
|
orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])}
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
return True
|
|
|
|
|
|
def accept_completion(task: dict, feedback: dict, *, recorded_at: datetime | None = None) -> bool:
|
|
context = task.get("handling_context") or {}
|
|
order_id = feedback["source_event_id"].removeprefix("peripheral_sample:")
|
|
order = context.get("work_orders", {}).get(order_id)
|
|
if order is None:
|
|
return False
|
|
order.update(status=feedback["status"], source_revision=feedback["source_revision"])
|
|
if feedback["status"] in {"failed", "cancelled"}:
|
|
if recorded_at is not None and not order.get("terminal_recorded_at"):
|
|
order["terminal_recorded_at"] = recorded_at.isoformat()
|
|
else:
|
|
order.pop("terminal_recorded_at", None)
|
|
# The synchronizer preserves the field activity's business time on ordinary
|
|
# edits. A revised field receipt can explicitly correct the verification start.
|
|
if feedback["status"] == "completed":
|
|
order["returned_at"] = feedback["occurred_at"]
|
|
if task.get("workflow") == "water_pollution":
|
|
order["feedback_evidence_id"] = (feedback.get("evidence") or {}).get("evidence_id")
|
|
else:
|
|
order.pop("returned_at", None)
|
|
if order.get("superseded_by"):
|
|
return True
|
|
orders = [item for item in context["work_orders"].values() if not item.get("superseded_by")]
|
|
if orders and all(item["status"] == "completed" for item in orders):
|
|
returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat)
|
|
if task.get("workflow") == "water_pollution":
|
|
# A completion receipt resumes review, never the pressure-based
|
|
# automatic recovery calculation. Source revisions may carry new
|
|
# sampling or retest facts even when the field time is unchanged.
|
|
context.update(phase="field_review", title="水污染处置反馈复核", returned_at=returned_at)
|
|
context.pop("verification_evidence_id", None)
|
|
return True
|
|
previous = context.get("returned_at")
|
|
if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at)
|
|
or context.get("phase") == "waiting_work_order"):
|
|
context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at)
|
|
for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"):
|
|
context.pop(key, None)
|
|
else:
|
|
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
|
context.pop("verification_evidence_id", None)
|
|
return True
|
|
|
|
|
|
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
|
|
incident_at: datetime, interval_seconds: int) -> dict:
|
|
"""Verify the first SCADA sample strictly after the field feedback time T."""
|
|
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
|
|
"required_samples": 1, "interval_seconds": interval_seconds,
|
|
"selection_method": "first_sample_after_feedback",
|
|
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
|
|
if not device_ids:
|
|
return {**base, "reason": "monitoring_configuration_missing"}
|
|
if returned_at < incident_at:
|
|
return {**base, "reason": "completion_precedes_incident"}
|
|
selected = set(device_ids)
|
|
baseline: dict[tuple[str, int], list[float]] = {}
|
|
observations: dict[datetime, dict[str, float]] = {}
|
|
origin = incident_at - timedelta(days=15)
|
|
for row in rows:
|
|
if row["device_id"] not in selected:
|
|
continue
|
|
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
|
|
if at > returned_at:
|
|
# Keep the nearest frame even when it contains null/invalid values;
|
|
# never skip it in order to find a later normal-looking sample.
|
|
observations.setdefault(at, {})
|
|
if row.get("quality") not in {None, "valid", "corrected"}:
|
|
continue
|
|
value = row.get("cleaned_value")
|
|
if value is None:
|
|
value = row.get("monitored_value")
|
|
if value is None or not isfinite(float(value)):
|
|
continue
|
|
value = float(value)
|
|
phase = int(at.timestamp()) % 86400
|
|
if origin <= at < incident_at:
|
|
baseline.setdefault((row["device_id"], phase), []).append(value)
|
|
elif at > returned_at:
|
|
observations[at][row["device_id"]] = value
|
|
if not observations:
|
|
return {**base, "reason": "post_completion_reading_missing"}
|
|
at = min(observations)
|
|
values = observations[at]
|
|
phase = int(at.timestamp()) % 86400
|
|
checks = []
|
|
for device in device_ids:
|
|
reference = baseline.get((device, phase), [])
|
|
if device not in values or len(reference) < 15:
|
|
checks.append({"device_id": device, "status": "missing_data"})
|
|
continue
|
|
center = median(reference)
|
|
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
|
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
|
lower, upper = min(reference) - margin, max(reference) + margin
|
|
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
|
"normal_day_count": len(reference),
|
|
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
|
sample = {"at": at.isoformat(), "devices": checks}
|
|
if all(item["status"] == "normal" for item in checks):
|
|
return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]}
|
|
return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample}
|
|
|
|
|
|
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
|
|
settings: ProjectSettings, *, limit: int = 20,
|
|
created_from: datetime | None = None) -> int:
|
|
"""Poll only persisted recovery waits; publish evidence before waking the Agent."""
|
|
from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord
|
|
from app.domain.schemas.operational_data import TaskDataSnapshot
|
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
|
|
|
async with service.repo.transaction():
|
|
candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope)
|
|
resumed = 0
|
|
for task in candidates:
|
|
if task.get("execution_scope") != service.execution_scope:
|
|
continue
|
|
context = task["handling_context"]
|
|
returned_at = datetime.fromisoformat(context["returned_at"])
|
|
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
|
|
interval = settings.scada_monitoring_interval_seconds
|
|
roles = settings.scada_monitoring
|
|
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
|
|
source = "scada_readings"
|
|
through = service._now()
|
|
report_revision = task.get("current_report_revision")
|
|
if (context.get("device_ids") and context.get("device_selection_version") == 2
|
|
and context.get("source_report_revision") == report_revision):
|
|
devices = context["device_ids"]
|
|
interval = context.get("interval_seconds", interval)
|
|
else:
|
|
# Full-network browsing is discovery, not a declaration that every
|
|
# device must recover for this incident. Prefer the report's scope.
|
|
report = await service.repo.get_latest_report(task["task_id"])
|
|
declared = ((report or {}).get("report") or {}).get("scada_device_ids") or []
|
|
devices.extend(declared)
|
|
for evidence in ([] if declared else await service.repo.list_evidence(task["task_id"])):
|
|
saved = evidence.get("data_snapshot") or {}
|
|
if saved.get("source") not in {"query_scada_readings", "scada_readings"}:
|
|
continue
|
|
data = saved.get("data")
|
|
if isinstance(data, dict):
|
|
data = data.get("data", data.get("items", []))
|
|
if isinstance(data, list):
|
|
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
|
|
devices = list(dict.fromkeys(devices))
|
|
async with timescale.transaction():
|
|
# Field completion authorizes a separate recovery observation even
|
|
# when the incident was found in a frozen replay. Keep that replay's
|
|
# query window and batch boundary unchanged.
|
|
rows = await ScadaRepository.get_recovery_scada_readings(
|
|
timescale, devices, incident_at, returned_at, through, baseline_limit=100_001,
|
|
) if through > returned_at and devices else []
|
|
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
|
|
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
|
|
incident_at=incident_at, interval_seconds=interval or 0))
|
|
async with service.repo.transaction():
|
|
await service.repo.lock(f"task:{task['task_id']}")
|
|
current = await service._task(UUID(task["task_id"]))
|
|
# A new receipt, cancellation or another poll may have won the race.
|
|
if (current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}
|
|
or current.get("current_report_revision") != report_revision):
|
|
continue
|
|
now = service._now()
|
|
if devices:
|
|
current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0,
|
|
device_selection_version=2, source_report_revision=report_revision)
|
|
current["handling_context"]["last_checked_at"] = now.isoformat()
|
|
current["handling_context"]["last_check"] = result
|
|
if result["status"] == "passed":
|
|
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
|
|
evidence = StoredEvidenceRecord(
|
|
evidence_id=evidence_id, kind="system_data",
|
|
summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常，等待 Agent 完成复核报告",
|
|
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
|
|
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
|
|
recorded_by=service._actor(), has_snapshot=True,
|
|
data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now,
|
|
origin="system_data", data=result, truncated=False),
|
|
)
|
|
if not await service.repo.get_evidence(task["task_id"], evidence_id):
|
|
await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json"))
|
|
current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id)
|
|
await service._requeue_task(current, now)
|
|
resumed += 1
|
|
else:
|
|
current.update(revision=current["revision"] + 1, updated_at=now)
|
|
await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump(
|
|
mode="json", exclude={"active_run", "latest_run"}))
|
|
await service._task_event(current, now)
|
|
return resumed
|