fix(scada): verify first sample after field completion
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
from typing import List, Any
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
from psycopg import AsyncConnection, Connection, sql
|
||||
from psycopg.rows import dict_row
|
||||
@@ -131,6 +131,29 @@ class ScadaRepository:
|
||||
)
|
||||
return await cur.fetchall()
|
||||
|
||||
@staticmethod
|
||||
async def get_recovery_scada_readings(
|
||||
conn: AsyncConnection, device_ids: list[str], incident_at: datetime,
|
||||
returned_at: datetime, through: datetime, *, baseline_limit: int = 100_001,
|
||||
) -> list[dict]:
|
||||
"""Read the causal reference and exactly the first stored frame after T."""
|
||||
if not device_ids or through <= returned_at:
|
||||
return []
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"WITH next_frame AS (SELECT min(time) AS time FROM scada.measurements "
|
||||
"WHERE device_id=ANY(%s) AND time>%s AND time<=%s), "
|
||||
"reference AS (SELECT time,device_id,monitored_value,cleaned_value "
|
||||
"FROM scada.measurements WHERE device_id=ANY(%s) AND time>=%s AND time<%s "
|
||||
"ORDER BY time,device_id LIMIT %s) "
|
||||
"SELECT time,device_id,monitored_value,cleaned_value FROM reference UNION ALL "
|
||||
"SELECT m.time,m.device_id,m.monitored_value,m.cleaned_value FROM scada.measurements m "
|
||||
"JOIN next_frame f ON m.time=f.time WHERE m.device_id=ANY(%s) ORDER BY time,device_id",
|
||||
(device_ids, returned_at, through, device_ids, incident_at - timedelta(days=15),
|
||||
incident_at, baseline_limit, device_ids),
|
||||
)
|
||||
return [dict(row) for row in await cur.fetchall()]
|
||||
|
||||
@staticmethod
|
||||
async def get_latest_scada_readings(
|
||||
conn: AsyncConnection,
|
||||
|
||||
@@ -65,16 +65,12 @@ def accept_completion(task: dict, feedback: dict) -> bool:
|
||||
|
||||
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
|
||||
incident_at: datetime, interval_seconds: int) -> dict:
|
||||
"""Require two fresh complete samples inside the pre-event normal envelope.
|
||||
|
||||
Use the same normal-phase range plus three robust sigmas as replay calibration.
|
||||
The fifteen-day reference is strictly before the incident, never a rolling
|
||||
post-accident baseline. Only finite, usable observations can prove recovery.
|
||||
"""
|
||||
"""Verify the first SCADA sample strictly after the field feedback time T."""
|
||||
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
|
||||
"required_samples": 2, "interval_seconds": interval_seconds,
|
||||
"required_samples": 1, "interval_seconds": interval_seconds,
|
||||
"selection_method": "first_sample_after_feedback",
|
||||
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
|
||||
if not device_ids or interval_seconds <= 0 or 86400 % interval_seconds:
|
||||
if not device_ids:
|
||||
return {**base, "reason": "monitoring_configuration_missing"}
|
||||
if returned_at < incident_at:
|
||||
return {**base, "reason": "completion_precedes_incident"}
|
||||
@@ -86,6 +82,10 @@ def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: d
|
||||
if row["device_id"] not in selected:
|
||||
continue
|
||||
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
|
||||
if at > returned_at:
|
||||
# Keep the nearest frame even when it contains null/invalid values;
|
||||
# never skip it in order to find a later normal-looking sample.
|
||||
observations.setdefault(at, {})
|
||||
if row.get("quality") not in {None, "valid", "corrected"}:
|
||||
continue
|
||||
value = row.get("cleaned_value")
|
||||
@@ -94,46 +94,33 @@ def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: d
|
||||
if value is None or not isfinite(float(value)):
|
||||
continue
|
||||
value = float(value)
|
||||
# Preserve the source sampling phase even for an off-grid event/receipt.
|
||||
phase = int(at.timestamp()) % 86400
|
||||
if origin <= at < incident_at:
|
||||
baseline.setdefault((row["device_id"], phase), []).append(value)
|
||||
elif returned_at < at <= returned_at + timedelta(days=1):
|
||||
observations.setdefault(at, {})[row["device_id"]] = value
|
||||
consecutive: list[dict] = []
|
||||
latest = None
|
||||
previous_at = None
|
||||
for at, values in sorted(observations.items()):
|
||||
phase = int(at.timestamp()) % 86400
|
||||
checks = []
|
||||
for device in device_ids:
|
||||
reference = baseline.get((device, phase), [])
|
||||
if device not in values or len(reference) < 15:
|
||||
checks.append({"device_id": device, "status": "missing_data"})
|
||||
continue
|
||||
center = median(reference)
|
||||
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
||||
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
||||
lower, upper = min(reference) - margin, max(reference) + margin
|
||||
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
||||
"normal_day_count": len(reference),
|
||||
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
||||
latest = {"at": at.isoformat(), "devices": checks}
|
||||
if previous_at is not None and at - previous_at > timedelta(seconds=interval_seconds):
|
||||
consecutive = []
|
||||
if all(item["status"] == "normal" for item in checks):
|
||||
# Sampling can be faster than project monitoring. Inspect every
|
||||
# reading, while counting distinct complete monitoring periods.
|
||||
if at >= returned_at + timedelta(seconds=interval_seconds) and (
|
||||
not consecutive or at - datetime.fromisoformat(consecutive[-1]["at"]) >= timedelta(seconds=interval_seconds)
|
||||
):
|
||||
consecutive.append(latest)
|
||||
else:
|
||||
consecutive = []
|
||||
previous_at = at
|
||||
if len(consecutive) >= 2:
|
||||
return {**base, "status": "passed", "verified_through": consecutive[-1]["at"], "samples": consecutive[-2:]}
|
||||
return {**base, "reason": "post_completion_data_not_normal_or_incomplete", "latest_sample": latest}
|
||||
elif at > returned_at:
|
||||
observations[at][row["device_id"]] = value
|
||||
if not observations:
|
||||
return {**base, "reason": "post_completion_reading_missing"}
|
||||
at = min(observations)
|
||||
values = observations[at]
|
||||
phase = int(at.timestamp()) % 86400
|
||||
checks = []
|
||||
for device in device_ids:
|
||||
reference = baseline.get((device, phase), [])
|
||||
if device not in values or len(reference) < 15:
|
||||
checks.append({"device_id": device, "status": "missing_data"})
|
||||
continue
|
||||
center = median(reference)
|
||||
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
||||
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
||||
lower, upper = min(reference) - margin, max(reference) + margin
|
||||
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
||||
"normal_day_count": len(reference),
|
||||
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
||||
sample = {"at": at.isoformat(), "devices": checks}
|
||||
if all(item["status"] == "normal" for item in checks):
|
||||
return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]}
|
||||
return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample}
|
||||
|
||||
|
||||
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
|
||||
@@ -155,7 +142,7 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
|
||||
roles = settings.scada_monitoring
|
||||
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
|
||||
source = "scada_readings"
|
||||
through = min(service._now(), returned_at + timedelta(days=1))
|
||||
through = service._now()
|
||||
if context.get("device_ids"):
|
||||
devices = context["device_ids"]
|
||||
interval = context.get("interval_seconds", interval)
|
||||
@@ -173,8 +160,8 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
|
||||
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
|
||||
devices = list(dict.fromkeys(devices))
|
||||
async with timescale.transaction():
|
||||
rows = await ScadaRepository.get_scada_by_ids_time_range(
|
||||
timescale, devices, incident_at - timedelta(days=15), through, row_limit=100_001,
|
||||
rows = await ScadaRepository.get_recovery_scada_readings(
|
||||
timescale, devices, incident_at, returned_at, through, baseline_limit=100_001,
|
||||
) if through > returned_at and devices else []
|
||||
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
|
||||
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
|
||||
@@ -186,15 +173,15 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
|
||||
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
|
||||
continue
|
||||
now = service._now()
|
||||
if devices and interval:
|
||||
current["handling_context"].update(device_ids=devices, interval_seconds=interval)
|
||||
if devices:
|
||||
current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0)
|
||||
current["handling_context"]["last_checked_at"] = now.isoformat()
|
||||
current["handling_context"]["last_check"] = result
|
||||
if result["status"] == "passed":
|
||||
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
|
||||
evidence = StoredEvidenceRecord(
|
||||
evidence_id=evidence_id, kind="system_data",
|
||||
summary="完工反馈后连续两个采样周期的 SCADA 恢复正常,等待 Agent 完成复核报告",
|
||||
summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常,等待 Agent 完成复核报告",
|
||||
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
|
||||
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
|
||||
recorded_by=service._actor(), has_snapshot=True,
|
||||
|
||||
Reference in New Issue
Block a user