fix(scada): verify first sample after field completion
Generic Container CI/CD / test-build-publish (push) Failing after 47s
Server CI/CD v2 / build-test-publish-and-deploy (push) Failing after 47s

This commit is contained in:
2026-11-07 19:50:31 +08:00
parent ff8aba4964
commit dc83cf5c88
7 changed files with 131 additions and 71 deletions
+38 -51
View File
@@ -65,16 +65,12 @@ def accept_completion(task: dict, feedback: dict) -> bool:
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
incident_at: datetime, interval_seconds: int) -> dict:
"""Require two fresh complete samples inside the pre-event normal envelope.
Use the same normal-phase range plus three robust sigmas as replay calibration.
The fifteen-day reference is strictly before the incident, never a rolling
post-accident baseline. Only finite, usable observations can prove recovery.
"""
"""Verify the first SCADA sample strictly after the field feedback time T."""
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
"required_samples": 2, "interval_seconds": interval_seconds,
"required_samples": 1, "interval_seconds": interval_seconds,
"selection_method": "first_sample_after_feedback",
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
if not device_ids or interval_seconds <= 0 or 86400 % interval_seconds:
if not device_ids:
return {**base, "reason": "monitoring_configuration_missing"}
if returned_at < incident_at:
return {**base, "reason": "completion_precedes_incident"}
@@ -86,6 +82,10 @@ def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: d
if row["device_id"] not in selected:
continue
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
if at > returned_at:
# Keep the nearest frame even when it contains null/invalid values;
# never skip it in order to find a later normal-looking sample.
observations.setdefault(at, {})
if row.get("quality") not in {None, "valid", "corrected"}:
continue
value = row.get("cleaned_value")
@@ -94,46 +94,33 @@ def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: d
if value is None or not isfinite(float(value)):
continue
value = float(value)
# Preserve the source sampling phase even for an off-grid event/receipt.
phase = int(at.timestamp()) % 86400
if origin <= at < incident_at:
baseline.setdefault((row["device_id"], phase), []).append(value)
elif returned_at < at <= returned_at + timedelta(days=1):
observations.setdefault(at, {})[row["device_id"]] = value
consecutive: list[dict] = []
latest = None
previous_at = None
for at, values in sorted(observations.items()):
phase = int(at.timestamp()) % 86400
checks = []
for device in device_ids:
reference = baseline.get((device, phase), [])
if device not in values or len(reference) < 15:
checks.append({"device_id": device, "status": "missing_data"})
continue
center = median(reference)
sigma = 1.4826 * median(abs(value - center) for value in reference)
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
lower, upper = min(reference) - margin, max(reference) + margin
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
"normal_day_count": len(reference),
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
latest = {"at": at.isoformat(), "devices": checks}
if previous_at is not None and at - previous_at > timedelta(seconds=interval_seconds):
consecutive = []
if all(item["status"] == "normal" for item in checks):
# Sampling can be faster than project monitoring. Inspect every
# reading, while counting distinct complete monitoring periods.
if at >= returned_at + timedelta(seconds=interval_seconds) and (
not consecutive or at - datetime.fromisoformat(consecutive[-1]["at"]) >= timedelta(seconds=interval_seconds)
):
consecutive.append(latest)
else:
consecutive = []
previous_at = at
if len(consecutive) >= 2:
return {**base, "status": "passed", "verified_through": consecutive[-1]["at"], "samples": consecutive[-2:]}
return {**base, "reason": "post_completion_data_not_normal_or_incomplete", "latest_sample": latest}
elif at > returned_at:
observations[at][row["device_id"]] = value
if not observations:
return {**base, "reason": "post_completion_reading_missing"}
at = min(observations)
values = observations[at]
phase = int(at.timestamp()) % 86400
checks = []
for device in device_ids:
reference = baseline.get((device, phase), [])
if device not in values or len(reference) < 15:
checks.append({"device_id": device, "status": "missing_data"})
continue
center = median(reference)
sigma = 1.4826 * median(abs(value - center) for value in reference)
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
lower, upper = min(reference) - margin, max(reference) + margin
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
"normal_day_count": len(reference),
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
sample = {"at": at.isoformat(), "devices": checks}
if all(item["status"] == "normal" for item in checks):
return {**base, "status": "passed", "verified_through": at.isoformat(), "samples": [sample]}
return {**base, "reason": "nearest_post_completion_reading_not_normal_or_incomplete", "sample": sample}
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
@@ -155,7 +142,7 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
roles = settings.scada_monitoring
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
source = "scada_readings"
through = min(service._now(), returned_at + timedelta(days=1))
through = service._now()
if context.get("device_ids"):
devices = context["device_ids"]
interval = context.get("interval_seconds", interval)
@@ -173,8 +160,8 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
devices = list(dict.fromkeys(devices))
async with timescale.transaction():
rows = await ScadaRepository.get_scada_by_ids_time_range(
timescale, devices, incident_at - timedelta(days=15), through, row_limit=100_001,
rows = await ScadaRepository.get_recovery_scada_readings(
timescale, devices, incident_at, returned_at, through, baseline_limit=100_001,
) if through > returned_at and devices else []
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
@@ -186,15 +173,15 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
continue
now = service._now()
if devices and interval:
current["handling_context"].update(device_ids=devices, interval_seconds=interval)
if devices:
current["handling_context"].update(device_ids=devices, interval_seconds=interval or 0)
current["handling_context"]["last_checked_at"] = now.isoformat()
current["handling_context"]["last_check"] = result
if result["status"] == "passed":
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
evidence = StoredEvidenceRecord(
evidence_id=evidence_id, kind="system_data",
summary="完工反馈后连续两个采样周期的 SCADA 恢复正常,等待 Agent 完成复核报告",
summary="完工反馈时间之后最近时刻的 SCADA 已恢复正常,等待 Agent 完成复核报告",
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
recorded_by=service._actor(), has_snapshot=True,