feat(scada): add calibrated monitoring and incident recovery

2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Compare replay-rule thresholds against frozen hydraulic outputs without rerunning hydraulics."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--calibration-dir", required=True, type=Path)
+    parser.add_argument("--thresholds", nargs="+", type=float, default=[3, 5, 10, 20, 30])
+    args = parser.parse_args()
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    os.umask(0o077)
+    from scripts.benchmark.calibration import digest, pressure_preview, save_json
+    from scripts.benchmark.pressure_thresholds import learn_pressure_thresholds
+    from scripts.benchmark.simulation_design import BurstSimulationDesign
+
+    folder = args.calibration_dir.resolve()
+    source = folder / "calibration.json"
+    report = json.loads(source.read_text())
+    if report.get("status") != "completed" or any(not 0 < value <= 100 for value in args.thresholds):
+        raise ValueError("a completed calibration and thresholds in (0,100] are required")
+    design = BurstSimulationDesign.model_validate(report["design"])
+    pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
+    cases = {case.case_id: case for case in design.cases}
+    index_by_time = {at: index for index, at in enumerate(design.event_times)}
+    output = {"version": "burst-threshold-sensitivity@1", "calibration_sha256": digest(source),
+              "scope": "pure cleaner/replay-rule sensitivity, not Agent or worker acceptance",
+              "thresholds_percent": args.thresholds, "rows": [], "summary": [],
+              "adaptive_fit": learn_pressure_thresholds([], ["calibration-history"], design.event_times[0]),
+              "adaptive_note": "this package has four warmup samples; fitting needs 1440 pre-event normal samples per device",
+              "absolute_drop_gate_applied": False}
+    cache = {}
+    controls = {}
+    for entry in report["cases"]:
+        case = cases[entry["case_id"]]
+        pipe = pipes[case.pipe_id]
+        index = index_by_time[case.occurred_at]
+        if index not in cache:
+            cache[index] = (json.loads((folder / f"normal-{index}.json").read_text()),
+                            json.loads((folder / f"warmup-{index}.json").read_text()))
+        normal, warmup = cache[index]
+        burst = json.loads((folder / f"{case.case_id}-{entry['severity']}.json").read_text())
+        for threshold in args.thresholds:
+            rule = SimpleNamespace(primary_pressure_drop_percent=threshold,
+                                   consecutive_samples=design.selection_rule.consecutive_samples)
+            key = (pipe.pipe_id, index, threshold)
+            if key not in controls:
+                controls[key] = pressure_preview(pipe, case.occurred_at, report["duration_seconds"], 900,
+                                                 warmup["nodes"], normal["nodes"], rule)
+            event = pressure_preview(pipe, case.occurred_at, report["duration_seconds"], 900,
+                                     warmup["nodes"], burst["nodes"], rule)
+            at = event["first_signal_at"]
+            output["rows"].append({"case_id": case.case_id, "pipe_id": pipe.pipe_id,
+                                   "severity": entry["severity"], "threshold_percent": threshold,
+                                   "event_signal": event["raised"], "first_signal_at": at,
+                                   "delay_minutes": (datetime.fromisoformat(at) - case.occurred_at).total_seconds() / 60 if at else None,
+                                   "normal_false_positive": controls[key]["raised"]})
+    for threshold in args.thresholds:
+        rows = [row for row in output["rows"] if row["threshold_percent"] == threshold]
+        normal = [value for key, value in controls.items() if key[2] == threshold]
+        output["summary"].append({"threshold_percent": threshold, "event_cases": len(rows),
+                                  "event_signals": sum(row["event_signal"] for row in rows),
+                                  "unique_normal_controls": len(normal),
+                                  "normal_false_positives": sum(row["raised"] for row in normal)})
+    path = folder / "threshold-sensitivity.json"
+    save_json(path, output)
+    for summary in output["summary"]:
+        print(summary)
+    print(f"threshold sensitivity complete: {path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())