feat(scada): add calibrated monitoring and incident recovery

2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
@@ -1,8 +1,11 @@
 from datetime import datetime, timedelta, timezone
 from uuid import uuid4
+import pytest

 from app.domain.schemas.scada_replay import ScadaReplayCreate
+from app.domain.schemas.project_settings import ScadaMonitoringSettings
 from app.services import scada_replay_monitoring
+from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY


 TARGET = datetime(2026, 10, 16, tzinfo=timezone.utc)
@@ -68,6 +71,14 @@ def test_multiple_indicators_create_suspected_burst_result(monkeypatch):
    assert result["pressure"]["hit_device_ids"] == PRESSURE_IDS
    assert result["outlet_flow"]["hit_device_ids"] == FLOW_IDS
    assert result["burst_detection"]["hit"] is True
+    pressure = result["pressure"]["devices"][0]
+    assert pressure["value_change"] == -30
+    assert pressure["unit"] == "m"
+    assert pressure["baseline_method"] == "recent_samples_median"
+    assert pressure["baseline_sample_count"] == 4
+    assert pressure["baseline_end"] == result["period_start"]
+    assert result["outlet_flow"]["devices"][0]["value_change"] == 50
+    assert result["burst_detection"]["score_interpretation"] == "lower_is_more_anomalous_not_probability"


 def test_single_indicator_stays_attention(monkeypatch):
@@ -106,3 +117,194 @@ def test_missing_project_roles_is_data_gap_not_normal():

    assert result["status"] == "insufficient_data"
    assert result["burst_detection"]["status"] == "not_run"
+
+
+def _weighted_document():
+    document = _document()
+    document["config"]["start_time"] = (TARGET - timedelta(minutes=30)).isoformat()
+    document["monitoring_config"] = ScadaMonitoringSettings(
+        pressure_device_ids=PRESSURE_IDS, outlet_flow_device_ids=FLOW_IDS,
+    ).model_dump(mode="json")
+    document["monitoring_calibration"] = {
+        "version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
+        "status": "ready", "knowledge_cutoff": (TARGET - timedelta(minutes=30)).isoformat(),
+        "pressure": {device: {"threshold_percent": 10, "method": "frozen_normal_history"} for device in PRESSURE_IDS},
+        "outlet_flow": {"F1": {"threshold_percent": 5, "method": "frozen_normal_history"}},
+        "burst_detection": {"threshold": -0.08},
+    }
+    profile = {"phase_origin": document["config"]["start_time"], "interval_seconds": 900,
+               "lower": [99.], "upper": [101.]}
+    calibration = document["monitoring_calibration"]
+    calibration["diagnostic"] = {device: profile for device in PRESSURE_IDS + FLOW_IDS}
+    for kind in ("pressure", "outlet_flow"):
+        for item in calibration[kind].values():
+            item["normal_profile"] = profile
+    return document
+
+
+@pytest.mark.parametrize("flow,expected_score", [("increase", 1.0), ("flat", 0.9), ("missing", 0.9)])
+def test_pressure_and_detection_can_trigger_with_low_or_unavailable_flow_weight(monkeypatch, flow, expected_score):
+    monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
+        "rows": [{"Score": -0.2}], "summary": {},
+    })
+    rows = _rows()
+    for row in rows:
+        if flow == "flat" and row["device_id"] == "F1":
+            row["cleaned_value"] = 100
+    if flow == "missing":
+        rows = [row for row in rows if not (row["device_id"] == "F1"
+                and datetime.fromisoformat(row["time"]) >= TARGET - timedelta(minutes=15))]
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        _weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
+        period_end=TARGET, batch_sequence=1440,
+    )
+    assert result["status"] == "suspected_burst"
+    assert result["weighted"]["score"] == expected_score
+    assert result["weighted"]["weights"]["outlet_flow"] == 0.1
+    assert result["pressure"]["devices"][0]["threshold_percent"] == 10
+    assert result["burst_detection"]["score_threshold"] == -0.08
+    if flow == "missing":
+        assert result["weighted"]["unavailable_indicators"] == ["outlet_flow"]
+
+
+@pytest.mark.parametrize("indicator", ["pressure", "outlet_flow", "burst_detection"])
+def test_a_single_indicator_cannot_create_a_weighted_burst_alert(monkeypatch, indicator):
+    monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
+        "rows": [{"Score": -0.2 if indicator == "burst_detection" else 0.02}], "summary": {},
+    })
+    rows = _rows()
+    for row in rows:
+        if (row["device_id"] in PRESSURE_IDS and indicator != "pressure"
+                or row["device_id"] == "F1" and indicator != "outlet_flow"):
+            row["cleaned_value"] = 100
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        _weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
+        period_end=TARGET, batch_sequence=1440,
+    )
+    assert result["status"] == "attention"
+    assert result["weighted"]["score"] < 0.8
+
+
+def test_weighted_policy_requires_a_persisted_causal_calibration():
+    document = _weighted_document()
+    document.pop("monitoring_calibration")
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
+    )
+    assert result["status"] == "insufficient_data"
+
+
+def test_calibration_from_another_or_future_cutoff_is_rejected():
+    document = _weighted_document()
+    document["monitoring_calibration"]["knowledge_cutoff"] = (TARGET + timedelta(minutes=15)).isoformat()
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
+    )
+    assert result["status"] == "insufficient_data"
+
+
+def test_ready_flow_anomaly_survives_missing_pressure_calibration(monkeypatch):
+    document = _weighted_document()
+    document["monitoring_calibration"].update(status="insufficient_data", pressure={})
+    document["monitoring_calibration"].pop("burst_detection")
+    def unexpected_detector(**_):
+        raise AssertionError("Unavailable pressure calibration must not run the detector")
+    monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", unexpected_detector)
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
+    )
+    assert result["status"] == "insufficient_data"
+    assert result["outlet_flow"]["hit"] is True
+    assert result["weighted"]["score"] == 0.1
+    assert result["burst_detection"]["status"] == "baseline_not_ready"
+
+
+def test_relative_baseline_matches_calibration_immediately_previous_samples():
+    config = ScadaReplayCreate.model_validate(_document()["config"])
+    rows = [{"device_id": "P1", "time": (TARGET - timedelta(minutes=15 * (5 - i))).isoformat(),
+             "cleaned_value": value, "quality": "valid"}
+            for i, value in enumerate((120., 115., 110., 105., 100., 95.))]
+    result = scada_replay_monitoring._relative_indicator(
+        rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
+    )
+    assert result["devices"][0]["baseline"] == 107.5
+    assert result["devices"][0]["baseline_end"] == (TARGET - timedelta(minutes=15)).isoformat()
+
+
+def test_period_start_reading_is_not_a_new_observation():
+    config = ScadaReplayCreate.model_validate(_document()["config"])
+    rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
+    result = scada_replay_monitoring._relative_indicator(
+        rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
+    )
+    assert result["status"] == "insufficient_data"
+    assert result["gaps"][0]["reason"] == "period_reading_missing"
+
+
+def test_detector_cannot_reuse_the_previous_period_score(monkeypatch):
+    monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection",
+                        lambda **_: pytest.fail("stale target reached detector"))
+    rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
+    result = scada_replay_monitoring._burst_detection(rows, PRESSURE_IDS, 900, TARGET)
+    assert result["status"] == "current_reading_missing"
+    assert result["hit"] is False
+
+
+def test_old_calibration_version_requires_a_new_replay():
+    document = _weighted_document()
+    document["monitoring_calibration"]["version"] = "scada-monitor-calibration@1"
+    result = scada_replay_monitoring.evaluate_monitoring_period(
+        document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
+    )
+    assert result["status"] == "insufficient_data"
+    assert "新建" in result["detail"]
+    assert not any(result[key]["hit"] for key in ("pressure", "outlet_flow", "burst_detection"))
+
+
+def test_normal_diurnal_change_then_t_and_t_plus_fifteen_share_one_incident(monkeypatch):
+    from app.services.scada_replay_processing import detect_frame
+    from app.services.scada_monitoring_incidents import update_incident
+    document = _weighted_document()
+    config = ScadaReplayCreate.model_validate(document["config"])
+    profile = {"phase_origin": config.start_time.isoformat(), "interval_seconds": 900,
+               "lower": [99., 199., 199., 199.], "upper": [101., 201., 201., 201.]}
+    document["monitoring_calibration"]["diagnostic"]["F1"] = profile
+    document["monitoring_calibration"]["outlet_flow"]["F1"]["normal_profile"] = profile
+    monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **kwargs: {
+        "rows": [{"Score": -0.2 if kwargs["observed_pressure_data"].iloc[-1].min() < 90 else 0.02}],
+        "summary": {},
+    })
+    rows = _rows()
+    rows += [{"time": (TARGET - timedelta(days=15)).isoformat(), "device_id": device,
+              "cleaned_value": 100., "quality": "valid"} for device in PRESSURE_IDS + FLOW_IDS]
+    for row in rows:
+        if row["device_id"] == "F1" and _time_for_test(row) >= TARGET - timedelta(minutes=15):
+            row["cleaned_value"] = 200.
+    rows += [{"time": (TARGET + timedelta(minutes=15)).isoformat(), "device_id": device,
+              "cleaned_value": 200. if device == "F1" else 70., "quality": "valid"}
+             for device in PRESSURE_IDS + FLOW_IDS]
+    document["histories"] = {device: [100.] * 4 for device in PRESSURE_IDS + FLOW_IDS}
+    actions, statuses = [], []
+    for sequence, offset in enumerate((-15, 0, 15), start=1):
+        at = TARGET + timedelta(minutes=offset)
+        frame = [row for row in rows if _time_for_test(row) == at]
+        document, changes = detect_frame(document, frame, config, at)
+        result = scada_replay_monitoring.evaluate_monitoring_period(
+            document, [row for row in rows if _time_for_test(row) <= at],
+            period_start=at - timedelta(minutes=15), period_end=at, batch_sequence=sequence,
+        )
+        result["anomaly_signals"] = [state["event"] for state in document["rule_states"].values()
+                                    if state.get("event")]
+        statuses.append(result["status"])
+        incident, action = update_incident(document, result, sequence)
+        actions.append(action)
+        if incident and not incident.get("task_id"):
+            incident["task_id"] = str(uuid4())
+        if offset == -15:
+            assert changes == []  # Normal doubling of flow does not launch diagnosis.
+    assert statuses == ["normal", "suspected_burst", "suspected_burst"]
+    assert actions == [None, "raised", "ongoing"]
+
+
+def _time_for_test(row):
+    return datetime.fromisoformat(row["time"])