99 lines
5.7 KiB
Python
99 lines
5.7 KiB
Python
"""Persist one accident across monitoring periods without moving its evidence cutoff."""
|
|
from datetime import datetime
|
|
from math import isfinite
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
|
|
|
|
def _anchors(monitoring: dict, indicator: str = "pressure") -> dict:
|
|
return {
|
|
item["device_id"]: {"baseline": item["baseline"], "threshold_percent": item["threshold_percent"]}
|
|
for item in monitoring.get(indicator, {}).get("devices", [])
|
|
if item.get("hit") and item.get("baseline", 0) > 0
|
|
and 0 < item.get("threshold_percent", 0) < 100
|
|
}
|
|
|
|
|
|
def update_incident(document: dict, monitoring: dict, sequence: int) -> tuple[dict | None, str | None]:
|
|
incident = document.get("monitoring_incident")
|
|
# Adopt the last task from an older replay document on its first new step.
|
|
previous = document.get("last_monitoring") or {}
|
|
if incident:
|
|
incident.setdefault("burst_diagnosed", previous.get("status") == "suspected_burst")
|
|
if not incident and previous.get("task_id"):
|
|
incident = {
|
|
"incident_id": str(uuid5(NAMESPACE_URL, f"scada_replay:{document['replay_id']}:monitor:"
|
|
f"{previous['period_end']}:{previous['rule_version']}")),
|
|
"started_at": previous["period_end"], "last_observed_at": previous["period_end"],
|
|
"status": "active", "task_id": previous["task_id"], "recovery_count": 0,
|
|
"pressure_anchors": _anchors(previous), "alert": None,
|
|
"flow_anchors": _anchors(previous, "outlet_flow"),
|
|
"burst_diagnosed": previous.get("status") == "suspected_burst",
|
|
}
|
|
document["monitoring_incident"] = incident
|
|
if incident and incident.get("last_monitoring_id") == monitoring.get("monitoring_id") and monitoring.get("monitoring_id"):
|
|
return incident, incident["last_action"]
|
|
anomaly = (monitoring["status"] in {"attention", "suspected_burst"}
|
|
or bool(monitoring.get("anomaly_signals"))
|
|
or any(monitoring.get(key, {}).get("hit") is True
|
|
for key in ("pressure", "outlet_flow", "burst_detection")))
|
|
if anomaly and (not incident or incident["status"] == "resolved"):
|
|
external = (f"scada_replay:{document['replay_id']}:monitor:"
|
|
f"{monitoring['period_end']}:{monitoring['rule_version']}")
|
|
incident = {
|
|
"incident_id": str(uuid5(NAMESPACE_URL, external)), "external_id": external,
|
|
"started_at": monitoring["period_end"], "status": "active", "task_id": None,
|
|
"recovery_count": 0, "pressure_anchors": _anchors(monitoring), "alert": None,
|
|
"flow_anchors": _anchors(monitoring, "outlet_flow"),
|
|
"batch_sequence": sequence,
|
|
}
|
|
document["monitoring_incident"] = incident
|
|
action = "raised"
|
|
elif incident and incident["status"] == "active":
|
|
# A later diagnostic stage may add pressure/flow evidence to the same task.
|
|
# Freeze each newly abnormal device once; retain earlier references.
|
|
for indicator, field in (("pressure", "pressure_anchors"), ("outlet_flow", "flow_anchors")):
|
|
for device, anchor in _anchors(monitoring, indicator).items():
|
|
incident.setdefault(field, {}).setdefault(device, anchor)
|
|
current = {item["device_id"]: item for item in monitoring.get("pressure", {}).get("devices", [])}
|
|
anchors = incident["pressure_anchors"]
|
|
flow = {item["device_id"]: item for item in monitoring.get("outlet_flow", {}).get("devices", [])}
|
|
# Rolling relative changes can become normal while absolute pressure stays low.
|
|
# Confirm recovery against the frozen pre-accident references as well.
|
|
recovered = monitoring["status"] == "normal" and not anomaly and all(
|
|
monitoring.get(key, {}).get("status") == "ready"
|
|
and monitoring[key].get("hit") is False for key in ("pressure", "burst_detection")
|
|
) and all(
|
|
device in current and isfinite(current[device]["value"])
|
|
and current[device].get("observed_at") is not None
|
|
and datetime.fromisoformat(current[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
|
|
and current[device]["value"] >= anchor["baseline"] * (1 - anchor["threshold_percent"] / 100)
|
|
for device, anchor in anchors.items()
|
|
) and all(
|
|
device in flow and isfinite(flow[device]["value"])
|
|
and flow[device].get("observed_at") is not None
|
|
and datetime.fromisoformat(flow[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
|
|
and flow[device]["value"] <= anchor["baseline"] * (1 + anchor["threshold_percent"] / 100)
|
|
for device, anchor in incident.get("flow_anchors", {}).items()
|
|
)
|
|
contiguous = bool(monitoring.get("period_start")) and (
|
|
datetime.fromisoformat(monitoring["period_start"]) == datetime.fromisoformat(incident["last_observed_at"])
|
|
)
|
|
incident["recovery_count"] = (incident["recovery_count"] + 1 if contiguous else 1) if recovered else 0
|
|
action = "resolved" if incident["recovery_count"] >= 2 else "ongoing"
|
|
if action == "resolved":
|
|
incident["status"] = "resolved"
|
|
else:
|
|
return None, None
|
|
incident.update(last_observed_at=monitoring["period_end"],
|
|
last_monitoring_id=monitoring.get("monitoring_id"), last_action=action)
|
|
return incident, action
|
|
|
|
|
|
def attach_incident(monitoring: dict, incident: dict, action: str) -> None:
|
|
monitoring["task_id"] = incident["task_id"]
|
|
monitoring["incident"] = {
|
|
key: incident[key] for key in (
|
|
"incident_id", "status", "started_at", "last_observed_at", "recovery_count",
|
|
)
|
|
} | {"action": action}
|