OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scada): add calibrated monitoring and incident recovery

Browse Source
This commit is contained in:
jiang
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/scada_replay.py
+245 -42
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import json
from uuid import NAMESPACE_URL, uuid4, uuid5
from fastapi import HTTPException
@@ -14,6 +15,18 @@ from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.content_digest import digest
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
from app.services.scada_replay_monitoring import evaluate_monitoring_period
from app.services.scada_normal_profile import current_calibration, diagnostic_signal, weighted_diagnosis
# 500 devices at 15-minute sampling need 720,000 rows for fifteen days.
# Leave headroom without allowing dense sources to make preparation unbounded.
BASELINE_PAGE_SIZE = 2000
BASELINE_MAX_ROWS = 1_000_000
BASELINE_MAX_BYTES = 512 * 1024 * 1024
class BaselineBudgetExceeded(RuntimeError):
"""The unpublished baseline exceeded its fixed preparation budget."""
def parse_time(value: str) -> datetime:
@@ -36,6 +49,10 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
step_pending=document.get("step_pending", False),
monitoring_count=document.get("monitoring_count", 0),
last_monitoring=document.get("last_monitoring"),
# Old persisted documents without a policy still use three-signal detection.
monitoring_config=({"decision_policy": "all_signals", **document["monitoring_config"]}
if document.get("monitoring_config") else None),
monitoring_calibration=document.get("monitoring_calibration"),
)
@@ -105,7 +122,7 @@ class ScadaReplayService:
await self.ensure_storage()
async with self.repo.control():
document = await self.repo.get(replay_id)
key, body = str(command.request_id), command.model_dump(mode="json")
key, body = str(command.request_id), command.model_dump(mode="json", exclude_none=True)
receipt = document.setdefault("receipts", {}).get(key)
if receipt:
if receipt != body:
@@ -114,9 +131,13 @@ class ScadaReplayService:
if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新，请刷新后重试")
action, status = command.action, document["status"]
if document.get("step_pending") and action == "step":
if (action in {"step", "monitor", "retry"} and document.get("warmed")
and weighted_diagnosis(document)
and current_calibration(document, parse_time(document["start_time"])) is None):
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
if document.get("step_pending") and action in {"step", "monitor"}:
raise HTTPException(409, "当前时间步正在处理，请完成后继续")
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
if action in {"step", "monitor", "retry"} and await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有另一条活动回放")
if action == "step" and status in {"ready", "paused"}:
interval = document.get("monitoring_interval_seconds")
@@ -126,8 +147,31 @@ class ScadaReplayService:
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
document["step_start"] = document["virtual_now"]
document["step_anomaly_signals"] = []
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
document["status"], document["step_pending"] = "paused", True
elif action == "monitor" and status in {"ready", "paused", "completed"}:
target = command.at_time
if target is None:
raise HTTPException(422, "选定时刻监测需要提供 at_time")
start = parse_time(document["start_time"])
current = parse_time(document["virtual_now"])
if not start <= target <= parse_time(document["end_time"]):
raise HTTPException(422, "监测时刻必须在本轮演示区间内")
if target < current:
raise HTTPException(409, "监测历史时刻需要新建一轮回放，避免使用后续状态")
interval = document.get("monitoring_interval_seconds")
if interval is None:
raise HTTPException(409, "本轮未固定监测周期，请停止后新建演示")
previous = document.get("last_monitoring") or {}
# Reusing an immutable result must not count as another normal
# period, resolve an incident, or create another investigation.
if not previous.get("period_end") or parse_time(previous["period_end"]) != target:
document["step_start"] = (current if target > current
else max(start, target - timedelta(seconds=interval))).isoformat()
document["step_target"] = target.isoformat()
document["step_anomaly_signals"] = []
document["status"], document["step_pending"] = "paused", True
elif action == "stop" and status not in {"completed", "stopped"}:
document["status"], document["step_pending"] = "stopped", False
document.pop("step_target", None)
@@ -161,11 +205,16 @@ class ScadaReplayService:
# Failure never discards the prepared batch; retry resumes it.
document = await self.repo.get(replay_id)
document["status"], document["step_pending"] = "failed", False
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变，需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
)
if isinstance(exc, HTTPException):
document["error"] = str(exc.detail)
elif isinstance(exc, BaselineBudgetExceeded):
document["error"] = str(exc)
else:
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变，需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
)
async with self.repo.connection.transaction():
await self.repo.save(document)
raise
@@ -174,23 +223,9 @@ class ScadaReplayService:
config = ScadaReplayCreate.model_validate(document["config"])
replay_id = document["replay_id"]
if not document.get("warmed"):
import math
window = max(rule.window_samples for rule in config.rules)
async with self.timescale.transaction():
warmup = await ScadaRepository.get_warmup_values(
self.timescale, config.device_ids, config.start_time, window,
)
for row in warmup:
value = row["monitored_value"]
bounds = config.quality_ranges.get(row["device_id"])
if value is not None and math.isfinite(value) and not (bounds and (
bounds.lower is not None and value < bounds.lower
or bounds.upper is not None and value > bounds.upper
)):
document["histories"].setdefault(row["device_id"], []).append(value)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.save(document)
await self._freeze_baseline(document, config)
if weighted_diagnosis(document) and current_calibration(document, config.start_time) is None:
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
batch = await self.repo.pending_batch(replay_id)
if batch is None:
step = document.get("step_pending", False)
@@ -252,6 +287,15 @@ class ScadaReplayService:
target = result.get("step_target")
was_step = bool(target)
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
if was_step:
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
if diagnostic_signal(result, item)}
for event in changes:
if event["change"] == "raised" and diagnostic_signal(result, event):
signals[event["event_id"]] = {key: event[key] for key in (
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
) if key in event}
result["step_anomaly_signals"] = list(signals.values())
monitoring = None
if was_step and not result["step_pending"]:
period_start = parse_time(result.get("step_start", result["start_time"]))
@@ -261,7 +305,7 @@ class ScadaReplayService:
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
))
previous = await self.repo.monitoring_outputs(
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
replay_id, role_ids, period_end - timedelta(days=15),
period_end, batch["sequence"],
) if role_ids else []
from app.mcp.analysis_service import bounded_calculation
@@ -273,11 +317,25 @@ class ScadaReplayService:
period_end=period_end,
batch_sequence=batch["sequence"],
)
result["last_monitoring"] = monitoring
# Any configured measurement rule can enter diagnosis, including
# indicators outside the pressure/outlet roles used for burst scoring.
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
if diagnostic_signal(result, item)}
for state in result.get("rule_states", {}).values():
event = state.get("event")
if event and diagnostic_signal(result, event):
signals[event["event_id"]] = {key: event[key] for key in (
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
) if key in event}
monitoring["anomaly_signals"] = list(signals.values())
if signals and monitoring["status"] == "normal":
monitoring["status"] = "attention"
monitoring["detail"] = "本周期曾检出测点异常，进入异常调查与爆管诊断"
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]:
result.pop("step_target", None)
result.pop("step_start", None)
result.pop("step_anomaly_signals", None)
if batch["refs"]:
last = batch["refs"][-1]
result["cursor"] = [last["time"], last["device_id"]]
@@ -295,6 +353,80 @@ class ScadaReplayService:
await self.repo.save_batch(replay_id, batch)
await self.repo.save(result)
async def _freeze_baseline(self, document, config):
"""Stage bounded pages, then atomically publish one causal source snapshot."""
if document.get("warmed"):
return
start = config.start_time - timedelta(days=15)
batch_count, row_count, byte_count, after = 0, 0, 0, None
counts = {device_id: 0 for device_id in config.device_ids}
valid_counts = dict(counts)
histories = {device_id: [] for device_id in config.device_ids}
window = max(rule.window_samples for rule in config.rules)
calibration = None
if (document.get("monitoring_config") or {}).get("decision_policy") == "weighted":
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder
calibration = MonitoringCalibrationBuilder(document, config)
async with self.repo.connection.transaction():
await self.repo.clear_unpublished_baseline(document["replay_id"])
# Keep only the current page in memory. Each BizDB write commits before
# the next source read; clean_prepared baseline pages remain invisible.
async with self.timescale.transaction():
await ScadaRepository.begin_replay_baseline_snapshot(self.timescale)
while True:
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, start, config.start_time,
config.start_time, min(BASELINE_PAGE_SIZE, BASELINE_MAX_ROWS - row_count + 1), after,
)
if not rows:
break
if row_count + len(rows) > BASELINE_MAX_ROWS:
raise BaselineBudgetExceeded(
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_ROWS:,} 条准备预算，"
"尚未发布；请减少回放设备或核查历史采样密度后重试"
)
outputs = prepare_cleaning(rows, config)
if calibration is not None:
calibration.add(outputs)
batch = {
"sequence": -batch_count - 1, "stage": "clean_prepared", "kind": "baseline",
"at": outputs[-1]["time"], "outputs": outputs, "events": [],
}
batch_bytes = len(json.dumps(batch, ensure_ascii=False, allow_nan=False,
separators=(",", ":")).encode("utf-8"))
if byte_count + batch_bytes > BASELINE_MAX_BYTES:
raise BaselineBudgetExceeded(
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_BYTES // (1024 * 1024)} MiB 准备预算，"
"尚未发布；请减少回放设备或核查历史采样密度后重试"
)
async with self.repo.connection.transaction():
await self.repo.save_batch(document["replay_id"], batch)
batch_count += 1
row_count += len(outputs)
byte_count += batch_bytes
for row in outputs:
device_id = row["device_id"]
counts[device_id] += 1
if row["quality"] == "valid":
valid_counts[device_id] += 1
histories[device_id].append(row["cleaned_value"])
histories[device_id] = histories[device_id][-window:]
last = rows[-1]
after = (last["time"], last["device_id"])
document["baseline"] = {
"start_time": start.isoformat(), "end_time": config.start_time.isoformat(),
"end_exclusive": True, "sample_counts": counts, "valid_sample_counts": valid_counts,
"row_count": row_count, "batch_count": batch_count, "byte_count": byte_count,
}
document["histories"] = histories
if calibration is not None:
from app.mcp.analysis_service import bounded_calculation
document["monitoring_calibration"] = await bounded_calculation(calibration.fit)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.publish_baseline(document["replay_id"])
await self.repo.save(document)
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
@@ -309,9 +441,20 @@ class ScadaReplayService:
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context))
for event in changes:
# Period diagnosis owns measurement notifications during manual steps.
# Retain rule events as evidence; quality alerts remain independent.
if (not create_tasks and event.get("category") != "data_quality"
and (event["change"] == "raised" or event.get("notification_suppressed"))):
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
event["notification_suppressed"] = True
document["rule_states"][event.get("state_key", event["rule_id"])]["event"].update(
batch_sequence=batch["sequence"], notification_suppressed=True,
)
continue
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["batch_sequence"] = batch["sequence"]
await alerts.push(TaskAlertEvent(
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
@@ -345,10 +488,30 @@ class ScadaReplayService:
)
task = await tasks.create(command)
event["task_id"] = str(task.task_id)
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["task_id"] = event["task_id"]
async def _create_monitoring_task(self, document, monitoring, batch):
if monitoring["status"] != "suspected_burst":
from app.services.scada_monitoring_incidents import attach_incident, update_incident
incident, action = update_incident(document, monitoring, batch["sequence"])
if incident is None:
return
if incident.get("task_id"):
if monitoring["status"] == "suspected_burst" and not incident.get("burst_diagnosed"):
from uuid import UUID
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
tasks = OperationalTaskService(
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context),
)
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
"batch_sequence": batch["sequence"],
}, uuid5(NAMESPACE_URL, incident["incident_id"] + ":burst-evidence"))
incident["burst_diagnosed"] = True
if action == "resolved" and incident.get("alert"):
await self._publish_monitoring_alert(document, incident, "resolved")
attach_incident(monitoring, incident, action)
return
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
@@ -358,29 +521,44 @@ class ScadaReplayService:
self.context,
permissions_for_context(self.context),
)
external = (
f"scada_replay:{document['replay_id']}:monitor:"
f"{monitoring['period_end']}:{monitoring['rule_version']}"
)
external = incident["external_id"]
pressure = monitoring["pressure"]
flow = monitoring["outlet_flow"]
detector = monitoring["burst_detection"]
score = detector.get("target_score")
score_text = f"{score:.4f}" if score is not None else "数据不足"
summary = (
f"步长监测疑似爆管：爆管侦测得分 {detector['target_score']:.4f}；"
f"压力下降测点 {len(pressure['hit_device_ids'])} 个；"
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
f"SCADA 数据诊断发现指标异常，进入异常调查与爆管诊断。"
f"爆管侦测得分 {score_text}；"
f"压力下降测点 {len(pressure.get('hit_device_ids', []))} 个；"
f"出厂边界流量增加测点 {len(flow.get('hit_device_ids', []))} 个。"
"任一指标异常即启动调查，加权分仅用于评估爆管嫌疑，不能据此提前确诊。"
)
if monitoring.get("anomaly_signals"):
signals = monitoring["anomaly_signals"]
devices = {device for item in signals for device in item.get("device_ids", [])}
summary += f" 单项异常 {len(signals)} 项，涉及 {len(devices)} 个测点；明细通过冻结 SCADA 查询核验。"
if monitoring.get("weighted"):
weighted = monitoring["weighted"]
weights = weighted["weights"]
summary += (
f" 综合评分 {weighted['score']:.2f}，爆管嫌疑门槛 {weighted['threshold']:.2f}"
f"（侦测 {weights['burst_detection']:.0%}、压力 {weights['pressure']:.0%}、"
f"流量 {weights['outlet_flow']:.0%}）。"
)
if detector.get("score_threshold") is not None:
summary += f" 侦测动态阈值 {detector['score_threshold']:.4f}。"
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external),
task_type="event_investigation",
workflow="burst_report",
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
workflow="scada_anomaly",
title=f"SCADA 异常调查 · {monitoring['period_end']}",
summary=summary,
source=TaskSource(
kind="external_system",
external_id=external,
external_revision=1,
event_kind="scada_burst_suspected",
event_kind="scada_anomaly",
replay_context={
"replay_id": document["replay_id"],
"knowledge_cutoff": monitoring["knowledge_cutoff"],
@@ -390,8 +568,33 @@ class ScadaReplayService:
trigger=TaskTrigger(
kind="event",
occurred_at=monitoring["period_end"],
detail="主动步长监测的多项指标相互支持，进入爆管调查；尚未现场确诊",
detail="SCADA 任一指标异常启动调查，核验数据后进行爆管诊断；异常不等于已确诊爆管",
),
)
task = await tasks.create(command)
monitoring["task_id"] = str(task.task_id)
incident["task_id"] = str(task.task_id)
incident["burst_diagnosed"] = monitoring["status"] == "suspected_burst"
incident["alert"] = {
"alert_id": incident["incident_id"],
"producer_id": "scada-replay:" + document["replay_id"],
"occurred_at": incident["started_at"],
"condition_id": f"operational-task:{task.task_id}",
"source": {"kind": "scada_replay", "replay_id": document["replay_id"],
"rule_id": "scada_diagnosis", "batch_sequence": batch["sequence"]},
"title": "SCADA 指标异常", "description": "已启动调查，详细诊断见调查报告。", "severity": "warning",
}
await self._publish_monitoring_alert(document, incident, "raised")
attach_incident(monitoring, incident, action)
async def _publish_monitoring_alert(self, document, incident, change):
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
alerts = TaskAlertService(
TaskAlertRepository(self.repo.connection, self.context.project_id), self.context,
internal_producer_id="scada-replay:" + document["replay_id"],
)
await alerts.push(TaskAlertEvent(
**incident["alert"], event_type=change,
event_id=uuid5(NAMESPACE_URL, incident["incident_id"] + ":" + change),
))