601 lines
35 KiB
Python
601 lines
35 KiB
Python
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta, timezone
|
||
import json
|
||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||
|
||
from fastapi import HTTPException
|
||
from app.auth.permissions import permissions_for_context
|
||
|
||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
|
||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||
from app.services.content_digest import digest
|
||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||
from app.services.scada_normal_profile import current_calibration, diagnostic_signal, weighted_diagnosis
|
||
|
||
|
||
# 500 devices at 15-minute sampling need 720,000 rows for fifteen days.
|
||
# Leave headroom without allowing dense sources to make preparation unbounded.
|
||
BASELINE_PAGE_SIZE = 2000
|
||
BASELINE_MAX_ROWS = 1_000_000
|
||
BASELINE_MAX_BYTES = 512 * 1024 * 1024
|
||
|
||
|
||
class BaselineBudgetExceeded(RuntimeError):
|
||
"""The unpublished baseline exceeded its fixed preparation budget."""
|
||
|
||
|
||
def parse_time(value: str) -> datetime:
|
||
return datetime.fromisoformat(value)
|
||
|
||
|
||
def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||
return ScadaReplaySnapshot(
|
||
**{key: document[key] for key in (
|
||
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
|
||
"virtual_now", "created_at", "updated_at",
|
||
)},
|
||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||
devices=document["devices"], rules=document["config"]["rules"],
|
||
latest=list(document.get("latest", {}).values()),
|
||
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
|
||
error=document.get("error"),
|
||
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
|
||
step_pending=document.get("step_pending", False),
|
||
monitoring_count=document.get("monitoring_count", 0),
|
||
last_monitoring=document.get("last_monitoring"),
|
||
# Old persisted documents without a policy still use three-signal detection.
|
||
monitoring_config=({"decision_policy": "all_signals", **document["monitoring_config"]}
|
||
if document.get("monitoring_config") else None),
|
||
monitoring_calibration=document.get("monitoring_calibration"),
|
||
)
|
||
|
||
|
||
class ScadaReplayService:
|
||
def __init__(self, repository, context, timescale=None, *, clock=None):
|
||
self.repo, self.context, self.timescale = repository, context, timescale
|
||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||
|
||
async def ensure_storage(self):
|
||
if not await self.repo.available():
|
||
raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql")
|
||
|
||
async def create(
|
||
self,
|
||
command: ScadaReplayCreate,
|
||
*,
|
||
monitoring_interval_seconds: int | None,
|
||
monitoring_config: ScadaMonitoringSettings | None = None,
|
||
):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
previous = await self.repo.by_request(command.request_id)
|
||
config = command.model_dump(mode="json")
|
||
if previous:
|
||
if previous["config"] != config:
|
||
raise HTTPException(409, "创建请求号已用于不同回放配置")
|
||
return snapshot(previous)
|
||
if monitoring_interval_seconds is None:
|
||
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示")
|
||
replay_id = str(uuid4())
|
||
if await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建")
|
||
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
|
||
if not set(command.device_ids) <= metadata.keys():
|
||
raise HTTPException(422, "设备不属于当前项目")
|
||
for rule in command.rules:
|
||
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
|
||
raise HTTPException(422, "规则单位与设备单位不一致，请分组配置")
|
||
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
|
||
raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数")
|
||
if monitoring_config:
|
||
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
|
||
if not role_ids <= metadata.keys():
|
||
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
|
||
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
|
||
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
|
||
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
|
||
for device in monitoring_config.outlet_flow_device_ids):
|
||
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
|
||
now = self.clock().isoformat()
|
||
document = {
|
||
"replay_id": replay_id, "project_id": str(self.context.project_id),
|
||
"created_by": str(self.context.user_id), "config": config,
|
||
"revision": 1, "status": "ready", "start_time": config["start_time"],
|
||
"end_time": config["end_time"], "virtual_now": config["start_time"],
|
||
"created_at": now, "updated_at": now,
|
||
"monitoring_interval_seconds": monitoring_interval_seconds,
|
||
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
|
||
"devices": [metadata[device] for device in command.device_ids],
|
||
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
|
||
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
|
||
}
|
||
await self.repo.insert(document)
|
||
return snapshot(document)
|
||
|
||
async def command(self, replay_id, command: ScadaReplayCommand):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
document = await self.repo.get(replay_id)
|
||
key, body = str(command.request_id), command.model_dump(mode="json", exclude_none=True)
|
||
receipt = document.setdefault("receipts", {}).get(key)
|
||
if receipt:
|
||
if receipt != body:
|
||
raise HTTPException(409, "命令请求号已用于不同内容")
|
||
return snapshot(document)
|
||
if command.expected_revision != document["revision"]:
|
||
raise HTTPException(409, "回放控制状态已更新，请刷新后重试")
|
||
action, status = command.action, document["status"]
|
||
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
||
and weighted_diagnosis(document)
|
||
and current_calibration(document, parse_time(document["start_time"])) is None):
|
||
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
|
||
if document.get("step_pending") and action in {"step", "monitor"}:
|
||
raise HTTPException(409, "当前时间步正在处理，请完成后继续")
|
||
if action in {"step", "monitor", "retry"} and await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||
if action == "step" and status in {"ready", "paused"}:
|
||
interval = document.get("monitoring_interval_seconds")
|
||
if interval is None:
|
||
raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示")
|
||
start = parse_time(document["start_time"])
|
||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||
document["step_start"] = document["virtual_now"]
|
||
document["step_anomaly_signals"] = []
|
||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||
document["status"], document["step_pending"] = "paused", True
|
||
elif action == "monitor" and status in {"ready", "paused", "completed"}:
|
||
target = command.at_time
|
||
if target is None:
|
||
raise HTTPException(422, "选定时刻监测需要提供 at_time")
|
||
start = parse_time(document["start_time"])
|
||
current = parse_time(document["virtual_now"])
|
||
if not start <= target <= parse_time(document["end_time"]):
|
||
raise HTTPException(422, "监测时刻必须在本轮演示区间内")
|
||
if target < current:
|
||
raise HTTPException(409, "监测历史时刻需要新建一轮回放，避免使用后续状态")
|
||
interval = document.get("monitoring_interval_seconds")
|
||
if interval is None:
|
||
raise HTTPException(409, "本轮未固定监测周期，请停止后新建演示")
|
||
previous = document.get("last_monitoring") or {}
|
||
# Reusing an immutable result must not count as another normal
|
||
# period, resolve an incident, or create another investigation.
|
||
if not previous.get("period_end") or parse_time(previous["period_end"]) != target:
|
||
document["step_start"] = (current if target > current
|
||
else max(start, target - timedelta(seconds=interval))).isoformat()
|
||
document["step_target"] = target.isoformat()
|
||
document["step_anomaly_signals"] = []
|
||
document["status"], document["step_pending"] = "paused", True
|
||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||
document["status"], document["step_pending"] = "stopped", False
|
||
document.pop("step_target", None)
|
||
document.pop("step_start", None)
|
||
elif action == "retry" and status == "failed":
|
||
if "SOURCE_CHANGED" in (document.get("error") or ""):
|
||
raise HTTPException(409, "原始数据发生变化，需要停止并新建回放")
|
||
document["status"], document["error"] = "paused", None
|
||
document["step_pending"] = bool(document.get("step_target"))
|
||
else:
|
||
raise HTTPException(409, "当前回放状态不支持此操作")
|
||
document["revision"] += 1
|
||
document["receipts"][key] = body
|
||
if len(document["receipts"]) > 256:
|
||
del document["receipts"][next(iter(document["receipts"]))]
|
||
await self.repo.save(document)
|
||
return snapshot(document)
|
||
|
||
async def process_pending_step(self, replay_id):
|
||
await self.ensure_storage()
|
||
async with self.repo.processing() as owned:
|
||
if not owned:
|
||
raise HTTPException(409, "当前时间步正在由另一请求处理")
|
||
while True:
|
||
document = await self.repo.get(replay_id)
|
||
if not document.get("step_pending"):
|
||
return snapshot(document)
|
||
try:
|
||
await self._advance(document)
|
||
except Exception as exc:
|
||
# Failure never discards the prepared batch; retry resumes it.
|
||
document = await self.repo.get(replay_id)
|
||
document["status"], document["step_pending"] = "failed", False
|
||
if isinstance(exc, HTTPException):
|
||
document["error"] = str(exc.detail)
|
||
elif isinstance(exc, BaselineBudgetExceeded):
|
||
document["error"] = str(exc)
|
||
else:
|
||
document["error"] = (
|
||
"SOURCE_CHANGED: 原始观测已改变，需要新建回放"
|
||
if "SOURCE_CHANGED" in str(exc)
|
||
else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
|
||
)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
raise
|
||
|
||
async def _advance(self, document):
|
||
config = ScadaReplayCreate.model_validate(document["config"])
|
||
replay_id = document["replay_id"]
|
||
if not document.get("warmed"):
|
||
await self._freeze_baseline(document, config)
|
||
if weighted_diagnosis(document) and current_calibration(document, config.start_time) is None:
|
||
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
|
||
batch = await self.repo.pending_batch(replay_id)
|
||
if batch is None:
|
||
step = document.get("step_pending", False)
|
||
virtual = parse_time(document["virtual_now"])
|
||
through = parse_time(document["step_target"]) if step else virtual
|
||
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
|
||
if document.get("cursor") else None)
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, config.start_time, config.end_time,
|
||
through, 501, after, frame=True,
|
||
)
|
||
if len(rows) > 500:
|
||
raise ValueError("SCADA frame exceeds selected device bound")
|
||
at = rows[0]["time"] if rows else through
|
||
if step:
|
||
virtual = through
|
||
document["virtual_now"] = virtual.isoformat()
|
||
if not rows and document.get("processed_until") == at.isoformat() and not step:
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
return
|
||
sequence = document["batch_sequence"] + 1
|
||
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
|
||
"refs": refs, "input_digest": digest(refs), "events": []}
|
||
async with self.repo.connection.transaction():
|
||
document["batch_sequence"] = sequence
|
||
document["delivered_until"] = at.isoformat()
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(document)
|
||
else:
|
||
rows = None
|
||
if batch["stage"] == "selected":
|
||
if rows is None:
|
||
# Only crash recovery reloads the selected raw frame.
|
||
at = parse_time(batch["at"])
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, at, config.end_time, at, 501,
|
||
) if batch["refs"] else []
|
||
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
if digest(actual) != batch["input_digest"]:
|
||
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
|
||
batch["outputs"] = prepare_cleaning(rows, config)
|
||
batch["stage"] = "clean_prepared"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "clean_prepared":
|
||
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
|
||
batch["stage"] = "cleaned"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "cleaned":
|
||
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
|
||
result["processed_until"] = batch["at"]
|
||
target = result.get("step_target")
|
||
was_step = bool(target)
|
||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||
if was_step:
|
||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||
if diagnostic_signal(result, item)}
|
||
for event in changes:
|
||
if event["change"] == "raised" and diagnostic_signal(result, event):
|
||
signals[event["event_id"]] = {key: event[key] for key in (
|
||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||
) if key in event}
|
||
result["step_anomaly_signals"] = list(signals.values())
|
||
monitoring = None
|
||
if was_step and not result["step_pending"]:
|
||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||
period_end = parse_time(target)
|
||
role = result.get("monitoring_config") or {}
|
||
role_ids = list(dict.fromkeys(
|
||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||
))
|
||
previous = await self.repo.monitoring_outputs(
|
||
replay_id, role_ids, period_end - timedelta(days=15),
|
||
period_end, batch["sequence"],
|
||
) if role_ids else []
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
monitoring = await bounded_calculation(
|
||
evaluate_monitoring_period,
|
||
result,
|
||
previous + batch["outputs"],
|
||
period_start=period_start,
|
||
period_end=period_end,
|
||
batch_sequence=batch["sequence"],
|
||
)
|
||
# Any configured measurement rule can enter diagnosis, including
|
||
# indicators outside the pressure/outlet roles used for burst scoring.
|
||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||
if diagnostic_signal(result, item)}
|
||
for state in result.get("rule_states", {}).values():
|
||
event = state.get("event")
|
||
if event and diagnostic_signal(result, event):
|
||
signals[event["event_id"]] = {key: event[key] for key in (
|
||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||
) if key in event}
|
||
monitoring["anomaly_signals"] = list(signals.values())
|
||
if signals and monitoring["status"] == "normal":
|
||
monitoring["status"] = "attention"
|
||
monitoring["detail"] = "本周期曾检出测点异常，进入异常调查与爆管诊断"
|
||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||
if not result["step_pending"]:
|
||
result.pop("step_target", None)
|
||
result.pop("step_start", None)
|
||
result.pop("step_anomaly_signals", None)
|
||
if batch["refs"]:
|
||
last = batch["refs"][-1]
|
||
result["cursor"] = [last["time"], last["device_id"]]
|
||
if parse_time(batch["at"]) >= config.end_time:
|
||
result["status"] = "completed"
|
||
batch["stage"], batch["events"] = "detected", changes
|
||
if monitoring is not None:
|
||
batch["monitoring"] = monitoring
|
||
async with self.repo.connection.transaction():
|
||
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
|
||
if monitoring is not None:
|
||
await self._create_monitoring_task(result, monitoring, batch)
|
||
batch["monitoring"] = monitoring
|
||
result["last_monitoring"] = monitoring
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(result)
|
||
|
||
async def _freeze_baseline(self, document, config):
|
||
"""Stage bounded pages, then atomically publish one causal source snapshot."""
|
||
if document.get("warmed"):
|
||
return
|
||
start = config.start_time - timedelta(days=15)
|
||
batch_count, row_count, byte_count, after = 0, 0, 0, None
|
||
counts = {device_id: 0 for device_id in config.device_ids}
|
||
valid_counts = dict(counts)
|
||
histories = {device_id: [] for device_id in config.device_ids}
|
||
window = max(rule.window_samples for rule in config.rules)
|
||
calibration = None
|
||
if (document.get("monitoring_config") or {}).get("decision_policy") == "weighted":
|
||
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder
|
||
calibration = MonitoringCalibrationBuilder(document, config)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.clear_unpublished_baseline(document["replay_id"])
|
||
# Keep only the current page in memory. Each BizDB write commits before
|
||
# the next source read; clean_prepared baseline pages remain invisible.
|
||
async with self.timescale.transaction():
|
||
await ScadaRepository.begin_replay_baseline_snapshot(self.timescale)
|
||
while True:
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, start, config.start_time,
|
||
config.start_time, min(BASELINE_PAGE_SIZE, BASELINE_MAX_ROWS - row_count + 1), after,
|
||
)
|
||
if not rows:
|
||
break
|
||
if row_count + len(rows) > BASELINE_MAX_ROWS:
|
||
raise BaselineBudgetExceeded(
|
||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_ROWS:,} 条准备预算，"
|
||
"尚未发布；请减少回放设备或核查历史采样密度后重试"
|
||
)
|
||
outputs = prepare_cleaning(rows, config)
|
||
if calibration is not None:
|
||
calibration.add(outputs)
|
||
batch = {
|
||
"sequence": -batch_count - 1, "stage": "clean_prepared", "kind": "baseline",
|
||
"at": outputs[-1]["time"], "outputs": outputs, "events": [],
|
||
}
|
||
batch_bytes = len(json.dumps(batch, ensure_ascii=False, allow_nan=False,
|
||
separators=(",", ":")).encode("utf-8"))
|
||
if byte_count + batch_bytes > BASELINE_MAX_BYTES:
|
||
raise BaselineBudgetExceeded(
|
||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_BYTES // (1024 * 1024)} MiB 准备预算，"
|
||
"尚未发布；请减少回放设备或核查历史采样密度后重试"
|
||
)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(document["replay_id"], batch)
|
||
batch_count += 1
|
||
row_count += len(outputs)
|
||
byte_count += batch_bytes
|
||
for row in outputs:
|
||
device_id = row["device_id"]
|
||
counts[device_id] += 1
|
||
if row["quality"] == "valid":
|
||
valid_counts[device_id] += 1
|
||
histories[device_id].append(row["cleaned_value"])
|
||
histories[device_id] = histories[device_id][-window:]
|
||
last = rows[-1]
|
||
after = (last["time"], last["device_id"])
|
||
document["baseline"] = {
|
||
"start_time": start.isoformat(), "end_time": config.start_time.isoformat(),
|
||
"end_exclusive": True, "sample_counts": counts, "valid_sample_counts": valid_counts,
|
||
"row_count": row_count, "batch_count": batch_count, "byte_count": byte_count,
|
||
}
|
||
document["histories"] = histories
|
||
if calibration is not None:
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
document["monitoring_calibration"] = await bounded_calculation(calibration.fit)
|
||
document["warmed"] = True
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.publish_baseline(document["replay_id"])
|
||
await self.repo.save(document)
|
||
|
||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||
from app.services.task_alerts import TaskAlertService
|
||
from uuid import UUID
|
||
producer = "scada-replay:" + document["replay_id"]
|
||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||
self.context, internal_producer_id=producer)
|
||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context, permissions_for_context(self.context))
|
||
for event in changes:
|
||
# Period diagnosis owns measurement notifications during manual steps.
|
||
# Retain rule events as evidence; quality alerts remain independent.
|
||
if (not create_tasks and event.get("category") != "data_quality"
|
||
and (event["change"] == "raised" or event.get("notification_suppressed"))):
|
||
if event["change"] == "raised":
|
||
event["batch_sequence"] = batch["sequence"]
|
||
event["notification_suppressed"] = True
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"].update(
|
||
batch_sequence=batch["sequence"], notification_suppressed=True,
|
||
)
|
||
continue
|
||
if event["change"] == "raised":
|
||
event["batch_sequence"] = batch["sequence"]
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["batch_sequence"] = batch["sequence"]
|
||
await alerts.push(TaskAlertEvent(
|
||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||
occurred_at=event["occurred_at"], condition_id=None,
|
||
source={"kind": "scada_replay", "replay_id": document["replay_id"],
|
||
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
|
||
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
|
||
))
|
||
if event["change"] != "raised":
|
||
if event.get("task_id"):
|
||
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
|
||
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
|
||
"batch_sequence": batch["sequence"],
|
||
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
|
||
continue
|
||
if event.get("category") == "data_quality":
|
||
continue
|
||
if not create_tasks:
|
||
continue
|
||
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
|
||
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
|
||
summary=event["detail"], source=TaskSource(
|
||
kind="external_system", external_id=external, external_revision=1,
|
||
event_kind="scada_anomaly",
|
||
replay_context={"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
|
||
),
|
||
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"),
|
||
)
|
||
task = await tasks.create(command)
|
||
event["task_id"] = str(task.task_id)
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["task_id"] = event["task_id"]
|
||
|
||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||
from app.services.scada_monitoring_incidents import attach_incident, update_incident
|
||
incident, action = update_incident(document, monitoring, batch["sequence"])
|
||
if incident is None:
|
||
return
|
||
if incident.get("task_id"):
|
||
if monitoring["status"] == "suspected_burst" and not incident.get("burst_diagnosed"):
|
||
from uuid import UUID
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
tasks = OperationalTaskService(
|
||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context, permissions_for_context(self.context),
|
||
)
|
||
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
|
||
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||
"batch_sequence": batch["sequence"],
|
||
}, uuid5(NAMESPACE_URL, incident["incident_id"] + ":burst-evidence"))
|
||
incident["burst_diagnosed"] = True
|
||
if action == "resolved" and incident.get("alert"):
|
||
await self._publish_monitoring_alert(document, incident, "resolved")
|
||
attach_incident(monitoring, incident, action)
|
||
return
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
tasks = OperationalTaskService(
|
||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context,
|
||
permissions_for_context(self.context),
|
||
)
|
||
external = incident["external_id"]
|
||
pressure = monitoring["pressure"]
|
||
flow = monitoring["outlet_flow"]
|
||
detector = monitoring["burst_detection"]
|
||
score = detector.get("target_score")
|
||
score_text = f"{score:.4f}" if score is not None else "数据不足"
|
||
summary = (
|
||
f"SCADA 数据诊断发现指标异常，进入异常调查与爆管诊断。"
|
||
f"爆管侦测得分 {score_text}；"
|
||
f"压力下降测点 {len(pressure.get('hit_device_ids', []))} 个；"
|
||
f"出厂边界流量增加测点 {len(flow.get('hit_device_ids', []))} 个。"
|
||
"任一指标异常即启动调查，加权分用于评估爆管嫌疑；达到门槛后继续候选定位、隔离与影响分析。"
|
||
)
|
||
if monitoring.get("anomaly_signals"):
|
||
signals = monitoring["anomaly_signals"]
|
||
devices = {device for item in signals for device in item.get("device_ids", [])}
|
||
summary += f" 单项异常 {len(signals)} 项，涉及 {len(devices)} 个测点；明细通过冻结 SCADA 查询核验。"
|
||
if monitoring.get("weighted"):
|
||
weighted = monitoring["weighted"]
|
||
weights = weighted["weights"]
|
||
summary += (
|
||
f" 综合评分 {weighted['score']:.2f}，爆管嫌疑门槛 {weighted['threshold']:.2f}"
|
||
f"（侦测 {weights['burst_detection']:.0%}、压力 {weights['pressure']:.0%}、"
|
||
f"流量 {weights['outlet_flow']:.0%}）。"
|
||
)
|
||
if detector.get("score_threshold") is not None:
|
||
summary += f" 侦测动态阈值 {detector['score_threshold']:.4f}。"
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external),
|
||
task_type="event_investigation",
|
||
workflow="scada_anomaly",
|
||
title=f"SCADA 异常调查 · {monitoring['period_end']}",
|
||
summary=summary,
|
||
source=TaskSource(
|
||
kind="external_system",
|
||
external_id=external,
|
||
external_revision=1,
|
||
event_kind="scada_anomaly",
|
||
replay_context={
|
||
"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||
"batch_sequence": batch["sequence"],
|
||
},
|
||
),
|
||
trigger=TaskTrigger(
|
||
kind="event",
|
||
occurred_at=monitoring["period_end"],
|
||
detail="SCADA 任一指标异常启动调查，核验数据后进行爆管诊断；异常不等于已确诊爆管",
|
||
),
|
||
)
|
||
task = await tasks.create(command)
|
||
incident["task_id"] = str(task.task_id)
|
||
incident["burst_diagnosed"] = monitoring["status"] == "suspected_burst"
|
||
incident["alert"] = {
|
||
"alert_id": incident["incident_id"],
|
||
"producer_id": "scada-replay:" + document["replay_id"],
|
||
"occurred_at": incident["started_at"],
|
||
"condition_id": f"operational-task:{task.task_id}",
|
||
"source": {"kind": "scada_replay", "replay_id": document["replay_id"],
|
||
"rule_id": "scada_diagnosis", "batch_sequence": batch["sequence"]},
|
||
"title": "SCADA 指标异常", "description": "已启动调查，详细诊断见调查报告。", "severity": "warning",
|
||
}
|
||
await self._publish_monitoring_alert(document, incident, "raised")
|
||
attach_incident(monitoring, incident, action)
|
||
|
||
async def _publish_monitoring_alert(self, document, incident, change):
|
||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||
from app.services.task_alerts import TaskAlertService
|
||
alerts = TaskAlertService(
|
||
TaskAlertRepository(self.repo.connection, self.context.project_id), self.context,
|
||
internal_producer_id="scada-replay:" + document["replay_id"],
|
||
)
|
||
await alerts.push(TaskAlertEvent(
|
||
**incident["alert"], event_type=change,
|
||
event_id=uuid5(NAMESPACE_URL, incident["incident_id"] + ":" + change),
|
||
))
|