jiang
33df25f22b
Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
605 lines
35 KiB
Python
605 lines
35 KiB
Python
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta, timezone
|
||
import json
|
||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||
|
||
from fastapi import HTTPException
|
||
from app.core.config import settings
|
||
from app.auth.permissions import permissions_for_context
|
||
|
||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
|
||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||
from app.services.content_digest import digest
|
||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||
from app.services.scada_normal_profile import current_calibration, diagnostic_signal, weighted_diagnosis
|
||
|
||
|
||
# 500 devices at 15-minute sampling need 720,000 rows for fifteen days.
|
||
# Leave headroom without allowing dense sources to make preparation unbounded.
|
||
BASELINE_PAGE_SIZE = 2000
|
||
BASELINE_MAX_ROWS = 1_000_000
|
||
BASELINE_MAX_BYTES = 512 * 1024 * 1024
|
||
|
||
|
||
class BaselineBudgetExceeded(RuntimeError):
|
||
"""The unpublished baseline exceeded its fixed preparation budget."""
|
||
|
||
|
||
def parse_time(value: str) -> datetime:
|
||
return datetime.fromisoformat(value)
|
||
|
||
|
||
def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||
return ScadaReplaySnapshot(
|
||
**{key: document[key] for key in (
|
||
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
|
||
"virtual_now", "created_at", "updated_at",
|
||
)},
|
||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||
devices=document["devices"], rules=document["config"]["rules"],
|
||
latest=list(document.get("latest", {}).values()),
|
||
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
|
||
error=document.get("error"),
|
||
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
|
||
step_pending=document.get("step_pending", False),
|
||
monitoring_count=document.get("monitoring_count", 0),
|
||
last_monitoring=document.get("last_monitoring"),
|
||
# Old persisted documents without a policy still use three-signal detection.
|
||
monitoring_config=({"decision_policy": "all_signals", **document["monitoring_config"]}
|
||
if document.get("monitoring_config") else None),
|
||
monitoring_calibration=document.get("monitoring_calibration"),
|
||
)
|
||
|
||
|
||
class ScadaReplayService:
|
||
def __init__(self, repository, context, timescale=None, *, clock=None):
|
||
self.repo, self.context, self.timescale = repository, context, timescale
|
||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||
|
||
async def ensure_storage(self):
|
||
if not await self.repo.available():
|
||
raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql")
|
||
|
||
async def create(
|
||
self,
|
||
command: ScadaReplayCreate,
|
||
*,
|
||
monitoring_interval_seconds: int | None,
|
||
monitoring_config: ScadaMonitoringSettings | None = None,
|
||
):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
previous = await self.repo.by_request(command.request_id)
|
||
config = command.model_dump(mode="json")
|
||
if previous:
|
||
if previous["config"] != config:
|
||
raise HTTPException(409, "创建请求号已用于不同回放配置")
|
||
return snapshot(previous)
|
||
if monitoring_interval_seconds is None:
|
||
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示")
|
||
replay_id = str(uuid4())
|
||
if await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建")
|
||
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
|
||
if not set(command.device_ids) <= metadata.keys():
|
||
raise HTTPException(422, "设备不属于当前项目")
|
||
for rule in command.rules:
|
||
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
|
||
raise HTTPException(422, "规则单位与设备单位不一致，请分组配置")
|
||
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
|
||
raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数")
|
||
if monitoring_config:
|
||
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
|
||
if not role_ids <= metadata.keys():
|
||
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
|
||
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
|
||
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
|
||
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
|
||
for device in monitoring_config.outlet_flow_device_ids):
|
||
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
|
||
now = self.clock().isoformat()
|
||
document = {
|
||
"replay_id": replay_id, "project_id": str(self.context.project_id),
|
||
"created_by": str(self.context.user_id), "config": config,
|
||
"revision": 1, "status": "ready", "start_time": config["start_time"],
|
||
"end_time": config["end_time"], "virtual_now": config["start_time"],
|
||
"created_at": now, "updated_at": now,
|
||
"monitoring_interval_seconds": monitoring_interval_seconds,
|
||
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
|
||
"devices": [metadata[device] for device in command.device_ids],
|
||
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
|
||
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
|
||
}
|
||
await self.repo.insert(document)
|
||
return snapshot(document)
|
||
|
||
async def command(self, replay_id, command: ScadaReplayCommand):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
document = await self.repo.get(replay_id)
|
||
key, body = str(command.request_id), command.model_dump(mode="json", exclude_none=True)
|
||
receipt = document.setdefault("receipts", {}).get(key)
|
||
if receipt:
|
||
if receipt != body:
|
||
raise HTTPException(409, "命令请求号已用于不同内容")
|
||
return snapshot(document)
|
||
if command.expected_revision != document["revision"]:
|
||
raise HTTPException(409, "回放控制状态已更新，请刷新后重试")
|
||
action, status = command.action, document["status"]
|
||
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
||
and weighted_diagnosis(document)
|
||
and current_calibration(document, parse_time(document["start_time"])) is None):
|
||
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
|
||
if document.get("step_pending") and action in {"step", "monitor"}:
|
||
raise HTTPException(409, "当前时间步正在处理，请完成后继续")
|
||
if action in {"step", "monitor", "retry"} and await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||
if action == "step" and status in {"ready", "paused"}:
|
||
interval = document.get("monitoring_interval_seconds")
|
||
if interval is None:
|
||
raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示")
|
||
start = parse_time(document["start_time"])
|
||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||
document["step_start"] = document["virtual_now"]
|
||
document["step_anomaly_signals"] = []
|
||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||
document["status"], document["step_pending"] = "paused", True
|
||
elif action == "monitor" and status in {"ready", "paused", "completed"}:
|
||
target = command.at_time
|
||
if target is None:
|
||
raise HTTPException(422, "选定时刻监测需要提供 at_time")
|
||
start = parse_time(document["start_time"])
|
||
current = parse_time(document["virtual_now"])
|
||
if not start <= target <= parse_time(document["end_time"]):
|
||
raise HTTPException(422, "监测时刻必须在本轮演示区间内")
|
||
if target < current:
|
||
raise HTTPException(409, "监测历史时刻需要新建一轮回放，避免使用后续状态")
|
||
interval = document.get("monitoring_interval_seconds")
|
||
if interval is None:
|
||
raise HTTPException(409, "本轮未固定监测周期，请停止后新建演示")
|
||
previous = document.get("last_monitoring") or {}
|
||
# Reusing an immutable result must not count as another normal
|
||
# period, resolve an incident, or create another investigation.
|
||
if not previous.get("period_end") or parse_time(previous["period_end"]) != target:
|
||
document["step_start"] = (current if target > current
|
||
else max(start, target - timedelta(seconds=interval))).isoformat()
|
||
document["step_target"] = target.isoformat()
|
||
document["step_anomaly_signals"] = []
|
||
document["status"], document["step_pending"] = "paused", True
|
||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||
document["status"], document["step_pending"] = "stopped", False
|
||
document.pop("step_target", None)
|
||
document.pop("step_start", None)
|
||
elif action == "retry" and status == "failed":
|
||
if "SOURCE_CHANGED" in (document.get("error") or ""):
|
||
raise HTTPException(409, "原始数据发生变化，需要停止并新建回放")
|
||
document["status"], document["error"] = "paused", None
|
||
document["step_pending"] = bool(document.get("step_target"))
|
||
else:
|
||
raise HTTPException(409, "当前回放状态不支持此操作")
|
||
document["revision"] += 1
|
||
document["receipts"][key] = body
|
||
if len(document["receipts"]) > 256:
|
||
del document["receipts"][next(iter(document["receipts"]))]
|
||
await self.repo.save(document)
|
||
return snapshot(document)
|
||
|
||
async def process_pending_step(self, replay_id):
|
||
await self.ensure_storage()
|
||
async with self.repo.processing() as owned:
|
||
if not owned:
|
||
raise HTTPException(409, "当前时间步正在由另一请求处理")
|
||
while True:
|
||
document = await self.repo.get(replay_id)
|
||
if not document.get("step_pending"):
|
||
return snapshot(document)
|
||
try:
|
||
await self._advance(document)
|
||
except Exception as exc:
|
||
# Failure never discards the prepared batch; retry resumes it.
|
||
document = await self.repo.get(replay_id)
|
||
document["status"], document["step_pending"] = "failed", False
|
||
if isinstance(exc, HTTPException):
|
||
document["error"] = str(exc.detail)
|
||
elif isinstance(exc, BaselineBudgetExceeded):
|
||
document["error"] = str(exc)
|
||
else:
|
||
document["error"] = (
|
||
"SOURCE_CHANGED: 原始观测已改变，需要新建回放"
|
||
if "SOURCE_CHANGED" in str(exc)
|
||
else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
|
||
)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
raise
|
||
|
||
async def _advance(self, document):
|
||
config = ScadaReplayCreate.model_validate(document["config"])
|
||
replay_id = document["replay_id"]
|
||
if not document.get("warmed"):
|
||
await self._freeze_baseline(document, config)
|
||
if weighted_diagnosis(document) and current_calibration(document, config.start_time) is None:
|
||
raise HTTPException(409, "旧回放缺少新版历史诊断校准，请停止后新建演示")
|
||
batch = await self.repo.pending_batch(replay_id)
|
||
if batch is None:
|
||
step = document.get("step_pending", False)
|
||
virtual = parse_time(document["virtual_now"])
|
||
through = parse_time(document["step_target"]) if step else virtual
|
||
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
|
||
if document.get("cursor") else None)
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, config.start_time, config.end_time,
|
||
through, 501, after, frame=True,
|
||
)
|
||
if len(rows) > 500:
|
||
raise ValueError("SCADA frame exceeds selected device bound")
|
||
at = rows[0]["time"] if rows else through
|
||
if step:
|
||
virtual = through
|
||
document["virtual_now"] = virtual.isoformat()
|
||
if not rows and document.get("processed_until") == at.isoformat() and not step:
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
return
|
||
sequence = document["batch_sequence"] + 1
|
||
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
|
||
"refs": refs, "input_digest": digest(refs), "events": []}
|
||
async with self.repo.connection.transaction():
|
||
document["batch_sequence"] = sequence
|
||
document["delivered_until"] = at.isoformat()
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(document)
|
||
else:
|
||
rows = None
|
||
if batch["stage"] == "selected":
|
||
if rows is None:
|
||
# Only crash recovery reloads the selected raw frame.
|
||
at = parse_time(batch["at"])
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, at, config.end_time, at, 501,
|
||
) if batch["refs"] else []
|
||
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
if digest(actual) != batch["input_digest"]:
|
||
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
|
||
batch["outputs"] = prepare_cleaning(rows, config)
|
||
batch["stage"] = "clean_prepared"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "clean_prepared":
|
||
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
|
||
batch["stage"] = "cleaned"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "cleaned":
|
||
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
|
||
result["processed_until"] = batch["at"]
|
||
target = result.get("step_target")
|
||
was_step = bool(target)
|
||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||
if was_step:
|
||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||
if diagnostic_signal(result, item)}
|
||
for event in changes:
|
||
if event["change"] == "raised" and diagnostic_signal(result, event):
|
||
signals[event["event_id"]] = {key: event[key] for key in (
|
||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||
) if key in event}
|
||
result["step_anomaly_signals"] = list(signals.values())
|
||
monitoring = None
|
||
if was_step and not result["step_pending"]:
|
||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||
period_end = parse_time(target)
|
||
role = result.get("monitoring_config") or {}
|
||
role_ids = list(dict.fromkeys(
|
||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||
))
|
||
previous = await self.repo.monitoring_outputs(
|
||
replay_id, role_ids, period_end - timedelta(days=15),
|
||
period_end, batch["sequence"],
|
||
) if role_ids else []
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
monitoring = await bounded_calculation(
|
||
evaluate_monitoring_period,
|
||
result,
|
||
previous + batch["outputs"],
|
||
period_start=period_start,
|
||
period_end=period_end,
|
||
batch_sequence=batch["sequence"],
|
||
)
|
||
# Any configured measurement rule can enter diagnosis, including
|
||
# indicators outside the pressure/outlet roles used for burst scoring.
|
||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||
if diagnostic_signal(result, item)}
|
||
for state in result.get("rule_states", {}).values():
|
||
event = state.get("event")
|
||
if event and diagnostic_signal(result, event):
|
||
signals[event["event_id"]] = {key: event[key] for key in (
|
||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||
) if key in event}
|
||
monitoring["anomaly_signals"] = list(signals.values())
|
||
if signals and monitoring["status"] == "normal":
|
||
monitoring["status"] = "attention"
|
||
monitoring["detail"] = "本周期曾检出测点异常，进入异常调查与爆管诊断"
|
||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||
if not result["step_pending"]:
|
||
result.pop("step_target", None)
|
||
result.pop("step_start", None)
|
||
result.pop("step_anomaly_signals", None)
|
||
if batch["refs"]:
|
||
last = batch["refs"][-1]
|
||
result["cursor"] = [last["time"], last["device_id"]]
|
||
if parse_time(batch["at"]) >= config.end_time:
|
||
result["status"] = "completed"
|
||
batch["stage"], batch["events"] = "detected", changes
|
||
if monitoring is not None:
|
||
batch["monitoring"] = monitoring
|
||
async with self.repo.connection.transaction():
|
||
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
|
||
if monitoring is not None:
|
||
await self._create_monitoring_task(result, monitoring, batch)
|
||
batch["monitoring"] = monitoring
|
||
result["last_monitoring"] = monitoring
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(result)
|
||
|
||
async def _freeze_baseline(self, document, config):
|
||
"""Stage bounded pages, then atomically publish one causal source snapshot."""
|
||
if document.get("warmed"):
|
||
return
|
||
start = config.start_time - timedelta(days=15)
|
||
batch_count, row_count, byte_count, after = 0, 0, 0, None
|
||
counts = {device_id: 0 for device_id in config.device_ids}
|
||
valid_counts = dict(counts)
|
||
histories = {device_id: [] for device_id in config.device_ids}
|
||
window = max(rule.window_samples for rule in config.rules)
|
||
calibration = None
|
||
if (document.get("monitoring_config") or {}).get("decision_policy") == "weighted":
|
||
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder
|
||
calibration = MonitoringCalibrationBuilder(document, config)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.clear_unpublished_baseline(document["replay_id"])
|
||
# Keep only the current page in memory. Each BizDB write commits before
|
||
# the next source read; clean_prepared baseline pages remain invisible.
|
||
async with self.timescale.transaction():
|
||
await ScadaRepository.begin_replay_baseline_snapshot(self.timescale)
|
||
while True:
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, start, config.start_time,
|
||
config.start_time, min(BASELINE_PAGE_SIZE, BASELINE_MAX_ROWS - row_count + 1), after,
|
||
)
|
||
if not rows:
|
||
break
|
||
if row_count + len(rows) > BASELINE_MAX_ROWS:
|
||
raise BaselineBudgetExceeded(
|
||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_ROWS:,} 条准备预算，"
|
||
"尚未发布；请减少回放设备或核查历史采样密度后重试"
|
||
)
|
||
outputs = prepare_cleaning(rows, config)
|
||
if calibration is not None:
|
||
calibration.add(outputs)
|
||
batch = {
|
||
"sequence": -batch_count - 1, "stage": "clean_prepared", "kind": "baseline",
|
||
"at": outputs[-1]["time"], "outputs": outputs, "events": [],
|
||
}
|
||
batch_bytes = len(json.dumps(batch, ensure_ascii=False, allow_nan=False,
|
||
separators=(",", ":")).encode("utf-8"))
|
||
if byte_count + batch_bytes > BASELINE_MAX_BYTES:
|
||
raise BaselineBudgetExceeded(
|
||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_BYTES // (1024 * 1024)} MiB 准备预算，"
|
||
"尚未发布；请减少回放设备或核查历史采样密度后重试"
|
||
)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(document["replay_id"], batch)
|
||
batch_count += 1
|
||
row_count += len(outputs)
|
||
byte_count += batch_bytes
|
||
for row in outputs:
|
||
device_id = row["device_id"]
|
||
counts[device_id] += 1
|
||
if row["quality"] == "valid":
|
||
valid_counts[device_id] += 1
|
||
histories[device_id].append(row["cleaned_value"])
|
||
histories[device_id] = histories[device_id][-window:]
|
||
last = rows[-1]
|
||
after = (last["time"], last["device_id"])
|
||
document["baseline"] = {
|
||
"start_time": start.isoformat(), "end_time": config.start_time.isoformat(),
|
||
"end_exclusive": True, "sample_counts": counts, "valid_sample_counts": valid_counts,
|
||
"row_count": row_count, "batch_count": batch_count, "byte_count": byte_count,
|
||
}
|
||
document["histories"] = histories
|
||
if calibration is not None:
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
document["monitoring_calibration"] = await bounded_calculation(calibration.fit)
|
||
document["warmed"] = True
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.publish_baseline(document["replay_id"])
|
||
await self.repo.save(document)
|
||
|
||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||
from app.services.task_alerts import TaskAlertService
|
||
from uuid import UUID
|
||
producer = "scada-replay:" + document["replay_id"]
|
||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||
self.context, internal_producer_id=producer)
|
||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context, permissions_for_context(self.context),
|
||
execution_scope=settings.operational_task_execution_scope)
|
||
for event in changes:
|
||
# Period diagnosis owns measurement notifications during manual steps.
|
||
# Retain rule events as evidence; quality alerts remain independent.
|
||
if (not create_tasks and event.get("category") != "data_quality"
|
||
and (event["change"] == "raised" or event.get("notification_suppressed"))):
|
||
if event["change"] == "raised":
|
||
event["batch_sequence"] = batch["sequence"]
|
||
event["notification_suppressed"] = True
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"].update(
|
||
batch_sequence=batch["sequence"], notification_suppressed=True,
|
||
)
|
||
continue
|
||
if event["change"] == "raised":
|
||
event["batch_sequence"] = batch["sequence"]
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["batch_sequence"] = batch["sequence"]
|
||
await alerts.push(TaskAlertEvent(
|
||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||
occurred_at=event["occurred_at"], condition_id=None,
|
||
source={"kind": "scada_replay", "replay_id": document["replay_id"],
|
||
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
|
||
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
|
||
))
|
||
if event["change"] != "raised":
|
||
if event.get("task_id"):
|
||
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
|
||
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
|
||
"batch_sequence": batch["sequence"],
|
||
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
|
||
continue
|
||
if event.get("category") == "data_quality":
|
||
continue
|
||
if not create_tasks:
|
||
continue
|
||
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
|
||
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
|
||
summary=event["detail"], source=TaskSource(
|
||
kind="external_system", external_id=external, external_revision=1,
|
||
event_kind="scada_anomaly",
|
||
replay_context={"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
|
||
),
|
||
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"),
|
||
)
|
||
task = await tasks.create(command)
|
||
event["task_id"] = str(task.task_id)
|
||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["task_id"] = event["task_id"]
|
||
|
||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||
from app.services.scada_monitoring_incidents import attach_incident, update_incident
|
||
incident, action = update_incident(document, monitoring, batch["sequence"])
|
||
if incident is None:
|
||
return
|
||
if incident.get("task_id"):
|
||
if monitoring["status"] == "suspected_burst" and not incident.get("burst_diagnosed"):
|
||
from uuid import UUID
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
tasks = OperationalTaskService(
|
||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context, permissions_for_context(self.context),
|
||
execution_scope=settings.operational_task_execution_scope,
|
||
)
|
||
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
|
||
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||
"batch_sequence": batch["sequence"],
|
||
}, uuid5(NAMESPACE_URL, incident["incident_id"] + ":burst-evidence"))
|
||
incident["burst_diagnosed"] = True
|
||
if action == "resolved" and incident.get("alert"):
|
||
await self._publish_monitoring_alert(document, incident, "resolved")
|
||
attach_incident(monitoring, incident, action)
|
||
return
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
tasks = OperationalTaskService(
|
||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context,
|
||
permissions_for_context(self.context),
|
||
execution_scope=settings.operational_task_execution_scope,
|
||
)
|
||
external = incident["external_id"]
|
||
pressure = monitoring["pressure"]
|
||
flow = monitoring["outlet_flow"]
|
||
detector = monitoring["burst_detection"]
|
||
score = detector.get("target_score")
|
||
score_text = f"{score:.4f}" if score is not None else "数据不足"
|
||
summary = (
|
||
f"SCADA 数据诊断发现指标异常，进入异常调查与爆管诊断。"
|
||
f"爆管侦测得分 {score_text}；"
|
||
f"压力下降测点 {len(pressure.get('hit_device_ids', []))} 个；"
|
||
f"出厂边界流量增加测点 {len(flow.get('hit_device_ids', []))} 个。"
|
||
"任一指标异常即启动调查，加权分用于评估爆管嫌疑；达到门槛后继续候选定位、隔离与影响分析。"
|
||
)
|
||
if monitoring.get("anomaly_signals"):
|
||
signals = monitoring["anomaly_signals"]
|
||
devices = {device for item in signals for device in item.get("device_ids", [])}
|
||
summary += f" 单项异常 {len(signals)} 项，涉及 {len(devices)} 个测点；明细通过冻结 SCADA 查询核验。"
|
||
if monitoring.get("weighted"):
|
||
weighted = monitoring["weighted"]
|
||
weights = weighted["weights"]
|
||
summary += (
|
||
f" 综合评分 {weighted['score']:.2f}，爆管嫌疑门槛 {weighted['threshold']:.2f}"
|
||
f"（侦测 {weights['burst_detection']:.0%}、压力 {weights['pressure']:.0%}、"
|
||
f"流量 {weights['outlet_flow']:.0%}）。"
|
||
)
|
||
if detector.get("score_threshold") is not None:
|
||
summary += f" 侦测动态阈值 {detector['score_threshold']:.4f}。"
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external),
|
||
task_type="event_investigation",
|
||
workflow="scada_anomaly",
|
||
title=f"SCADA 异常调查 · {monitoring['period_end']}",
|
||
summary=summary,
|
||
source=TaskSource(
|
||
kind="external_system",
|
||
external_id=external,
|
||
external_revision=1,
|
||
event_kind="scada_anomaly",
|
||
replay_context={
|
||
"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||
"batch_sequence": batch["sequence"],
|
||
},
|
||
),
|
||
trigger=TaskTrigger(
|
||
kind="event",
|
||
occurred_at=monitoring["period_end"],
|
||
detail="SCADA 任一指标异常启动调查，核验数据后进行爆管诊断；异常不等于已确诊爆管",
|
||
),
|
||
)
|
||
task = await tasks.create(command)
|
||
incident["task_id"] = str(task.task_id)
|
||
incident["burst_diagnosed"] = monitoring["status"] == "suspected_burst"
|
||
incident["alert"] = {
|
||
"alert_id": incident["incident_id"],
|
||
"producer_id": "scada-replay:" + document["replay_id"],
|
||
"occurred_at": incident["started_at"],
|
||
"condition_id": f"operational-task:{task.task_id}",
|
||
"source": {"kind": "scada_replay", "replay_id": document["replay_id"],
|
||
"rule_id": "scada_diagnosis", "batch_sequence": batch["sequence"]},
|
||
"title": "SCADA 指标异常", "description": "已启动调查，详细诊断见调查报告。", "severity": "warning",
|
||
}
|
||
await self._publish_monitoring_alert(document, incident, "raised")
|
||
attach_incident(monitoring, incident, action)
|
||
|
||
async def _publish_monitoring_alert(self, document, incident, change):
|
||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||
from app.services.task_alerts import TaskAlertService
|
||
alerts = TaskAlertService(
|
||
TaskAlertRepository(self.repo.connection, self.context.project_id), self.context,
|
||
internal_producer_id="scada-replay:" + document["replay_id"],
|
||
)
|
||
await alerts.push(TaskAlertEvent(
|
||
**incident["alert"], event_type=change,
|
||
event_id=uuid5(NAMESPACE_URL, incident["incident_id"] + ":" + change),
|
||
))
|