398 lines
22 KiB
Python
398 lines
22 KiB
Python
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta, timezone
|
||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||
|
||
from fastapi import HTTPException
|
||
from app.auth.permissions import permissions_for_context
|
||
|
||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
|
||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||
from app.services.content_digest import digest
|
||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||
|
||
|
||
def parse_time(value: str) -> datetime:
|
||
return datetime.fromisoformat(value)
|
||
|
||
|
||
def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||
return ScadaReplaySnapshot(
|
||
**{key: document[key] for key in (
|
||
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
|
||
"virtual_now", "created_at", "updated_at",
|
||
)},
|
||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||
devices=document["devices"], rules=document["config"]["rules"],
|
||
latest=list(document.get("latest", {}).values()),
|
||
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
|
||
error=document.get("error"),
|
||
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
|
||
step_pending=document.get("step_pending", False),
|
||
monitoring_count=document.get("monitoring_count", 0),
|
||
last_monitoring=document.get("last_monitoring"),
|
||
)
|
||
|
||
|
||
class ScadaReplayService:
|
||
def __init__(self, repository, context, timescale=None, *, clock=None):
|
||
self.repo, self.context, self.timescale = repository, context, timescale
|
||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||
|
||
async def ensure_storage(self):
|
||
if not await self.repo.available():
|
||
raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql")
|
||
|
||
async def create(
|
||
self,
|
||
command: ScadaReplayCreate,
|
||
*,
|
||
monitoring_interval_seconds: int | None,
|
||
monitoring_config: ScadaMonitoringSettings | None = None,
|
||
):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
previous = await self.repo.by_request(command.request_id)
|
||
config = command.model_dump(mode="json")
|
||
if previous:
|
||
if previous["config"] != config:
|
||
raise HTTPException(409, "创建请求号已用于不同回放配置")
|
||
return snapshot(previous)
|
||
if monitoring_interval_seconds is None:
|
||
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示")
|
||
replay_id = str(uuid4())
|
||
if await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建")
|
||
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
|
||
if not set(command.device_ids) <= metadata.keys():
|
||
raise HTTPException(422, "设备不属于当前项目")
|
||
for rule in command.rules:
|
||
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
|
||
raise HTTPException(422, "规则单位与设备单位不一致，请分组配置")
|
||
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
|
||
raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数")
|
||
if monitoring_config:
|
||
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
|
||
if not role_ids <= metadata.keys():
|
||
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
|
||
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
|
||
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
|
||
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
|
||
for device in monitoring_config.outlet_flow_device_ids):
|
||
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
|
||
now = self.clock().isoformat()
|
||
document = {
|
||
"replay_id": replay_id, "project_id": str(self.context.project_id),
|
||
"created_by": str(self.context.user_id), "config": config,
|
||
"revision": 1, "status": "ready", "start_time": config["start_time"],
|
||
"end_time": config["end_time"], "virtual_now": config["start_time"],
|
||
"created_at": now, "updated_at": now,
|
||
"monitoring_interval_seconds": monitoring_interval_seconds,
|
||
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
|
||
"devices": [metadata[device] for device in command.device_ids],
|
||
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
|
||
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
|
||
}
|
||
await self.repo.insert(document)
|
||
return snapshot(document)
|
||
|
||
async def command(self, replay_id, command: ScadaReplayCommand):
|
||
await self.ensure_storage()
|
||
async with self.repo.control():
|
||
document = await self.repo.get(replay_id)
|
||
key, body = str(command.request_id), command.model_dump(mode="json")
|
||
receipt = document.setdefault("receipts", {}).get(key)
|
||
if receipt:
|
||
if receipt != body:
|
||
raise HTTPException(409, "命令请求号已用于不同内容")
|
||
return snapshot(document)
|
||
if command.expected_revision != document["revision"]:
|
||
raise HTTPException(409, "回放控制状态已更新，请刷新后重试")
|
||
action, status = command.action, document["status"]
|
||
if document.get("step_pending") and action == "step":
|
||
raise HTTPException(409, "当前时间步正在处理，请完成后继续")
|
||
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
|
||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||
if action == "step" and status in {"ready", "paused"}:
|
||
interval = document.get("monitoring_interval_seconds")
|
||
if interval is None:
|
||
raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示")
|
||
start = parse_time(document["start_time"])
|
||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||
document["step_start"] = document["virtual_now"]
|
||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||
document["status"], document["step_pending"] = "paused", True
|
||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||
document["status"], document["step_pending"] = "stopped", False
|
||
document.pop("step_target", None)
|
||
document.pop("step_start", None)
|
||
elif action == "retry" and status == "failed":
|
||
if "SOURCE_CHANGED" in (document.get("error") or ""):
|
||
raise HTTPException(409, "原始数据发生变化，需要停止并新建回放")
|
||
document["status"], document["error"] = "paused", None
|
||
document["step_pending"] = bool(document.get("step_target"))
|
||
else:
|
||
raise HTTPException(409, "当前回放状态不支持此操作")
|
||
document["revision"] += 1
|
||
document["receipts"][key] = body
|
||
if len(document["receipts"]) > 256:
|
||
del document["receipts"][next(iter(document["receipts"]))]
|
||
await self.repo.save(document)
|
||
return snapshot(document)
|
||
|
||
async def process_pending_step(self, replay_id):
|
||
await self.ensure_storage()
|
||
async with self.repo.processing() as owned:
|
||
if not owned:
|
||
raise HTTPException(409, "当前时间步正在由另一请求处理")
|
||
while True:
|
||
document = await self.repo.get(replay_id)
|
||
if not document.get("step_pending"):
|
||
return snapshot(document)
|
||
try:
|
||
await self._advance(document)
|
||
except Exception as exc:
|
||
# Failure never discards the prepared batch; retry resumes it.
|
||
document = await self.repo.get(replay_id)
|
||
document["status"], document["step_pending"] = "failed", False
|
||
document["error"] = (
|
||
"SOURCE_CHANGED: 原始观测已改变，需要新建回放"
|
||
if "SOURCE_CHANGED" in str(exc)
|
||
else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
|
||
)
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
raise
|
||
|
||
async def _advance(self, document):
|
||
config = ScadaReplayCreate.model_validate(document["config"])
|
||
replay_id = document["replay_id"]
|
||
if not document.get("warmed"):
|
||
import math
|
||
window = max(rule.window_samples for rule in config.rules)
|
||
async with self.timescale.transaction():
|
||
warmup = await ScadaRepository.get_warmup_values(
|
||
self.timescale, config.device_ids, config.start_time, window,
|
||
)
|
||
for row in warmup:
|
||
value = row["monitored_value"]
|
||
bounds = config.quality_ranges.get(row["device_id"])
|
||
if value is not None and math.isfinite(value) and not (bounds and (
|
||
bounds.lower is not None and value < bounds.lower
|
||
or bounds.upper is not None and value > bounds.upper
|
||
)):
|
||
document["histories"].setdefault(row["device_id"], []).append(value)
|
||
document["warmed"] = True
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
batch = await self.repo.pending_batch(replay_id)
|
||
if batch is None:
|
||
step = document.get("step_pending", False)
|
||
virtual = parse_time(document["virtual_now"])
|
||
through = parse_time(document["step_target"]) if step else virtual
|
||
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
|
||
if document.get("cursor") else None)
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, config.start_time, config.end_time,
|
||
through, 501, after, frame=True,
|
||
)
|
||
if len(rows) > 500:
|
||
raise ValueError("SCADA frame exceeds selected device bound")
|
||
at = rows[0]["time"] if rows else through
|
||
if step:
|
||
virtual = through
|
||
document["virtual_now"] = virtual.isoformat()
|
||
if not rows and document.get("processed_until") == at.isoformat() and not step:
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save(document)
|
||
return
|
||
sequence = document["batch_sequence"] + 1
|
||
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
|
||
"refs": refs, "input_digest": digest(refs), "events": []}
|
||
async with self.repo.connection.transaction():
|
||
document["batch_sequence"] = sequence
|
||
document["delivered_until"] = at.isoformat()
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(document)
|
||
else:
|
||
rows = None
|
||
if batch["stage"] == "selected":
|
||
if rows is None:
|
||
# Only crash recovery reloads the selected raw frame.
|
||
at = parse_time(batch["at"])
|
||
async with self.timescale.transaction():
|
||
rows = await ScadaRepository.get_raw_page(
|
||
self.timescale, config.device_ids, at, config.end_time, at, 501,
|
||
) if batch["refs"] else []
|
||
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||
"source_hash": row["source_hash"]} for row in rows]
|
||
if digest(actual) != batch["input_digest"]:
|
||
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
|
||
batch["outputs"] = prepare_cleaning(rows, config)
|
||
batch["stage"] = "clean_prepared"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "clean_prepared":
|
||
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
|
||
batch["stage"] = "cleaned"
|
||
async with self.repo.connection.transaction():
|
||
await self.repo.save_batch(replay_id, batch)
|
||
if batch["stage"] == "cleaned":
|
||
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
|
||
result["processed_until"] = batch["at"]
|
||
target = result.get("step_target")
|
||
was_step = bool(target)
|
||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||
monitoring = None
|
||
if was_step and not result["step_pending"]:
|
||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||
period_end = parse_time(target)
|
||
role = result.get("monitoring_config") or {}
|
||
role_ids = list(dict.fromkeys(
|
||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||
))
|
||
previous = await self.repo.monitoring_outputs(
|
||
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
|
||
period_end, batch["sequence"],
|
||
) if role_ids else []
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
monitoring = await bounded_calculation(
|
||
evaluate_monitoring_period,
|
||
result,
|
||
previous + batch["outputs"],
|
||
period_start=period_start,
|
||
period_end=period_end,
|
||
batch_sequence=batch["sequence"],
|
||
)
|
||
result["last_monitoring"] = monitoring
|
||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||
if not result["step_pending"]:
|
||
result.pop("step_target", None)
|
||
result.pop("step_start", None)
|
||
if batch["refs"]:
|
||
last = batch["refs"][-1]
|
||
result["cursor"] = [last["time"], last["device_id"]]
|
||
if parse_time(batch["at"]) >= config.end_time:
|
||
result["status"] = "completed"
|
||
batch["stage"], batch["events"] = "detected", changes
|
||
if monitoring is not None:
|
||
batch["monitoring"] = monitoring
|
||
async with self.repo.connection.transaction():
|
||
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
|
||
if monitoring is not None:
|
||
await self._create_monitoring_task(result, monitoring, batch)
|
||
batch["monitoring"] = monitoring
|
||
result["last_monitoring"] = monitoring
|
||
await self.repo.save_batch(replay_id, batch)
|
||
await self.repo.save(result)
|
||
|
||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||
from app.services.task_alerts import TaskAlertService
|
||
from uuid import UUID
|
||
producer = "scada-replay:" + document["replay_id"]
|
||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||
self.context, internal_producer_id=producer)
|
||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context, permissions_for_context(self.context))
|
||
for event in changes:
|
||
if event["change"] == "raised":
|
||
event["batch_sequence"] = batch["sequence"]
|
||
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
|
||
await alerts.push(TaskAlertEvent(
|
||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||
occurred_at=event["occurred_at"], condition_id=None,
|
||
source={"kind": "scada_replay", "replay_id": document["replay_id"],
|
||
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
|
||
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
|
||
))
|
||
if event["change"] != "raised":
|
||
if event.get("task_id"):
|
||
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
|
||
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
|
||
"batch_sequence": batch["sequence"],
|
||
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
|
||
continue
|
||
if event.get("category") == "data_quality":
|
||
continue
|
||
if not create_tasks:
|
||
continue
|
||
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
|
||
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
|
||
summary=event["detail"], source=TaskSource(
|
||
kind="external_system", external_id=external, external_revision=1,
|
||
event_kind="scada_anomaly",
|
||
replay_context={"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
|
||
),
|
||
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"),
|
||
)
|
||
task = await tasks.create(command)
|
||
event["task_id"] = str(task.task_id)
|
||
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
|
||
|
||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||
if monitoring["status"] != "suspected_burst":
|
||
return
|
||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||
from app.services.operational_tasks import OperationalTaskService
|
||
tasks = OperationalTaskService(
|
||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||
self.context,
|
||
permissions_for_context(self.context),
|
||
)
|
||
external = (
|
||
f"scada_replay:{document['replay_id']}:monitor:"
|
||
f"{monitoring['period_end']}:{monitoring['rule_version']}"
|
||
)
|
||
pressure = monitoring["pressure"]
|
||
flow = monitoring["outlet_flow"]
|
||
detector = monitoring["burst_detection"]
|
||
summary = (
|
||
f"步长监测疑似爆管：爆管侦测得分 {detector['target_score']:.4f}；"
|
||
f"压力下降测点 {len(pressure['hit_device_ids'])} 个；"
|
||
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
|
||
)
|
||
command = OperationalTaskCreate(
|
||
request_id=uuid5(NAMESPACE_URL, external),
|
||
task_type="event_investigation",
|
||
workflow="burst_report",
|
||
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
|
||
summary=summary,
|
||
source=TaskSource(
|
||
kind="external_system",
|
||
external_id=external,
|
||
external_revision=1,
|
||
event_kind="scada_burst_suspected",
|
||
replay_context={
|
||
"replay_id": document["replay_id"],
|
||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||
"batch_sequence": batch["sequence"],
|
||
},
|
||
),
|
||
trigger=TaskTrigger(
|
||
kind="event",
|
||
occurred_at=monitoring["period_end"],
|
||
detail="主动步长监测的多项指标相互支持，进入爆管调查；尚未现场确诊",
|
||
),
|
||
)
|
||
task = await tasks.create(command)
|
||
monitoring["task_id"] = str(task.task_id)
|