feat(scada): add project monitoring replay

app/services/scada_replay.py

@@ -0,0 +1,397 @@
+"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from uuid import NAMESPACE_URL, uuid4, uuid5
+
+from fastapi import HTTPException
+from app.auth.permissions import permissions_for_context
+
+from app.domain.schemas.project_settings import ScadaMonitoringSettings
+from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
+from app.infra.db.postgresql.scada import ScadaInfoRepository
+from app.infra.db.timescaledb.repositories.scada import ScadaRepository
+from app.services.content_digest import digest
+from app.services.scada_replay_processing import prepare_cleaning, detect_frame
+from app.services.scada_replay_monitoring import evaluate_monitoring_period
+
+
+def parse_time(value: str) -> datetime:
+    return datetime.fromisoformat(value)
+
+
+def snapshot(document: dict) -> ScadaReplaySnapshot:
+    return ScadaReplaySnapshot(
+        **{key: document[key] for key in (
+            "replay_id", "project_id", "revision", "status", "start_time", "end_time",
+            "virtual_now", "created_at", "updated_at",
+        )},
+        delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
+        processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
+        devices=document["devices"], rules=document["config"]["rules"],
+        latest=list(document.get("latest", {}).values()),
+        active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
+        error=document.get("error"),
+        monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
+        step_pending=document.get("step_pending", False),
+        monitoring_count=document.get("monitoring_count", 0),
+        last_monitoring=document.get("last_monitoring"),
+    )
+
+
+class ScadaReplayService:
+    def __init__(self, repository, context, timescale=None, *, clock=None):
+        self.repo, self.context, self.timescale = repository, context, timescale
+        self.clock = clock or (lambda: datetime.now(timezone.utc))
+
+    async def ensure_storage(self):
+        if not await self.repo.available():
+            raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql")
+
+    async def create(
+        self,
+        command: ScadaReplayCreate,
+        *,
+        monitoring_interval_seconds: int | None,
+        monitoring_config: ScadaMonitoringSettings | None = None,
+    ):
+        await self.ensure_storage()
+        async with self.repo.control():
+            previous = await self.repo.by_request(command.request_id)
+            config = command.model_dump(mode="json")
+            if previous:
+                if previous["config"] != config:
+                    raise HTTPException(409, "创建请求号已用于不同回放配置")
+                return snapshot(previous)
+            if monitoring_interval_seconds is None:
+                raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示")
+            replay_id = str(uuid4())
+            if await self.repo.active_other(replay_id):
+                raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建")
+            metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
+            if not set(command.device_ids) <= metadata.keys():
+                raise HTTPException(422, "设备不属于当前项目")
+            for rule in command.rules:
+                if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
+                    raise HTTPException(422, "规则单位与设备单位不一致，请分组配置")
+                if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
+                    raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数")
+            if monitoring_config:
+                role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
+                if not role_ids <= metadata.keys():
+                    raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
+                if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
+                    raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
+                if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
+                       for device in monitoring_config.outlet_flow_device_ids):
+                    raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
+            now = self.clock().isoformat()
+            document = {
+                "replay_id": replay_id, "project_id": str(self.context.project_id),
+                "created_by": str(self.context.user_id), "config": config,
+                "revision": 1, "status": "ready", "start_time": config["start_time"],
+                "end_time": config["end_time"], "virtual_now": config["start_time"],
+                "created_at": now, "updated_at": now,
+                "monitoring_interval_seconds": monitoring_interval_seconds,
+                "monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
+                "devices": [metadata[device] for device in command.device_ids],
+                "batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
+                "monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
+            }
+            await self.repo.insert(document)
+            return snapshot(document)
+
+    async def command(self, replay_id, command: ScadaReplayCommand):
+        await self.ensure_storage()
+        async with self.repo.control():
+            document = await self.repo.get(replay_id)
+            key, body = str(command.request_id), command.model_dump(mode="json")
+            receipt = document.setdefault("receipts", {}).get(key)
+            if receipt:
+                if receipt != body:
+                    raise HTTPException(409, "命令请求号已用于不同内容")
+                return snapshot(document)
+            if command.expected_revision != document["revision"]:
+                raise HTTPException(409, "回放控制状态已更新，请刷新后重试")
+            action, status = command.action, document["status"]
+            if document.get("step_pending") and action == "step":
+                raise HTTPException(409, "当前时间步正在处理，请完成后继续")
+            if action in {"step", "retry"} and await self.repo.active_other(replay_id):
+                raise HTTPException(409, "当前项目已有另一条活动回放")
+            if action == "step" and status in {"ready", "paused"}:
+                interval = document.get("monitoring_interval_seconds")
+                if interval is None:
+                    raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示")
+                start = parse_time(document["start_time"])
+                elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
+                target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
+                document["step_start"] = document["virtual_now"]
+                document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
+                document["status"], document["step_pending"] = "paused", True
+            elif action == "stop" and status not in {"completed", "stopped"}:
+                document["status"], document["step_pending"] = "stopped", False
+                document.pop("step_target", None)
+                document.pop("step_start", None)
+            elif action == "retry" and status == "failed":
+                if "SOURCE_CHANGED" in (document.get("error") or ""):
+                    raise HTTPException(409, "原始数据发生变化，需要停止并新建回放")
+                document["status"], document["error"] = "paused", None
+                document["step_pending"] = bool(document.get("step_target"))
+            else:
+                raise HTTPException(409, "当前回放状态不支持此操作")
+            document["revision"] += 1
+            document["receipts"][key] = body
+            if len(document["receipts"]) > 256:
+                del document["receipts"][next(iter(document["receipts"]))]
+            await self.repo.save(document)
+            return snapshot(document)
+
+    async def process_pending_step(self, replay_id):
+        await self.ensure_storage()
+        async with self.repo.processing() as owned:
+            if not owned:
+                raise HTTPException(409, "当前时间步正在由另一请求处理")
+            while True:
+                document = await self.repo.get(replay_id)
+                if not document.get("step_pending"):
+                    return snapshot(document)
+                try:
+                    await self._advance(document)
+                except Exception as exc:
+                    # Failure never discards the prepared batch; retry resumes it.
+                    document = await self.repo.get(replay_id)
+                    document["status"], document["step_pending"] = "failed", False
+                    document["error"] = (
+                        "SOURCE_CHANGED: 原始观测已改变，需要新建回放"
+                        if "SOURCE_CHANGED" in str(exc)
+                        else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试"
+                    )
+                    async with self.repo.connection.transaction():
+                        await self.repo.save(document)
+                    raise
+
+    async def _advance(self, document):
+        config = ScadaReplayCreate.model_validate(document["config"])
+        replay_id = document["replay_id"]
+        if not document.get("warmed"):
+            import math
+            window = max(rule.window_samples for rule in config.rules)
+            async with self.timescale.transaction():
+                warmup = await ScadaRepository.get_warmup_values(
+                    self.timescale, config.device_ids, config.start_time, window,
+                )
+            for row in warmup:
+                value = row["monitored_value"]
+                bounds = config.quality_ranges.get(row["device_id"])
+                if value is not None and math.isfinite(value) and not (bounds and (
+                    bounds.lower is not None and value < bounds.lower
+                    or bounds.upper is not None and value > bounds.upper
+                )):
+                    document["histories"].setdefault(row["device_id"], []).append(value)
+            document["warmed"] = True
+            async with self.repo.connection.transaction():
+                await self.repo.save(document)
+        batch = await self.repo.pending_batch(replay_id)
+        if batch is None:
+            step = document.get("step_pending", False)
+            virtual = parse_time(document["virtual_now"])
+            through = parse_time(document["step_target"]) if step else virtual
+            after = ((parse_time(document["cursor"][0]), document["cursor"][1])
+                     if document.get("cursor") else None)
+            async with self.timescale.transaction():
+                rows = await ScadaRepository.get_raw_page(
+                    self.timescale, config.device_ids, config.start_time, config.end_time,
+                    through, 501, after, frame=True,
+                )
+            if len(rows) > 500:
+                raise ValueError("SCADA frame exceeds selected device bound")
+            at = rows[0]["time"] if rows else through
+            if step:
+                virtual = through
+            document["virtual_now"] = virtual.isoformat()
+            if not rows and document.get("processed_until") == at.isoformat() and not step:
+                async with self.repo.connection.transaction():
+                    await self.repo.save(document)
+                return
+            sequence = document["batch_sequence"] + 1
+            refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
+                     "source_hash": row["source_hash"]} for row in rows]
+            batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
+                     "refs": refs, "input_digest": digest(refs), "events": []}
+            async with self.repo.connection.transaction():
+                document["batch_sequence"] = sequence
+                document["delivered_until"] = at.isoformat()
+                await self.repo.save_batch(replay_id, batch)
+                await self.repo.save(document)
+        else:
+            rows = None
+        if batch["stage"] == "selected":
+            if rows is None:
+                # Only crash recovery reloads the selected raw frame.
+                at = parse_time(batch["at"])
+                async with self.timescale.transaction():
+                    rows = await ScadaRepository.get_raw_page(
+                        self.timescale, config.device_ids, at, config.end_time, at, 501,
+                    ) if batch["refs"] else []
+                actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
+                           "source_hash": row["source_hash"]} for row in rows]
+                if digest(actual) != batch["input_digest"]:
+                    raise ValueError("SOURCE_CHANGED: selected raw frame differs")
+            batch["outputs"] = prepare_cleaning(rows, config)
+            batch["stage"] = "clean_prepared"
+            async with self.repo.connection.transaction():
+                await self.repo.save_batch(replay_id, batch)
+        if batch["stage"] == "clean_prepared":
+            await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
+            batch["stage"] = "cleaned"
+            async with self.repo.connection.transaction():
+                await self.repo.save_batch(replay_id, batch)
+        if batch["stage"] == "cleaned":
+            result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
+            result["processed_until"] = batch["at"]
+            target = result.get("step_target")
+            was_step = bool(target)
+            result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
+            monitoring = None
+            if was_step and not result["step_pending"]:
+                period_start = parse_time(result.get("step_start", result["start_time"]))
+                period_end = parse_time(target)
+                role = result.get("monitoring_config") or {}
+                role_ids = list(dict.fromkeys(
+                    role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
+                ))
+                previous = await self.repo.monitoring_outputs(
+                    replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
+                    period_end, batch["sequence"],
+                ) if role_ids else []
+                from app.mcp.analysis_service import bounded_calculation
+                monitoring = await bounded_calculation(
+                    evaluate_monitoring_period,
+                    result,
+                    previous + batch["outputs"],
+                    period_start=period_start,
+                    period_end=period_end,
+                    batch_sequence=batch["sequence"],
+                )
+                result["last_monitoring"] = monitoring
+                result["monitoring_count"] = result.get("monitoring_count", 0) + 1
+            if not result["step_pending"]:
+                result.pop("step_target", None)
+                result.pop("step_start", None)
+            if batch["refs"]:
+                last = batch["refs"][-1]
+                result["cursor"] = [last["time"], last["device_id"]]
+            if parse_time(batch["at"]) >= config.end_time:
+                result["status"] = "completed"
+            batch["stage"], batch["events"] = "detected", changes
+            if monitoring is not None:
+                batch["monitoring"] = monitoring
+            async with self.repo.connection.transaction():
+                await self._create_investigations(result, changes, batch, create_tasks=not was_step)
+                if monitoring is not None:
+                    await self._create_monitoring_task(result, monitoring, batch)
+                    batch["monitoring"] = monitoring
+                    result["last_monitoring"] = monitoring
+                await self.repo.save_batch(replay_id, batch)
+                await self.repo.save(result)
+
+    async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
+        from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
+        from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
+        from app.services.operational_tasks import OperationalTaskService
+        from app.domain.schemas.task_alert import TaskAlertEvent
+        from app.infra.db.postgresql.task_alerts import TaskAlertRepository
+        from app.services.task_alerts import TaskAlertService
+        from uuid import UUID
+        producer = "scada-replay:" + document["replay_id"]
+        alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
+                                  self.context, internal_producer_id=producer)
+        tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
+                                       self.context, permissions_for_context(self.context))
+        for event in changes:
+            if event["change"] == "raised":
+                event["batch_sequence"] = batch["sequence"]
+                document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
+            await alerts.push(TaskAlertEvent(
+                event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
+                alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
+                occurred_at=event["occurred_at"], condition_id=None,
+                source={"kind": "scada_replay", "replay_id": document["replay_id"],
+                        "rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
+                title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
+            ))
+            if event["change"] != "raised":
+                if event.get("task_id"):
+                    await tasks.advance_replay_investigation(UUID(event["task_id"]), {
+                        "replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
+                        "batch_sequence": batch["sequence"],
+                    }, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
+                continue
+            if event.get("category") == "data_quality":
+                continue
+            if not create_tasks:
+                continue
+            external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
+            command = OperationalTaskCreate(
+                request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
+                workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
+                summary=event["detail"], source=TaskSource(
+                    kind="external_system", external_id=external, external_revision=1,
+                    event_kind="scada_anomaly",
+                    replay_context={"replay_id": document["replay_id"],
+                                    "knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
+                ),
+                trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"),
+            )
+            task = await tasks.create(command)
+            event["task_id"] = str(task.task_id)
+            document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
+
+    async def _create_monitoring_task(self, document, monitoring, batch):
+        if monitoring["status"] != "suspected_burst":
+            return
+        from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
+        from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
+        from app.services.operational_tasks import OperationalTaskService
+        tasks = OperationalTaskService(
+            OperationalTaskRepository(self.repo.connection, self.context.project_id),
+            self.context,
+            permissions_for_context(self.context),
+        )
+        external = (
+            f"scada_replay:{document['replay_id']}:monitor:"
+            f"{monitoring['period_end']}:{monitoring['rule_version']}"
+        )
+        pressure = monitoring["pressure"]
+        flow = monitoring["outlet_flow"]
+        detector = monitoring["burst_detection"]
+        summary = (
+            f"步长监测疑似爆管：爆管侦测得分 {detector['target_score']:.4f}；"
+            f"压力下降测点 {len(pressure['hit_device_ids'])} 个；"
+            f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
+        )
+        command = OperationalTaskCreate(
+            request_id=uuid5(NAMESPACE_URL, external),
+            task_type="event_investigation",
+            workflow="burst_report",
+            title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
+            summary=summary,
+            source=TaskSource(
+                kind="external_system",
+                external_id=external,
+                external_revision=1,
+                event_kind="scada_burst_suspected",
+                replay_context={
+                    "replay_id": document["replay_id"],
+                    "knowledge_cutoff": monitoring["knowledge_cutoff"],
+                    "batch_sequence": batch["sequence"],
+                },
+            ),
+            trigger=TaskTrigger(
+                kind="event",
+                occurred_at=monitoring["period_end"],
+                detail="主动步长监测的多项指标相互支持，进入爆管调查；尚未现场确诊",
+            ),
+        )
+        task = await tasks.create(command)
+        monitoring["task_id"] = str(task.task_id)