"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten.""" from __future__ import annotations from datetime import datetime, timedelta, timezone from uuid import NAMESPACE_URL, uuid4, uuid5 from fastapi import HTTPException from app.auth.permissions import permissions_for_context from app.domain.schemas.project_settings import ScadaMonitoringSettings from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot from app.infra.db.postgresql.scada import ScadaInfoRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository from app.services.content_digest import digest from app.services.scada_replay_processing import prepare_cleaning, detect_frame from app.services.scada_replay_monitoring import evaluate_monitoring_period def parse_time(value: str) -> datetime: return datetime.fromisoformat(value) def snapshot(document: dict) -> ScadaReplaySnapshot: return ScadaReplaySnapshot( **{key: document[key] for key in ( "replay_id", "project_id", "revision", "status", "start_time", "end_time", "virtual_now", "created_at", "updated_at", )}, delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"), processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0), devices=document["devices"], rules=document["config"]["rules"], latest=list(document.get("latest", {}).values()), active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")], error=document.get("error"), monitoring_interval_seconds=document.get("monitoring_interval_seconds"), step_pending=document.get("step_pending", False), monitoring_count=document.get("monitoring_count", 0), last_monitoring=document.get("last_monitoring"), ) class ScadaReplayService: def __init__(self, repository, context, timescale=None, *, clock=None): self.repo, self.context, self.timescale = repository, context, timescale self.clock = clock or (lambda: datetime.now(timezone.utc)) async def ensure_storage(self): if not await self.repo.available(): raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql") async def create( self, command: ScadaReplayCreate, *, monitoring_interval_seconds: int | None, monitoring_config: ScadaMonitoringSettings | None = None, ): await self.ensure_storage() async with self.repo.control(): previous = await self.repo.by_request(command.request_id) config = command.model_dump(mode="json") if previous: if previous["config"] != config: raise HTTPException(409, "创建请求号已用于不同回放配置") return snapshot(previous) if monitoring_interval_seconds is None: raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示") replay_id = str(uuid4()) if await self.repo.active_other(replay_id): raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建") metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)} if not set(command.device_ids) <= metadata.keys(): raise HTTPException(422, "设备不属于当前项目") for rule in command.rules: if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids): raise HTTPException(422, "规则单位与设备单位不一致，请分组配置") if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip(): raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数") if monitoring_config: role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids) if not role_ids <= metadata.keys(): raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备") if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids): raise HTTPException(422, "压力监测角色必须引用 pressure 设备") if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"} for device in monitoring_config.outlet_flow_device_ids): raise HTTPException(422, "出厂边界流量角色必须引用流量设备") now = self.clock().isoformat() document = { "replay_id": replay_id, "project_id": str(self.context.project_id), "created_by": str(self.context.user_id), "config": config, "revision": 1, "status": "ready", "start_time": config["start_time"], "end_time": config["end_time"], "virtual_now": config["start_time"], "created_at": now, "updated_at": now, "monitoring_interval_seconds": monitoring_interval_seconds, "monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None, "devices": [metadata[device] for device in command.device_ids], "batch_sequence": 0, "processed_count": 0, "anomaly_count": 0, "monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {}, } await self.repo.insert(document) return snapshot(document) async def command(self, replay_id, command: ScadaReplayCommand): await self.ensure_storage() async with self.repo.control(): document = await self.repo.get(replay_id) key, body = str(command.request_id), command.model_dump(mode="json") receipt = document.setdefault("receipts", {}).get(key) if receipt: if receipt != body: raise HTTPException(409, "命令请求号已用于不同内容") return snapshot(document) if command.expected_revision != document["revision"]: raise HTTPException(409, "回放控制状态已更新，请刷新后重试") action, status = command.action, document["status"] if document.get("step_pending") and action == "step": raise HTTPException(409, "当前时间步正在处理，请完成后继续") if action in {"step", "retry"} and await self.repo.active_other(replay_id): raise HTTPException(409, "当前项目已有另一条活动回放") if action == "step" and status in {"ready", "paused"}: interval = document.get("monitoring_interval_seconds") if interval is None: raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示") start = parse_time(document["start_time"]) elapsed = (parse_time(document["virtual_now"]) - start).total_seconds() target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval) document["step_start"] = document["virtual_now"] document["step_target"] = min(parse_time(document["end_time"]), target).isoformat() document["status"], document["step_pending"] = "paused", True elif action == "stop" and status not in {"completed", "stopped"}: document["status"], document["step_pending"] = "stopped", False document.pop("step_target", None) document.pop("step_start", None) elif action == "retry" and status == "failed": if "SOURCE_CHANGED" in (document.get("error") or ""): raise HTTPException(409, "原始数据发生变化，需要停止并新建回放") document["status"], document["error"] = "paused", None document["step_pending"] = bool(document.get("step_target")) else: raise HTTPException(409, "当前回放状态不支持此操作") document["revision"] += 1 document["receipts"][key] = body if len(document["receipts"]) > 256: del document["receipts"][next(iter(document["receipts"]))] await self.repo.save(document) return snapshot(document) async def process_pending_step(self, replay_id): await self.ensure_storage() async with self.repo.processing() as owned: if not owned: raise HTTPException(409, "当前时间步正在由另一请求处理") while True: document = await self.repo.get(replay_id) if not document.get("step_pending"): return snapshot(document) try: await self._advance(document) except Exception as exc: # Failure never discards the prepared batch; retry resumes it. document = await self.repo.get(replay_id) document["status"], document["step_pending"] = "failed", False document["error"] = ( "SOURCE_CHANGED: 原始观测已改变，需要新建回放" if "SOURCE_CHANGED" in str(exc) else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试" ) async with self.repo.connection.transaction(): await self.repo.save(document) raise async def _advance(self, document): config = ScadaReplayCreate.model_validate(document["config"]) replay_id = document["replay_id"] if not document.get("warmed"): import math window = max(rule.window_samples for rule in config.rules) async with self.timescale.transaction(): warmup = await ScadaRepository.get_warmup_values( self.timescale, config.device_ids, config.start_time, window, ) for row in warmup: value = row["monitored_value"] bounds = config.quality_ranges.get(row["device_id"]) if value is not None and math.isfinite(value) and not (bounds and ( bounds.lower is not None and value < bounds.lower or bounds.upper is not None and value > bounds.upper )): document["histories"].setdefault(row["device_id"], []).append(value) document["warmed"] = True async with self.repo.connection.transaction(): await self.repo.save(document) batch = await self.repo.pending_batch(replay_id) if batch is None: step = document.get("step_pending", False) virtual = parse_time(document["virtual_now"]) through = parse_time(document["step_target"]) if step else virtual after = ((parse_time(document["cursor"][0]), document["cursor"][1]) if document.get("cursor") else None) async with self.timescale.transaction(): rows = await ScadaRepository.get_raw_page( self.timescale, config.device_ids, config.start_time, config.end_time, through, 501, after, frame=True, ) if len(rows) > 500: raise ValueError("SCADA frame exceeds selected device bound") at = rows[0]["time"] if rows else through if step: virtual = through document["virtual_now"] = virtual.isoformat() if not rows and document.get("processed_until") == at.isoformat() and not step: async with self.repo.connection.transaction(): await self.repo.save(document) return sequence = document["batch_sequence"] + 1 refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"], "source_hash": row["source_hash"]} for row in rows] batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(), "refs": refs, "input_digest": digest(refs), "events": []} async with self.repo.connection.transaction(): document["batch_sequence"] = sequence document["delivered_until"] = at.isoformat() await self.repo.save_batch(replay_id, batch) await self.repo.save(document) else: rows = None if batch["stage"] == "selected": if rows is None: # Only crash recovery reloads the selected raw frame. at = parse_time(batch["at"]) async with self.timescale.transaction(): rows = await ScadaRepository.get_raw_page( self.timescale, config.device_ids, at, config.end_time, at, 501, ) if batch["refs"] else [] actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"], "source_hash": row["source_hash"]} for row in rows] if digest(actual) != batch["input_digest"]: raise ValueError("SOURCE_CHANGED: selected raw frame differs") batch["outputs"] = prepare_cleaning(rows, config) batch["stage"] = "clean_prepared" async with self.repo.connection.transaction(): await self.repo.save_batch(replay_id, batch) if batch["stage"] == "clean_prepared": await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"]) batch["stage"] = "cleaned" async with self.repo.connection.transaction(): await self.repo.save_batch(replay_id, batch) if batch["stage"] == "cleaned": result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"])) result["processed_until"] = batch["at"] target = result.get("step_target") was_step = bool(target) result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target)) monitoring = None if was_step and not result["step_pending"]: period_start = parse_time(result.get("step_start", result["start_time"])) period_end = parse_time(target) role = result.get("monitoring_config") or {} role_ids = list(dict.fromkeys( role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", []) )) previous = await self.repo.monitoring_outputs( replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)), period_end, batch["sequence"], ) if role_ids else [] from app.mcp.analysis_service import bounded_calculation monitoring = await bounded_calculation( evaluate_monitoring_period, result, previous + batch["outputs"], period_start=period_start, period_end=period_end, batch_sequence=batch["sequence"], ) result["last_monitoring"] = monitoring result["monitoring_count"] = result.get("monitoring_count", 0) + 1 if not result["step_pending"]: result.pop("step_target", None) result.pop("step_start", None) if batch["refs"]: last = batch["refs"][-1] result["cursor"] = [last["time"], last["device_id"]] if parse_time(batch["at"]) >= config.end_time: result["status"] = "completed" batch["stage"], batch["events"] = "detected", changes if monitoring is not None: batch["monitoring"] = monitoring async with self.repo.connection.transaction(): await self._create_investigations(result, changes, batch, create_tasks=not was_step) if monitoring is not None: await self._create_monitoring_task(result, monitoring, batch) batch["monitoring"] = monitoring result["last_monitoring"] = monitoring await self.repo.save_batch(replay_id, batch) await self.repo.save(result) async def _create_investigations(self, document, changes, batch, *, create_tasks=True): from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository from app.services.operational_tasks import OperationalTaskService from app.domain.schemas.task_alert import TaskAlertEvent from app.infra.db.postgresql.task_alerts import TaskAlertRepository from app.services.task_alerts import TaskAlertService from uuid import UUID producer = "scada-replay:" + document["replay_id"] alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id), self.context, internal_producer_id=producer) tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id), self.context, permissions_for_context(self.context)) for event in changes: if event["change"] == "raised": event["batch_sequence"] = batch["sequence"] document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"] await alerts.push(TaskAlertEvent( event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]), alert_id=event["event_id"], producer_id=producer, event_type=event["change"], occurred_at=event["occurred_at"], condition_id=None, source={"kind": "scada_replay", "replay_id": document["replay_id"], "rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]}, title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning", )) if event["change"] != "raised": if event.get("task_id"): await tasks.advance_replay_investigation(UUID(event["task_id"]), { "replay_id": document["replay_id"], "knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"], }, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery")) continue if event.get("category") == "data_quality": continue if not create_tasks: continue external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}" command = OperationalTaskCreate( request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation", workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}", summary=event["detail"], source=TaskSource( kind="external_system", external_id=external, external_revision=1, event_kind="scada_anomaly", replay_context={"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]}, ), trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"), ) task = await tasks.create(command) event["task_id"] = str(task.task_id) document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"] async def _create_monitoring_task(self, document, monitoring, batch): if monitoring["status"] != "suspected_burst": return from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository from app.services.operational_tasks import OperationalTaskService tasks = OperationalTaskService( OperationalTaskRepository(self.repo.connection, self.context.project_id), self.context, permissions_for_context(self.context), ) external = ( f"scada_replay:{document['replay_id']}:monitor:" f"{monitoring['period_end']}:{monitoring['rule_version']}" ) pressure = monitoring["pressure"] flow = monitoring["outlet_flow"] detector = monitoring["burst_detection"] summary = ( f"步长监测疑似爆管：爆管侦测得分 {detector['target_score']:.4f}；" f"压力下降测点 {len(pressure['hit_device_ids'])} 个；" f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。" ) command = OperationalTaskCreate( request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation", workflow="burst_report", title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}", summary=summary, source=TaskSource( kind="external_system", external_id=external, external_revision=1, event_kind="scada_burst_suspected", replay_context={ "replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"], "batch_sequence": batch["sequence"], }, ), trigger=TaskTrigger( kind="event", occurred_at=monitoring["period_end"], detail="主动步长监测的多项指标相互支持，进入爆管调查；尚未现场确诊", ), ) task = await tasks.create(command) monitoring["task_id"] = str(task.task_id)