From 389892cb3f76444a6e456ae54545bdc8ac8637ca Mon Sep 17 00:00:00 2001 From: Huarch Date: Mon, 16 Nov 2026 15:13:22 +0800 Subject: [PATCH] feat(replay): execute periodic diagnosis and hydraulic checks --- app/algorithms/periodic_hydraulics.py | 167 +++++ app/api/v1/endpoints/scada_replay.py | 39 +- app/domain/schemas/periodic_task.py | 56 ++ app/domain/schemas/scada_replay.py | 4 + app/infra/db/postgresql/scada_replays.py | 96 +++ app/infra/epanet/model_snapshot.py | 51 ++ app/services/calculation_worker.py | 15 +- app/services/periodic_diagnosis.py | 122 ++++ app/services/periodic_tasks.py | 246 ++++++++ app/services/scada_replay.py | 12 +- contracts/manifest.json | 2 +- contracts/server-v1.openapi.json | 579 ++++++++++++++++++ docs/README.md | 1 + docs/periodic-task-execution.md | 49 ++ docs/scada-peripheral-replay-design.md | 2 + docs/workbench-runtime.md | 2 + .../work-order-skills/scada-anomaly/SKILL.md | 6 +- tests/api/test_scada_replay_api.py | 2 +- .../integration/test_periodic_task_storage.py | 95 +++ tests/unit/test_model_snapshot.py | 27 + tests/unit/test_periodic_hydraulics.py | 67 ++ tests/unit/test_periodic_tasks.py | 306 +++++++++ 22 files changed, 1934 insertions(+), 12 deletions(-) create mode 100644 app/algorithms/periodic_hydraulics.py create mode 100644 app/domain/schemas/periodic_task.py create mode 100644 app/infra/epanet/model_snapshot.py create mode 100644 app/services/periodic_diagnosis.py create mode 100644 app/services/periodic_tasks.py create mode 100644 docs/periodic-task-execution.md create mode 100644 tests/integration/test_periodic_task_storage.py create mode 100644 tests/unit/test_model_snapshot.py create mode 100644 tests/unit/test_periodic_hydraulics.py create mode 100644 tests/unit/test_periodic_tasks.py diff --git a/app/algorithms/periodic_hydraulics.py b/app/algorithms/periodic_hydraulics.py new file mode 100644 index 0000000..3715de6 --- /dev/null +++ b/app/algorithms/periodic_hydraulics.py @@ -0,0 +1,167 @@ +"""Rolling hydraulic checks using immutable INP and observed input only. + +Each observed timestamp is a steady hydraulic snapshot, matching the online +recalculation's semantics. This module never opens a business/telemetry database. +""" +from datetime import datetime +from math import isfinite +from pathlib import Path +from statistics import mean +from tempfile import TemporaryDirectory +from zoneinfo import ZoneInfo +import re + + +def _sampling_seconds(device: dict) -> int: + frequency = str(device.get("transmission_frequency") or "") + clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency) + minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency) + return max(1, int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock + else int(minutes[1]) * 60 if minutes else 900) + + +def pressure_metres(value: float, unit: str) -> float | None: + factors = {"m": 1.0, "mH2O": 1.0, "MPa": 101.9716213, + "kPa": .1019716213, "bar": 10.19716213} + return value * factors[unit] if unit in factors else None + + +def flow_cms(value: float, unit: str) -> float | None: + factors = {"m3/s": 1.0, "m³/s": 1.0, "m3/h": 1 / 3600, + "m³/h": 1 / 3600, "L/s": .001, "l/s": .001, + "LPS": .001, "CMH": 1 / 3600, "CMS": 1.0} + return value * factors[unit] if unit in factors else None + + +def run_periodic_hydraulics(model_input: str, devices: list[dict], rows: list[dict], + period_start: str, period_end: str) -> dict: + import wntr + + start, end = datetime.fromisoformat(period_start), datetime.fromisoformat(period_end) + usable = [row for row in rows if row.get("quality") in {"valid", "corrected"} + and row.get("cleaned_value") is not None and isfinite(row["cleaned_value"]) + and start <= datetime.fromisoformat(row["time"]) <= end] + sample_times = sorted({row["time"] for row in usable}, key=datetime.fromisoformat) + if len(sample_times) > 100: + raise ValueError("PERIOD_SAMPLE_LIMIT_EXCEEDED") + if not sample_times: + return {"conclusion": "insufficient_data", "comparisons": [], "snapshots": [], + "data_gaps": ["本周期没有有效观测,无法开展模拟与实测复核。"], "metrics": []} + comparisons, snapshots, gaps = [], [], set() + directory_devices = {device["device_id"]: device for device in devices} + with TemporaryDirectory(prefix="tjwater-periodic-hydraulics-") as directory: + source = Path(directory) / "frozen.inp" + source.write_text(model_input, encoding="utf-8") + for index, at in enumerate(sample_times): + model = wntr.network.WaterNetworkModel(str(source)) + clock = datetime.fromisoformat(at).astimezone(ZoneInfo("Asia/Shanghai")) + model.options.time.duration = 0 + model.options.time.pattern_start = clock.hour * 3600 + clock.minute * 60 + clock.second + model.options.time.start_clocktime = model.options.time.pattern_start + frame = {row["device_id"]: row for row in usable if row["time"] == at} + # Only exact-timestamp boundary observations can override the frozen + # model. Missing boundaries retain explicit model assumptions. + applied = [] + for device_id, row in frame.items(): + device = directory_devices.get(device_id, {}) + node_id, link_id = device.get("node_id"), device.get("link_id") + value, kind = row["cleaned_value"], device.get("device_type") + if kind == "tank_liquid_level" and node_id in model.tank_name_list: + if device.get("measurement_unit") != "m": + gaps.add(f"{device_id} 水位单位未适配。") + continue + model.get_node(node_id).init_level = value + applied.append(device_id) + elif kind == "fixed_pump" and link_id in model.pump_name_list: + if value not in (0, 1): + gaps.add(f"{device_id} 工频泵状态不是 0/1。") + continue + model.get_link(link_id).initial_status = "OPEN" if value else "CLOSED" + applied.append(device_id) + elif kind == "demand" and node_id in model.junction_name_list: + demand = flow_cms(value, device.get("measurement_unit", "")) + if demand is None: + gaps.add(f"{device_id} 需水量单位未适配。") + continue + node = model.get_node(node_id) + node.demand_timeseries_list[0].base_value = demand + node.demand_timeseries_list[0].pattern_name = None + applied.append(device_id) + elif kind in {"variable_pump", "reservoir_liquid_level"}: + gaps.add(f"{device_id} 缺少已核实的频率基准或水位高程转换,沿用冻结模型边界。") + result = wntr.sim.EpanetSimulator(model).run_sim( + file_prefix=str(Path(directory) / f"snapshot-{index}"), convergence_error=True) + pressure = result.node["pressure"].iloc[-1] + flow = result.link["flowrate"].iloc[-1] + if not all(isfinite(float(value)) for value in pressure) or not all(isfinite(float(value)) for value in flow): + raise ValueError("NONFINITE_HYDRAULIC_RESULT") + snapshots.append({"time": at, "node_count": len(pressure), "link_count": len(flow), + "minimum_pressure_m": float(pressure.min()), + "maximum_pressure_m": float(pressure.max()), + "applied_boundary_device_ids": applied}) + for device_id, row in frame.items(): + device = directory_devices.get(device_id, {}) + kind, node_id, link_id = device.get("device_type"), device.get("node_id"), device.get("link_id") + if kind == "pressure" and node_id in pressure: + observed = pressure_metres(row["cleaned_value"], device.get("measurement_unit", "")) + predicted, unit = float(pressure[node_id]), "m" + elif kind in {"flow", "pipe_flow"} and link_id in flow: + observed = flow_cms(row["cleaned_value"], device.get("measurement_unit", "")) + predicted, unit = float(flow[link_id]), "m3/s" + else: + if kind in {"pressure", "flow", "pipe_flow"}: + gaps.add(f"{device_id} 缺少有效模型关联。") + continue + if observed is None: + gaps.add(f"{device_id} 观测单位未适配。") + continue + delta = abs(predicted - observed) + comparisons.append({"time": at, "device_id": device_id, "kind": kind, + "element_id": node_id if kind == "pressure" else link_id, + "observed": observed, "predicted": predicted, "unit": unit, + "absolute_deviation": delta, + "relative_deviation_percent": delta / abs(observed) * 100 + if abs(observed) > 1e-8 else None}) + metrics = [] + for kind, label, attention, critical in (("pressure", "模拟-实测压力偏差", 8, 12), + ("flow", "模拟-实测流量偏差", 10, 15)): + values = [item["relative_deviation_percent"] for item in comparisons + if (item["kind"] == "pressure") == (kind == "pressure") + and item["relative_deviation_percent"] is not None] + if values: + value = mean(values) + metrics.append({"id": kind + "-deviation", "label": label, "value": f"{value:.2f}", + "unit": "%", "status": "critical" if value >= critical else "attention" if value >= attention else "normal", + "detail": f"有效比较 {len(values)} 项;近零实测只报告绝对偏差。"}) + else: + gaps.add(f"缺少可计算相对偏差的{label}数据。") + pressure_deltas = [item["absolute_deviation"] for item in comparisons if item["kind"] == "pressure"] + if pressure_deltas: + maximum = max(pressure_deltas) + metrics.append({"id": "max-pressure-delta", "label": "最大节点压差", "value": f"{maximum:.2f}", + "unit": "m", "status": "critical" if maximum >= 8 else "attention" if maximum >= 5 else "normal", + "detail": "有实测映射节点的最大绝对压力偏差。"}) + compared_ids = {item["device_id"] for item in comparisons} + valid_ids = {item["device_id"] for item in comparisons + if (end - datetime.fromisoformat(item["time"])).total_seconds() + < _sampling_seconds(directory_devices.get(item["device_id"], {}))} + expected = {device["device_id"] for device in devices if device.get("device_type") in {"pressure", "flow", "pipe_flow"}} + missing = sorted(expected - valid_ids) + if missing: + gaps.add("缺少周期终点可用观测或模型映射的测点:" + "、".join(missing)) + gaps.add("未取得有效边界遥测的要素采用本轮冻结模型设定;本结果为逐时刻稳态水力复核。") + anomaly = any(metric["status"] != "normal" for metric in metrics) + complete = not missing and all(any(metric["id"] == kind + "-deviation" for metric in metrics) for kind in ("pressure", "flow")) + deviations = sorted( + [item for item in comparisons if (item["relative_deviation_percent"] is not None + and item["relative_deviation_percent"] >= (8 if item["kind"] == "pressure" else 10)) + or (item["kind"] == "pressure" and item["absolute_deviation"] >= 5)], + key=lambda item: item["relative_deviation_percent"] or 0, reverse=True, + ) + causes = (["先核查偏差测点的数据质量与模型关联,再复核冻结模型中的泵阀状态、供水边界和需水模式。", + "当前证据仅定位偏差测点和关联要素,尚不能单独确定边界滞后或模型参数错误。"] if deviations else []) + return {"conclusion": "attention" if anomaly else "normal" if complete else "insufficient_data", + "metrics": metrics, "comparisons": comparisons, "snapshots": snapshots, + "data_gaps": sorted(gaps), "compared_device_count": len(compared_ids), "current_device_count": len(valid_ids), + "missing_device_ids": missing, "model_scope": "all_nodes_and_links", + "deviation_targets": deviations[:50], "cause_review": causes} diff --git a/app/api/v1/endpoints/scada_replay.py b/app/api/v1/endpoints/scada_replay.py index b637803..abd3373 100644 --- a/app/api/v1/endpoints/scada_replay.py +++ b/app/api/v1/endpoints/scada_replay.py @@ -14,6 +14,11 @@ from app.domain.schemas.scada_replay import ( from app.infra.db.postgresql.scada_replays import ScadaReplayRepository from app.domain.schemas.project_settings import ProjectSettings from app.services.scada_replay import ScadaReplayService, snapshot +from app.domain.schemas.periodic_task import PeriodicTaskPage, PeriodicTaskExecution, PeriodicTaskRetry +from app.services.periodic_tasks import ( + ensure_periodic_tasks, initialize_periodic_tasks, project_periodic_state, + project_periodic_tasks, retry_periodic_task, require_execution_permissions, +) router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"]) @@ -40,6 +45,8 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm settings=Depends(project_settings)): available = await repo.available() current = await repo.current() if available else None + if current and current.get("config", {}).get("scheduled_tasks_enabled"): + current = await project_periodic_state(repo, current) monitoring = settings.scada_monitoring monitoring_ready = bool( monitoring @@ -68,17 +75,22 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm @router.post("", response_model=ScadaReplaySnapshot, status_code=201) async def create_replay(command: ScadaReplayCreate, repo=Depends(repository), ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)): - return await ScadaReplayService(repo, ctx).create( + if command.scheduled_tasks_enabled: + require_execution_permissions(ctx) + result = await ScadaReplayService(repo, ctx).create( command, monitoring_interval_seconds=settings.scada_monitoring_interval_seconds, monitoring_config=settings.scada_monitoring, ) + if command.scheduled_tasks_enabled: + await initialize_periodic_tasks(repo, ctx, result.replay_id) + return result @router.get("/{replay_id}", response_model=ScadaReplaySnapshot) async def get_replay(replay_id: UUID, repo=Depends(repository)): await ScadaReplayService(repo, None).ensure_storage() - return snapshot(await repo.get(replay_id)) + return snapshot(await project_periodic_state(repo, await repo.get(replay_id))) @router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot) @@ -86,12 +98,33 @@ async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depe ctx=Depends(require_permission(SCADA_CLEAN)), timescale=Depends(get_project_timescale_connection)): service = ScadaReplayService(repo, ctx, timescale) + document = await repo.get(replay_id) + if command.action != "stop" and document.get("config", {}).get("scheduled_tasks_enabled"): + await ensure_periodic_tasks(repo, ctx, replay_id) result = await service.command(replay_id, command) if command.action in {"step", "monitor", "retry"} and result.step_pending: - return await service.process_pending_step(replay_id) + result = await service.process_pending_step(replay_id) + if command.action != "stop" and getattr(result, "scheduled_tasks_enabled", False): + await ensure_periodic_tasks(repo, ctx, replay_id) + return snapshot(await project_periodic_state(repo, await repo.get(replay_id))) return result +@router.get("/{replay_id}/scheduled-tasks", response_model=PeriodicTaskPage) +async def scheduled_tasks(replay_id: UUID, offset: int = Query(0, ge=0), + limit: int = Query(100, ge=1, le=200), repo=Depends(repository)): + await ScadaReplayService(repo, None).ensure_storage() + await repo.get(replay_id) + items = await project_periodic_tasks(repo, replay_id, offset, limit + 1) + return PeriodicTaskPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None) + + +@router.post("/{replay_id}/scheduled-tasks/{execution_id}/retry", response_model=PeriodicTaskExecution) +async def retry_scheduled_task(replay_id: UUID, execution_id: UUID, command: PeriodicTaskRetry, + repo=Depends(repository), ctx=Depends(require_permission(SCADA_CLEAN))): + return await retry_periodic_task(repo, ctx, replay_id, execution_id, command.request_id) + + @router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage) async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200), repo=Depends(repository)): diff --git a/app/domain/schemas/periodic_task.py b/app/domain/schemas/periodic_task.py new file mode 100644 index 0000000..131c9a6 --- /dev/null +++ b/app/domain/schemas/periodic_task.py @@ -0,0 +1,56 @@ +"""Persisted business-clock task executions, independent of Agent sessions.""" +from typing import Literal +from uuid import UUID + +from pydantic import AwareDatetime, Field + +from app.domain.schemas.shared import StrictModel + + +class PeriodicTaskStep(StrictModel): + id: str + title: str + status: Literal["pending", "running", "completed", "failed", "skipped"] + detail: str = "" + + +class PeriodicTaskMetric(StrictModel): + id: str + label: str + value: str + unit: str = "" + status: Literal["normal", "attention", "critical"] = "normal" + detail: str = "" + + +class PeriodicTaskExecution(StrictModel): + execution_id: UUID + replay_id: UUID + task_key: Literal["scada-diagnosis", "network-simulation"] + title: str + period_start: AwareDatetime + period_end: AwareDatetime + knowledge_cutoff: AwareDatetime + batch_sequence: int = Field(ge=1) + status: Literal["pending", "running", "completed", "failed"] + conclusion: Literal["normal", "attention", "insufficient_data", "suspected_burst", "suspected_pollution"] + summary: str + steps: list[PeriodicTaskStep] + metrics: list[PeriodicTaskMetric] = Field(default_factory=list) + evidence: list[str] = Field(default_factory=list) + data_gaps: list[str] = Field(default_factory=list) + risk_signals: list[dict] = Field(default_factory=list) + investigation_task_id: UUID | None = None + calculation_job_id: UUID | None = None + attempt: int = Field(default=1, ge=1) + updated_at: AwareDatetime + result: dict | None = None + + +class PeriodicTaskPage(StrictModel): + items: list[PeriodicTaskExecution] + next_offset: int | None + + +class PeriodicTaskRetry(StrictModel): + request_id: UUID diff --git a/app/domain/schemas/scada_replay.py b/app/domain/schemas/scada_replay.py index c9e1392..4135e7c 100644 --- a/app/domain/schemas/scada_replay.py +++ b/app/domain/schemas/scada_replay.py @@ -61,6 +61,7 @@ class ScadaReplayCreate(ReplayModel): device_ids: list[str] = Field(min_length=1, max_length=500) rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30) quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict) + scheduled_tasks_enabled: bool = False @model_validator(mode="after") def configuration(self): @@ -130,6 +131,9 @@ class ScadaReplaySnapshot(ReplayModel): end_time: datetime monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000) step_pending: bool = False + scheduled_tasks_enabled: bool = False + periodic_pending: bool = False + periodic_error: str | None = None monitoring_count: int = 0 last_monitoring: ScadaMonitoringResult | None = None monitoring_config: ScadaMonitoringSettings | None = None diff --git a/app/infra/db/postgresql/scada_replays.py b/app/infra/db/postgresql/scada_replays.py index b8aa59e..25f05e8 100644 --- a/app/infra/db/postgresql/scada_replays.py +++ b/app/infra/db/postgresql/scada_replays.py @@ -139,6 +139,102 @@ class ScadaReplayRepository: ) return [r["event"] for r in await cur.fetchall()] + async def queue_available(self) -> bool: + cursor = await self.connection.execute("SELECT to_regclass('workbench.calculation_jobs') IS NOT NULL AS ready") + return bool((await cursor.fetchone())["ready"]) + + async def periodic_events(self, replay_id, start, end, through_sequence: int) -> list[dict]: + cursor = await self.connection.execute( + "SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' " + "AND b.sequence>0 AND b.sequence<=%s " + "AND (b.document->>'at')::timestamptz>=%s::timestamptz " + "AND (b.document->>'at')::timestamptz<=%s::timestamptz ORDER BY b.sequence", + (self.project_id, str(replay_id), through_sequence, start, end), + ) + return [row["event"] for row in await cursor.fetchall()] + + async def batch(self, replay_id, sequence: int) -> dict | None: + cursor = await self.connection.execute( + "SELECT document FROM peripheral_sample.scada_replay_batches WHERE project_id=%s AND replay_id=%s AND sequence=%s", + (self.project_id, str(replay_id), sequence), + ) + row = await cursor.fetchone() + return row["document"] if row else None + + async def periodic_tasks(self, replay_id, offset=0, limit=100) -> list[dict]: + cursor = await self.connection.execute( + "SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' " + "ORDER BY b.sequence DESC,t.value->>'task_key' LIMIT %s OFFSET %s", + (self.project_id, str(replay_id), limit, offset), + ) + return [row["execution"] for row in await cursor.fetchall()] + + async def periodic_task(self, replay_id, execution_id) -> dict | None: + cursor = await self.connection.execute( + "SELECT document->'scheduled_tasks'->%s AS execution FROM peripheral_sample.scada_replay_batches " + "WHERE project_id=%s AND replay_id=%s AND stage='detected' " + "AND document->'scheduled_tasks' ? %s LIMIT 1", + (str(execution_id), self.project_id, str(replay_id), str(execution_id)), + ) + row = await cursor.fetchone() + return row["execution"] if row else None + + async def previous_simulations(self, replay_id, before, limit=2) -> list[dict]: + cursor = await self.connection.execute( + "SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' " + "AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' " + "AND (t.value->>'period_end')::timestamptz<%s::timestamptz " + "ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT %s", + (self.project_id, str(replay_id), before, limit), + ) + return [row["execution"] for row in await cursor.fetchall()] + + async def reference_simulations(self, replay_id, period_ends: list[datetime]) -> list[dict]: + cursor = await self.connection.execute( + "SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' " + "AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' " + "AND (t.value->>'period_end')::timestamptz=ANY(%s::timestamptz[]) " + "ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT 7", + (self.project_id, str(replay_id), period_ends), + ) + return [row["execution"] for row in await cursor.fetchall()] + + async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id) -> None: + await self.connection.execute( + "UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{periodic_retry_receipts}'," + "COALESCE(document->'periodic_retry_receipts','{}'::jsonb) || jsonb_build_object(%s::text,%s::text)) " + "WHERE project_id=%s AND replay_id=%s AND sequence=%s", + (str(request_id), str(execution_id), self.project_id, str(replay_id), sequence), + ) + + async def save_periodic_task(self, replay_id, sequence: int, execution: dict) -> None: + cursor = await self.connection.execute( + "UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{scheduled_tasks}'," + "COALESCE(document->'scheduled_tasks','{}'::jsonb) || jsonb_build_object(%s::text,%s::jsonb)) " + "WHERE project_id=%s AND replay_id=%s AND sequence=%s AND stage='detected' RETURNING sequence", + (str(execution["execution_id"]), Jsonb(execution), self.project_id, str(replay_id), sequence), + ) + if await cursor.fetchone() is None: + raise ValueError("PERIOD_BATCH_UNAVAILABLE") + + async def simulation_state(self, replay_id, execution: dict) -> None: + state = {"execution_id": str(execution["execution_id"]), "status": execution["status"], + "error": execution["summary"] if execution["status"] == "failed" else None} + await self.connection.execute( + "UPDATE peripheral_sample.scada_replays SET document=jsonb_set(document,'{last_scheduled_simulation}',%s),updated_at=now() " + "WHERE project_id=%s AND replay_id=%s " + "AND (document->'last_monitoring'->>'period_end')::timestamptz=%s::timestamptz", + (Jsonb(state), self.project_id, str(replay_id), execution["period_end"]), + ) + async def monitoring_outputs( self, replay_id, diff --git a/app/infra/epanet/model_snapshot.py b/app/infra/epanet/model_snapshot.py new file mode 100644 index 0000000..7ad81b2 --- /dev/null +++ b/app/infra/epanet/model_snapshot.py @@ -0,0 +1,51 @@ +"""Read one consistent model without modifying the online network.""" +from pathlib import Path +from tempfile import TemporaryDirectory +import base64 +import gzip +from io import BytesIO + +from app.native.wndb.core.connection import project_transaction +from app.native.wndb.inp.exporter import dump_inp + +MAX_MODEL_BYTES = 64 * 1024 * 1024 +MAX_COMPRESSED_BYTES = 8 * 1024 * 1024 + + +class ModelSnapshotBudgetExceeded(ValueError): + pass + + +def pack_model_input(content: str) -> str: + raw = content.encode("utf-8") + if len(raw) > MAX_MODEL_BYTES: + raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED") + compressed = gzip.compress(raw, compresslevel=6, mtime=0) + if len(compressed) > MAX_COMPRESSED_BYTES: + raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED") + return base64.b64encode(compressed).decode("ascii") + + +def unpack_model_input(content: str) -> str: + if len(content) > (MAX_COMPRESSED_BYTES + 2) // 3 * 4: + raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED") + compressed = base64.b64decode(content, validate=True) + with gzip.GzipFile(fileobj=BytesIO(compressed)) as stream: + raw = stream.read(MAX_MODEL_BYTES + 1) + if len(raw) > MAX_MODEL_BYTES: + raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED") + return raw.decode("utf-8") + + +def capture_model_input(project: str) -> str: + with TemporaryDirectory(prefix="tjwater-model-snapshot-") as directory: + path = Path(directory) / "network.inp" + with project_transaction(project) as connection: + connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + dump_inp(project, str(path), "2") + if path.stat().st_size > MAX_MODEL_BYTES: + raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED") + content = path.read_text(encoding="utf-8") + if not content.strip(): + raise ValueError("MODEL_SNAPSHOT_UNAVAILABLE") + return content diff --git a/app/services/calculation_worker.py b/app/services/calculation_worker.py index 55b2003..0181d9f 100644 --- a/app/services/calculation_worker.py +++ b/app/services/calculation_worker.py @@ -80,7 +80,11 @@ async def execute_child(input_path: str, output_path: str): row = json.loads(Path(input_path).read_text()) try: async with execution_scope(row) as scope: - result = await execute_calculation_job(scope, row) + if row.get('tool') == 'scheduled_network_simulation': + from app.services.periodic_tasks import execute_periodic_calculation + result = await execute_periodic_calculation(scope, row) + else: + result = await execute_calculation_job(scope, row) payload = {'status': 'succeeded', 'result': result.model_dump(mode='json')} except Exception as exc: # Do not persist exception text: DSNs, SQL, credentials or model data can occur there. @@ -224,7 +228,14 @@ class CalculationWorker: terminal = {'status': 'failed', 'error': {'code': 'CLEANUP_FAILED'}} cleaned = True async with self.repository(row['project_id']) as repository: - finished = await repository.finish(row, self.owner, **terminal) + if row.get('tool') == 'scheduled_network_simulation': + from app.services.periodic_tasks import persist_periodic_terminal + async with repository.connection.transaction(): + finished = await repository.finish(row, self.owner, **terminal) + if finished: + await persist_periodic_terminal(repository.connection, row, terminal) + else: + finished = await repository.finish(row, self.owner, **terminal) if not finished: status = await repository.heartbeat(row, self.owner, self.settings.lease_seconds) if status == 'cancel_requested': diff --git a/app/services/periodic_diagnosis.py b/app/services/periodic_diagnosis.py new file mode 100644 index 0000000..d16b349 --- /dev/null +++ b/app/services/periodic_diagnosis.py @@ -0,0 +1,122 @@ +"""Project completed SCADA monitoring into an auditable scheduled execution.""" +from collections.abc import Sequence +from datetime import datetime, timezone +import re +from uuid import NAMESPACE_URL, uuid5 + +from app.domain.schemas.periodic_task import PeriodicTaskExecution +from app.services.scada_normal_profile import pollution_signal + + +STEP_TITLES = { + "scada-diagnosis": ["读取 SCADA 遥测", "检查新鲜度完整率", "诊断压力流量与水质", "判断潜在事件", "整理诊断结论"], + "network-simulation": ["冻结模型与边界", "运行水力模型", "比对实测偏差", "定位偏差区域", "输出模型复核"], +} + + +def execution_record(document: dict, monitoring: dict, task_key: str) -> dict: + return PeriodicTaskExecution( + execution_id=uuid5(NAMESPACE_URL, f"{document['replay_id']}:{task_key}:{monitoring['period_end']}"), + replay_id=document["replay_id"], task_key=task_key, + title="SCADA 数据诊断" if task_key == "scada-diagnosis" else "管网定时模拟", + period_start=monitoring["period_start"], period_end=monitoring["period_end"], + knowledge_cutoff=monitoring["knowledge_cutoff"], batch_sequence=monitoring["batch_sequence"], + status="pending", conclusion="insufficient_data", summary="等待本周期执行。", + steps=[{"id": f"step-{index}", "title": title, "status": "pending"} + for index, title in enumerate(STEP_TITLES[task_key])], + updated_at=datetime.now(timezone.utc), + ).model_dump(mode="json") + + +def diagnosis_execution(document: dict, monitoring: dict, pollution_events: Sequence[dict] = ()) -> dict: + record = execution_record(document, monitoring, "scada-diagnosis") + end = datetime.fromisoformat(monitoring["period_end"]) + devices = {device["device_id"]: device for device in document["devices"]} + rules = {rule["rule_id"]: rule for rule in document["config"]["rules"]} + signals = [] + water_events = {} + active_events = [state["event"] for state in document.get("rule_states", {}).values() if state.get("event")] + for event in [*pollution_events, *active_events]: + if not pollution_signal(document, event): + continue + if event.get("occurred_at") and datetime.fromisoformat(event["occurred_at"]) > end: + continue + if event.get("batch_sequence", 0) > monitoring["batch_sequence"]: + continue + water_events[event["event_id"]] = event + evidence = {event["event_id"]: event for event in monitoring.get("anomaly_signals", [])} + evidence.update(water_events) + for signal in evidence.values(): + rule = rules.get(signal["rule_id"], {}) + water = pollution_signal(document, signal) + signals.append({"kind": "suspected_pollution" if water else "hydraulic_anomaly", + "device_ids": signal["device_ids"], "detail": signal["detail"], + "metric_name": rule.get("metric_name", ""), "rule_id": signal["rule_id"], + **({"task_id": signal["task_id"]} if water and signal.get("task_id") else {})}) + if monitoring["status"] == "suspected_burst": + signals.append({"kind": "suspected_burst", "detail": "压力与爆管侦测等联合证据达到项目嫌疑门槛。", + "device_ids": monitoring.get("pressure", {}).get("hit_device_ids", [])}) + missing, invalid, fresh_ids = [], [], set() + for device_id, device in devices.items(): + row = document.get("latest", {}).get(device_id) + frequency = str(device.get("transmission_frequency") or "") + clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency) + minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency) + seconds = (int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock + else int(minutes[1]) * 60 if minutes else document["monitoring_interval_seconds"]) + age = (end - datetime.fromisoformat(row["time"])).total_seconds() if row else None + if age is None or age < 0 or age >= 2 * max(seconds, 1): + missing.append(device_id) + elif row["quality"] not in {"valid", "corrected"}: + invalid.append(device_id) + else: + fresh_ids.add(device_id) + conclusion = ("suspected_burst" if monitoring["status"] == "suspected_burst" else + "suspected_pollution" if any(signal["kind"] == "suspected_pollution" for signal in signals) else + "attention" if signals or monitoring["status"] == "attention" or invalid else monitoring["status"]) + gaps = [] + if missing: + gaps.append("缺少及时观测的测点:" + "、".join(missing)) + if invalid: + gaps.append("无效读数测点:" + "、".join(invalid)) + if monitoring["status"] == "insufficient_data": + gaps.append(monitoring["detail"]) + water_ids = {key for key, device in devices.items() if device.get("device_type") in {"quality", "water_quality"}} + configured_water = {device for rule in rules.values() if rule.get("kind") == "bounds" and rule.get("metric_name") + for device in rule.get("device_ids", [])} + if water_ids - configured_water: + gaps.append("尚未配置明确水质指标与业务阈值的测点,污染风险未评估:" + "、".join(sorted(water_ids - configured_water))) + if gaps and conclusion == "normal": + conclusion = "insufficient_data" + summary = ("发现水质异常,存在污染风险,已进入原因核验。" if conclusion == "suspected_pollution" + else monitoring["detail"]) + coverage = len(fresh_ids) / len(devices) * 100 if devices else 0 + metrics = [{"id": "usable-coverage", "label": "及时有效测点占比", "value": f"{coverage:.1f}", "unit": "%", + "status": "attention" if missing or invalid else "normal", "detail": "按设备采样周期检查;缺测或无效读数单列。"}, + {"id": "risk-signals", "label": "运行异常信号", "value": str(len(signals)), "unit": "项", + "status": "attention" if signals else "normal", "detail": "压力、流量与水质分别判别,异常不等于现场确诊。"}] + for key, title, kinds in (("pressure", "压力", {"pressure"}), ("flow", "流量", {"flow", "pipe_flow"}), + ("water", "水质", {"quality", "water_quality"})): + ids = {device for device, metadata in devices.items() if metadata.get("device_type") in kinds} + if ids: + valid_count = len(ids & fresh_ids) + metrics.append({"id": key + "-freshness", "label": title + "测点及时有效率", + "value": f"{valid_count / len(ids) * 100:.1f}", "unit": "%", + "status": "normal" if valid_count == len(ids) else "attention", + "detail": f"有效 {valid_count} / {len(ids)} 个测点;按设备采样周期核验。"}) + weighted = monitoring.get("weighted") + if weighted: + metrics.append({"id": "burst-score", "label": "爆管嫌疑综合评分", "value": f"{weighted['score']:.2f}", + "unit": "", "status": "attention" if weighted["score"] >= weighted["threshold"] else "normal", + "detail": f"项目嫌疑门槛 {weighted['threshold']:.2f};综合分不代表概率。"}) + record.update(status="completed", conclusion=conclusion, summary=summary, metrics=metrics, + data_gaps=gaps, risk_signals=signals, + investigation_task_id=monitoring.get("task_id") or next( + (signal["task_id"] for signal in signals if signal.get("task_id")), None), + evidence=[f"周期 {monitoring['period_start']} 至 {monitoring['period_end']}", + f"冻结批次 {monitoring['batch_sequence']};监测回执 {monitoring['monitoring_id']}", + *[signal["detail"] for signal in signals]], + result={"monitoring": monitoring, "missing_device_ids": missing, "invalid_device_ids": invalid}) + for step in record["steps"]: + step.update(status="completed") + return PeriodicTaskExecution.model_validate(record).model_dump(mode="json") diff --git a/app/services/periodic_tasks.py b/app/services/periodic_tasks.py new file mode 100644 index 0000000..efe7c30 --- /dev/null +++ b/app/services/periodic_tasks.py @@ -0,0 +1,246 @@ +"""Business-clock executions reuse replay batches and the durable calculation queue.""" +from datetime import datetime, timedelta, timezone +from uuid import UUID, NAMESPACE_URL, uuid5 + +from fastapi import HTTPException + +from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context +from app.domain.schemas.periodic_task import PeriodicTaskExecution +from app.mcp.errors import CalculationCapacityExceeded +from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository +from app.infra.db.postgresql.scada_replays import ScadaReplayRepository +from app.services.periodic_diagnosis import diagnosis_execution, execution_record + + +PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation" +ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"} + + +def require_execution_permissions(context) -> None: + if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context): + raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限") + + +async def initialize_periodic_tasks(repo, context, replay_id) -> None: + document = await repo.get(replay_id) + if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"): + return + require_execution_permissions(context) + if not await repo.queue_available(): + raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构") + from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded + from app.services.content_digest import digest + from app.mcp.analysis_service import bounded_calculation + try: + model_input = await bounded_calculation(capture_model_input, context.project_code) + packed = await bounded_calculation(pack_model_input, model_input) + except ModelSnapshotBudgetExceeded as error: + raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error + async with repo.control(): + document = await repo.get(replay_id) + if not document.get("scheduled_model_ready"): + document["scheduled_model_digest"] = digest({"inp": model_input}) + await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot", + "model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]}) + document["scheduled_model_ready"] = True + await repo.save(document) + + +async def ensure_periodic_tasks(repo, context, replay_id) -> None: + document = await repo.get(replay_id) + if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"): + return + require_execution_permissions(context) + await initialize_periodic_tasks(repo, context, replay_id) + async with repo.control(): + document = await repo.get(replay_id) + monitoring = document["last_monitoring"] + batch = await repo.batch(replay_id, monitoring["batch_sequence"]) + if not batch: + raise HTTPException(409, "本周期冻结数据批次尚未提交") + diagnosis = execution_record(document, monitoring, "scada-diagnosis") + if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}): + events = await repo.periodic_events( + replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"], + ) + diagnosis = diagnosis_execution(document, monitoring, events) + await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis) + simulation = execution_record(document, monitoring, "network-simulation") + if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}): + await enqueue_simulation(repo, context, document, simulation) + + +async def enqueue_simulation(repo, context, document, execution) -> None: + queue = CalculationJobRepository(repo.connection, repo.project_id) + identity = f"periodic:{execution['execution_id']}:{execution['attempt']}" + try: + row = await queue.enqueue( + owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)), + tool=PERIODIC_SIMULATION_TOOL, + parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"], + "execution_id": execution["execution_id"], "attempt": execution["attempt"]}, + binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600, + ) + execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。") + except CalculationCapacityExceeded: + execution.update(calculation_job_id=None, status="failed", summary="计算队列已满,水力计算尚未启动,可按原输入重试。") + execution["steps"][1].update(status="failed", detail="计算队列容量不足。") + execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。") + await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution) + await repo.simulation_state(document["replay_id"], execution) + + +def project_job(execution: dict, job: dict | None) -> dict: + result = dict(execution) + if execution["status"] in {"completed", "failed"}: + return result + if not job: + result.update(status="failed", summary="本周期计算记录不可用,请重试。") + elif job["status"] in ACTIVE_JOB_STATES: + result.update(status="running" if job["status"] != "queued" else "pending") + result["steps"] = [dict(step) for step in execution["steps"]] + result["steps"][1].update(status="running" if job["status"] == "running" else "pending") + elif job["status"] == "succeeded" and job.get("result"): + return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json") + else: + code = (job.get("error") or {}).get("code", job["status"]) + result.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。") + return result + + +async def project_periodic_tasks(repo, replay_id, offset, limit): + items = await repo.periodic_tasks(replay_id, offset, limit) + queue = CalculationJobRepository(repo.connection, repo.project_id) + result = [] + for execution in items: + job = (await queue.get(execution["calculation_job_id"], include_result=True) + if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None) + result.append(project_job(execution, job) if execution.get("calculation_job_id") else execution) + return result + + +async def project_periodic_state(repo, document: dict) -> dict: + state = document.get("last_scheduled_simulation") + if not state or state["status"] not in {"pending", "running"}: + return document + batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"]) + execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"]) + if execution: + queue = CalculationJobRepository(repo.connection, repo.project_id) + execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True)) + return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"], + "status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}} + return document + + +async def retry_periodic_task(repo, context, replay_id, execution_id, request_id): + require_execution_permissions(context) + async with repo.control(): + document = await repo.get(replay_id) + # A retry is scoped to one frozen execution, including older periods. + execution = await repo.periodic_task(replay_id, execution_id) + if not execution or execution["task_key"] != "network-simulation": + raise HTTPException(404, "当前回放不存在此水力计算任务") + batch = await repo.batch(replay_id, execution["batch_sequence"]) + receipts = batch.get("periodic_retry_receipts", {}) + prior = receipts.get(str(request_id)) + if prior: + if prior != str(execution_id): + raise HTTPException(409, "重试请求号已用于另一条执行记录") + return execution + queue = CalculationJobRepository(repo.connection, repo.project_id) + current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True)) + if current["status"] != "failed": + raise HTTPException(409, "仅失败的水力计算可重试") + execution.update(attempt=execution["attempt"] + 1, status="pending", result=None, + updated_at=datetime.now(timezone.utc).isoformat()) + for step in execution["steps"]: + step.update(status="pending", detail="") + await enqueue_simulation(repo, context, document, execution) + await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id) + return execution + + +async def execute_periodic_calculation(scope, row): + """Internal worker job; admission and execution both recheck project rights.""" + from app.mcp.models import QueryResult + from app.algorithms.periodic_hydraulics import run_periodic_hydraulics + from app.infra.epanet.model_snapshot import unpack_model_input + from app.mcp.analysis_service import bounded_calculation + require_execution_permissions(scope.project) + parameters = row["parameters"] + if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}: + raise HTTPException(403, "周期任务所属项目不匹配") + repo = ScadaReplayRepository(scope.connection, scope.project.project_id) + document = await repo.get(UUID(parameters["replay_id"])) + batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"]) + execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"]) + if (not execution or execution["calculation_job_id"] != str(row["job_id"]) + or str(document["created_by"]) != str(row["owner_user_id"]) + or execution["attempt"] != parameters["attempt"]): + raise HTTPException(409, "周期计算执行身份已失效") + observations = await repo.detected_outputs( + document["replay_id"], document["config"]["device_ids"], + datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]), + execution["batch_sequence"], limit=50_001, + ) + if len(observations) > 50_000: + raise HTTPException(413, "本周期观测超过水力复核预算") + previous = await repo.previous_simulations(document["replay_id"], execution["period_end"]) + reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)] + references = await repo.reference_simulations(document["replay_id"], reference_times) + model_snapshot = await repo.batch(document["replay_id"], 0) + await scope.connection.commit() + if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot" + or model_snapshot["model_digest"] != document["scheduled_model_digest"]): + raise HTTPException(409, "冻结模型缺失或摘要不一致") + model_input = unpack_model_input(model_snapshot["model_input_gzip"]) + result = await bounded_calculation(run_periodic_hydraulics, model_input, + document["devices"], observations, + execution["period_start"], execution["period_end"]) + result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous] + current_metrics = {item["id"]: item for item in result["metrics"]} + contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"]) + prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {} + worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics + and float(metric["value"]) > float(prior_metrics[key]["value"]) + and metric["status"] != "normal"] + if worsening: + result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。") + result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references] + observed_references = {datetime.fromisoformat(item["period_end"]) for item in references} + missing_references = [time.isoformat() for time in reference_times if time not in observed_references] + if missing_references: + result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references)) + execution.update(status="completed", conclusion=result["conclusion"], result=result, + metrics=result["metrics"], data_gaps=result["data_gaps"], + summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention" + else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data" + else "本周期水力计算与实测偏差复核已完成。"), + updated_at=datetime.now(timezone.utc).isoformat(), + evidence=[f"模型快照 {document['scheduled_model_digest']}", + f"冻结观测批次不超过 {execution['batch_sequence']}", + f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"]) + for index, step in enumerate(execution["steps"]): + step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed") + execution["steps"][3]["detail"] = ";".join(result.get("cause_review", [])) + return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"], + queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution) + + +async def persist_periodic_terminal(connection, row, terminal): + """Called inside the supervisor's fenced terminal transaction.""" + repo = ScadaReplayRepository(connection, row["project_id"]) + parameters = row["parameters"] + execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"]) + if not execution or execution["calculation_job_id"] != str(row["job_id"]): + return + if terminal["status"] == "succeeded": + execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json") + else: + code = (terminal.get("error") or {}).get("code", terminal["status"]) + execution.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。", + updated_at=datetime.now(timezone.utc).isoformat()) + execution["steps"][1].update(status="failed", detail=execution["summary"]) + await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution) + await repo.simulation_state(parameters["replay_id"], execution) diff --git a/app/services/scada_replay.py b/app/services/scada_replay.py index 882fdc7..33ae904 100644 --- a/app/services/scada_replay.py +++ b/app/services/scada_replay.py @@ -42,6 +42,9 @@ def snapshot(document: dict) -> ScadaReplaySnapshot: "virtual_now", "created_at", "updated_at", )}, delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"), + scheduled_tasks_enabled=document.get("config", {}).get("scheduled_tasks_enabled", False), + periodic_pending=(document.get("last_scheduled_simulation") or {}).get("status") in {"pending", "running"}, + periodic_error=(document.get("last_scheduled_simulation") or {}).get("error"), processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0), devices=document["devices"], rules=document["config"]["rules"], latest=list(document.get("latest", {}).values()), @@ -132,6 +135,11 @@ class ScadaReplayService: return snapshot(document) if command.expected_revision != document["revision"]: raise HTTPException(409, "回放控制状态已更新,请刷新后重试") + if document["config"].get("scheduled_tasks_enabled") and command.action in {"step", "monitor"}: + from app.services.periodic_tasks import project_periodic_state + document = await project_periodic_state(self.repo, document) + if snapshot(document).periodic_pending: + raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放") action, status = command.action, document["status"] if (action in {"step", "monitor", "retry"} and document.get("warmed") and weighted_diagnosis(document) @@ -332,7 +340,7 @@ class ScadaReplayService: monitoring["anomaly_signals"] = list(signals.values()) if signals and monitoring["status"] == "normal": monitoring["status"] = "attention" - monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断" + monitoring["detail"] = "本周期曾检出测点异常,进入运行异常与潜在事件诊断" result["monitoring_count"] = result.get("monitoring_count", 0) + 1 if not result["step_pending"]: result.pop("step_target", None) @@ -565,7 +573,7 @@ class ScadaReplayService: trigger=TaskTrigger( kind="event", occurred_at=monitoring["period_end"], - detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管", + detail="SCADA 任一指标异常启动调查,核验爆管或污染等潜在事件;异常不等于现场确诊", ), ) task = await tasks.create(command) diff --git a/contracts/manifest.json b/contracts/manifest.json index ddd981b..f38beeb 100644 --- a/contracts/manifest.json +++ b/contracts/manifest.json @@ -3,7 +3,7 @@ "contracts": { "server": { "file": "server-v1.openapi.json", - "sha256": "04aeed6f1a7414b37267d84141d036d00d2f8579ee28af37590bd4b92137e712" + "sha256": "90e0ac0fb466f0a67c9a8a909252a610cb8a089fd4289fa23232d4646d0a8849" } } } diff --git a/contracts/server-v1.openapi.json b/contracts/server-v1.openapi.json index d03cf83..8f54c08 100644 --- a/contracts/server-v1.openapi.json +++ b/contracts/server-v1.openapi.json @@ -6519,6 +6519,302 @@ "title": "Page[str]", "type": "object" }, + "PeriodicTaskExecution": { + "additionalProperties": false, + "properties": { + "attempt": { + "default": 1, + "minimum": 1.0, + "title": "Attempt", + "type": "integer" + }, + "batch_sequence": { + "minimum": 1.0, + "title": "Batch Sequence", + "type": "integer" + }, + "calculation_job_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Calculation Job Id" + }, + "conclusion": { + "enum": [ + "normal", + "attention", + "insufficient_data", + "suspected_burst", + "suspected_pollution" + ], + "title": "Conclusion", + "type": "string" + }, + "data_gaps": { + "items": { + "type": "string" + }, + "title": "Data Gaps", + "type": "array" + }, + "evidence": { + "items": { + "type": "string" + }, + "title": "Evidence", + "type": "array" + }, + "execution_id": { + "format": "uuid", + "title": "Execution Id", + "type": "string" + }, + "investigation_task_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Investigation Task Id" + }, + "knowledge_cutoff": { + "format": "date-time", + "title": "Knowledge Cutoff", + "type": "string" + }, + "metrics": { + "items": { + "$ref": "#/components/schemas/PeriodicTaskMetric" + }, + "title": "Metrics", + "type": "array" + }, + "period_end": { + "format": "date-time", + "title": "Period End", + "type": "string" + }, + "period_start": { + "format": "date-time", + "title": "Period Start", + "type": "string" + }, + "replay_id": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + }, + "result": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Result" + }, + "risk_signals": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Risk Signals", + "type": "array" + }, + "status": { + "enum": [ + "pending", + "running", + "completed", + "failed" + ], + "title": "Status", + "type": "string" + }, + "steps": { + "items": { + "$ref": "#/components/schemas/PeriodicTaskStep" + }, + "title": "Steps", + "type": "array" + }, + "summary": { + "title": "Summary", + "type": "string" + }, + "task_key": { + "enum": [ + "scada-diagnosis", + "network-simulation" + ], + "title": "Task Key", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "execution_id", + "replay_id", + "task_key", + "title", + "period_start", + "period_end", + "knowledge_cutoff", + "batch_sequence", + "status", + "conclusion", + "summary", + "steps", + "updated_at" + ], + "title": "PeriodicTaskExecution", + "type": "object" + }, + "PeriodicTaskMetric": { + "additionalProperties": false, + "properties": { + "detail": { + "default": "", + "title": "Detail", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "status": { + "default": "normal", + "enum": [ + "normal", + "attention", + "critical" + ], + "title": "Status", + "type": "string" + }, + "unit": { + "default": "", + "title": "Unit", + "type": "string" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "id", + "label", + "value" + ], + "title": "PeriodicTaskMetric", + "type": "object" + }, + "PeriodicTaskPage": { + "additionalProperties": false, + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/PeriodicTaskExecution" + }, + "title": "Items", + "type": "array" + }, + "next_offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Next Offset" + } + }, + "required": [ + "items", + "next_offset" + ], + "title": "PeriodicTaskPage", + "type": "object" + }, + "PeriodicTaskRetry": { + "additionalProperties": false, + "properties": { + "request_id": { + "format": "uuid", + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id" + ], + "title": "PeriodicTaskRetry", + "type": "object" + }, + "PeriodicTaskStep": { + "additionalProperties": false, + "properties": { + "detail": { + "default": "", + "title": "Detail", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "pending", + "running", + "completed", + "failed", + "skipped" + ], + "title": "Status", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + } + }, + "required": [ + "id", + "title", + "status" + ], + "title": "PeriodicTaskStep", + "type": "object" + }, "PlaceSearchConfig": { "additionalProperties": false, "properties": { @@ -8453,6 +8749,11 @@ "title": "Rules", "type": "array" }, + "scheduled_tasks_enabled": { + "default": false, + "title": "Scheduled Tasks Enabled", + "type": "boolean" + }, "start_time": { "format": "date-time", "title": "Start Time", @@ -8766,6 +9067,22 @@ ], "title": "Monitoring Interval Seconds" }, + "periodic_error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Periodic Error" + }, + "periodic_pending": { + "default": false, + "title": "Periodic Pending", + "type": "boolean" + }, "processed_count": { "default": 0, "title": "Processed Count", @@ -8804,6 +9121,11 @@ "title": "Rules", "type": "array" }, + "scheduled_tasks_enabled": { + "default": false, + "title": "Scheduled Tasks Enabled", + "type": "boolean" + }, "start_time": { "format": "date-time", "title": "Start Time", @@ -43962,6 +44284,263 @@ ] } }, + "/api/v1/scada-replays/{replay_id}/scheduled-tasks": { + "get": { + "operationId": "get_scada_replays_replay_id_scheduled_tasks", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "default": 0, + "minimum": 0, + "title": "Offset", + "type": "integer" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 100, + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PeriodicTaskPage" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Scheduled Tasks", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry": { + "post": { + "operationId": "post_scada_replays_replay_id_scheduled_tasks_execution_id_retry", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "path", + "name": "execution_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Execution Id", + "type": "string" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PeriodicTaskRetry" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PeriodicTaskExecution" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Retry Scheduled Task", + "tags": [ + "SCADA Replay" + ] + } + }, "/api/v1/sensor-placement-candidates/{node_id}": { "get": { "operationId": "get_sensor_placement_candidates_node_id", diff --git a/docs/README.md b/docs/README.md index 43633b8..3962788 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,6 +36,7 @@ | 主题 | 维护入口 | | --- | --- | +| 播放时间驱动的诊断、定时模拟及模板动作 | [周期任务执行](periodic-task-execution.md) | | 回放、时间轴、周期监测与调查边界 | [SCADA 回放设计与实现](scada-peripheral-replay-design.md) | | 外围测试样本与数据再现 | [合成样本](peripheral-generated-data.md) | | 凤阳历史爆管模拟 | [凤阳 SCADA 复制记录](fengyang-burst-scada.md) | diff --git a/docs/periodic-task-execution.md b/docs/periodic-task-execution.md new file mode 100644 index 0000000..aedc8f5 --- /dev/null +++ b/docs/periodic-task-execution.md @@ -0,0 +1,49 @@ +# 播放时间驱动的周期任务 + +2026-11-11:SCADA 数据诊断与管网定时模拟接入实际处理,管网智能调度与泵站能耗检查保留现有动作模板。任务名称、列表、报告及 Agent 指令使用业务表述;内部执行来源与证据限制分别保存。 + +## 时钟与周期 + +新建回放请求 `scheduled_tasks_enabled: true` 启用两类实际任务,供水前端默认发送该字段。省略字段的旧客户端与已有回放继续保持原有处理行为。项目监测周期读取 MetaDB `projects.settings.scada_monitoring_interval_seconds`,本业务约定为 `900` 秒;前端保留的 SCADA 任务模板也改为 15 分钟。5 分钟时间轴定位刻度、180 倍播放速度、设备采样周期和接口轮询间隔各自独立。 + +播放或单步在业务周期终点 T 读取并提交本周期观测、诊断回执和调查关联,随后创建周期执行记录并提交水力计算。水力任务未结束时暂停时间推进,已有调查可以独立执行。关闭页面停止继续推进;已提交的计算仍由现有 calculation worker 完成。跨日期推进保留原始业务时间,查看历史指针不触发处理;向已处理时刻重新监测沿用新建回放的因果边界。 + +## SCADA 数据诊断 + +执行步骤为读取观测、检查新鲜度完整率、诊断压力流量与水质、判断潜在事件、整理诊断结论。复用现有清洗、动态压力阈值、爆管侦测及指标规则,不按已知事件日期生成异常。 + +周期记录分别保留数据质量、压力/流量运行异常、疑似爆管与疑似污染线索。水质必须有指标名称、单位及明确业务阈值;只知道 `mg/L` 不能推断测量对象。水质规则尚未配置时明确记录污染风险未评估。水质有效异常进入现有独立 `water_pollution` 调查,不等待压力基线或爆管模型就绪,纯水质异常不强制进入爆管定位。周期报告从已冻结周期批次事件和持续异常状态读取水质证据,保留关联调查;周期内发生后恢复的异常仍记入本周期,水质证据不计入爆管支持分。 + +仪表量程用于读数有效性校验,业务阈值用于异常判断;可靠采集的异常值保留。异常持续期间复用已有调查;恢复与再次发生沿用现有异常事件生命周期。正常周期由 Server 生成报告,不启动 Agent。 + +## 管网定时模拟 + +在创建本轮回放时,以只读一致事务导出 EPANET 2 输入并保存摘要。完整模型按 gzip 压缩保存,原文限制为 64 MiB、压缩后限制为 8 MiB,冻结设备目录与周期输入后,计算 worker 使用相同模型版本和不超过本周期 `batch_sequence`、`knowledge_cutoff` 的观测。计算不重新查询实时 TSDB,不更新在线模型、正式 raw 或实时基线。 + +现有 `wntr==1.3.2` 与 EPANET 内核运行逐观测时刻的稳态水力复核,与原在线滚动重算的计算语义一致。每个时刻计算全部节点和管段,将已关联测点的压力、流量按单位转换后与实测比较。水箱水位、工频泵状态和需水量只有在单位及关联明确时覆盖模型边界;变频频率与水源液位缺少已核实换算依据时保留冻结模型设定,并明确报告限制。 + +输出压力/流量平均相对偏差、最大节点压差、偏差测点及关联要素、全模型规模、原始比较明细与缺项。近零实测不计算百分比。模板压力 8%/12%、流量 10%/15%、最大压差 5m/8m 的关注/异常阈值用于本版复核,不能解释为爆管确诊阈值。比较已完成的相邻周期及本轮模型下此前七天同窗结果,缺少参考时列出实际日期,不补造基线。偏差来源输出待核查原因,不将参数或边界问题当作已证实事实。 + +## 持久化与重试 + +不新增数据库表。压缩模型输入保存在 `peripheral_sample.scada_replay_batches` 的零号 `model_snapshot` 批次,回放主文档只保存摘要和就绪标记,避免轮询和每帧处理重复加载、重写完整模型。执行记录保存在业务批次的 `document.scheduled_tasks`,不通过回放快照返回完整模型。 + +每条执行身份由回放、任务类型和周期终点确定,同周期重复命令复用记录。水力计算复用 `workbench.calculation_jobs`,内部工具键为 `scheduled_network_simulation`。提交与执行分别核验项目成员和 SCADA 读取、清洗、水力计算权限。任务直接由 Server 执行,不伪造 Agent 会话、运行租约或 OAuth 上下文。 + +worker 沿用现有并发限制、子进程隔离、超时、租约及代次栅栏。成功结果和终态在同一栅栏事务内回写周期记录,计算队列结果过期不删除已保存的业务报告。失败或中断可按原模型、批次和时间边界重试,增加 attempt,保留同一执行身份。 + +## 接口与前端 + +- `GET /api/v1/scada-replays/{replay_id}/scheduled-tasks?offset=0&limit=100`:分页查询周期记录,仅读取已提交结果;不创建任务。 +- `POST /api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry`:提交 `{ "request_id": "UUID" }`,重试失败的水力计算。 +- 回放快照增加 `scheduled_tasks_enabled`、`periodic_pending`、`periodic_error`;轮询实际队列状态,不把排队视为完成。 + +前端以当前回放业务时间生成保留的调度与能耗任务,暂停后进度冻结。实际回放启用时,SCADA 和定时模拟由真实周期记录替代对应生成条目;旧客户端与未启用的回放仍可使用完整原模板。列表显示最近六小时,接口保留分页历史。执行状态和诊断结论分开,报告展示实际步骤、指标、证据、缺项、调查与重试入口;刷新保留当前选中报告。 + +## 验证与启用 + +后端使用 `tests/unit/test_periodic_tasks.py`、`test_periodic_hydraulics.py` 及现有回放、计算 worker 和架构边界测试;真实 SQL 与 EPANET 联合检查为 `PERIODIC_TASK_STORAGE_TEST=1 python -m pytest tests/integration/test_periodic_task_storage.py -q`。该集成检查只使用独立项目 UUID,所有记录强制回滚,不保留测试观测或任务。 + +前端运行 `pnpm lint`、`pnpm typecheck`、`pnpm test`、`pnpm build`,聚焦浏览器检查为 `pnpm test:browser tests/browser/periodic-tasks.e2e.ts`,覆盖桌面、手机、失败重试、刷新恢复与水平溢出。接口变更后重新导出 Server OpenAPI 并同步 Agent 的生成契约。 + +部署使用现有 API 与 calculation worker,不增加服务或迁移。两个进程都需要加载本轮代码,旧 worker 无法执行新增内部工具。未运行实际 Agent、现场审批或设备操作时,不将静态检查、固定浏览器响应和原生计算验证报告为现场业务验收。 diff --git a/docs/scada-peripheral-replay-design.md b/docs/scada-peripheral-replay-design.md index 3fc621b..c6917e9 100644 --- a/docs/scada-peripheral-replay-design.md +++ b/docs/scada-peripheral-replay-design.md @@ -1,5 +1,7 @@ # SCADA 外围系统时段回放、主动监测与爆管调查设计 +2026-11-11 增补:[周期任务执行](periodic-task-execution.md) 已将 SCADA 综合诊断与管网定时模拟接入外围播放时间,诊断覆盖疑似爆管、污染及其他异常;智能调度和能耗保留动作模板。以下“本阶段不创建定时任务”等表述记录原单步版本边界,新接入范围以增补文档为准。SCADA 读取清洗仍由命令请求执行,水力复核由现有 calculation worker 异步完成。 + 状态:时段回放、主动周期监测、条件式 `burst_report` 调查及回放安全的爆管定位适配已实现。单步处理由用户操作触发并在当前 API 请求内完成,不部署 SCADA 回放 worker。`tjwater_v2` 与 `tjwater2_v2` 尚未确认出厂边界流量角色;仍待完整月度数据和真实 Agent 联调。更新日期:2026-11-03。下面明确区分已实现范围、后续目标与验收要求。 本次将外围 SCADA 定义为一个模拟外部数据源:本地 TSDB 已保存原始观测,通过只读 API 按选定时间区间和后台时钟分批读取 `scada.measurements.monitored_value`,将这一批直接交给清洗流水线,保存 `cleaned_value` 后执行异常检测及后续 Agent 调查。原始值、修正值、真实工况异常和处理结论分别保留。浏览器负责控制和展示,关闭页面不停止后台回放。 diff --git a/docs/workbench-runtime.md b/docs/workbench-runtime.md index e3a806c..d82b3f3 100644 --- a/docs/workbench-runtime.md +++ b/docs/workbench-runtime.md @@ -1,5 +1,7 @@ # 工作台运行态 +播放周期任务通过[独立执行接口](periodic-task-execution.md)读取,前端将其与原有运行态组合;下文兼容字段 `conditions` 不承担回放周期记录的存储或投影。 + `GET /api/v1/workbench/runtime` 按当前项目读取 BizDB 中的分析成果和任务告警。外围样板工单经同步器进入 `workbench.operational_tasks`,由工况任务接口和 `sync-snapshot` 提供给前端;定时任务表暂时保留,但不进入当前产品主路径。 - `analysis.runs/results` 只用于分析成果、历史成果和兼容场景字段。 diff --git a/resources/work-order-skills/scada-anomaly/SKILL.md b/resources/work-order-skills/scada-anomaly/SKILL.md index 0d7539f..f4547e8 100644 --- a/resources/work-order-skills/scada-anomaly/SKILL.md +++ b/resources/work-order-skills/scada-anomaly/SKILL.md @@ -1,8 +1,8 @@ --- name: scada-anomaly -description: 从 SCADA 指标异常开展数据核验、爆管诊断、候选定位与影响分析。 +description: 从 SCADA 压力、流量和水质异常开展数据核验、潜在爆管或污染诊断及影响分析。 metadata: - version: "1.0.21" + version: "1.0.22" --- # SCADA 数据异常调查 @@ -21,7 +21,7 @@ metadata: 全量devices含完整目录,readings含观测,coverage含各设备读数数量、缺测及冻结范围外清单。不得由代表测点推断全网。单查list_scada_devices默认每页100,读至page.has_more=false,属性缩略不代表完整信息。全量后可按device_ids局部复核并持续翻页,不能替代全量输入。final引用本轮knowledge_cutoff及batch_sequence的全量证据;Server拒绝局部或旧批次证据。 -核对模型得分、压力变化与出厂边界流量的联合证据;权重、门槛、阈值、测点与基线留在依据。得分越低越异常,相似度非发生概率。区分边界与其他管流测点,规则零命中不代表无流量变化;其他异常按证据选择原因。 +核对模型得分、压力绝对下降及变化率与出厂边界流量的联合证据;权重、门槛、阈值、测点与各自基线留在依据,目标告警与参考日分别解释。得分越低越异常,相似度非发生概率。区分边界与其他管流测点,规则零命中不代表无流量变化;其他异常按证据选择原因。水质按指标、单位、业务阈值与相邻趋势核查,区别仪表量程,不要求爆管定位。 ## 候选定位与隔离 diff --git a/tests/api/test_scada_replay_api.py b/tests/api/test_scada_replay_api.py index dc4148c..c142d49 100644 --- a/tests/api/test_scada_replay_api.py +++ b/tests/api/test_scada_replay_api.py @@ -110,7 +110,7 @@ def test_monitoring_command_processes_pending_period_in_same_request(monkeypatch ) result = asyncio.run(scada_replay.replay_command( - replay_id, command, repo="repo", ctx=context, timescale="timescale" + replay_id, command, repo=SimpleNamespace(get=AsyncMock(return_value={"config": {}})), ctx=context, timescale="timescale" )) assert result == "completed-period" diff --git a/tests/integration/test_periodic_task_storage.py b/tests/integration/test_periodic_task_storage.py new file mode 100644 index 0000000..aca18eb --- /dev/null +++ b/tests/integration/test_periodic_task_storage.py @@ -0,0 +1,95 @@ +"""Native calculation and real SQL under an isolated project UUID, always rolled back.""" +import os +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from psycopg.rows import dict_row +import pytest + +from app.auth.project_dependencies import ProjectContext +from app.infra.db.metadb.database import SessionLocal, engine +from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository +from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository +from app.infra.db.postgresql.scada_replays import ScadaReplayRepository +from app.services.periodic_tasks import ensure_periodic_tasks, execute_periodic_calculation, persist_periodic_terminal, project_periodic_tasks +from app.infra.epanet.model_snapshot import pack_model_input +from types import SimpleNamespace + + +pytestmark = pytest.mark.skipif(os.getenv("PERIODIC_TASK_STORAGE_TEST") != "1", reason="explicit rollback-only SQL test required") + + +@pytest.fixture +def anyio_backend(): return "asyncio" + + +@pytest.mark.anyio +async def test_frozen_period_queue_native_result_and_retry_storage(): + from tests.unit.test_periodic_hydraulics import MODEL + async with SessionLocal() as session: + metadata = MetadataRepository(session) + routing = await metadata.get_project_db_routing(UUID("871d7538-589c-47de-a6d5-3185ccb9f0e1"), "biz_data") + await engine.dispose() + if not routing: + pytest.skip("Project route unavailable") + connection = await psycopg.AsyncConnection.connect(routing.dsn, row_factory=dict_row, connect_timeout=5) + project_id, user_id, replay_id, request_id = uuid4(), uuid4(), uuid4(), uuid4() + try: + async with connection.transaction(force_rollback=True): + repo = ScadaReplayRepository(connection, project_id) + if not await repo.available() or not await repo.queue_available(): + pytest.skip("Replay or calculation storage is not provisioned") + start, end = "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00" + devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m", "transmission_frequency": "0:15:00"}, + {"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h", "transmission_frequency": "0:15:00"}] + rows = [{"device_id": device, "time": at, "quality": "valid", "cleaned_value": value} + for at in (start, end) for device, value in (("P1", 39.8), ("F1", 3.6))] + monitoring = {"monitoring_id": str(uuid4()), "period_start": start, "period_end": end, "knowledge_cutoff": end, + "batch_sequence": 1, "status": "normal", "detail": "本周期正常", "task_id": None} + document = {"project_id": str(project_id), "replay_id": str(replay_id), "created_by": str(user_id), "status": "paused", + "config": {"request_id": str(request_id), "scheduled_tasks_enabled": True, "device_ids": ["P1", "F1"], "rules": []}, + "devices": devices, "latest": {row["device_id"]: row for row in rows}, + "monitoring_interval_seconds": 900, "last_monitoring": monitoring, + "scheduled_model_ready": True, "scheduled_model_digest": "frozen-test-model", + "updated_at": datetime.now(timezone.utc).isoformat()} + await repo.insert(document) + await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot", + "model_input_gzip": pack_model_input(MODEL), "model_digest": "frozen-test-model"}) + await repo.save_batch(replay_id, {"sequence": 1, "stage": "detected", "monitoring": monitoring, "outputs": rows}) + context = ProjectContext(project_id, "tjwater2_v2", user_id, "member") + await ensure_periodic_tasks(repo, context, replay_id) + await ensure_periodic_tasks(repo, context, replay_id) + page = await project_periodic_tasks(repo, replay_id, 0, 100) + assert len(page) == 2 + simulation = next(item for item in page if item["task_key"] == "network-simulation") + assert simulation["status"] == "pending" + queue = CalculationJobRepository(connection, project_id) + owner = str(uuid4()) + job = await queue.claim(owner, 30) + # Avoid committing the enclosing test rollback transaction: production + # releases its read transaction before computation, tested separately. + class ReadScopeConnection: + def __getattr__(self, key): return getattr(connection, key) + async def commit(self): pass + result = await execute_periodic_calculation(SimpleNamespace(project=context, connection=ReadScopeConnection()), job) + assert result.data["conclusion"] == "normal" + assert len(result.data["result"]["comparisons"]) == 4 + terminal = {"status": "succeeded", "result": result.model_dump(mode="json")} + assert await queue.finish(job, owner, **terminal) + await persist_periodic_terminal(connection, job, terminal) + # A stale worker must not republish another result after its fence closed. + assert not await queue.finish(job, owner, **terminal) + stored = await repo.periodic_task(replay_id, simulation["execution_id"]) + assert stored["status"] == "completed" + assert len(stored["result"]["comparisons"]) == 4 + await repo.save_retry_receipt(replay_id, 1, request_id, simulation["execution_id"]) + assert (await repo.batch(replay_id, 1))["periodic_retry_receipts"][str(request_id)] == simulation["execution_id"] + assert len(await repo.periodic_tasks(replay_id, 0, 1)) == 1 + assert len(await repo.periodic_tasks(replay_id, 1, 1)) == 1 + assert (await repo.get(replay_id))["last_scheduled_simulation"]["status"] == "completed" + # Check rollback on the same connection, with no application rows retained. + row = await (await connection.execute("SELECT count(*) AS count FROM peripheral_sample.scada_replays WHERE project_id=%s", (project_id,))).fetchone() + assert row["count"] == 0 + finally: + await connection.close() diff --git a/tests/unit/test_model_snapshot.py b/tests/unit/test_model_snapshot.py new file mode 100644 index 0000000..9649d64 --- /dev/null +++ b/tests/unit/test_model_snapshot.py @@ -0,0 +1,27 @@ +import base64 +import gzip + +import pytest + +from app.infra.epanet import model_snapshot + + +def test_full_model_larger_than_response_budget_round_trips_compressed(): + content = "[JUNCTIONS]\n" + "节点 12.345\n" * 900_000 + assert len(content.encode()) > 8 * 1024 * 1024 + packed = model_snapshot.pack_model_input(content) + assert len(packed) < len(content) + assert model_snapshot.unpack_model_input(packed) == content + + +def test_decoding_enforces_expanded_model_budget(monkeypatch): + packed = base64.b64encode(gzip.compress(b"A" * 1025)).decode() + monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 1024) + with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded): + model_snapshot.unpack_model_input(packed) + + +def test_oversized_model_is_rejected_without_compressing(monkeypatch): + monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 3) + with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded): + model_snapshot.pack_model_input("four") diff --git a/tests/unit/test_periodic_hydraulics.py b/tests/unit/test_periodic_hydraulics.py new file mode 100644 index 0000000..a672ace --- /dev/null +++ b/tests/unit/test_periodic_hydraulics.py @@ -0,0 +1,67 @@ +from datetime import datetime + +import pytest + +from app.algorithms.periodic_hydraulics import run_periodic_hydraulics + + +MODEL = """[JUNCTIONS] + J1 10 1 +[RESERVOIRS] + R1 50 +[PIPES] + L1 R1 J1 500 150 100 0 OPEN +[OPTIONS] + UNITS LPS + HEADLOSS H-W +[TIMES] + DURATION 0:15 + HYDRAULIC TIMESTEP 0:15 + REPORT TIMESTEP 0:15 +[END] +""" + + +def test_real_epanet_checks_all_model_elements_without_a_source_database(): + devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"}, + {"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h"}] + rows = [{"time": at, "device_id": device, "quality": "valid", "cleaned_value": value} + for at in ("2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00", "2026-07-16T08:15:00+08:00") + for device, value in (("P1", 39.8), ("F1", 3.6))] + result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00") + assert len(result["comparisons"]) == 4 + assert result["conclusion"] == "normal" + assert result["compared_device_count"] == 2 + assert result["missing_device_ids"] == [] + assert all(snapshot["node_count"] == 2 and snapshot["link_count"] == 1 for snapshot in result["snapshots"]) + assert all(datetime.fromisoformat(item["time"]).hour <= 8 for item in result["comparisons"]) + assert {item["time"] for item in result["comparisons"]} == {row["time"] for row in rows[:4]} + assert next(item for item in result["comparisons"] if item["device_id"] == "F1")["observed"] == .001 + + +def test_no_valid_measurement_is_a_data_gap_not_normal(): + result = run_periodic_hydraulics(MODEL, [], [], "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00") + assert result["conclusion"] == "insufficient_data" + assert result["comparisons"] == [] + + +def test_missing_mapping_and_zero_reference_are_not_silently_compared(): + devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"}, + {"device_id": "F1", "device_type": "pipe_flow", "measurement_unit": "m3/h"}] + rows = [{"time": "2026-07-16T08:00:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 0.}] + result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00") + assert result["comparisons"][0]["relative_deviation_percent"] is None + assert result["missing_device_ids"] == ["F1"] + assert result["conclusion"] != "normal" + + +def test_previous_sample_does_not_claim_current_period_is_normal(): + devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"}] + rows = [{"time": "2026-07-16T07:45:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 39.8}] + result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00") + assert result["compared_device_count"] == 1 + assert result["current_device_count"] == 0 + assert result["conclusion"] == "insufficient_data" + devices[0]["transmission_frequency"] = "0:30:00" + slower = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00") + assert slower["current_device_count"] == 1 diff --git a/tests/unit/test_periodic_tasks.py b/tests/unit/test_periodic_tasks.py new file mode 100644 index 0000000..0c624c2 --- /dev/null +++ b/tests/unit/test_periodic_tasks.py @@ -0,0 +1,306 @@ +import asyncio +import json +from contextlib import asynccontextmanager +from copy import deepcopy +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import HTTPException + +from app.domain.schemas.periodic_task import PeriodicTaskExecution +from app.services.periodic_diagnosis import diagnosis_execution, execution_record +from app.services import periodic_tasks + + +def document(): + return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()), + "monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest", + "devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"}, + {"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}], + "config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"], + "rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]}, + "latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}} + + +def monitoring(): + return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00", + "period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00", + "status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None} + + +def test_water_risk_is_independent_of_incomplete_pressure_model(): + source, period = document(), monitoring() + period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[ + {"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}]) + result = diagnosis_execution(source, period) + assert result["conclusion"] == "suspected_pollution" + assert result["investigation_task_id"] == period["task_id"] + assert result["risk_signals"][0]["metric_name"] == "余氯" + assert result["data_gaps"] == ["压力基线不足"] + assert all(step["status"] == "completed" for step in result["steps"]) + + +def detected_water_event(): + from datetime import timedelta + from app.domain.schemas.scada_replay import ScadaReplayCreate + from app.services.scada_replay_processing import detect_frame, prepare_cleaning + + source = document() + at = datetime.fromisoformat(monitoring()["period_end"]) + config = ScadaReplayCreate( + request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15), + device_ids=["Q1"], rules=[{ + "rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], + "lower": 0.2, "unit": "mg/L", "metric_name": "余氯", + }], + ) + source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules] + source, events = detect_frame(source, prepare_cleaning([ + {"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"} + ], config), config, at) + event = next(event for event in events if event["rule_id"] == "water") + task_id = str(uuid4()) + event["task_id"] = task_id + source["rule_states"]["water"]["event"]["task_id"] = task_id + return source, event + + +def test_periodic_report_includes_water_event_excluded_from_burst_monitoring(): + from app.services.scada_normal_profile import diagnostic_signal + + source, event = detected_water_event() + period = monitoring() + assert not diagnostic_signal(source, event) + original = deepcopy(period) + result = diagnosis_execution(source, period) + + assert result["conclusion"] == "suspected_pollution" + assert result["investigation_task_id"] == event["task_id"] + water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution") + assert water["device_ids"] == ["Q1"] + assert water["task_id"] == event["task_id"] + assert period == original + + +def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence(): + source, event = detected_water_event() + source["rule_states"]["water"]["event"] = None + event.update(status="resolved", resolved_at=monitoring()["period_end"]) + period = monitoring() + hydraulic_task_id = str(uuid4()) + hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()), + "device_ids": ["P1"], "detail": "压力变化需要复核"} + period.update(status="attention", task_id=hydraulic_task_id, + weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal]) + original = deepcopy(period) + result = diagnosis_execution(source, period, [event, deepcopy(event)]) + + assert result["conclusion"] == "suspected_pollution" + assert result["investigation_task_id"] == hydraulic_task_id + water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"] + assert len(water) == 1 + assert water[0]["task_id"] == event["task_id"] + assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"] + for signal in result["risk_signals"]) + assert period == original + assert result["result"]["monitoring"] == original + + +def test_periodic_water_projection_excludes_telemetry_quality_events(): + source, event = detected_water_event() + event["category"] = "data_quality" + source["rule_states"]["water"]["event"] = deepcopy(event) + result = diagnosis_execution(source, monitoring(), [event]) + + assert result["conclusion"] == "normal" + assert result["risk_signals"] == [] + assert result["investigation_task_id"] is None + + +@pytest.mark.parametrize("future", [ + {"occurred_at": "2026-07-16T08:15:00+08:00"}, + {"batch_sequence": 3}, +]) +def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future): + source, event = detected_water_event() + event.update(future) + source["rule_states"]["water"]["event"] = event + result = diagnosis_execution(source, monitoring(), [event]) + + assert result["conclusion"] == "normal" + assert result["risk_signals"] == [] + assert result["investigation_task_id"] is None + + +def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal(): + source, period = document(), monitoring() + source["latest"]["P1"]["quality"] = "invalid" + first = diagnosis_execution(source, period) + assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"] + assert first["conclusion"] == "attention" + assert first["result"]["invalid_device_ids"] == ["P1"] + source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00" + assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"] + other = dict(source, replay_id=str(uuid4())) + assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"] + + +class Repository: + def __init__(self): + self.document = document() + self.document["last_monitoring"] = monitoring() + self.project_id = self.document["project_id"] + self.connection = object() + self.tasks = {} + self.receipts = {} + self.state = None + self.events = [] + + @asynccontextmanager + async def control(self): yield + async def get(self, _): return deepcopy(self.document) + async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)} + async def periodic_events(self, *_): return deepcopy(self.events) + async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution) + async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution) + async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id))) + async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id) + + +def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch): + repo = Repository() + enqueue = AsyncMock(return_value={"job_id": uuid4()}) + monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue)) + context = SimpleNamespace(project_id=repo.project_id) + monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) + + async def run(): + await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"]) + await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"]) + asyncio.run(run()) + assert len(repo.tasks) == 2 + assert enqueue.await_count == 1 + payload = enqueue.call_args.kwargs + assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id} + assert payload["owner_user_id"] == repo.document["created_by"] + assert repo.state["status"] == "pending" + for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task) + + +def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch): + source, event = detected_water_event() + source["rule_states"]["water"]["event"] = None + source["last_monitoring"] = monitoring() + repo = Repository() + repo.document = source + repo.project_id = source["project_id"] + repo.events = [event] + read_events = AsyncMock(wraps=repo.periodic_events) + repo.periodic_events = read_events + monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( + enqueue=AsyncMock(return_value={"job_id": uuid4()}))) + monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) + + asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"])) + + read_events.assert_awaited_once() + diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis") + assert diagnosis["conclusion"] == "suspected_pollution" + assert diagnosis["investigation_task_id"] == event["task_id"] + + +def test_worker_interruption_is_reported_as_retryable_failure(): + record = execution_record(document(), monitoring(), "network-simulation") + result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}}) + assert result["status"] == "failed" + assert "WORKER_INTERRUPTED" in result["summary"] + assert record["status"] == "pending" + + +def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch): + repo = Repository() + monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( + enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded()))) + monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) + asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"])) + assert len(repo.tasks) == 2 + assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed" + assert repo.state["status"] == "failed" + assert repo.state["calculation_job_id"] is None + assert "尚未启动" in repo.state["summary"] + + +def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch): + repo = Repository() + record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation") + record.update(calculation_job_id=str(uuid4()), status="failed") + repo.tasks[record["execution_id"]] = deepcopy(record) + enqueue = AsyncMock(return_value={"job_id": uuid4()}) + monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace( + enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"}))) + monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"}) + request = uuid4() + async def run(): + for _ in range(2): + await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id), + repo.document["replay_id"], record["execution_id"], request) + asyncio.run(run()) + retried = repo.tasks[record["execution_id"]] + assert retried["attempt"] == 2 + assert retried["period_end"] == record["period_end"] + assert retried["batch_sequence"] == record["batch_sequence"] + assert enqueue.await_count == 1 + + +def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch): + monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"}) + with pytest.raises(HTTPException) as error: + asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {})) + assert error.value.status_code == 403 + + +def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch): + from tests.unit.test_scada_replay_service import setup + from app.domain.schemas.scada_replay import ScadaReplayCommand + repo, service, _, _, create = setup(monkeypatch) + monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value)) + + async def run(): + replay = await service.create(create, monitoring_interval_seconds=900) + command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step") + accepted = await service.command(replay.replay_id, command) + repo.document["config"]["scheduled_tasks_enabled"] = True + repo.document["last_scheduled_simulation"] = {"status": "pending"} + # A transport retry returns the original accepted period, even while its + # calculation is queued; a different step must wait. + repeated = await service.command(replay.replay_id, command) + assert repeated.revision == accepted.revision + with pytest.raises(HTTPException) as error: + await service.command(replay.replay_id, ScadaReplayCommand( + request_id=uuid4(), expected_revision=accepted.revision, action="step")) + assert error.value.status_code == 409 + assert "水力计算" in error.value.detail + asyncio.run(run()) + + +def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path): + from app.services import calculation_worker + from app.mcp.models import QueryResult + @asynccontextmanager + async def scope(_): yield SimpleNamespace(project="authorized-project") + monkeypatch.setattr(calculation_worker, "execution_scope", scope) + internal = AsyncMock(return_value=QueryResult( + source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test", + queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"})) + monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal) + ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding")) + monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary) + source, target = tmp_path / "job.json", tmp_path / "result.json" + source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL})) + asyncio.run(calculation_worker.execute_child(str(source), str(target))) + assert json.loads(target.read_text())["status"] == "succeeded" + internal.assert_awaited_once() + ordinary.assert_not_awaited()