Compare commits

...
Author SHA1 Message Date
jiang 9cd1400723 fix(replay): preserve failed periodic execution boundaries
Generic Container CI/CD / test-build-publish (push) Successful in 4m23s
Server CI/CD v2 / build-test-publish-and-deploy (push) Successful in 4m23s
2026-11-16 16:21:58 +08:00
jiang 389892cb3f feat(replay): execute periodic diagnosis and hydraulic checks 2026-11-16 15:13:22 +08:00
22 changed files with 2042 additions and 12 deletions
+167
View File
@@ -0,0 +1,167 @@
"""Rolling hydraulic checks using immutable INP and observed input only.
Each observed timestamp is a steady hydraulic snapshot, matching the online
recalculation's semantics. This module never opens a business/telemetry database.
"""
from datetime import datetime
from math import isfinite
from pathlib import Path
from statistics import mean
from tempfile import TemporaryDirectory
from zoneinfo import ZoneInfo
import re
def _sampling_seconds(device: dict) -> int:
frequency = str(device.get("transmission_frequency") or "")
clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
return max(1, int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock
else int(minutes[1]) * 60 if minutes else 900)
def pressure_metres(value: float, unit: str) -> float | None:
factors = {"m": 1.0, "mH2O": 1.0, "MPa": 101.9716213,
"kPa": .1019716213, "bar": 10.19716213}
return value * factors[unit] if unit in factors else None
def flow_cms(value: float, unit: str) -> float | None:
factors = {"m3/s": 1.0, "m³/s": 1.0, "m3/h": 1 / 3600,
"m³/h": 1 / 3600, "L/s": .001, "l/s": .001,
"LPS": .001, "CMH": 1 / 3600, "CMS": 1.0}
return value * factors[unit] if unit in factors else None
def run_periodic_hydraulics(model_input: str, devices: list[dict], rows: list[dict],
period_start: str, period_end: str) -> dict:
import wntr
start, end = datetime.fromisoformat(period_start), datetime.fromisoformat(period_end)
usable = [row for row in rows if row.get("quality") in {"valid", "corrected"}
and row.get("cleaned_value") is not None and isfinite(row["cleaned_value"])
and start <= datetime.fromisoformat(row["time"]) <= end]
sample_times = sorted({row["time"] for row in usable}, key=datetime.fromisoformat)
if len(sample_times) > 100:
raise ValueError("PERIOD_SAMPLE_LIMIT_EXCEEDED")
if not sample_times:
return {"conclusion": "insufficient_data", "comparisons": [], "snapshots": [],
"data_gaps": ["本周期没有有效观测,无法开展模拟与实测复核。"], "metrics": []}
comparisons, snapshots, gaps = [], [], set()
directory_devices = {device["device_id"]: device for device in devices}
with TemporaryDirectory(prefix="tjwater-periodic-hydraulics-") as directory:
source = Path(directory) / "frozen.inp"
source.write_text(model_input, encoding="utf-8")
for index, at in enumerate(sample_times):
model = wntr.network.WaterNetworkModel(str(source))
clock = datetime.fromisoformat(at).astimezone(ZoneInfo("Asia/Shanghai"))
model.options.time.duration = 0
model.options.time.pattern_start = clock.hour * 3600 + clock.minute * 60 + clock.second
model.options.time.start_clocktime = model.options.time.pattern_start
frame = {row["device_id"]: row for row in usable if row["time"] == at}
# Only exact-timestamp boundary observations can override the frozen
# model. Missing boundaries retain explicit model assumptions.
applied = []
for device_id, row in frame.items():
device = directory_devices.get(device_id, {})
node_id, link_id = device.get("node_id"), device.get("link_id")
value, kind = row["cleaned_value"], device.get("device_type")
if kind == "tank_liquid_level" and node_id in model.tank_name_list:
if device.get("measurement_unit") != "m":
gaps.add(f"{device_id} 水位单位未适配。")
continue
model.get_node(node_id).init_level = value
applied.append(device_id)
elif kind == "fixed_pump" and link_id in model.pump_name_list:
if value not in (0, 1):
gaps.add(f"{device_id} 工频泵状态不是 0/1。")
continue
model.get_link(link_id).initial_status = "OPEN" if value else "CLOSED"
applied.append(device_id)
elif kind == "demand" and node_id in model.junction_name_list:
demand = flow_cms(value, device.get("measurement_unit", ""))
if demand is None:
gaps.add(f"{device_id} 需水量单位未适配。")
continue
node = model.get_node(node_id)
node.demand_timeseries_list[0].base_value = demand
node.demand_timeseries_list[0].pattern_name = None
applied.append(device_id)
elif kind in {"variable_pump", "reservoir_liquid_level"}:
gaps.add(f"{device_id} 缺少已核实的频率基准或水位高程转换,沿用冻结模型边界。")
result = wntr.sim.EpanetSimulator(model).run_sim(
file_prefix=str(Path(directory) / f"snapshot-{index}"), convergence_error=True)
pressure = result.node["pressure"].iloc[-1]
flow = result.link["flowrate"].iloc[-1]
if not all(isfinite(float(value)) for value in pressure) or not all(isfinite(float(value)) for value in flow):
raise ValueError("NONFINITE_HYDRAULIC_RESULT")
snapshots.append({"time": at, "node_count": len(pressure), "link_count": len(flow),
"minimum_pressure_m": float(pressure.min()),
"maximum_pressure_m": float(pressure.max()),
"applied_boundary_device_ids": applied})
for device_id, row in frame.items():
device = directory_devices.get(device_id, {})
kind, node_id, link_id = device.get("device_type"), device.get("node_id"), device.get("link_id")
if kind == "pressure" and node_id in pressure:
observed = pressure_metres(row["cleaned_value"], device.get("measurement_unit", ""))
predicted, unit = float(pressure[node_id]), "m"
elif kind in {"flow", "pipe_flow"} and link_id in flow:
observed = flow_cms(row["cleaned_value"], device.get("measurement_unit", ""))
predicted, unit = float(flow[link_id]), "m3/s"
else:
if kind in {"pressure", "flow", "pipe_flow"}:
gaps.add(f"{device_id} 缺少有效模型关联。")
continue
if observed is None:
gaps.add(f"{device_id} 观测单位未适配。")
continue
delta = abs(predicted - observed)
comparisons.append({"time": at, "device_id": device_id, "kind": kind,
"element_id": node_id if kind == "pressure" else link_id,
"observed": observed, "predicted": predicted, "unit": unit,
"absolute_deviation": delta,
"relative_deviation_percent": delta / abs(observed) * 100
if abs(observed) > 1e-8 else None})
metrics = []
for kind, label, attention, critical in (("pressure", "模拟-实测压力偏差", 8, 12),
("flow", "模拟-实测流量偏差", 10, 15)):
values = [item["relative_deviation_percent"] for item in comparisons
if (item["kind"] == "pressure") == (kind == "pressure")
and item["relative_deviation_percent"] is not None]
if values:
value = mean(values)
metrics.append({"id": kind + "-deviation", "label": label, "value": f"{value:.2f}",
"unit": "%", "status": "critical" if value >= critical else "attention" if value >= attention else "normal",
"detail": f"有效比较 {len(values)} 项;近零实测只报告绝对偏差。"})
else:
gaps.add(f"缺少可计算相对偏差的{label}数据。")
pressure_deltas = [item["absolute_deviation"] for item in comparisons if item["kind"] == "pressure"]
if pressure_deltas:
maximum = max(pressure_deltas)
metrics.append({"id": "max-pressure-delta", "label": "最大节点压差", "value": f"{maximum:.2f}",
"unit": "m", "status": "critical" if maximum >= 8 else "attention" if maximum >= 5 else "normal",
"detail": "有实测映射节点的最大绝对压力偏差。"})
compared_ids = {item["device_id"] for item in comparisons}
valid_ids = {item["device_id"] for item in comparisons
if (end - datetime.fromisoformat(item["time"])).total_seconds()
< _sampling_seconds(directory_devices.get(item["device_id"], {}))}
expected = {device["device_id"] for device in devices if device.get("device_type") in {"pressure", "flow", "pipe_flow"}}
missing = sorted(expected - valid_ids)
if missing:
gaps.add("缺少周期终点可用观测或模型映射的测点:" + "、".join(missing))
gaps.add("未取得有效边界遥测的要素采用本轮冻结模型设定;本结果为逐时刻稳态水力复核。")
anomaly = any(metric["status"] != "normal" for metric in metrics)
complete = not missing and all(any(metric["id"] == kind + "-deviation" for metric in metrics) for kind in ("pressure", "flow"))
deviations = sorted(
[item for item in comparisons if (item["relative_deviation_percent"] is not None
and item["relative_deviation_percent"] >= (8 if item["kind"] == "pressure" else 10))
or (item["kind"] == "pressure" and item["absolute_deviation"] >= 5)],
key=lambda item: item["relative_deviation_percent"] or 0, reverse=True,
)
causes = (["先核查偏差测点的数据质量与模型关联,再复核冻结模型中的泵阀状态、供水边界和需水模式。",
"当前证据仅定位偏差测点和关联要素,尚不能单独确定边界滞后或模型参数错误。"] if deviations else [])
return {"conclusion": "attention" if anomaly else "normal" if complete else "insufficient_data",
"metrics": metrics, "comparisons": comparisons, "snapshots": snapshots,
"data_gaps": sorted(gaps), "compared_device_count": len(compared_ids), "current_device_count": len(valid_ids),
"missing_device_ids": missing, "model_scope": "all_nodes_and_links",
"deviation_targets": deviations[:50], "cause_review": causes}
+36 -3
View File
@@ -14,6 +14,11 @@ from app.domain.schemas.scada_replay import (
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.domain.schemas.project_settings import ProjectSettings from app.domain.schemas.project_settings import ProjectSettings
from app.services.scada_replay import ScadaReplayService, snapshot from app.services.scada_replay import ScadaReplayService, snapshot
from app.domain.schemas.periodic_task import PeriodicTaskPage, PeriodicTaskExecution, PeriodicTaskRetry
from app.services.periodic_tasks import (
ensure_periodic_tasks, initialize_periodic_tasks, project_periodic_state,
project_periodic_tasks, retry_periodic_task, require_execution_permissions,
)
router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"]) router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"])
@@ -40,6 +45,8 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm
settings=Depends(project_settings)): settings=Depends(project_settings)):
available = await repo.available() available = await repo.available()
current = await repo.current() if available else None current = await repo.current() if available else None
if current and current.get("config", {}).get("scheduled_tasks_enabled"):
current = await project_periodic_state(repo, current)
monitoring = settings.scada_monitoring monitoring = settings.scada_monitoring
monitoring_ready = bool( monitoring_ready = bool(
monitoring monitoring
@@ -68,17 +75,22 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm
@router.post("", response_model=ScadaReplaySnapshot, status_code=201) @router.post("", response_model=ScadaReplaySnapshot, status_code=201)
async def create_replay(command: ScadaReplayCreate, repo=Depends(repository), async def create_replay(command: ScadaReplayCreate, repo=Depends(repository),
ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)): ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)):
return await ScadaReplayService(repo, ctx).create( if command.scheduled_tasks_enabled:
require_execution_permissions(ctx)
result = await ScadaReplayService(repo, ctx).create(
command, command,
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds, monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
monitoring_config=settings.scada_monitoring, monitoring_config=settings.scada_monitoring,
) )
if command.scheduled_tasks_enabled:
await initialize_periodic_tasks(repo, ctx, result.replay_id)
return result
@router.get("/{replay_id}", response_model=ScadaReplaySnapshot) @router.get("/{replay_id}", response_model=ScadaReplaySnapshot)
async def get_replay(replay_id: UUID, repo=Depends(repository)): async def get_replay(replay_id: UUID, repo=Depends(repository)):
await ScadaReplayService(repo, None).ensure_storage() await ScadaReplayService(repo, None).ensure_storage()
return snapshot(await repo.get(replay_id)) return snapshot(await project_periodic_state(repo, await repo.get(replay_id)))
@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot) @router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot)
@@ -86,12 +98,33 @@ async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depe
ctx=Depends(require_permission(SCADA_CLEAN)), ctx=Depends(require_permission(SCADA_CLEAN)),
timescale=Depends(get_project_timescale_connection)): timescale=Depends(get_project_timescale_connection)):
service = ScadaReplayService(repo, ctx, timescale) service = ScadaReplayService(repo, ctx, timescale)
document = await repo.get(replay_id)
if command.action != "stop" and document.get("config", {}).get("scheduled_tasks_enabled"):
await ensure_periodic_tasks(repo, ctx, replay_id)
result = await service.command(replay_id, command) result = await service.command(replay_id, command)
if command.action in {"step", "monitor", "retry"} and result.step_pending: if command.action in {"step", "monitor", "retry"} and result.step_pending:
return await service.process_pending_step(replay_id) result = await service.process_pending_step(replay_id)
if command.action != "stop" and getattr(result, "scheduled_tasks_enabled", False):
await ensure_periodic_tasks(repo, ctx, replay_id)
return snapshot(await project_periodic_state(repo, await repo.get(replay_id)))
return result return result
@router.get("/{replay_id}/scheduled-tasks", response_model=PeriodicTaskPage)
async def scheduled_tasks(replay_id: UUID, offset: int = Query(0, ge=0),
limit: int = Query(100, ge=1, le=200), repo=Depends(repository)):
await ScadaReplayService(repo, None).ensure_storage()
await repo.get(replay_id)
items = await project_periodic_tasks(repo, replay_id, offset, limit + 1)
return PeriodicTaskPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None)
@router.post("/{replay_id}/scheduled-tasks/{execution_id}/retry", response_model=PeriodicTaskExecution)
async def retry_scheduled_task(replay_id: UUID, execution_id: UUID, command: PeriodicTaskRetry,
repo=Depends(repository), ctx=Depends(require_permission(SCADA_CLEAN))):
return await retry_periodic_task(repo, ctx, replay_id, execution_id, command.request_id)
@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage) @router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage)
async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200), async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200),
repo=Depends(repository)): repo=Depends(repository)):
+56
View File
@@ -0,0 +1,56 @@
"""Persisted business-clock task executions, independent of Agent sessions."""
from typing import Literal
from uuid import UUID
from pydantic import AwareDatetime, Field
from app.domain.schemas.shared import StrictModel
class PeriodicTaskStep(StrictModel):
id: str
title: str
status: Literal["pending", "running", "completed", "failed", "skipped"]
detail: str = ""
class PeriodicTaskMetric(StrictModel):
id: str
label: str
value: str
unit: str = ""
status: Literal["normal", "attention", "critical"] = "normal"
detail: str = ""
class PeriodicTaskExecution(StrictModel):
execution_id: UUID
replay_id: UUID
task_key: Literal["scada-diagnosis", "network-simulation"]
title: str
period_start: AwareDatetime
period_end: AwareDatetime
knowledge_cutoff: AwareDatetime
batch_sequence: int = Field(ge=1)
status: Literal["pending", "running", "completed", "failed"]
conclusion: Literal["normal", "attention", "insufficient_data", "suspected_burst", "suspected_pollution"]
summary: str
steps: list[PeriodicTaskStep]
metrics: list[PeriodicTaskMetric] = Field(default_factory=list)
evidence: list[str] = Field(default_factory=list)
data_gaps: list[str] = Field(default_factory=list)
risk_signals: list[dict] = Field(default_factory=list)
investigation_task_id: UUID | None = None
calculation_job_id: UUID | None = None
attempt: int = Field(default=1, ge=1)
updated_at: AwareDatetime
result: dict | None = None
class PeriodicTaskPage(StrictModel):
items: list[PeriodicTaskExecution]
next_offset: int | None
class PeriodicTaskRetry(StrictModel):
request_id: UUID
+4
View File
@@ -61,6 +61,7 @@ class ScadaReplayCreate(ReplayModel):
device_ids: list[str] = Field(min_length=1, max_length=500) device_ids: list[str] = Field(min_length=1, max_length=500)
rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30) rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30)
quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict) quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict)
scheduled_tasks_enabled: bool = False
@model_validator(mode="after") @model_validator(mode="after")
def configuration(self): def configuration(self):
@@ -130,6 +131,9 @@ class ScadaReplaySnapshot(ReplayModel):
end_time: datetime end_time: datetime
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000) monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
step_pending: bool = False step_pending: bool = False
scheduled_tasks_enabled: bool = False
periodic_pending: bool = False
periodic_error: str | None = None
monitoring_count: int = 0 monitoring_count: int = 0
last_monitoring: ScadaMonitoringResult | None = None last_monitoring: ScadaMonitoringResult | None = None
monitoring_config: ScadaMonitoringSettings | None = None monitoring_config: ScadaMonitoringSettings | None = None
+96
View File
@@ -139,6 +139,102 @@ class ScadaReplayRepository:
) )
return [r["event"] for r in await cur.fetchall()] return [r["event"] for r in await cur.fetchall()]
async def queue_available(self) -> bool:
cursor = await self.connection.execute("SELECT to_regclass('workbench.calculation_jobs') IS NOT NULL AS ready")
return bool((await cursor.fetchone())["ready"])
async def periodic_events(self, replay_id, start, end, through_sequence: int) -> list[dict]:
cursor = await self.connection.execute(
"SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
"AND b.sequence>0 AND b.sequence<=%s "
"AND (b.document->>'at')::timestamptz>=%s::timestamptz "
"AND (b.document->>'at')::timestamptz<=%s::timestamptz ORDER BY b.sequence",
(self.project_id, str(replay_id), through_sequence, start, end),
)
return [row["event"] for row in await cursor.fetchall()]
async def batch(self, replay_id, sequence: int) -> dict | None:
cursor = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replay_batches WHERE project_id=%s AND replay_id=%s AND sequence=%s",
(self.project_id, str(replay_id), sequence),
)
row = await cursor.fetchone()
return row["document"] if row else None
async def periodic_tasks(self, replay_id, offset=0, limit=100) -> list[dict]:
cursor = await self.connection.execute(
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
"ORDER BY b.sequence DESC,t.value->>'task_key' LIMIT %s OFFSET %s",
(self.project_id, str(replay_id), limit, offset),
)
return [row["execution"] for row in await cursor.fetchall()]
async def periodic_task(self, replay_id, execution_id) -> dict | None:
cursor = await self.connection.execute(
"SELECT document->'scheduled_tasks'->%s AS execution FROM peripheral_sample.scada_replay_batches "
"WHERE project_id=%s AND replay_id=%s AND stage='detected' "
"AND document->'scheduled_tasks' ? %s LIMIT 1",
(str(execution_id), self.project_id, str(replay_id), str(execution_id)),
)
row = await cursor.fetchone()
return row["execution"] if row else None
async def previous_simulations(self, replay_id, before, limit=2) -> list[dict]:
cursor = await self.connection.execute(
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
"AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' "
"AND (t.value->>'period_end')::timestamptz<%s::timestamptz "
"ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT %s",
(self.project_id, str(replay_id), before, limit),
)
return [row["execution"] for row in await cursor.fetchall()]
async def reference_simulations(self, replay_id, period_ends: list[datetime]) -> list[dict]:
cursor = await self.connection.execute(
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
"AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' "
"AND (t.value->>'period_end')::timestamptz=ANY(%s::timestamptz[]) "
"ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT 7",
(self.project_id, str(replay_id), period_ends),
)
return [row["execution"] for row in await cursor.fetchall()]
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id) -> None:
await self.connection.execute(
"UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{periodic_retry_receipts}',"
"COALESCE(document->'periodic_retry_receipts','{}'::jsonb) || jsonb_build_object(%s::text,%s::text)) "
"WHERE project_id=%s AND replay_id=%s AND sequence=%s",
(str(request_id), str(execution_id), self.project_id, str(replay_id), sequence),
)
async def save_periodic_task(self, replay_id, sequence: int, execution: dict) -> None:
cursor = await self.connection.execute(
"UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{scheduled_tasks}',"
"COALESCE(document->'scheduled_tasks','{}'::jsonb) || jsonb_build_object(%s::text,%s::jsonb)) "
"WHERE project_id=%s AND replay_id=%s AND sequence=%s AND stage='detected' RETURNING sequence",
(str(execution["execution_id"]), Jsonb(execution), self.project_id, str(replay_id), sequence),
)
if await cursor.fetchone() is None:
raise ValueError("PERIOD_BATCH_UNAVAILABLE")
async def simulation_state(self, replay_id, execution: dict) -> None:
state = {"execution_id": str(execution["execution_id"]), "status": execution["status"],
"error": execution["summary"] if execution["status"] == "failed" else None}
await self.connection.execute(
"UPDATE peripheral_sample.scada_replays SET document=jsonb_set(document,'{last_scheduled_simulation}',%s),updated_at=now() "
"WHERE project_id=%s AND replay_id=%s "
"AND (document->'last_monitoring'->>'period_end')::timestamptz=%s::timestamptz",
(Jsonb(state), self.project_id, str(replay_id), execution["period_end"]),
)
async def monitoring_outputs( async def monitoring_outputs(
self, self,
replay_id, replay_id,
+51
View File
@@ -0,0 +1,51 @@
"""Read one consistent model without modifying the online network."""
from pathlib import Path
from tempfile import TemporaryDirectory
import base64
import gzip
from io import BytesIO
from app.native.wndb.core.connection import project_transaction
from app.native.wndb.inp.exporter import dump_inp
MAX_MODEL_BYTES = 64 * 1024 * 1024
MAX_COMPRESSED_BYTES = 8 * 1024 * 1024
class ModelSnapshotBudgetExceeded(ValueError):
pass
def pack_model_input(content: str) -> str:
raw = content.encode("utf-8")
if len(raw) > MAX_MODEL_BYTES:
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
compressed = gzip.compress(raw, compresslevel=6, mtime=0)
if len(compressed) > MAX_COMPRESSED_BYTES:
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
return base64.b64encode(compressed).decode("ascii")
def unpack_model_input(content: str) -> str:
if len(content) > (MAX_COMPRESSED_BYTES + 2) // 3 * 4:
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
compressed = base64.b64decode(content, validate=True)
with gzip.GzipFile(fileobj=BytesIO(compressed)) as stream:
raw = stream.read(MAX_MODEL_BYTES + 1)
if len(raw) > MAX_MODEL_BYTES:
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
return raw.decode("utf-8")
def capture_model_input(project: str) -> str:
with TemporaryDirectory(prefix="tjwater-model-snapshot-") as directory:
path = Path(directory) / "network.inp"
with project_transaction(project) as connection:
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
dump_inp(project, str(path), "2")
if path.stat().st_size > MAX_MODEL_BYTES:
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
content = path.read_text(encoding="utf-8")
if not content.strip():
raise ValueError("MODEL_SNAPSHOT_UNAVAILABLE")
return content
+13 -2
View File
@@ -80,7 +80,11 @@ async def execute_child(input_path: str, output_path: str):
row = json.loads(Path(input_path).read_text()) row = json.loads(Path(input_path).read_text())
try: try:
async with execution_scope(row) as scope: async with execution_scope(row) as scope:
result = await execute_calculation_job(scope, row) if row.get('tool') == 'scheduled_network_simulation':
from app.services.periodic_tasks import execute_periodic_calculation
result = await execute_periodic_calculation(scope, row)
else:
result = await execute_calculation_job(scope, row)
payload = {'status': 'succeeded', 'result': result.model_dump(mode='json')} payload = {'status': 'succeeded', 'result': result.model_dump(mode='json')}
except Exception as exc: except Exception as exc:
# Do not persist exception text: DSNs, SQL, credentials or model data can occur there. # Do not persist exception text: DSNs, SQL, credentials or model data can occur there.
@@ -224,7 +228,14 @@ class CalculationWorker:
terminal = {'status': 'failed', 'error': {'code': 'CLEANUP_FAILED'}} terminal = {'status': 'failed', 'error': {'code': 'CLEANUP_FAILED'}}
cleaned = True cleaned = True
async with self.repository(row['project_id']) as repository: async with self.repository(row['project_id']) as repository:
finished = await repository.finish(row, self.owner, **terminal) if row.get('tool') == 'scheduled_network_simulation':
from app.services.periodic_tasks import persist_periodic_terminal
async with repository.connection.transaction():
finished = await repository.finish(row, self.owner, **terminal)
if finished:
await persist_periodic_terminal(repository.connection, row, terminal)
else:
finished = await repository.finish(row, self.owner, **terminal)
if not finished: if not finished:
status = await repository.heartbeat(row, self.owner, self.settings.lease_seconds) status = await repository.heartbeat(row, self.owner, self.settings.lease_seconds)
if status == 'cancel_requested': if status == 'cancel_requested':
+122
View File
@@ -0,0 +1,122 @@
"""Project completed SCADA monitoring into an auditable scheduled execution."""
from collections.abc import Sequence
from datetime import datetime, timezone
import re
from uuid import NAMESPACE_URL, uuid5
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.services.scada_normal_profile import pollution_signal
STEP_TITLES = {
"scada-diagnosis": ["读取 SCADA 遥测", "检查新鲜度完整率", "诊断压力流量与水质", "判断潜在事件", "整理诊断结论"],
"network-simulation": ["冻结模型与边界", "运行水力模型", "比对实测偏差", "定位偏差区域", "输出模型复核"],
}
def execution_record(document: dict, monitoring: dict, task_key: str) -> dict:
return PeriodicTaskExecution(
execution_id=uuid5(NAMESPACE_URL, f"{document['replay_id']}:{task_key}:{monitoring['period_end']}"),
replay_id=document["replay_id"], task_key=task_key,
title="SCADA 数据诊断" if task_key == "scada-diagnosis" else "管网定时模拟",
period_start=monitoring["period_start"], period_end=monitoring["period_end"],
knowledge_cutoff=monitoring["knowledge_cutoff"], batch_sequence=monitoring["batch_sequence"],
status="pending", conclusion="insufficient_data", summary="等待本周期执行。",
steps=[{"id": f"step-{index}", "title": title, "status": "pending"}
for index, title in enumerate(STEP_TITLES[task_key])],
updated_at=datetime.now(timezone.utc),
).model_dump(mode="json")
def diagnosis_execution(document: dict, monitoring: dict, pollution_events: Sequence[dict] = ()) -> dict:
record = execution_record(document, monitoring, "scada-diagnosis")
end = datetime.fromisoformat(monitoring["period_end"])
devices = {device["device_id"]: device for device in document["devices"]}
rules = {rule["rule_id"]: rule for rule in document["config"]["rules"]}
signals = []
water_events = {}
active_events = [state["event"] for state in document.get("rule_states", {}).values() if state.get("event")]
for event in [*pollution_events, *active_events]:
if not pollution_signal(document, event):
continue
if event.get("occurred_at") and datetime.fromisoformat(event["occurred_at"]) > end:
continue
if event.get("batch_sequence", 0) > monitoring["batch_sequence"]:
continue
water_events[event["event_id"]] = event
evidence = {event["event_id"]: event for event in monitoring.get("anomaly_signals", [])}
evidence.update(water_events)
for signal in evidence.values():
rule = rules.get(signal["rule_id"], {})
water = pollution_signal(document, signal)
signals.append({"kind": "suspected_pollution" if water else "hydraulic_anomaly",
"device_ids": signal["device_ids"], "detail": signal["detail"],
"metric_name": rule.get("metric_name", ""), "rule_id": signal["rule_id"],
**({"task_id": signal["task_id"]} if water and signal.get("task_id") else {})})
if monitoring["status"] == "suspected_burst":
signals.append({"kind": "suspected_burst", "detail": "压力与爆管侦测等联合证据达到项目嫌疑门槛。",
"device_ids": monitoring.get("pressure", {}).get("hit_device_ids", [])})
missing, invalid, fresh_ids = [], [], set()
for device_id, device in devices.items():
row = document.get("latest", {}).get(device_id)
frequency = str(device.get("transmission_frequency") or "")
clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
seconds = (int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock
else int(minutes[1]) * 60 if minutes else document["monitoring_interval_seconds"])
age = (end - datetime.fromisoformat(row["time"])).total_seconds() if row else None
if age is None or age < 0 or age >= 2 * max(seconds, 1):
missing.append(device_id)
elif row["quality"] not in {"valid", "corrected"}:
invalid.append(device_id)
else:
fresh_ids.add(device_id)
conclusion = ("suspected_burst" if monitoring["status"] == "suspected_burst" else
"suspected_pollution" if any(signal["kind"] == "suspected_pollution" for signal in signals) else
"attention" if signals or monitoring["status"] == "attention" or invalid else monitoring["status"])
gaps = []
if missing:
gaps.append("缺少及时观测的测点:" + "、".join(missing))
if invalid:
gaps.append("无效读数测点:" + "、".join(invalid))
if monitoring["status"] == "insufficient_data":
gaps.append(monitoring["detail"])
water_ids = {key for key, device in devices.items() if device.get("device_type") in {"quality", "water_quality"}}
configured_water = {device for rule in rules.values() if rule.get("kind") == "bounds" and rule.get("metric_name")
for device in rule.get("device_ids", [])}
if water_ids - configured_water:
gaps.append("尚未配置明确水质指标与业务阈值的测点,污染风险未评估:" + "、".join(sorted(water_ids - configured_water)))
if gaps and conclusion == "normal":
conclusion = "insufficient_data"
summary = ("发现水质异常,存在污染风险,已进入原因核验。" if conclusion == "suspected_pollution"
else monitoring["detail"])
coverage = len(fresh_ids) / len(devices) * 100 if devices else 0
metrics = [{"id": "usable-coverage", "label": "及时有效测点占比", "value": f"{coverage:.1f}", "unit": "%",
"status": "attention" if missing or invalid else "normal", "detail": "按设备采样周期检查;缺测或无效读数单列。"},
{"id": "risk-signals", "label": "运行异常信号", "value": str(len(signals)), "unit": "项",
"status": "attention" if signals else "normal", "detail": "压力、流量与水质分别判别,异常不等于现场确诊。"}]
for key, title, kinds in (("pressure", "压力", {"pressure"}), ("flow", "流量", {"flow", "pipe_flow"}),
("water", "水质", {"quality", "water_quality"})):
ids = {device for device, metadata in devices.items() if metadata.get("device_type") in kinds}
if ids:
valid_count = len(ids & fresh_ids)
metrics.append({"id": key + "-freshness", "label": title + "测点及时有效率",
"value": f"{valid_count / len(ids) * 100:.1f}", "unit": "%",
"status": "normal" if valid_count == len(ids) else "attention",
"detail": f"有效 {valid_count} / {len(ids)} 个测点;按设备采样周期核验。"})
weighted = monitoring.get("weighted")
if weighted:
metrics.append({"id": "burst-score", "label": "爆管嫌疑综合评分", "value": f"{weighted['score']:.2f}",
"unit": "", "status": "attention" if weighted["score"] >= weighted["threshold"] else "normal",
"detail": f"项目嫌疑门槛 {weighted['threshold']:.2f};综合分不代表概率。"})
record.update(status="completed", conclusion=conclusion, summary=summary, metrics=metrics,
data_gaps=gaps, risk_signals=signals,
investigation_task_id=monitoring.get("task_id") or next(
(signal["task_id"] for signal in signals if signal.get("task_id")), None),
evidence=[f"周期 {monitoring['period_start']} 至 {monitoring['period_end']}",
f"冻结批次 {monitoring['batch_sequence']};监测回执 {monitoring['monitoring_id']}",
*[signal["detail"] for signal in signals]],
result={"monitoring": monitoring, "missing_device_ids": missing, "invalid_device_ids": invalid})
for step in record["steps"]:
step.update(status="completed")
return PeriodicTaskExecution.model_validate(record).model_dump(mode="json")
+256
View File
@@ -0,0 +1,256 @@
"""Business-clock executions reuse replay batches and the durable calculation queue."""
from datetime import datetime, timedelta, timezone
from uuid import UUID, NAMESPACE_URL, uuid5
from fastapi import HTTPException
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.mcp.errors import CalculationCapacityExceeded
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation"
ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"}
def require_execution_permissions(context) -> None:
if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context):
raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限")
async def initialize_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"):
return
require_execution_permissions(context)
if not await repo.queue_available():
raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构")
from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded
from app.services.content_digest import digest
from app.mcp.analysis_service import bounded_calculation
try:
model_input = await bounded_calculation(capture_model_input, context.project_code)
packed = await bounded_calculation(pack_model_input, model_input)
except ModelSnapshotBudgetExceeded as error:
raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error
async with repo.control():
document = await repo.get(replay_id)
if not document.get("scheduled_model_ready"):
document["scheduled_model_digest"] = digest({"inp": model_input})
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
"model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]})
document["scheduled_model_ready"] = True
await repo.save(document)
async def ensure_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"):
return
require_execution_permissions(context)
await initialize_periodic_tasks(repo, context, replay_id)
async with repo.control():
document = await repo.get(replay_id)
monitoring = document["last_monitoring"]
batch = await repo.batch(replay_id, monitoring["batch_sequence"])
if not batch:
raise HTTPException(409, "本周期冻结数据批次尚未提交")
diagnosis = execution_record(document, monitoring, "scada-diagnosis")
if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}):
events = await repo.periodic_events(
replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"],
)
diagnosis = diagnosis_execution(document, monitoring, events)
await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis)
simulation = execution_record(document, monitoring, "network-simulation")
if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}):
await enqueue_simulation(repo, context, document, simulation)
async def enqueue_simulation(repo, context, document, execution) -> None:
queue = CalculationJobRepository(repo.connection, repo.project_id)
identity = f"periodic:{execution['execution_id']}:{execution['attempt']}"
try:
row = await queue.enqueue(
owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)),
tool=PERIODIC_SIMULATION_TOOL,
parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"],
"execution_id": execution["execution_id"], "attempt": execution["attempt"]},
binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600,
)
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
except CalculationCapacityExceeded:
execution.update(failed_execution(execution, "计算队列已满,水力计算尚未启动,可按原输入重试。"), calculation_job_id=None)
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
await repo.simulation_state(document["replay_id"], execution)
def failed_execution(execution: dict, summary: str) -> dict:
result = {**execution, "status": "failed", "summary": summary,
"steps": [dict(step) for step in execution["steps"]]}
result["steps"][1].update(status="failed", detail=summary)
for step in result["steps"][2:]:
step.update(status="skipped", detail="本周期计算未完成,此步骤尚未执行。")
return result
def project_job(execution: dict, job: dict | None) -> dict:
result = dict(execution)
if execution["status"] == "completed":
return result
if execution["status"] == "failed":
return failed_execution(execution, execution["summary"])
if not job:
return failed_execution(execution, "本周期计算记录不可用,请重试。")
elif job["status"] in ACTIVE_JOB_STATES:
result.update(status="running" if job["status"] != "queued" else "pending")
result["steps"] = [dict(step) for step in execution["steps"]]
result["steps"][1].update(status="running" if job["status"] == "running" else "pending")
elif job["status"] == "succeeded" and job.get("result"):
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
else:
code = (job.get("error") or {}).get("code", job["status"])
return failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
return result
async def project_periodic_tasks(repo, replay_id, offset, limit):
items = await repo.periodic_tasks(replay_id, offset, limit)
queue = CalculationJobRepository(repo.connection, repo.project_id)
result = []
for execution in items:
job = (await queue.get(execution["calculation_job_id"], include_result=True)
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
result.append(project_job(execution, job))
return result
async def project_periodic_state(repo, document: dict) -> dict:
state = document.get("last_scheduled_simulation")
if not state or state["status"] not in {"pending", "running"}:
return document
batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"])
if execution:
queue = CalculationJobRepository(repo.connection, repo.project_id)
execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"],
"status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}}
return document
async def retry_periodic_task(repo, context, replay_id, execution_id, request_id):
require_execution_permissions(context)
async with repo.control():
document = await repo.get(replay_id)
# A retry is scoped to one frozen execution, including older periods.
execution = await repo.periodic_task(replay_id, execution_id)
if not execution or execution["task_key"] != "network-simulation":
raise HTTPException(404, "当前回放不存在此水力计算任务")
batch = await repo.batch(replay_id, execution["batch_sequence"])
receipts = batch.get("periodic_retry_receipts", {})
prior = receipts.get(str(request_id))
if prior:
if prior != str(execution_id):
raise HTTPException(409, "重试请求号已用于另一条执行记录")
return execution
queue = CalculationJobRepository(repo.connection, repo.project_id)
current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
if current["status"] != "failed":
raise HTTPException(409, "仅失败的水力计算可重试")
execution.update(attempt=execution["attempt"] + 1, status="pending", result=None,
updated_at=datetime.now(timezone.utc).isoformat())
for step in execution["steps"]:
step.update(status="pending", detail="")
await enqueue_simulation(repo, context, document, execution)
await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id)
return execution
async def execute_periodic_calculation(scope, row):
"""Internal worker job; admission and execution both recheck project rights."""
from app.mcp.models import QueryResult
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
from app.infra.epanet.model_snapshot import unpack_model_input
from app.mcp.analysis_service import bounded_calculation
require_execution_permissions(scope.project)
parameters = row["parameters"]
if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}:
raise HTTPException(403, "周期任务所属项目不匹配")
repo = ScadaReplayRepository(scope.connection, scope.project.project_id)
document = await repo.get(UUID(parameters["replay_id"]))
batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"])
if (not execution or execution["calculation_job_id"] != str(row["job_id"])
or str(document["created_by"]) != str(row["owner_user_id"])
or execution["attempt"] != parameters["attempt"]):
raise HTTPException(409, "周期计算执行身份已失效")
observations = await repo.detected_outputs(
document["replay_id"], document["config"]["device_ids"],
datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]),
execution["batch_sequence"], limit=50_001,
)
if len(observations) > 50_000:
raise HTTPException(413, "本周期观测超过水力复核预算")
previous = await repo.previous_simulations(document["replay_id"], execution["period_end"])
reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)]
references = await repo.reference_simulations(document["replay_id"], reference_times)
model_snapshot = await repo.batch(document["replay_id"], 0)
await scope.connection.commit()
if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot"
or model_snapshot["model_digest"] != document["scheduled_model_digest"]):
raise HTTPException(409, "冻结模型缺失或摘要不一致")
model_input = unpack_model_input(model_snapshot["model_input_gzip"])
result = await bounded_calculation(run_periodic_hydraulics, model_input,
document["devices"], observations,
execution["period_start"], execution["period_end"])
result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous]
current_metrics = {item["id"]: item for item in result["metrics"]}
contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"])
prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {}
worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics
and float(metric["value"]) > float(prior_metrics[key]["value"])
and metric["status"] != "normal"]
if worsening:
result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。")
result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references]
observed_references = {datetime.fromisoformat(item["period_end"]) for item in references}
missing_references = [time.isoformat() for time in reference_times if time not in observed_references]
if missing_references:
result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references))
execution.update(status="completed", conclusion=result["conclusion"], result=result,
metrics=result["metrics"], data_gaps=result["data_gaps"],
summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention"
else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data"
else "本周期水力计算与实测偏差复核已完成。"),
updated_at=datetime.now(timezone.utc).isoformat(),
evidence=[f"模型快照 {document['scheduled_model_digest']}",
f"冻结观测批次不超过 {execution['batch_sequence']}",
f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"])
for index, step in enumerate(execution["steps"]):
step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed")
execution["steps"][3]["detail"] = ";".join(result.get("cause_review", []))
return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"],
queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution)
async def persist_periodic_terminal(connection, row, terminal):
"""Called inside the supervisor's fenced terminal transaction."""
repo = ScadaReplayRepository(connection, row["project_id"])
parameters = row["parameters"]
execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"])
if not execution or execution["calculation_job_id"] != str(row["job_id"]):
return
if terminal["status"] == "succeeded":
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
else:
code = (terminal.get("error") or {}).get("code", terminal["status"])
execution = failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
execution["updated_at"] = datetime.now(timezone.utc).isoformat()
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
await repo.simulation_state(parameters["replay_id"], execution)
+12 -2
View File
@@ -42,6 +42,9 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
"virtual_now", "created_at", "updated_at", "virtual_now", "created_at", "updated_at",
)}, )},
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"), delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
scheduled_tasks_enabled=document.get("config", {}).get("scheduled_tasks_enabled", False),
periodic_pending=(document.get("last_scheduled_simulation") or {}).get("status") in {"pending", "running"},
periodic_error=(document.get("last_scheduled_simulation") or {}).get("error"),
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0), processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
devices=document["devices"], rules=document["config"]["rules"], devices=document["devices"], rules=document["config"]["rules"],
latest=list(document.get("latest", {}).values()), latest=list(document.get("latest", {}).values()),
@@ -132,6 +135,13 @@ class ScadaReplayService:
return snapshot(document) return snapshot(document)
if command.expected_revision != document["revision"]: if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新,请刷新后重试") raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
if document["config"].get("scheduled_tasks_enabled") and command.action in {"step", "monitor"}:
from app.services.periodic_tasks import project_periodic_state
document = await project_periodic_state(self.repo, document)
if snapshot(document).periodic_pending:
raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放")
if (document.get("last_scheduled_simulation") or {}).get("status") == "failed":
raise HTTPException(409, "本周期水力计算未完成,请在工况任务中重试,或停止后重新开始本轮")
action, status = command.action, document["status"] action, status = command.action, document["status"]
if (action in {"step", "monitor", "retry"} and document.get("warmed") if (action in {"step", "monitor", "retry"} and document.get("warmed")
and weighted_diagnosis(document) and weighted_diagnosis(document)
@@ -332,7 +342,7 @@ class ScadaReplayService:
monitoring["anomaly_signals"] = list(signals.values()) monitoring["anomaly_signals"] = list(signals.values())
if signals and monitoring["status"] == "normal": if signals and monitoring["status"] == "normal":
monitoring["status"] = "attention" monitoring["status"] = "attention"
monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断" monitoring["detail"] = "本周期曾检出测点异常,进入运行异常与潜在事件诊断"
result["monitoring_count"] = result.get("monitoring_count", 0) + 1 result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]: if not result["step_pending"]:
result.pop("step_target", None) result.pop("step_target", None)
@@ -565,7 +575,7 @@ class ScadaReplayService:
trigger=TaskTrigger( trigger=TaskTrigger(
kind="event", kind="event",
occurred_at=monitoring["period_end"], occurred_at=monitoring["period_end"],
detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管", detail="SCADA 任一指标异常启动调查,核验爆管或污染等潜在事件;异常不等于现场确诊",
), ),
) )
task = await tasks.create(command) task = await tasks.create(command)
+1 -1
View File
@@ -3,7 +3,7 @@
"contracts": { "contracts": {
"server": { "server": {
"file": "server-v1.openapi.json", "file": "server-v1.openapi.json",
"sha256": "04aeed6f1a7414b37267d84141d036d00d2f8579ee28af37590bd4b92137e712" "sha256": "90e0ac0fb466f0a67c9a8a909252a610cb8a089fd4289fa23232d4646d0a8849"
} }
} }
} }
+579
View File
@@ -6519,6 +6519,302 @@
"title": "Page[str]", "title": "Page[str]",
"type": "object" "type": "object"
}, },
"PeriodicTaskExecution": {
"additionalProperties": false,
"properties": {
"attempt": {
"default": 1,
"minimum": 1.0,
"title": "Attempt",
"type": "integer"
},
"batch_sequence": {
"minimum": 1.0,
"title": "Batch Sequence",
"type": "integer"
},
"calculation_job_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Calculation Job Id"
},
"conclusion": {
"enum": [
"normal",
"attention",
"insufficient_data",
"suspected_burst",
"suspected_pollution"
],
"title": "Conclusion",
"type": "string"
},
"data_gaps": {
"items": {
"type": "string"
},
"title": "Data Gaps",
"type": "array"
},
"evidence": {
"items": {
"type": "string"
},
"title": "Evidence",
"type": "array"
},
"execution_id": {
"format": "uuid",
"title": "Execution Id",
"type": "string"
},
"investigation_task_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Investigation Task Id"
},
"knowledge_cutoff": {
"format": "date-time",
"title": "Knowledge Cutoff",
"type": "string"
},
"metrics": {
"items": {
"$ref": "#/components/schemas/PeriodicTaskMetric"
},
"title": "Metrics",
"type": "array"
},
"period_end": {
"format": "date-time",
"title": "Period End",
"type": "string"
},
"period_start": {
"format": "date-time",
"title": "Period Start",
"type": "string"
},
"replay_id": {
"format": "uuid",
"title": "Replay Id",
"type": "string"
},
"result": {
"anyOf": [
{
"additionalProperties": true,
"type": "object"
},
{
"type": "null"
}
],
"title": "Result"
},
"risk_signals": {
"items": {
"additionalProperties": true,
"type": "object"
},
"title": "Risk Signals",
"type": "array"
},
"status": {
"enum": [
"pending",
"running",
"completed",
"failed"
],
"title": "Status",
"type": "string"
},
"steps": {
"items": {
"$ref": "#/components/schemas/PeriodicTaskStep"
},
"title": "Steps",
"type": "array"
},
"summary": {
"title": "Summary",
"type": "string"
},
"task_key": {
"enum": [
"scada-diagnosis",
"network-simulation"
],
"title": "Task Key",
"type": "string"
},
"title": {
"title": "Title",
"type": "string"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
"type": "string"
}
},
"required": [
"execution_id",
"replay_id",
"task_key",
"title",
"period_start",
"period_end",
"knowledge_cutoff",
"batch_sequence",
"status",
"conclusion",
"summary",
"steps",
"updated_at"
],
"title": "PeriodicTaskExecution",
"type": "object"
},
"PeriodicTaskMetric": {
"additionalProperties": false,
"properties": {
"detail": {
"default": "",
"title": "Detail",
"type": "string"
},
"id": {
"title": "Id",
"type": "string"
},
"label": {
"title": "Label",
"type": "string"
},
"status": {
"default": "normal",
"enum": [
"normal",
"attention",
"critical"
],
"title": "Status",
"type": "string"
},
"unit": {
"default": "",
"title": "Unit",
"type": "string"
},
"value": {
"title": "Value",
"type": "string"
}
},
"required": [
"id",
"label",
"value"
],
"title": "PeriodicTaskMetric",
"type": "object"
},
"PeriodicTaskPage": {
"additionalProperties": false,
"properties": {
"items": {
"items": {
"$ref": "#/components/schemas/PeriodicTaskExecution"
},
"title": "Items",
"type": "array"
},
"next_offset": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Next Offset"
}
},
"required": [
"items",
"next_offset"
],
"title": "PeriodicTaskPage",
"type": "object"
},
"PeriodicTaskRetry": {
"additionalProperties": false,
"properties": {
"request_id": {
"format": "uuid",
"title": "Request Id",
"type": "string"
}
},
"required": [
"request_id"
],
"title": "PeriodicTaskRetry",
"type": "object"
},
"PeriodicTaskStep": {
"additionalProperties": false,
"properties": {
"detail": {
"default": "",
"title": "Detail",
"type": "string"
},
"id": {
"title": "Id",
"type": "string"
},
"status": {
"enum": [
"pending",
"running",
"completed",
"failed",
"skipped"
],
"title": "Status",
"type": "string"
},
"title": {
"title": "Title",
"type": "string"
}
},
"required": [
"id",
"title",
"status"
],
"title": "PeriodicTaskStep",
"type": "object"
},
"PlaceSearchConfig": { "PlaceSearchConfig": {
"additionalProperties": false, "additionalProperties": false,
"properties": { "properties": {
@@ -8453,6 +8749,11 @@
"title": "Rules", "title": "Rules",
"type": "array" "type": "array"
}, },
"scheduled_tasks_enabled": {
"default": false,
"title": "Scheduled Tasks Enabled",
"type": "boolean"
},
"start_time": { "start_time": {
"format": "date-time", "format": "date-time",
"title": "Start Time", "title": "Start Time",
@@ -8766,6 +9067,22 @@
], ],
"title": "Monitoring Interval Seconds" "title": "Monitoring Interval Seconds"
}, },
"periodic_error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Periodic Error"
},
"periodic_pending": {
"default": false,
"title": "Periodic Pending",
"type": "boolean"
},
"processed_count": { "processed_count": {
"default": 0, "default": 0,
"title": "Processed Count", "title": "Processed Count",
@@ -8804,6 +9121,11 @@
"title": "Rules", "title": "Rules",
"type": "array" "type": "array"
}, },
"scheduled_tasks_enabled": {
"default": false,
"title": "Scheduled Tasks Enabled",
"type": "boolean"
},
"start_time": { "start_time": {
"format": "date-time", "format": "date-time",
"title": "Start Time", "title": "Start Time",
@@ -43962,6 +44284,263 @@
] ]
} }
}, },
"/api/v1/scada-replays/{replay_id}/scheduled-tasks": {
"get": {
"operationId": "get_scada_replays_replay_id_scheduled_tasks",
"parameters": [
{
"in": "path",
"name": "replay_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Replay Id",
"type": "string"
}
},
{
"in": "query",
"name": "offset",
"required": false,
"schema": {
"default": 0,
"minimum": 0,
"title": "Offset",
"type": "integer"
}
},
{
"in": "query",
"name": "limit",
"required": false,
"schema": {
"default": 100,
"maximum": 200,
"minimum": 1,
"title": "Limit",
"type": "integer"
}
},
{
"in": "header",
"name": "X-Project-Id",
"required": true,
"schema": {
"title": "X-Project-Id",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PeriodicTaskPage"
}
}
},
"description": "Successful Response"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Authentication required"
},
"403": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Insufficient permission"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Resource not found"
},
"409": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Resource conflict"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Validation error"
},
"503": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Dependency unavailable"
}
},
"security": [
{
"OAuth2PasswordBearer": []
}
],
"summary": "Scheduled Tasks",
"tags": [
"SCADA Replay"
]
}
},
"/api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry": {
"post": {
"operationId": "post_scada_replays_replay_id_scheduled_tasks_execution_id_retry",
"parameters": [
{
"in": "path",
"name": "replay_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Replay Id",
"type": "string"
}
},
{
"in": "path",
"name": "execution_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Execution Id",
"type": "string"
}
},
{
"in": "header",
"name": "X-Project-Id",
"required": true,
"schema": {
"title": "X-Project-Id",
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PeriodicTaskRetry"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PeriodicTaskExecution"
}
}
},
"description": "Successful Response"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Authentication required"
},
"403": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Insufficient permission"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Resource not found"
},
"409": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Resource conflict"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Validation error"
},
"503": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
},
"description": "Dependency unavailable"
}
},
"security": [
{
"OAuth2PasswordBearer": []
}
],
"summary": "Retry Scheduled Task",
"tags": [
"SCADA Replay"
]
}
},
"/api/v1/sensor-placement-candidates/{node_id}": { "/api/v1/sensor-placement-candidates/{node_id}": {
"get": { "get": {
"operationId": "get_sensor_placement_candidates_node_id", "operationId": "get_sensor_placement_candidates_node_id",
+1
View File
@@ -36,6 +36,7 @@
| 主题 | 维护入口 | | 主题 | 维护入口 |
| --- | --- | | --- | --- |
| 播放时间驱动的诊断、定时模拟及模板动作 | [周期任务执行](periodic-task-execution.md) |
| 回放、时间轴、周期监测与调查边界 | [SCADA 回放设计与实现](scada-peripheral-replay-design.md) | | 回放、时间轴、周期监测与调查边界 | [SCADA 回放设计与实现](scada-peripheral-replay-design.md) |
| 外围测试样本与数据再现 | [合成样本](peripheral-generated-data.md) | | 外围测试样本与数据再现 | [合成样本](peripheral-generated-data.md) |
| 凤阳历史爆管模拟 | [凤阳 SCADA 复制记录](fengyang-burst-scada.md) | | 凤阳历史爆管模拟 | [凤阳 SCADA 复制记录](fengyang-burst-scada.md) |
+53
View File
@@ -0,0 +1,53 @@
# 播放时间驱动的周期任务
2026-11-11:SCADA 数据诊断与管网定时模拟接入实际处理,管网智能调度与泵站能耗检查保留现有动作模板。任务名称、列表、报告及 Agent 指令使用业务表述;内部执行来源与证据限制分别保存。
## 时钟与周期
新建回放请求 `scheduled_tasks_enabled: true` 启用两类实际任务,供水前端默认发送该字段。省略字段的旧客户端与已有回放继续保持原有处理行为。项目监测周期读取 MetaDB `projects.settings.scada_monitoring_interval_seconds`,本业务约定为 `900` 秒;前端保留的 SCADA 任务模板也改为 15 分钟。5 分钟时间轴定位刻度、180 倍播放速度、设备采样周期和接口轮询间隔各自独立。
播放或单步在业务周期终点 T 读取并提交本周期观测、诊断回执和调查关联,随后创建周期执行记录并提交水力计算。水力任务未结束时暂停时间推进,已有调查可以独立执行。关闭页面停止继续推进;已提交的计算仍由现有 calculation worker 完成。跨日期推进保留原始业务时间,查看历史指针不触发处理;向已处理时刻重新监测沿用新建回放的因果边界。
## SCADA 数据诊断
执行步骤为读取观测、检查新鲜度完整率、诊断压力流量与水质、判断潜在事件、整理诊断结论。复用现有清洗、动态压力阈值、爆管侦测及指标规则,不按已知事件日期生成异常。
周期记录分别保留数据质量、压力/流量运行异常、疑似爆管与疑似污染线索。水质必须有指标名称、单位及明确业务阈值;只知道 `mg/L` 不能推断测量对象。水质规则尚未配置时明确记录污染风险未评估。水质有效异常进入现有独立 `water_pollution` 调查,不等待压力基线或爆管模型就绪,纯水质异常不强制进入爆管定位。周期报告从已冻结周期批次事件和持续异常状态读取水质证据,保留关联调查;周期内发生后恢复的异常仍记入本周期,水质证据不计入爆管支持分。
仪表量程用于读数有效性校验,业务阈值用于异常判断;可靠采集的异常值保留。异常持续期间复用已有调查;恢复与再次发生沿用现有异常事件生命周期。正常周期由 Server 生成报告,不启动 Agent。
## 管网定时模拟
在创建本轮回放时,以只读一致事务导出 EPANET 2 输入并保存摘要。完整模型按 gzip 压缩保存,原文限制为 64 MiB、压缩后限制为 8 MiB,冻结设备目录与周期输入后,计算 worker 使用相同模型版本和不超过本周期 `batch_sequence`、`knowledge_cutoff` 的观测。计算不重新查询实时 TSDB,不更新在线模型、正式 raw 或实时基线。
现有 `wntr==1.3.2` 与 EPANET 内核运行逐观测时刻的稳态水力复核,与原在线滚动重算的计算语义一致。每个时刻计算全部节点和管段,将已关联测点的压力、流量按单位转换后与实测比较。水箱水位、工频泵状态和需水量只有在单位及关联明确时覆盖模型边界;变频频率与水源液位缺少已核实换算依据时保留冻结模型设定,并明确报告限制。
输出压力/流量平均相对偏差、最大节点压差、偏差测点及关联要素、全模型规模、原始比较明细与缺项。近零实测不计算百分比。模板压力 8%/12%、流量 10%/15%、最大压差 5m/8m 的关注/异常阈值用于本版复核,不能解释为爆管确诊阈值。比较已完成的相邻周期及本轮模型下此前七天同窗结果,缺少参考时列出实际日期,不补造基线。偏差来源输出待核查原因,不将参数或边界问题当作已证实事实。
## 持久化与重试
不新增数据库表。压缩模型输入保存在 `peripheral_sample.scada_replay_batches` 的零号 `model_snapshot` 批次,回放主文档只保存摘要和就绪标记,避免轮询和每帧处理重复加载、重写完整模型。执行记录保存在业务批次的 `document.scheduled_tasks`,不通过回放快照返回完整模型。
每条执行身份由回放、任务类型和周期终点确定,同周期重复命令复用记录。水力计算复用 `workbench.calculation_jobs`,内部工具键为 `scheduled_network_simulation`。提交与执行分别核验项目成员和 SCADA 读取、清洗、水力计算权限。任务直接由 Server 执行,不伪造 Agent 会话、运行租约或 OAuth 上下文。
worker 沿用现有并发限制、子进程隔离、超时、租约及代次栅栏。成功结果和终态在同一栅栏事务内回写周期记录,计算队列结果过期不删除已保存的业务报告。失败或中断可按原模型、批次和时间边界重试,增加 attempt,保留同一执行身份。
水力计算排队或运行期间保留当前周期;失败、超时、中断或计算记录丢失后,播放与单步均停止推进。报告保留已完成的冻结步骤,将计算步骤标为失败、后续比较和复核步骤标为跳过;队列容量不足的失败同样适用。可在工况任务中重试本周期计算,成功后再继续播放,或停止后重新开始一轮。重复发送已接受命令仍返回原回执,不重新执行。SCADA 诊断已提交的正常、异常或数据不足结论均保留。
管网智能调度与泵站能耗检查继续使用原有 mock 动作模板,尚未接入执行算法。
## 接口与前端
- `GET /api/v1/scada-replays/{replay_id}/scheduled-tasks?offset=0&limit=100`:分页查询周期记录,仅读取已提交结果;不创建任务。
- `POST /api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry`:提交 `{ "request_id": "UUID" }`,重试失败的水力计算。
- 回放快照增加 `scheduled_tasks_enabled`、`periodic_pending`、`periodic_error`;轮询实际队列状态,不把排队视为完成。
前端以当前回放业务时间生成保留的调度与能耗任务,暂停后进度冻结。SCADA 和管网定时模拟仅使用后端持久化周期记录,不因未启动回放、未启用周期作业或接口失败而退回生成模板。无回放时提示启动监测,旧回放未启用时提示重新开始;读取失败显示错误并保留已读取的后端记录。列表显示最近六小时,接口保留分页历史。执行状态和诊断结论分开,报告展示实际步骤、指标、证据、缺项、调查与重试入口;刷新保留当前选中报告。
## 验证与启用
后端使用 `tests/unit/test_periodic_tasks.py`、`test_periodic_hydraulics.py` 及现有回放、计算 worker 和架构边界测试;真实 SQL 与 EPANET 联合检查为 `PERIODIC_TASK_STORAGE_TEST=1 python -m pytest tests/integration/test_periodic_task_storage.py -q`。该集成检查只使用独立项目 UUID,所有记录强制回滚,不保留测试观测或任务。
前端运行 `pnpm lint`、`pnpm typecheck`、`pnpm test`、`pnpm build`,聚焦浏览器检查为 `pnpm test:browser tests/browser/periodic-tasks.e2e.ts`,覆盖桌面、手机、失败重试、刷新恢复与水平溢出。接口变更后重新导出 Server OpenAPI 并同步 Agent 的生成契约。
部署使用现有 API 与 calculation worker,不增加服务或迁移。两个进程都需要加载本轮代码,旧 worker 无法执行新增内部工具。未运行实际 Agent、现场审批或设备操作时,不将静态检查、固定浏览器响应和原生计算验证报告为现场业务验收。
+2
View File
@@ -1,5 +1,7 @@
# SCADA 外围系统时段回放、主动监测与爆管调查设计 # SCADA 外围系统时段回放、主动监测与爆管调查设计
2026-11-11 增补:[周期任务执行](periodic-task-execution.md) 已将 SCADA 综合诊断与管网定时模拟接入外围播放时间,诊断覆盖疑似爆管、污染及其他异常;智能调度和能耗保留动作模板。以下“本阶段不创建定时任务”等表述记录原单步版本边界,新接入范围以增补文档为准。SCADA 读取清洗仍由命令请求执行,水力复核由现有 calculation worker 异步完成。
状态:时段回放、主动周期监测、条件式 `burst_report` 调查及回放安全的爆管定位适配已实现。单步处理由用户操作触发并在当前 API 请求内完成,不部署 SCADA 回放 worker。`tjwater_v2` 与 `tjwater2_v2` 尚未确认出厂边界流量角色;仍待完整月度数据和真实 Agent 联调。更新日期:2026-11-03。下面明确区分已实现范围、后续目标与验收要求。 状态:时段回放、主动周期监测、条件式 `burst_report` 调查及回放安全的爆管定位适配已实现。单步处理由用户操作触发并在当前 API 请求内完成,不部署 SCADA 回放 worker。`tjwater_v2` 与 `tjwater2_v2` 尚未确认出厂边界流量角色;仍待完整月度数据和真实 Agent 联调。更新日期:2026-11-03。下面明确区分已实现范围、后续目标与验收要求。
本次将外围 SCADA 定义为一个模拟外部数据源:本地 TSDB 已保存原始观测,通过只读 API 按选定时间区间和后台时钟分批读取 `scada.measurements.monitored_value`,将这一批直接交给清洗流水线,保存 `cleaned_value` 后执行异常检测及后续 Agent 调查。原始值、修正值、真实工况异常和处理结论分别保留。浏览器负责控制和展示,关闭页面不停止后台回放。 本次将外围 SCADA 定义为一个模拟外部数据源:本地 TSDB 已保存原始观测,通过只读 API 按选定时间区间和后台时钟分批读取 `scada.measurements.monitored_value`,将这一批直接交给清洗流水线,保存 `cleaned_value` 后执行异常检测及后续 Agent 调查。原始值、修正值、真实工况异常和处理结论分别保留。浏览器负责控制和展示,关闭页面不停止后台回放。
+2
View File
@@ -1,5 +1,7 @@
# 工作台运行态 # 工作台运行态
播放周期任务通过[独立执行接口](periodic-task-execution.md)读取,前端将其与原有运行态组合;下文兼容字段 `conditions` 不承担回放周期记录的存储或投影。
`GET /api/v1/workbench/runtime` 按当前项目读取 BizDB 中的分析成果和任务告警。外围样板工单经同步器进入 `workbench.operational_tasks`,由工况任务接口和 `sync-snapshot` 提供给前端;定时任务表暂时保留,但不进入当前产品主路径。 `GET /api/v1/workbench/runtime` 按当前项目读取 BizDB 中的分析成果和任务告警。外围样板工单经同步器进入 `workbench.operational_tasks`,由工况任务接口和 `sync-snapshot` 提供给前端;定时任务表暂时保留,但不进入当前产品主路径。
- `analysis.runs/results` 只用于分析成果、历史成果和兼容场景字段。 - `analysis.runs/results` 只用于分析成果、历史成果和兼容场景字段。
@@ -1,8 +1,8 @@
--- ---
name: scada-anomaly name: scada-anomaly
description: 从 SCADA 指标异常开展数据核验、爆管诊断、候选定位与影响分析。 description: 从 SCADA 压力、流量和水质异常开展数据核验、潜在爆管或污染诊断及影响分析。
metadata: metadata:
version: "1.0.21" version: "1.0.22"
--- ---
# SCADA 数据异常调查 # SCADA 数据异常调查
@@ -21,7 +21,7 @@ metadata:
全量devices含完整目录,readings含观测,coverage含各设备读数数量、缺测及冻结范围外清单。不得由代表测点推断全网。单查list_scada_devices默认每页100,读至page.has_more=false,属性缩略不代表完整信息。全量后可按device_ids局部复核并持续翻页,不能替代全量输入。final引用本轮knowledge_cutoff及batch_sequence的全量证据;Server拒绝局部或旧批次证据。 全量devices含完整目录,readings含观测,coverage含各设备读数数量、缺测及冻结范围外清单。不得由代表测点推断全网。单查list_scada_devices默认每页100,读至page.has_more=false,属性缩略不代表完整信息。全量后可按device_ids局部复核并持续翻页,不能替代全量输入。final引用本轮knowledge_cutoff及batch_sequence的全量证据;Server拒绝局部或旧批次证据。
核对模型得分、压力变化与出厂边界流量的联合证据;权重、门槛、阈值、测点与基线留在依据。得分越低越异常,相似度非发生概率。区分边界与其他管流测点,规则零命中不代表无流量变化;其他异常按证据选择原因。 核对模型得分、压力绝对下降及变化率与出厂边界流量的联合证据;权重、门槛、阈值、测点与各自基线留在依据,目标告警与参考日分别解释。得分越低越异常,相似度非发生概率。区分边界与其他管流测点,规则零命中不代表无流量变化;其他异常按证据选择原因。水质按指标、单位、业务阈值与相邻趋势核查,区别仪表量程,不要求爆管定位。
## 候选定位与隔离 ## 候选定位与隔离
+34 -1
View File
@@ -110,7 +110,7 @@ def test_monitoring_command_processes_pending_period_in_same_request(monkeypatch
) )
result = asyncio.run(scada_replay.replay_command( result = asyncio.run(scada_replay.replay_command(
replay_id, command, repo="repo", ctx=context, timescale="timescale" replay_id, command, repo=SimpleNamespace(get=AsyncMock(return_value={"config": {}})), ctx=context, timescale="timescale"
)) ))
assert result == "completed-period" assert result == "completed-period"
@@ -127,3 +127,36 @@ def test_client_cannot_override_project_frequency_on_creation(replay_api):
}) })
assert response.status_code == 422 assert response.status_code == 422
create.assert_not_awaited() create.assert_not_awaited()
def test_periodic_endpoint_returns_only_persisted_diagnosis_and_simulation_without_creating_runs(replay_api, monkeypatch):
from app.services import periodic_tasks
from app.services.periodic_diagnosis import execution_record
client, _, _, context, _ = replay_api
replay_id = uuid4()
document = {"replay_id": str(replay_id)}
monitoring = {"period_start": "2026-07-16T07:45:00+08:00", "period_end": "2026-07-16T08:00:00+08:00",
"knowledge_cutoff": "2026-07-16T08:00:00+08:00", "batch_sequence": 2}
records = [execution_record(document, monitoring, key) for key in ("scada-diagnosis", "network-simulation")]
for record in records:
record.update(status="completed", conclusion="normal", summary="已完成本周期实际处理。")
repo = SimpleNamespace(connection=object(), project_id=context.project_id,
available=AsyncMock(return_value=True), get=AsyncMock(return_value=document),
periodic_tasks=AsyncMock(return_value=records))
ensure = AsyncMock()
monkeypatch.setattr(scada_replay, "ensure_periodic_tasks", ensure)
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace())
client.app.dependency_overrides[scada_replay.repository] = lambda: repo
response = client.get(f"/scada-replays/{replay_id}/scheduled-tasks")
assert response.status_code == 200
items = response.json()["items"]
assert {item["task_key"] for item in items} == {"scada-diagnosis", "network-simulation"}
assert [item["execution_id"] for item in items] == [item["execution_id"] for item in records]
assert all(item["status"] == "completed" for item in items)
assert response.json()["next_offset"] is None
repo.get.assert_awaited_once_with(replay_id)
repo.periodic_tasks.assert_awaited_once_with(replay_id, 0, 101)
ensure.assert_not_awaited()
@@ -0,0 +1,95 @@
"""Native calculation and real SQL under an isolated project UUID, always rolled back."""
import os
from datetime import datetime, timezone
from uuid import UUID, uuid4
import psycopg
from psycopg.rows import dict_row
import pytest
from app.auth.project_dependencies import ProjectContext
from app.infra.db.metadb.database import SessionLocal, engine
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.services.periodic_tasks import ensure_periodic_tasks, execute_periodic_calculation, persist_periodic_terminal, project_periodic_tasks
from app.infra.epanet.model_snapshot import pack_model_input
from types import SimpleNamespace
pytestmark = pytest.mark.skipif(os.getenv("PERIODIC_TASK_STORAGE_TEST") != "1", reason="explicit rollback-only SQL test required")
@pytest.fixture
def anyio_backend(): return "asyncio"
@pytest.mark.anyio
async def test_frozen_period_queue_native_result_and_retry_storage():
from tests.unit.test_periodic_hydraulics import MODEL
async with SessionLocal() as session:
metadata = MetadataRepository(session)
routing = await metadata.get_project_db_routing(UUID("871d7538-589c-47de-a6d5-3185ccb9f0e1"), "biz_data")
await engine.dispose()
if not routing:
pytest.skip("Project route unavailable")
connection = await psycopg.AsyncConnection.connect(routing.dsn, row_factory=dict_row, connect_timeout=5)
project_id, user_id, replay_id, request_id = uuid4(), uuid4(), uuid4(), uuid4()
try:
async with connection.transaction(force_rollback=True):
repo = ScadaReplayRepository(connection, project_id)
if not await repo.available() or not await repo.queue_available():
pytest.skip("Replay or calculation storage is not provisioned")
start, end = "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00"
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m", "transmission_frequency": "0:15:00"},
{"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h", "transmission_frequency": "0:15:00"}]
rows = [{"device_id": device, "time": at, "quality": "valid", "cleaned_value": value}
for at in (start, end) for device, value in (("P1", 39.8), ("F1", 3.6))]
monitoring = {"monitoring_id": str(uuid4()), "period_start": start, "period_end": end, "knowledge_cutoff": end,
"batch_sequence": 1, "status": "normal", "detail": "本周期正常", "task_id": None}
document = {"project_id": str(project_id), "replay_id": str(replay_id), "created_by": str(user_id), "status": "paused",
"config": {"request_id": str(request_id), "scheduled_tasks_enabled": True, "device_ids": ["P1", "F1"], "rules": []},
"devices": devices, "latest": {row["device_id"]: row for row in rows},
"monitoring_interval_seconds": 900, "last_monitoring": monitoring,
"scheduled_model_ready": True, "scheduled_model_digest": "frozen-test-model",
"updated_at": datetime.now(timezone.utc).isoformat()}
await repo.insert(document)
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
"model_input_gzip": pack_model_input(MODEL), "model_digest": "frozen-test-model"})
await repo.save_batch(replay_id, {"sequence": 1, "stage": "detected", "monitoring": monitoring, "outputs": rows})
context = ProjectContext(project_id, "tjwater2_v2", user_id, "member")
await ensure_periodic_tasks(repo, context, replay_id)
await ensure_periodic_tasks(repo, context, replay_id)
page = await project_periodic_tasks(repo, replay_id, 0, 100)
assert len(page) == 2
simulation = next(item for item in page if item["task_key"] == "network-simulation")
assert simulation["status"] == "pending"
queue = CalculationJobRepository(connection, project_id)
owner = str(uuid4())
job = await queue.claim(owner, 30)
# Avoid committing the enclosing test rollback transaction: production
# releases its read transaction before computation, tested separately.
class ReadScopeConnection:
def __getattr__(self, key): return getattr(connection, key)
async def commit(self): pass
result = await execute_periodic_calculation(SimpleNamespace(project=context, connection=ReadScopeConnection()), job)
assert result.data["conclusion"] == "normal"
assert len(result.data["result"]["comparisons"]) == 4
terminal = {"status": "succeeded", "result": result.model_dump(mode="json")}
assert await queue.finish(job, owner, **terminal)
await persist_periodic_terminal(connection, job, terminal)
# A stale worker must not republish another result after its fence closed.
assert not await queue.finish(job, owner, **terminal)
stored = await repo.periodic_task(replay_id, simulation["execution_id"])
assert stored["status"] == "completed"
assert len(stored["result"]["comparisons"]) == 4
await repo.save_retry_receipt(replay_id, 1, request_id, simulation["execution_id"])
assert (await repo.batch(replay_id, 1))["periodic_retry_receipts"][str(request_id)] == simulation["execution_id"]
assert len(await repo.periodic_tasks(replay_id, 0, 1)) == 1
assert len(await repo.periodic_tasks(replay_id, 1, 1)) == 1
assert (await repo.get(replay_id))["last_scheduled_simulation"]["status"] == "completed"
# Check rollback on the same connection, with no application rows retained.
row = await (await connection.execute("SELECT count(*) AS count FROM peripheral_sample.scada_replays WHERE project_id=%s", (project_id,))).fetchone()
assert row["count"] == 0
finally:
await connection.close()
+27
View File
@@ -0,0 +1,27 @@
import base64
import gzip
import pytest
from app.infra.epanet import model_snapshot
def test_full_model_larger_than_response_budget_round_trips_compressed():
content = "[JUNCTIONS]\n" + "节点 12.345\n" * 900_000
assert len(content.encode()) > 8 * 1024 * 1024
packed = model_snapshot.pack_model_input(content)
assert len(packed) < len(content)
assert model_snapshot.unpack_model_input(packed) == content
def test_decoding_enforces_expanded_model_budget(monkeypatch):
packed = base64.b64encode(gzip.compress(b"A" * 1025)).decode()
monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 1024)
with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded):
model_snapshot.unpack_model_input(packed)
def test_oversized_model_is_rejected_without_compressing(monkeypatch):
monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 3)
with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded):
model_snapshot.pack_model_input("four")
+67
View File
@@ -0,0 +1,67 @@
from datetime import datetime
import pytest
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
MODEL = """[JUNCTIONS]
J1 10 1
[RESERVOIRS]
R1 50
[PIPES]
L1 R1 J1 500 150 100 0 OPEN
[OPTIONS]
UNITS LPS
HEADLOSS H-W
[TIMES]
DURATION 0:15
HYDRAULIC TIMESTEP 0:15
REPORT TIMESTEP 0:15
[END]
"""
def test_real_epanet_checks_all_model_elements_without_a_source_database():
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"},
{"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h"}]
rows = [{"time": at, "device_id": device, "quality": "valid", "cleaned_value": value}
for at in ("2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00", "2026-07-16T08:15:00+08:00")
for device, value in (("P1", 39.8), ("F1", 3.6))]
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
assert len(result["comparisons"]) == 4
assert result["conclusion"] == "normal"
assert result["compared_device_count"] == 2
assert result["missing_device_ids"] == []
assert all(snapshot["node_count"] == 2 and snapshot["link_count"] == 1 for snapshot in result["snapshots"])
assert all(datetime.fromisoformat(item["time"]).hour <= 8 for item in result["comparisons"])
assert {item["time"] for item in result["comparisons"]} == {row["time"] for row in rows[:4]}
assert next(item for item in result["comparisons"] if item["device_id"] == "F1")["observed"] == .001
def test_no_valid_measurement_is_a_data_gap_not_normal():
result = run_periodic_hydraulics(MODEL, [], [], "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
assert result["conclusion"] == "insufficient_data"
assert result["comparisons"] == []
def test_missing_mapping_and_zero_reference_are_not_silently_compared():
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"},
{"device_id": "F1", "device_type": "pipe_flow", "measurement_unit": "m3/h"}]
rows = [{"time": "2026-07-16T08:00:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 0.}]
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
assert result["comparisons"][0]["relative_deviation_percent"] is None
assert result["missing_device_ids"] == ["F1"]
assert result["conclusion"] != "normal"
def test_previous_sample_does_not_claim_current_period_is_normal():
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"}]
rows = [{"time": "2026-07-16T07:45:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 39.8}]
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
assert result["compared_device_count"] == 1
assert result["current_device_count"] == 0
assert result["conclusion"] == "insufficient_data"
devices[0]["transmission_frequency"] = "0:30:00"
slower = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
assert slower["current_device_count"] == 1
+365
View File
@@ -0,0 +1,365 @@
import asyncio
import json
from contextlib import asynccontextmanager
from copy import deepcopy
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock
from uuid import uuid4
import pytest
from fastapi import HTTPException
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
from app.services import periodic_tasks
def document():
return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()),
"monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest",
"devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"},
{"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}],
"config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"],
"rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]},
"latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}}
def monitoring():
return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00",
"period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00",
"status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None}
def test_water_risk_is_independent_of_incomplete_pressure_model():
source, period = document(), monitoring()
period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[
{"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}])
result = diagnosis_execution(source, period)
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == period["task_id"]
assert result["risk_signals"][0]["metric_name"] == "余氯"
assert result["data_gaps"] == ["压力基线不足"]
assert all(step["status"] == "completed" for step in result["steps"])
def detected_water_event():
from datetime import timedelta
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
source = document()
at = datetime.fromisoformat(monitoring()["period_end"])
config = ScadaReplayCreate(
request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15),
device_ids=["Q1"], rules=[{
"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"],
"lower": 0.2, "unit": "mg/L", "metric_name": "余氯",
}],
)
source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules]
source, events = detect_frame(source, prepare_cleaning([
{"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"}
], config), config, at)
event = next(event for event in events if event["rule_id"] == "water")
task_id = str(uuid4())
event["task_id"] = task_id
source["rule_states"]["water"]["event"]["task_id"] = task_id
return source, event
def test_periodic_report_includes_water_event_excluded_from_burst_monitoring():
from app.services.scada_normal_profile import diagnostic_signal
source, event = detected_water_event()
period = monitoring()
assert not diagnostic_signal(source, event)
original = deepcopy(period)
result = diagnosis_execution(source, period)
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == event["task_id"]
water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution")
assert water["device_ids"] == ["Q1"]
assert water["task_id"] == event["task_id"]
assert period == original
def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence():
source, event = detected_water_event()
source["rule_states"]["water"]["event"] = None
event.update(status="resolved", resolved_at=monitoring()["period_end"])
period = monitoring()
hydraulic_task_id = str(uuid4())
hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()),
"device_ids": ["P1"], "detail": "压力变化需要复核"}
period.update(status="attention", task_id=hydraulic_task_id,
weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal])
original = deepcopy(period)
result = diagnosis_execution(source, period, [event, deepcopy(event)])
assert result["conclusion"] == "suspected_pollution"
assert result["investigation_task_id"] == hydraulic_task_id
water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"]
assert len(water) == 1
assert water[0]["task_id"] == event["task_id"]
assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"]
for signal in result["risk_signals"])
assert period == original
assert result["result"]["monitoring"] == original
def test_periodic_water_projection_excludes_telemetry_quality_events():
source, event = detected_water_event()
event["category"] = "data_quality"
source["rule_states"]["water"]["event"] = deepcopy(event)
result = diagnosis_execution(source, monitoring(), [event])
assert result["conclusion"] == "normal"
assert result["risk_signals"] == []
assert result["investigation_task_id"] is None
@pytest.mark.parametrize("future", [
{"occurred_at": "2026-07-16T08:15:00+08:00"},
{"batch_sequence": 3},
])
def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future):
source, event = detected_water_event()
event.update(future)
source["rule_states"]["water"]["event"] = event
result = diagnosis_execution(source, monitoring(), [event])
assert result["conclusion"] == "normal"
assert result["risk_signals"] == []
assert result["investigation_task_id"] is None
def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal():
source, period = document(), monitoring()
source["latest"]["P1"]["quality"] = "invalid"
first = diagnosis_execution(source, period)
assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"]
assert first["conclusion"] == "attention"
assert first["result"]["invalid_device_ids"] == ["P1"]
source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00"
assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"]
other = dict(source, replay_id=str(uuid4()))
assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"]
class Repository:
def __init__(self):
self.document = document()
self.document["last_monitoring"] = monitoring()
self.project_id = self.document["project_id"]
self.connection = object()
self.tasks = {}
self.receipts = {}
self.state = None
self.events = []
@asynccontextmanager
async def control(self): yield
async def get(self, _): return deepcopy(self.document)
async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)}
async def periodic_events(self, *_): return deepcopy(self.events)
async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution)
async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution)
async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id)))
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id)
def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch):
repo = Repository()
enqueue = AsyncMock(return_value={"job_id": uuid4()})
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue))
context = SimpleNamespace(project_id=repo.project_id)
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
async def run():
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
asyncio.run(run())
assert len(repo.tasks) == 2
assert enqueue.await_count == 1
payload = enqueue.call_args.kwargs
assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id}
assert payload["owner_user_id"] == repo.document["created_by"]
assert repo.state["status"] == "pending"
for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task)
def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch):
source, event = detected_water_event()
source["rule_states"]["water"]["event"] = None
source["last_monitoring"] = monitoring()
repo = Repository()
repo.document = source
repo.project_id = source["project_id"]
repo.events = [event]
read_events = AsyncMock(wraps=repo.periodic_events)
repo.periodic_events = read_events
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=AsyncMock(return_value={"job_id": uuid4()})))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"]))
read_events.assert_awaited_once()
diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")
assert diagnosis["conclusion"] == "suspected_pollution"
assert diagnosis["investigation_task_id"] == event["task_id"]
def test_worker_interruption_is_reported_as_retryable_failure():
record = execution_record(document(), monitoring(), "network-simulation")
result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}})
assert result["status"] == "failed"
assert "WORKER_INTERRUPTED" in result["summary"]
assert record["status"] == "pending"
@pytest.mark.parametrize("job", [None, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}},
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}])
def test_failed_calculation_marks_its_execution_step_and_skips_dependent_steps(job):
record = execution_record(document(), monitoring(), "network-simulation")
record["steps"][0]["status"] = "completed"
original = deepcopy(record)
result = periodic_tasks.project_job(record, job)
assert result["status"] == "failed"
assert result["steps"][0]["status"] == "completed"
assert result["steps"][1]["status"] == "failed"
assert result["steps"][1]["detail"] == result["summary"]
assert all(step["status"] == "skipped" for step in result["steps"][2:])
assert record == original
def test_worker_failure_persists_failed_and_skipped_steps_in_the_business_report(monkeypatch):
repo = Repository()
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
record["steps"][0]["status"] = "completed"
record["calculation_job_id"] = str(uuid4())
repo.tasks[record["execution_id"]] = record
monkeypatch.setattr(periodic_tasks, "ScadaReplayRepository", lambda *_: repo)
row = {"project_id": repo.project_id, "job_id": record["calculation_job_id"], "parameters": {
"replay_id": record["replay_id"], "execution_id": record["execution_id"], "batch_sequence": record["batch_sequence"]}}
asyncio.run(periodic_tasks.persist_periodic_terminal(repo.connection, row,
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}))
saved = repo.tasks[record["execution_id"]]
assert saved["status"] == repo.state["status"] == "failed"
assert [step["status"] for step in saved["steps"]] == ["completed", "failed", "skipped", "skipped", "skipped"]
assert saved["execution_id"] == record["execution_id"]
assert saved["batch_sequence"] == record["batch_sequence"]
@pytest.mark.parametrize("action", ["step", "monitor"])
def test_failed_period_must_be_retried_before_advancing_but_can_be_stopped(monkeypatch, action):
from tests.unit.test_scada_replay_service import setup
from app.domain.schemas.scada_replay import ScadaReplayCommand
repo, service, _, _, create = setup(monkeypatch)
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
async def run():
replay = await service.create(create, monitoring_interval_seconds=900)
repo.document["config"]["scheduled_tasks_enabled"] = True
repo.document["last_scheduled_simulation"] = {"status": "failed", "error": "EXECUTION_TIMEOUT"}
original = deepcopy(repo.document)
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action=action,
**({"at_time": replay.start_time} if action == "monitor" else {}))
with pytest.raises(HTTPException) as error:
await service.command(replay.replay_id, command)
assert error.value.status_code == 409
assert "重试" in error.value.detail
assert repo.document == original
stopped = await service.command(replay.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=replay.revision, action="stop"))
assert stopped.status == "stopped"
asyncio.run(run())
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
repo = Repository()
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded())))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"]))
assert len(repo.tasks) == 2
assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed"
assert repo.state["status"] == "failed"
assert repo.state["calculation_job_id"] is None
assert "尚未启动" in repo.state["summary"]
def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch):
repo = Repository()
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
record.update(calculation_job_id=str(uuid4()), status="failed")
repo.tasks[record["execution_id"]] = deepcopy(record)
enqueue = AsyncMock(return_value={"job_id": uuid4()})
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"})))
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
request = uuid4()
async def run():
for _ in range(2):
await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id),
repo.document["replay_id"], record["execution_id"], request)
asyncio.run(run())
retried = repo.tasks[record["execution_id"]]
assert retried["attempt"] == 2
assert retried["period_end"] == record["period_end"]
assert retried["batch_sequence"] == record["batch_sequence"]
assert enqueue.await_count == 1
def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"})
with pytest.raises(HTTPException) as error:
asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {}))
assert error.value.status_code == 403
@pytest.mark.parametrize("calculation_status", ["pending", "failed"])
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch, calculation_status):
from tests.unit.test_scada_replay_service import setup
from app.domain.schemas.scada_replay import ScadaReplayCommand
repo, service, _, _, create = setup(monkeypatch)
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
async def run():
replay = await service.create(create, monitoring_interval_seconds=900)
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
accepted = await service.command(replay.replay_id, command)
repo.document["config"]["scheduled_tasks_enabled"] = True
repo.document["last_scheduled_simulation"] = {"status": calculation_status}
# A transport retry returns the original accepted period, even while its
# calculation is queued; a different step must wait.
repeated = await service.command(replay.replay_id, command)
assert repeated.revision == accepted.revision
with pytest.raises(HTTPException) as error:
await service.command(replay.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=accepted.revision, action="step"))
assert error.value.status_code == 409
assert "水力计算" in error.value.detail
asyncio.run(run())
def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path):
from app.services import calculation_worker
from app.mcp.models import QueryResult
@asynccontextmanager
async def scope(_): yield SimpleNamespace(project="authorized-project")
monkeypatch.setattr(calculation_worker, "execution_scope", scope)
internal = AsyncMock(return_value=QueryResult(
source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test",
queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"}))
monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal)
ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding"))
monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary)
source, target = tmp_path / "job.json", tmp_path / "result.json"
source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL}))
asyncio.run(calculation_worker.execute_child(str(source), str(target)))
assert json.loads(target.read_text())["status"] == "succeeded"
internal.assert_awaited_once()
ordinary.assert_not_awaited()