feat(replay): execute periodic diagnosis and hydraulic checks
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
"""Rolling hydraulic checks using immutable INP and observed input only.
|
||||
|
||||
Each observed timestamp is a steady hydraulic snapshot, matching the online
|
||||
recalculation's semantics. This module never opens a business/telemetry database.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from math import isfinite
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from tempfile import TemporaryDirectory
|
||||
from zoneinfo import ZoneInfo
|
||||
import re
|
||||
|
||||
|
||||
def _sampling_seconds(device: dict) -> int:
|
||||
frequency = str(device.get("transmission_frequency") or "")
|
||||
clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
|
||||
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
|
||||
return max(1, int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock
|
||||
else int(minutes[1]) * 60 if minutes else 900)
|
||||
|
||||
|
||||
def pressure_metres(value: float, unit: str) -> float | None:
|
||||
factors = {"m": 1.0, "mH2O": 1.0, "MPa": 101.9716213,
|
||||
"kPa": .1019716213, "bar": 10.19716213}
|
||||
return value * factors[unit] if unit in factors else None
|
||||
|
||||
|
||||
def flow_cms(value: float, unit: str) -> float | None:
|
||||
factors = {"m3/s": 1.0, "m³/s": 1.0, "m3/h": 1 / 3600,
|
||||
"m³/h": 1 / 3600, "L/s": .001, "l/s": .001,
|
||||
"LPS": .001, "CMH": 1 / 3600, "CMS": 1.0}
|
||||
return value * factors[unit] if unit in factors else None
|
||||
|
||||
|
||||
def run_periodic_hydraulics(model_input: str, devices: list[dict], rows: list[dict],
|
||||
period_start: str, period_end: str) -> dict:
|
||||
import wntr
|
||||
|
||||
start, end = datetime.fromisoformat(period_start), datetime.fromisoformat(period_end)
|
||||
usable = [row for row in rows if row.get("quality") in {"valid", "corrected"}
|
||||
and row.get("cleaned_value") is not None and isfinite(row["cleaned_value"])
|
||||
and start <= datetime.fromisoformat(row["time"]) <= end]
|
||||
sample_times = sorted({row["time"] for row in usable}, key=datetime.fromisoformat)
|
||||
if len(sample_times) > 100:
|
||||
raise ValueError("PERIOD_SAMPLE_LIMIT_EXCEEDED")
|
||||
if not sample_times:
|
||||
return {"conclusion": "insufficient_data", "comparisons": [], "snapshots": [],
|
||||
"data_gaps": ["本周期没有有效观测,无法开展模拟与实测复核。"], "metrics": []}
|
||||
comparisons, snapshots, gaps = [], [], set()
|
||||
directory_devices = {device["device_id"]: device for device in devices}
|
||||
with TemporaryDirectory(prefix="tjwater-periodic-hydraulics-") as directory:
|
||||
source = Path(directory) / "frozen.inp"
|
||||
source.write_text(model_input, encoding="utf-8")
|
||||
for index, at in enumerate(sample_times):
|
||||
model = wntr.network.WaterNetworkModel(str(source))
|
||||
clock = datetime.fromisoformat(at).astimezone(ZoneInfo("Asia/Shanghai"))
|
||||
model.options.time.duration = 0
|
||||
model.options.time.pattern_start = clock.hour * 3600 + clock.minute * 60 + clock.second
|
||||
model.options.time.start_clocktime = model.options.time.pattern_start
|
||||
frame = {row["device_id"]: row for row in usable if row["time"] == at}
|
||||
# Only exact-timestamp boundary observations can override the frozen
|
||||
# model. Missing boundaries retain explicit model assumptions.
|
||||
applied = []
|
||||
for device_id, row in frame.items():
|
||||
device = directory_devices.get(device_id, {})
|
||||
node_id, link_id = device.get("node_id"), device.get("link_id")
|
||||
value, kind = row["cleaned_value"], device.get("device_type")
|
||||
if kind == "tank_liquid_level" and node_id in model.tank_name_list:
|
||||
if device.get("measurement_unit") != "m":
|
||||
gaps.add(f"{device_id} 水位单位未适配。")
|
||||
continue
|
||||
model.get_node(node_id).init_level = value
|
||||
applied.append(device_id)
|
||||
elif kind == "fixed_pump" and link_id in model.pump_name_list:
|
||||
if value not in (0, 1):
|
||||
gaps.add(f"{device_id} 工频泵状态不是 0/1。")
|
||||
continue
|
||||
model.get_link(link_id).initial_status = "OPEN" if value else "CLOSED"
|
||||
applied.append(device_id)
|
||||
elif kind == "demand" and node_id in model.junction_name_list:
|
||||
demand = flow_cms(value, device.get("measurement_unit", ""))
|
||||
if demand is None:
|
||||
gaps.add(f"{device_id} 需水量单位未适配。")
|
||||
continue
|
||||
node = model.get_node(node_id)
|
||||
node.demand_timeseries_list[0].base_value = demand
|
||||
node.demand_timeseries_list[0].pattern_name = None
|
||||
applied.append(device_id)
|
||||
elif kind in {"variable_pump", "reservoir_liquid_level"}:
|
||||
gaps.add(f"{device_id} 缺少已核实的频率基准或水位高程转换,沿用冻结模型边界。")
|
||||
result = wntr.sim.EpanetSimulator(model).run_sim(
|
||||
file_prefix=str(Path(directory) / f"snapshot-{index}"), convergence_error=True)
|
||||
pressure = result.node["pressure"].iloc[-1]
|
||||
flow = result.link["flowrate"].iloc[-1]
|
||||
if not all(isfinite(float(value)) for value in pressure) or not all(isfinite(float(value)) for value in flow):
|
||||
raise ValueError("NONFINITE_HYDRAULIC_RESULT")
|
||||
snapshots.append({"time": at, "node_count": len(pressure), "link_count": len(flow),
|
||||
"minimum_pressure_m": float(pressure.min()),
|
||||
"maximum_pressure_m": float(pressure.max()),
|
||||
"applied_boundary_device_ids": applied})
|
||||
for device_id, row in frame.items():
|
||||
device = directory_devices.get(device_id, {})
|
||||
kind, node_id, link_id = device.get("device_type"), device.get("node_id"), device.get("link_id")
|
||||
if kind == "pressure" and node_id in pressure:
|
||||
observed = pressure_metres(row["cleaned_value"], device.get("measurement_unit", ""))
|
||||
predicted, unit = float(pressure[node_id]), "m"
|
||||
elif kind in {"flow", "pipe_flow"} and link_id in flow:
|
||||
observed = flow_cms(row["cleaned_value"], device.get("measurement_unit", ""))
|
||||
predicted, unit = float(flow[link_id]), "m3/s"
|
||||
else:
|
||||
if kind in {"pressure", "flow", "pipe_flow"}:
|
||||
gaps.add(f"{device_id} 缺少有效模型关联。")
|
||||
continue
|
||||
if observed is None:
|
||||
gaps.add(f"{device_id} 观测单位未适配。")
|
||||
continue
|
||||
delta = abs(predicted - observed)
|
||||
comparisons.append({"time": at, "device_id": device_id, "kind": kind,
|
||||
"element_id": node_id if kind == "pressure" else link_id,
|
||||
"observed": observed, "predicted": predicted, "unit": unit,
|
||||
"absolute_deviation": delta,
|
||||
"relative_deviation_percent": delta / abs(observed) * 100
|
||||
if abs(observed) > 1e-8 else None})
|
||||
metrics = []
|
||||
for kind, label, attention, critical in (("pressure", "模拟-实测压力偏差", 8, 12),
|
||||
("flow", "模拟-实测流量偏差", 10, 15)):
|
||||
values = [item["relative_deviation_percent"] for item in comparisons
|
||||
if (item["kind"] == "pressure") == (kind == "pressure")
|
||||
and item["relative_deviation_percent"] is not None]
|
||||
if values:
|
||||
value = mean(values)
|
||||
metrics.append({"id": kind + "-deviation", "label": label, "value": f"{value:.2f}",
|
||||
"unit": "%", "status": "critical" if value >= critical else "attention" if value >= attention else "normal",
|
||||
"detail": f"有效比较 {len(values)} 项;近零实测只报告绝对偏差。"})
|
||||
else:
|
||||
gaps.add(f"缺少可计算相对偏差的{label}数据。")
|
||||
pressure_deltas = [item["absolute_deviation"] for item in comparisons if item["kind"] == "pressure"]
|
||||
if pressure_deltas:
|
||||
maximum = max(pressure_deltas)
|
||||
metrics.append({"id": "max-pressure-delta", "label": "最大节点压差", "value": f"{maximum:.2f}",
|
||||
"unit": "m", "status": "critical" if maximum >= 8 else "attention" if maximum >= 5 else "normal",
|
||||
"detail": "有实测映射节点的最大绝对压力偏差。"})
|
||||
compared_ids = {item["device_id"] for item in comparisons}
|
||||
valid_ids = {item["device_id"] for item in comparisons
|
||||
if (end - datetime.fromisoformat(item["time"])).total_seconds()
|
||||
< _sampling_seconds(directory_devices.get(item["device_id"], {}))}
|
||||
expected = {device["device_id"] for device in devices if device.get("device_type") in {"pressure", "flow", "pipe_flow"}}
|
||||
missing = sorted(expected - valid_ids)
|
||||
if missing:
|
||||
gaps.add("缺少周期终点可用观测或模型映射的测点:" + "、".join(missing))
|
||||
gaps.add("未取得有效边界遥测的要素采用本轮冻结模型设定;本结果为逐时刻稳态水力复核。")
|
||||
anomaly = any(metric["status"] != "normal" for metric in metrics)
|
||||
complete = not missing and all(any(metric["id"] == kind + "-deviation" for metric in metrics) for kind in ("pressure", "flow"))
|
||||
deviations = sorted(
|
||||
[item for item in comparisons if (item["relative_deviation_percent"] is not None
|
||||
and item["relative_deviation_percent"] >= (8 if item["kind"] == "pressure" else 10))
|
||||
or (item["kind"] == "pressure" and item["absolute_deviation"] >= 5)],
|
||||
key=lambda item: item["relative_deviation_percent"] or 0, reverse=True,
|
||||
)
|
||||
causes = (["先核查偏差测点的数据质量与模型关联,再复核冻结模型中的泵阀状态、供水边界和需水模式。",
|
||||
"当前证据仅定位偏差测点和关联要素,尚不能单独确定边界滞后或模型参数错误。"] if deviations else [])
|
||||
return {"conclusion": "attention" if anomaly else "normal" if complete else "insufficient_data",
|
||||
"metrics": metrics, "comparisons": comparisons, "snapshots": snapshots,
|
||||
"data_gaps": sorted(gaps), "compared_device_count": len(compared_ids), "current_device_count": len(valid_ids),
|
||||
"missing_device_ids": missing, "model_scope": "all_nodes_and_links",
|
||||
"deviation_targets": deviations[:50], "cause_review": causes}
|
||||
@@ -14,6 +14,11 @@ from app.domain.schemas.scada_replay import (
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
from app.services.scada_replay import ScadaReplayService, snapshot
|
||||
from app.domain.schemas.periodic_task import PeriodicTaskPage, PeriodicTaskExecution, PeriodicTaskRetry
|
||||
from app.services.periodic_tasks import (
|
||||
ensure_periodic_tasks, initialize_periodic_tasks, project_periodic_state,
|
||||
project_periodic_tasks, retry_periodic_task, require_execution_permissions,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"])
|
||||
|
||||
@@ -40,6 +45,8 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm
|
||||
settings=Depends(project_settings)):
|
||||
available = await repo.available()
|
||||
current = await repo.current() if available else None
|
||||
if current and current.get("config", {}).get("scheduled_tasks_enabled"):
|
||||
current = await project_periodic_state(repo, current)
|
||||
monitoring = settings.scada_monitoring
|
||||
monitoring_ready = bool(
|
||||
monitoring
|
||||
@@ -68,17 +75,22 @@ async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_perm
|
||||
@router.post("", response_model=ScadaReplaySnapshot, status_code=201)
|
||||
async def create_replay(command: ScadaReplayCreate, repo=Depends(repository),
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)):
|
||||
return await ScadaReplayService(repo, ctx).create(
|
||||
if command.scheduled_tasks_enabled:
|
||||
require_execution_permissions(ctx)
|
||||
result = await ScadaReplayService(repo, ctx).create(
|
||||
command,
|
||||
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
|
||||
monitoring_config=settings.scada_monitoring,
|
||||
)
|
||||
if command.scheduled_tasks_enabled:
|
||||
await initialize_periodic_tasks(repo, ctx, result.replay_id)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/{replay_id}", response_model=ScadaReplaySnapshot)
|
||||
async def get_replay(replay_id: UUID, repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
return snapshot(await repo.get(replay_id))
|
||||
return snapshot(await project_periodic_state(repo, await repo.get(replay_id)))
|
||||
|
||||
|
||||
@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot)
|
||||
@@ -86,12 +98,33 @@ async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depe
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)),
|
||||
timescale=Depends(get_project_timescale_connection)):
|
||||
service = ScadaReplayService(repo, ctx, timescale)
|
||||
document = await repo.get(replay_id)
|
||||
if command.action != "stop" and document.get("config", {}).get("scheduled_tasks_enabled"):
|
||||
await ensure_periodic_tasks(repo, ctx, replay_id)
|
||||
result = await service.command(replay_id, command)
|
||||
if command.action in {"step", "monitor", "retry"} and result.step_pending:
|
||||
return await service.process_pending_step(replay_id)
|
||||
result = await service.process_pending_step(replay_id)
|
||||
if command.action != "stop" and getattr(result, "scheduled_tasks_enabled", False):
|
||||
await ensure_periodic_tasks(repo, ctx, replay_id)
|
||||
return snapshot(await project_periodic_state(repo, await repo.get(replay_id)))
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/{replay_id}/scheduled-tasks", response_model=PeriodicTaskPage)
|
||||
async def scheduled_tasks(replay_id: UUID, offset: int = Query(0, ge=0),
|
||||
limit: int = Query(100, ge=1, le=200), repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
await repo.get(replay_id)
|
||||
items = await project_periodic_tasks(repo, replay_id, offset, limit + 1)
|
||||
return PeriodicTaskPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None)
|
||||
|
||||
|
||||
@router.post("/{replay_id}/scheduled-tasks/{execution_id}/retry", response_model=PeriodicTaskExecution)
|
||||
async def retry_scheduled_task(replay_id: UUID, execution_id: UUID, command: PeriodicTaskRetry,
|
||||
repo=Depends(repository), ctx=Depends(require_permission(SCADA_CLEAN))):
|
||||
return await retry_periodic_task(repo, ctx, replay_id, execution_id, command.request_id)
|
||||
|
||||
|
||||
@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage)
|
||||
async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200),
|
||||
repo=Depends(repository)):
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Persisted business-clock task executions, independent of Agent sessions."""
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AwareDatetime, Field
|
||||
|
||||
from app.domain.schemas.shared import StrictModel
|
||||
|
||||
|
||||
class PeriodicTaskStep(StrictModel):
|
||||
id: str
|
||||
title: str
|
||||
status: Literal["pending", "running", "completed", "failed", "skipped"]
|
||||
detail: str = ""
|
||||
|
||||
|
||||
class PeriodicTaskMetric(StrictModel):
|
||||
id: str
|
||||
label: str
|
||||
value: str
|
||||
unit: str = ""
|
||||
status: Literal["normal", "attention", "critical"] = "normal"
|
||||
detail: str = ""
|
||||
|
||||
|
||||
class PeriodicTaskExecution(StrictModel):
|
||||
execution_id: UUID
|
||||
replay_id: UUID
|
||||
task_key: Literal["scada-diagnosis", "network-simulation"]
|
||||
title: str
|
||||
period_start: AwareDatetime
|
||||
period_end: AwareDatetime
|
||||
knowledge_cutoff: AwareDatetime
|
||||
batch_sequence: int = Field(ge=1)
|
||||
status: Literal["pending", "running", "completed", "failed"]
|
||||
conclusion: Literal["normal", "attention", "insufficient_data", "suspected_burst", "suspected_pollution"]
|
||||
summary: str
|
||||
steps: list[PeriodicTaskStep]
|
||||
metrics: list[PeriodicTaskMetric] = Field(default_factory=list)
|
||||
evidence: list[str] = Field(default_factory=list)
|
||||
data_gaps: list[str] = Field(default_factory=list)
|
||||
risk_signals: list[dict] = Field(default_factory=list)
|
||||
investigation_task_id: UUID | None = None
|
||||
calculation_job_id: UUID | None = None
|
||||
attempt: int = Field(default=1, ge=1)
|
||||
updated_at: AwareDatetime
|
||||
result: dict | None = None
|
||||
|
||||
|
||||
class PeriodicTaskPage(StrictModel):
|
||||
items: list[PeriodicTaskExecution]
|
||||
next_offset: int | None
|
||||
|
||||
|
||||
class PeriodicTaskRetry(StrictModel):
|
||||
request_id: UUID
|
||||
@@ -61,6 +61,7 @@ class ScadaReplayCreate(ReplayModel):
|
||||
device_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30)
|
||||
quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict)
|
||||
scheduled_tasks_enabled: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def configuration(self):
|
||||
@@ -130,6 +131,9 @@ class ScadaReplaySnapshot(ReplayModel):
|
||||
end_time: datetime
|
||||
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
|
||||
step_pending: bool = False
|
||||
scheduled_tasks_enabled: bool = False
|
||||
periodic_pending: bool = False
|
||||
periodic_error: str | None = None
|
||||
monitoring_count: int = 0
|
||||
last_monitoring: ScadaMonitoringResult | None = None
|
||||
monitoring_config: ScadaMonitoringSettings | None = None
|
||||
|
||||
@@ -139,6 +139,102 @@ class ScadaReplayRepository:
|
||||
)
|
||||
return [r["event"] for r in await cur.fetchall()]
|
||||
|
||||
async def queue_available(self) -> bool:
|
||||
cursor = await self.connection.execute("SELECT to_regclass('workbench.calculation_jobs') IS NOT NULL AS ready")
|
||||
return bool((await cursor.fetchone())["ready"])
|
||||
|
||||
async def periodic_events(self, replay_id, start, end, through_sequence: int) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
|
||||
"AND b.sequence>0 AND b.sequence<=%s "
|
||||
"AND (b.document->>'at')::timestamptz>=%s::timestamptz "
|
||||
"AND (b.document->>'at')::timestamptz<=%s::timestamptz ORDER BY b.sequence",
|
||||
(self.project_id, str(replay_id), through_sequence, start, end),
|
||||
)
|
||||
return [row["event"] for row in await cursor.fetchall()]
|
||||
|
||||
async def batch(self, replay_id, sequence: int) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replay_batches WHERE project_id=%s AND replay_id=%s AND sequence=%s",
|
||||
(self.project_id, str(replay_id), sequence),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def periodic_tasks(self, replay_id, offset=0, limit=100) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
|
||||
"ORDER BY b.sequence DESC,t.value->>'task_key' LIMIT %s OFFSET %s",
|
||||
(self.project_id, str(replay_id), limit, offset),
|
||||
)
|
||||
return [row["execution"] for row in await cursor.fetchall()]
|
||||
|
||||
async def periodic_task(self, replay_id, execution_id) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT document->'scheduled_tasks'->%s AS execution FROM peripheral_sample.scada_replay_batches "
|
||||
"WHERE project_id=%s AND replay_id=%s AND stage='detected' "
|
||||
"AND document->'scheduled_tasks' ? %s LIMIT 1",
|
||||
(str(execution_id), self.project_id, str(replay_id), str(execution_id)),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["execution"] if row else None
|
||||
|
||||
async def previous_simulations(self, replay_id, before, limit=2) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
|
||||
"AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' "
|
||||
"AND (t.value->>'period_end')::timestamptz<%s::timestamptz "
|
||||
"ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT %s",
|
||||
(self.project_id, str(replay_id), before, limit),
|
||||
)
|
||||
return [row["execution"] for row in await cursor.fetchall()]
|
||||
|
||||
async def reference_simulations(self, replay_id, period_ends: list[datetime]) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT t.value AS execution FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_each(COALESCE(b.document->'scheduled_tasks','{}'::jsonb)) t "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' "
|
||||
"AND t.value->>'task_key'='network-simulation' AND t.value->>'status'='completed' "
|
||||
"AND (t.value->>'period_end')::timestamptz=ANY(%s::timestamptz[]) "
|
||||
"ORDER BY (t.value->>'period_end')::timestamptz DESC LIMIT 7",
|
||||
(self.project_id, str(replay_id), period_ends),
|
||||
)
|
||||
return [row["execution"] for row in await cursor.fetchall()]
|
||||
|
||||
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id) -> None:
|
||||
await self.connection.execute(
|
||||
"UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{periodic_retry_receipts}',"
|
||||
"COALESCE(document->'periodic_retry_receipts','{}'::jsonb) || jsonb_build_object(%s::text,%s::text)) "
|
||||
"WHERE project_id=%s AND replay_id=%s AND sequence=%s",
|
||||
(str(request_id), str(execution_id), self.project_id, str(replay_id), sequence),
|
||||
)
|
||||
|
||||
async def save_periodic_task(self, replay_id, sequence: int, execution: dict) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"UPDATE peripheral_sample.scada_replay_batches SET document=jsonb_set(document,'{scheduled_tasks}',"
|
||||
"COALESCE(document->'scheduled_tasks','{}'::jsonb) || jsonb_build_object(%s::text,%s::jsonb)) "
|
||||
"WHERE project_id=%s AND replay_id=%s AND sequence=%s AND stage='detected' RETURNING sequence",
|
||||
(str(execution["execution_id"]), Jsonb(execution), self.project_id, str(replay_id), sequence),
|
||||
)
|
||||
if await cursor.fetchone() is None:
|
||||
raise ValueError("PERIOD_BATCH_UNAVAILABLE")
|
||||
|
||||
async def simulation_state(self, replay_id, execution: dict) -> None:
|
||||
state = {"execution_id": str(execution["execution_id"]), "status": execution["status"],
|
||||
"error": execution["summary"] if execution["status"] == "failed" else None}
|
||||
await self.connection.execute(
|
||||
"UPDATE peripheral_sample.scada_replays SET document=jsonb_set(document,'{last_scheduled_simulation}',%s),updated_at=now() "
|
||||
"WHERE project_id=%s AND replay_id=%s "
|
||||
"AND (document->'last_monitoring'->>'period_end')::timestamptz=%s::timestamptz",
|
||||
(Jsonb(state), self.project_id, str(replay_id), execution["period_end"]),
|
||||
)
|
||||
|
||||
async def monitoring_outputs(
|
||||
self,
|
||||
replay_id,
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Read one consistent model without modifying the online network."""
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import base64
|
||||
import gzip
|
||||
from io import BytesIO
|
||||
|
||||
from app.native.wndb.core.connection import project_transaction
|
||||
from app.native.wndb.inp.exporter import dump_inp
|
||||
|
||||
MAX_MODEL_BYTES = 64 * 1024 * 1024
|
||||
MAX_COMPRESSED_BYTES = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class ModelSnapshotBudgetExceeded(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def pack_model_input(content: str) -> str:
|
||||
raw = content.encode("utf-8")
|
||||
if len(raw) > MAX_MODEL_BYTES:
|
||||
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
|
||||
compressed = gzip.compress(raw, compresslevel=6, mtime=0)
|
||||
if len(compressed) > MAX_COMPRESSED_BYTES:
|
||||
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
|
||||
return base64.b64encode(compressed).decode("ascii")
|
||||
|
||||
|
||||
def unpack_model_input(content: str) -> str:
|
||||
if len(content) > (MAX_COMPRESSED_BYTES + 2) // 3 * 4:
|
||||
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
|
||||
compressed = base64.b64decode(content, validate=True)
|
||||
with gzip.GzipFile(fileobj=BytesIO(compressed)) as stream:
|
||||
raw = stream.read(MAX_MODEL_BYTES + 1)
|
||||
if len(raw) > MAX_MODEL_BYTES:
|
||||
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
|
||||
return raw.decode("utf-8")
|
||||
|
||||
|
||||
def capture_model_input(project: str) -> str:
|
||||
with TemporaryDirectory(prefix="tjwater-model-snapshot-") as directory:
|
||||
path = Path(directory) / "network.inp"
|
||||
with project_transaction(project) as connection:
|
||||
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
dump_inp(project, str(path), "2")
|
||||
if path.stat().st_size > MAX_MODEL_BYTES:
|
||||
raise ModelSnapshotBudgetExceeded("MODEL_SNAPSHOT_BUDGET_EXCEEDED")
|
||||
content = path.read_text(encoding="utf-8")
|
||||
if not content.strip():
|
||||
raise ValueError("MODEL_SNAPSHOT_UNAVAILABLE")
|
||||
return content
|
||||
@@ -80,7 +80,11 @@ async def execute_child(input_path: str, output_path: str):
|
||||
row = json.loads(Path(input_path).read_text())
|
||||
try:
|
||||
async with execution_scope(row) as scope:
|
||||
result = await execute_calculation_job(scope, row)
|
||||
if row.get('tool') == 'scheduled_network_simulation':
|
||||
from app.services.periodic_tasks import execute_periodic_calculation
|
||||
result = await execute_periodic_calculation(scope, row)
|
||||
else:
|
||||
result = await execute_calculation_job(scope, row)
|
||||
payload = {'status': 'succeeded', 'result': result.model_dump(mode='json')}
|
||||
except Exception as exc:
|
||||
# Do not persist exception text: DSNs, SQL, credentials or model data can occur there.
|
||||
@@ -224,7 +228,14 @@ class CalculationWorker:
|
||||
terminal = {'status': 'failed', 'error': {'code': 'CLEANUP_FAILED'}}
|
||||
cleaned = True
|
||||
async with self.repository(row['project_id']) as repository:
|
||||
finished = await repository.finish(row, self.owner, **terminal)
|
||||
if row.get('tool') == 'scheduled_network_simulation':
|
||||
from app.services.periodic_tasks import persist_periodic_terminal
|
||||
async with repository.connection.transaction():
|
||||
finished = await repository.finish(row, self.owner, **terminal)
|
||||
if finished:
|
||||
await persist_periodic_terminal(repository.connection, row, terminal)
|
||||
else:
|
||||
finished = await repository.finish(row, self.owner, **terminal)
|
||||
if not finished:
|
||||
status = await repository.heartbeat(row, self.owner, self.settings.lease_seconds)
|
||||
if status == 'cancel_requested':
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Project completed SCADA monitoring into an auditable scheduled execution."""
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
import re
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from app.domain.schemas.periodic_task import PeriodicTaskExecution
|
||||
from app.services.scada_normal_profile import pollution_signal
|
||||
|
||||
|
||||
STEP_TITLES = {
|
||||
"scada-diagnosis": ["读取 SCADA 遥测", "检查新鲜度完整率", "诊断压力流量与水质", "判断潜在事件", "整理诊断结论"],
|
||||
"network-simulation": ["冻结模型与边界", "运行水力模型", "比对实测偏差", "定位偏差区域", "输出模型复核"],
|
||||
}
|
||||
|
||||
|
||||
def execution_record(document: dict, monitoring: dict, task_key: str) -> dict:
|
||||
return PeriodicTaskExecution(
|
||||
execution_id=uuid5(NAMESPACE_URL, f"{document['replay_id']}:{task_key}:{monitoring['period_end']}"),
|
||||
replay_id=document["replay_id"], task_key=task_key,
|
||||
title="SCADA 数据诊断" if task_key == "scada-diagnosis" else "管网定时模拟",
|
||||
period_start=monitoring["period_start"], period_end=monitoring["period_end"],
|
||||
knowledge_cutoff=monitoring["knowledge_cutoff"], batch_sequence=monitoring["batch_sequence"],
|
||||
status="pending", conclusion="insufficient_data", summary="等待本周期执行。",
|
||||
steps=[{"id": f"step-{index}", "title": title, "status": "pending"}
|
||||
for index, title in enumerate(STEP_TITLES[task_key])],
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
def diagnosis_execution(document: dict, monitoring: dict, pollution_events: Sequence[dict] = ()) -> dict:
|
||||
record = execution_record(document, monitoring, "scada-diagnosis")
|
||||
end = datetime.fromisoformat(monitoring["period_end"])
|
||||
devices = {device["device_id"]: device for device in document["devices"]}
|
||||
rules = {rule["rule_id"]: rule for rule in document["config"]["rules"]}
|
||||
signals = []
|
||||
water_events = {}
|
||||
active_events = [state["event"] for state in document.get("rule_states", {}).values() if state.get("event")]
|
||||
for event in [*pollution_events, *active_events]:
|
||||
if not pollution_signal(document, event):
|
||||
continue
|
||||
if event.get("occurred_at") and datetime.fromisoformat(event["occurred_at"]) > end:
|
||||
continue
|
||||
if event.get("batch_sequence", 0) > monitoring["batch_sequence"]:
|
||||
continue
|
||||
water_events[event["event_id"]] = event
|
||||
evidence = {event["event_id"]: event for event in monitoring.get("anomaly_signals", [])}
|
||||
evidence.update(water_events)
|
||||
for signal in evidence.values():
|
||||
rule = rules.get(signal["rule_id"], {})
|
||||
water = pollution_signal(document, signal)
|
||||
signals.append({"kind": "suspected_pollution" if water else "hydraulic_anomaly",
|
||||
"device_ids": signal["device_ids"], "detail": signal["detail"],
|
||||
"metric_name": rule.get("metric_name", ""), "rule_id": signal["rule_id"],
|
||||
**({"task_id": signal["task_id"]} if water and signal.get("task_id") else {})})
|
||||
if monitoring["status"] == "suspected_burst":
|
||||
signals.append({"kind": "suspected_burst", "detail": "压力与爆管侦测等联合证据达到项目嫌疑门槛。",
|
||||
"device_ids": monitoring.get("pressure", {}).get("hit_device_ids", [])})
|
||||
missing, invalid, fresh_ids = [], [], set()
|
||||
for device_id, device in devices.items():
|
||||
row = document.get("latest", {}).get(device_id)
|
||||
frequency = str(device.get("transmission_frequency") or "")
|
||||
clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
|
||||
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
|
||||
seconds = (int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock
|
||||
else int(minutes[1]) * 60 if minutes else document["monitoring_interval_seconds"])
|
||||
age = (end - datetime.fromisoformat(row["time"])).total_seconds() if row else None
|
||||
if age is None or age < 0 or age >= 2 * max(seconds, 1):
|
||||
missing.append(device_id)
|
||||
elif row["quality"] not in {"valid", "corrected"}:
|
||||
invalid.append(device_id)
|
||||
else:
|
||||
fresh_ids.add(device_id)
|
||||
conclusion = ("suspected_burst" if monitoring["status"] == "suspected_burst" else
|
||||
"suspected_pollution" if any(signal["kind"] == "suspected_pollution" for signal in signals) else
|
||||
"attention" if signals or monitoring["status"] == "attention" or invalid else monitoring["status"])
|
||||
gaps = []
|
||||
if missing:
|
||||
gaps.append("缺少及时观测的测点:" + "、".join(missing))
|
||||
if invalid:
|
||||
gaps.append("无效读数测点:" + "、".join(invalid))
|
||||
if monitoring["status"] == "insufficient_data":
|
||||
gaps.append(monitoring["detail"])
|
||||
water_ids = {key for key, device in devices.items() if device.get("device_type") in {"quality", "water_quality"}}
|
||||
configured_water = {device for rule in rules.values() if rule.get("kind") == "bounds" and rule.get("metric_name")
|
||||
for device in rule.get("device_ids", [])}
|
||||
if water_ids - configured_water:
|
||||
gaps.append("尚未配置明确水质指标与业务阈值的测点,污染风险未评估:" + "、".join(sorted(water_ids - configured_water)))
|
||||
if gaps and conclusion == "normal":
|
||||
conclusion = "insufficient_data"
|
||||
summary = ("发现水质异常,存在污染风险,已进入原因核验。" if conclusion == "suspected_pollution"
|
||||
else monitoring["detail"])
|
||||
coverage = len(fresh_ids) / len(devices) * 100 if devices else 0
|
||||
metrics = [{"id": "usable-coverage", "label": "及时有效测点占比", "value": f"{coverage:.1f}", "unit": "%",
|
||||
"status": "attention" if missing or invalid else "normal", "detail": "按设备采样周期检查;缺测或无效读数单列。"},
|
||||
{"id": "risk-signals", "label": "运行异常信号", "value": str(len(signals)), "unit": "项",
|
||||
"status": "attention" if signals else "normal", "detail": "压力、流量与水质分别判别,异常不等于现场确诊。"}]
|
||||
for key, title, kinds in (("pressure", "压力", {"pressure"}), ("flow", "流量", {"flow", "pipe_flow"}),
|
||||
("water", "水质", {"quality", "water_quality"})):
|
||||
ids = {device for device, metadata in devices.items() if metadata.get("device_type") in kinds}
|
||||
if ids:
|
||||
valid_count = len(ids & fresh_ids)
|
||||
metrics.append({"id": key + "-freshness", "label": title + "测点及时有效率",
|
||||
"value": f"{valid_count / len(ids) * 100:.1f}", "unit": "%",
|
||||
"status": "normal" if valid_count == len(ids) else "attention",
|
||||
"detail": f"有效 {valid_count} / {len(ids)} 个测点;按设备采样周期核验。"})
|
||||
weighted = monitoring.get("weighted")
|
||||
if weighted:
|
||||
metrics.append({"id": "burst-score", "label": "爆管嫌疑综合评分", "value": f"{weighted['score']:.2f}",
|
||||
"unit": "", "status": "attention" if weighted["score"] >= weighted["threshold"] else "normal",
|
||||
"detail": f"项目嫌疑门槛 {weighted['threshold']:.2f};综合分不代表概率。"})
|
||||
record.update(status="completed", conclusion=conclusion, summary=summary, metrics=metrics,
|
||||
data_gaps=gaps, risk_signals=signals,
|
||||
investigation_task_id=monitoring.get("task_id") or next(
|
||||
(signal["task_id"] for signal in signals if signal.get("task_id")), None),
|
||||
evidence=[f"周期 {monitoring['period_start']} 至 {monitoring['period_end']}",
|
||||
f"冻结批次 {monitoring['batch_sequence']};监测回执 {monitoring['monitoring_id']}",
|
||||
*[signal["detail"] for signal in signals]],
|
||||
result={"monitoring": monitoring, "missing_device_ids": missing, "invalid_device_ids": invalid})
|
||||
for step in record["steps"]:
|
||||
step.update(status="completed")
|
||||
return PeriodicTaskExecution.model_validate(record).model_dump(mode="json")
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Business-clock executions reuse replay batches and the durable calculation queue."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID, NAMESPACE_URL, uuid5
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context
|
||||
from app.domain.schemas.periodic_task import PeriodicTaskExecution
|
||||
from app.mcp.errors import CalculationCapacityExceeded
|
||||
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
|
||||
|
||||
|
||||
PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation"
|
||||
ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"}
|
||||
|
||||
|
||||
def require_execution_permissions(context) -> None:
|
||||
if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context):
|
||||
raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限")
|
||||
|
||||
|
||||
async def initialize_periodic_tasks(repo, context, replay_id) -> None:
|
||||
document = await repo.get(replay_id)
|
||||
if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"):
|
||||
return
|
||||
require_execution_permissions(context)
|
||||
if not await repo.queue_available():
|
||||
raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构")
|
||||
from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded
|
||||
from app.services.content_digest import digest
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
try:
|
||||
model_input = await bounded_calculation(capture_model_input, context.project_code)
|
||||
packed = await bounded_calculation(pack_model_input, model_input)
|
||||
except ModelSnapshotBudgetExceeded as error:
|
||||
raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error
|
||||
async with repo.control():
|
||||
document = await repo.get(replay_id)
|
||||
if not document.get("scheduled_model_ready"):
|
||||
document["scheduled_model_digest"] = digest({"inp": model_input})
|
||||
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
|
||||
"model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]})
|
||||
document["scheduled_model_ready"] = True
|
||||
await repo.save(document)
|
||||
|
||||
|
||||
async def ensure_periodic_tasks(repo, context, replay_id) -> None:
|
||||
document = await repo.get(replay_id)
|
||||
if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"):
|
||||
return
|
||||
require_execution_permissions(context)
|
||||
await initialize_periodic_tasks(repo, context, replay_id)
|
||||
async with repo.control():
|
||||
document = await repo.get(replay_id)
|
||||
monitoring = document["last_monitoring"]
|
||||
batch = await repo.batch(replay_id, monitoring["batch_sequence"])
|
||||
if not batch:
|
||||
raise HTTPException(409, "本周期冻结数据批次尚未提交")
|
||||
diagnosis = execution_record(document, monitoring, "scada-diagnosis")
|
||||
if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}):
|
||||
events = await repo.periodic_events(
|
||||
replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"],
|
||||
)
|
||||
diagnosis = diagnosis_execution(document, monitoring, events)
|
||||
await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis)
|
||||
simulation = execution_record(document, monitoring, "network-simulation")
|
||||
if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}):
|
||||
await enqueue_simulation(repo, context, document, simulation)
|
||||
|
||||
|
||||
async def enqueue_simulation(repo, context, document, execution) -> None:
|
||||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||||
identity = f"periodic:{execution['execution_id']}:{execution['attempt']}"
|
||||
try:
|
||||
row = await queue.enqueue(
|
||||
owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)),
|
||||
tool=PERIODIC_SIMULATION_TOOL,
|
||||
parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"],
|
||||
"execution_id": execution["execution_id"], "attempt": execution["attempt"]},
|
||||
binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600,
|
||||
)
|
||||
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
|
||||
except CalculationCapacityExceeded:
|
||||
execution.update(calculation_job_id=None, status="failed", summary="计算队列已满,水力计算尚未启动,可按原输入重试。")
|
||||
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
|
||||
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
|
||||
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
|
||||
await repo.simulation_state(document["replay_id"], execution)
|
||||
|
||||
|
||||
def project_job(execution: dict, job: dict | None) -> dict:
|
||||
result = dict(execution)
|
||||
if execution["status"] in {"completed", "failed"}:
|
||||
return result
|
||||
if not job:
|
||||
result.update(status="failed", summary="本周期计算记录不可用,请重试。")
|
||||
elif job["status"] in ACTIVE_JOB_STATES:
|
||||
result.update(status="running" if job["status"] != "queued" else "pending")
|
||||
result["steps"] = [dict(step) for step in execution["steps"]]
|
||||
result["steps"][1].update(status="running" if job["status"] == "running" else "pending")
|
||||
elif job["status"] == "succeeded" and job.get("result"):
|
||||
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
|
||||
else:
|
||||
code = (job.get("error") or {}).get("code", job["status"])
|
||||
result.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
||||
return result
|
||||
|
||||
|
||||
async def project_periodic_tasks(repo, replay_id, offset, limit):
|
||||
items = await repo.periodic_tasks(replay_id, offset, limit)
|
||||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||||
result = []
|
||||
for execution in items:
|
||||
job = (await queue.get(execution["calculation_job_id"], include_result=True)
|
||||
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
|
||||
result.append(project_job(execution, job) if execution.get("calculation_job_id") else execution)
|
||||
return result
|
||||
|
||||
|
||||
async def project_periodic_state(repo, document: dict) -> dict:
|
||||
state = document.get("last_scheduled_simulation")
|
||||
if not state or state["status"] not in {"pending", "running"}:
|
||||
return document
|
||||
batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"])
|
||||
execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"])
|
||||
if execution:
|
||||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||||
execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
|
||||
return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"],
|
||||
"status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}}
|
||||
return document
|
||||
|
||||
|
||||
async def retry_periodic_task(repo, context, replay_id, execution_id, request_id):
|
||||
require_execution_permissions(context)
|
||||
async with repo.control():
|
||||
document = await repo.get(replay_id)
|
||||
# A retry is scoped to one frozen execution, including older periods.
|
||||
execution = await repo.periodic_task(replay_id, execution_id)
|
||||
if not execution or execution["task_key"] != "network-simulation":
|
||||
raise HTTPException(404, "当前回放不存在此水力计算任务")
|
||||
batch = await repo.batch(replay_id, execution["batch_sequence"])
|
||||
receipts = batch.get("periodic_retry_receipts", {})
|
||||
prior = receipts.get(str(request_id))
|
||||
if prior:
|
||||
if prior != str(execution_id):
|
||||
raise HTTPException(409, "重试请求号已用于另一条执行记录")
|
||||
return execution
|
||||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||||
current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
|
||||
if current["status"] != "failed":
|
||||
raise HTTPException(409, "仅失败的水力计算可重试")
|
||||
execution.update(attempt=execution["attempt"] + 1, status="pending", result=None,
|
||||
updated_at=datetime.now(timezone.utc).isoformat())
|
||||
for step in execution["steps"]:
|
||||
step.update(status="pending", detail="")
|
||||
await enqueue_simulation(repo, context, document, execution)
|
||||
await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id)
|
||||
return execution
|
||||
|
||||
|
||||
async def execute_periodic_calculation(scope, row):
|
||||
"""Internal worker job; admission and execution both recheck project rights."""
|
||||
from app.mcp.models import QueryResult
|
||||
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
|
||||
from app.infra.epanet.model_snapshot import unpack_model_input
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
require_execution_permissions(scope.project)
|
||||
parameters = row["parameters"]
|
||||
if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}:
|
||||
raise HTTPException(403, "周期任务所属项目不匹配")
|
||||
repo = ScadaReplayRepository(scope.connection, scope.project.project_id)
|
||||
document = await repo.get(UUID(parameters["replay_id"]))
|
||||
batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"])
|
||||
execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"])
|
||||
if (not execution or execution["calculation_job_id"] != str(row["job_id"])
|
||||
or str(document["created_by"]) != str(row["owner_user_id"])
|
||||
or execution["attempt"] != parameters["attempt"]):
|
||||
raise HTTPException(409, "周期计算执行身份已失效")
|
||||
observations = await repo.detected_outputs(
|
||||
document["replay_id"], document["config"]["device_ids"],
|
||||
datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]),
|
||||
execution["batch_sequence"], limit=50_001,
|
||||
)
|
||||
if len(observations) > 50_000:
|
||||
raise HTTPException(413, "本周期观测超过水力复核预算")
|
||||
previous = await repo.previous_simulations(document["replay_id"], execution["period_end"])
|
||||
reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)]
|
||||
references = await repo.reference_simulations(document["replay_id"], reference_times)
|
||||
model_snapshot = await repo.batch(document["replay_id"], 0)
|
||||
await scope.connection.commit()
|
||||
if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot"
|
||||
or model_snapshot["model_digest"] != document["scheduled_model_digest"]):
|
||||
raise HTTPException(409, "冻结模型缺失或摘要不一致")
|
||||
model_input = unpack_model_input(model_snapshot["model_input_gzip"])
|
||||
result = await bounded_calculation(run_periodic_hydraulics, model_input,
|
||||
document["devices"], observations,
|
||||
execution["period_start"], execution["period_end"])
|
||||
result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous]
|
||||
current_metrics = {item["id"]: item for item in result["metrics"]}
|
||||
contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"])
|
||||
prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {}
|
||||
worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics
|
||||
and float(metric["value"]) > float(prior_metrics[key]["value"])
|
||||
and metric["status"] != "normal"]
|
||||
if worsening:
|
||||
result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。")
|
||||
result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references]
|
||||
observed_references = {datetime.fromisoformat(item["period_end"]) for item in references}
|
||||
missing_references = [time.isoformat() for time in reference_times if time not in observed_references]
|
||||
if missing_references:
|
||||
result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references))
|
||||
execution.update(status="completed", conclusion=result["conclusion"], result=result,
|
||||
metrics=result["metrics"], data_gaps=result["data_gaps"],
|
||||
summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention"
|
||||
else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data"
|
||||
else "本周期水力计算与实测偏差复核已完成。"),
|
||||
updated_at=datetime.now(timezone.utc).isoformat(),
|
||||
evidence=[f"模型快照 {document['scheduled_model_digest']}",
|
||||
f"冻结观测批次不超过 {execution['batch_sequence']}",
|
||||
f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"])
|
||||
for index, step in enumerate(execution["steps"]):
|
||||
step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed")
|
||||
execution["steps"][3]["detail"] = ";".join(result.get("cause_review", []))
|
||||
return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"],
|
||||
queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution)
|
||||
|
||||
|
||||
async def persist_periodic_terminal(connection, row, terminal):
|
||||
"""Called inside the supervisor's fenced terminal transaction."""
|
||||
repo = ScadaReplayRepository(connection, row["project_id"])
|
||||
parameters = row["parameters"]
|
||||
execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"])
|
||||
if not execution or execution["calculation_job_id"] != str(row["job_id"]):
|
||||
return
|
||||
if terminal["status"] == "succeeded":
|
||||
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
|
||||
else:
|
||||
code = (terminal.get("error") or {}).get("code", terminal["status"])
|
||||
execution.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。",
|
||||
updated_at=datetime.now(timezone.utc).isoformat())
|
||||
execution["steps"][1].update(status="failed", detail=execution["summary"])
|
||||
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
|
||||
await repo.simulation_state(parameters["replay_id"], execution)
|
||||
@@ -42,6 +42,9 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||||
"virtual_now", "created_at", "updated_at",
|
||||
)},
|
||||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||||
scheduled_tasks_enabled=document.get("config", {}).get("scheduled_tasks_enabled", False),
|
||||
periodic_pending=(document.get("last_scheduled_simulation") or {}).get("status") in {"pending", "running"},
|
||||
periodic_error=(document.get("last_scheduled_simulation") or {}).get("error"),
|
||||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||||
devices=document["devices"], rules=document["config"]["rules"],
|
||||
latest=list(document.get("latest", {}).values()),
|
||||
@@ -132,6 +135,11 @@ class ScadaReplayService:
|
||||
return snapshot(document)
|
||||
if command.expected_revision != document["revision"]:
|
||||
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
|
||||
if document["config"].get("scheduled_tasks_enabled") and command.action in {"step", "monitor"}:
|
||||
from app.services.periodic_tasks import project_periodic_state
|
||||
document = await project_periodic_state(self.repo, document)
|
||||
if snapshot(document).periodic_pending:
|
||||
raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放")
|
||||
action, status = command.action, document["status"]
|
||||
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
||||
and weighted_diagnosis(document)
|
||||
@@ -332,7 +340,7 @@ class ScadaReplayService:
|
||||
monitoring["anomaly_signals"] = list(signals.values())
|
||||
if signals and monitoring["status"] == "normal":
|
||||
monitoring["status"] = "attention"
|
||||
monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断"
|
||||
monitoring["detail"] = "本周期曾检出测点异常,进入运行异常与潜在事件诊断"
|
||||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||||
if not result["step_pending"]:
|
||||
result.pop("step_target", None)
|
||||
@@ -565,7 +573,7 @@ class ScadaReplayService:
|
||||
trigger=TaskTrigger(
|
||||
kind="event",
|
||||
occurred_at=monitoring["period_end"],
|
||||
detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管",
|
||||
detail="SCADA 任一指标异常启动调查,核验爆管或污染等潜在事件;异常不等于现场确诊",
|
||||
),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
|
||||
Reference in New Issue
Block a user