feat(replay): execute periodic diagnosis and hydraulic checks

This commit is contained in:
2026-11-16 15:13:22 +08:00
parent 4c441fcbf2
commit 389892cb3f
22 changed files with 1934 additions and 12 deletions
+13 -2
View File
@@ -80,7 +80,11 @@ async def execute_child(input_path: str, output_path: str):
row = json.loads(Path(input_path).read_text())
try:
async with execution_scope(row) as scope:
result = await execute_calculation_job(scope, row)
if row.get('tool') == 'scheduled_network_simulation':
from app.services.periodic_tasks import execute_periodic_calculation
result = await execute_periodic_calculation(scope, row)
else:
result = await execute_calculation_job(scope, row)
payload = {'status': 'succeeded', 'result': result.model_dump(mode='json')}
except Exception as exc:
# Do not persist exception text: DSNs, SQL, credentials or model data can occur there.
@@ -224,7 +228,14 @@ class CalculationWorker:
terminal = {'status': 'failed', 'error': {'code': 'CLEANUP_FAILED'}}
cleaned = True
async with self.repository(row['project_id']) as repository:
finished = await repository.finish(row, self.owner, **terminal)
if row.get('tool') == 'scheduled_network_simulation':
from app.services.periodic_tasks import persist_periodic_terminal
async with repository.connection.transaction():
finished = await repository.finish(row, self.owner, **terminal)
if finished:
await persist_periodic_terminal(repository.connection, row, terminal)
else:
finished = await repository.finish(row, self.owner, **terminal)
if not finished:
status = await repository.heartbeat(row, self.owner, self.settings.lease_seconds)
if status == 'cancel_requested':
+122
View File
@@ -0,0 +1,122 @@
"""Project completed SCADA monitoring into an auditable scheduled execution."""
from collections.abc import Sequence
from datetime import datetime, timezone
import re
from uuid import NAMESPACE_URL, uuid5
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.services.scada_normal_profile import pollution_signal
STEP_TITLES = {
"scada-diagnosis": ["读取 SCADA 遥测", "检查新鲜度完整率", "诊断压力流量与水质", "判断潜在事件", "整理诊断结论"],
"network-simulation": ["冻结模型与边界", "运行水力模型", "比对实测偏差", "定位偏差区域", "输出模型复核"],
}
def execution_record(document: dict, monitoring: dict, task_key: str) -> dict:
return PeriodicTaskExecution(
execution_id=uuid5(NAMESPACE_URL, f"{document['replay_id']}:{task_key}:{monitoring['period_end']}"),
replay_id=document["replay_id"], task_key=task_key,
title="SCADA 数据诊断" if task_key == "scada-diagnosis" else "管网定时模拟",
period_start=monitoring["period_start"], period_end=monitoring["period_end"],
knowledge_cutoff=monitoring["knowledge_cutoff"], batch_sequence=monitoring["batch_sequence"],
status="pending", conclusion="insufficient_data", summary="等待本周期执行。",
steps=[{"id": f"step-{index}", "title": title, "status": "pending"}
for index, title in enumerate(STEP_TITLES[task_key])],
updated_at=datetime.now(timezone.utc),
).model_dump(mode="json")
def diagnosis_execution(document: dict, monitoring: dict, pollution_events: Sequence[dict] = ()) -> dict:
record = execution_record(document, monitoring, "scada-diagnosis")
end = datetime.fromisoformat(monitoring["period_end"])
devices = {device["device_id"]: device for device in document["devices"]}
rules = {rule["rule_id"]: rule for rule in document["config"]["rules"]}
signals = []
water_events = {}
active_events = [state["event"] for state in document.get("rule_states", {}).values() if state.get("event")]
for event in [*pollution_events, *active_events]:
if not pollution_signal(document, event):
continue
if event.get("occurred_at") and datetime.fromisoformat(event["occurred_at"]) > end:
continue
if event.get("batch_sequence", 0) > monitoring["batch_sequence"]:
continue
water_events[event["event_id"]] = event
evidence = {event["event_id"]: event for event in monitoring.get("anomaly_signals", [])}
evidence.update(water_events)
for signal in evidence.values():
rule = rules.get(signal["rule_id"], {})
water = pollution_signal(document, signal)
signals.append({"kind": "suspected_pollution" if water else "hydraulic_anomaly",
"device_ids": signal["device_ids"], "detail": signal["detail"],
"metric_name": rule.get("metric_name", ""), "rule_id": signal["rule_id"],
**({"task_id": signal["task_id"]} if water and signal.get("task_id") else {})})
if monitoring["status"] == "suspected_burst":
signals.append({"kind": "suspected_burst", "detail": "压力与爆管侦测等联合证据达到项目嫌疑门槛。",
"device_ids": monitoring.get("pressure", {}).get("hit_device_ids", [])})
missing, invalid, fresh_ids = [], [], set()
for device_id, device in devices.items():
row = document.get("latest", {}).get(device_id)
frequency = str(device.get("transmission_frequency") or "")
clock = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
seconds = (int(clock[1]) * 3600 + int(clock[2]) * 60 + int(clock[3]) if clock
else int(minutes[1]) * 60 if minutes else document["monitoring_interval_seconds"])
age = (end - datetime.fromisoformat(row["time"])).total_seconds() if row else None
if age is None or age < 0 or age >= 2 * max(seconds, 1):
missing.append(device_id)
elif row["quality"] not in {"valid", "corrected"}:
invalid.append(device_id)
else:
fresh_ids.add(device_id)
conclusion = ("suspected_burst" if monitoring["status"] == "suspected_burst" else
"suspected_pollution" if any(signal["kind"] == "suspected_pollution" for signal in signals) else
"attention" if signals or monitoring["status"] == "attention" or invalid else monitoring["status"])
gaps = []
if missing:
gaps.append("缺少及时观测的测点:" + "、".join(missing))
if invalid:
gaps.append("无效读数测点:" + "、".join(invalid))
if monitoring["status"] == "insufficient_data":
gaps.append(monitoring["detail"])
water_ids = {key for key, device in devices.items() if device.get("device_type") in {"quality", "water_quality"}}
configured_water = {device for rule in rules.values() if rule.get("kind") == "bounds" and rule.get("metric_name")
for device in rule.get("device_ids", [])}
if water_ids - configured_water:
gaps.append("尚未配置明确水质指标与业务阈值的测点,污染风险未评估:" + "、".join(sorted(water_ids - configured_water)))
if gaps and conclusion == "normal":
conclusion = "insufficient_data"
summary = ("发现水质异常,存在污染风险,已进入原因核验。" if conclusion == "suspected_pollution"
else monitoring["detail"])
coverage = len(fresh_ids) / len(devices) * 100 if devices else 0
metrics = [{"id": "usable-coverage", "label": "及时有效测点占比", "value": f"{coverage:.1f}", "unit": "%",
"status": "attention" if missing or invalid else "normal", "detail": "按设备采样周期检查;缺测或无效读数单列。"},
{"id": "risk-signals", "label": "运行异常信号", "value": str(len(signals)), "unit": "项",
"status": "attention" if signals else "normal", "detail": "压力、流量与水质分别判别,异常不等于现场确诊。"}]
for key, title, kinds in (("pressure", "压力", {"pressure"}), ("flow", "流量", {"flow", "pipe_flow"}),
("water", "水质", {"quality", "water_quality"})):
ids = {device for device, metadata in devices.items() if metadata.get("device_type") in kinds}
if ids:
valid_count = len(ids & fresh_ids)
metrics.append({"id": key + "-freshness", "label": title + "测点及时有效率",
"value": f"{valid_count / len(ids) * 100:.1f}", "unit": "%",
"status": "normal" if valid_count == len(ids) else "attention",
"detail": f"有效 {valid_count} / {len(ids)} 个测点;按设备采样周期核验。"})
weighted = monitoring.get("weighted")
if weighted:
metrics.append({"id": "burst-score", "label": "爆管嫌疑综合评分", "value": f"{weighted['score']:.2f}",
"unit": "", "status": "attention" if weighted["score"] >= weighted["threshold"] else "normal",
"detail": f"项目嫌疑门槛 {weighted['threshold']:.2f};综合分不代表概率。"})
record.update(status="completed", conclusion=conclusion, summary=summary, metrics=metrics,
data_gaps=gaps, risk_signals=signals,
investigation_task_id=monitoring.get("task_id") or next(
(signal["task_id"] for signal in signals if signal.get("task_id")), None),
evidence=[f"周期 {monitoring['period_start']} 至 {monitoring['period_end']}",
f"冻结批次 {monitoring['batch_sequence']};监测回执 {monitoring['monitoring_id']}",
*[signal["detail"] for signal in signals]],
result={"monitoring": monitoring, "missing_device_ids": missing, "invalid_device_ids": invalid})
for step in record["steps"]:
step.update(status="completed")
return PeriodicTaskExecution.model_validate(record).model_dump(mode="json")
+246
View File
@@ -0,0 +1,246 @@
"""Business-clock executions reuse replay batches and the durable calculation queue."""
from datetime import datetime, timedelta, timezone
from uuid import UUID, NAMESPACE_URL, uuid5
from fastapi import HTTPException
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.mcp.errors import CalculationCapacityExceeded
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation"
ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"}
def require_execution_permissions(context) -> None:
if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context):
raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限")
async def initialize_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"):
return
require_execution_permissions(context)
if not await repo.queue_available():
raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构")
from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded
from app.services.content_digest import digest
from app.mcp.analysis_service import bounded_calculation
try:
model_input = await bounded_calculation(capture_model_input, context.project_code)
packed = await bounded_calculation(pack_model_input, model_input)
except ModelSnapshotBudgetExceeded as error:
raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error
async with repo.control():
document = await repo.get(replay_id)
if not document.get("scheduled_model_ready"):
document["scheduled_model_digest"] = digest({"inp": model_input})
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
"model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]})
document["scheduled_model_ready"] = True
await repo.save(document)
async def ensure_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"):
return
require_execution_permissions(context)
await initialize_periodic_tasks(repo, context, replay_id)
async with repo.control():
document = await repo.get(replay_id)
monitoring = document["last_monitoring"]
batch = await repo.batch(replay_id, monitoring["batch_sequence"])
if not batch:
raise HTTPException(409, "本周期冻结数据批次尚未提交")
diagnosis = execution_record(document, monitoring, "scada-diagnosis")
if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}):
events = await repo.periodic_events(
replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"],
)
diagnosis = diagnosis_execution(document, monitoring, events)
await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis)
simulation = execution_record(document, monitoring, "network-simulation")
if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}):
await enqueue_simulation(repo, context, document, simulation)
async def enqueue_simulation(repo, context, document, execution) -> None:
queue = CalculationJobRepository(repo.connection, repo.project_id)
identity = f"periodic:{execution['execution_id']}:{execution['attempt']}"
try:
row = await queue.enqueue(
owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)),
tool=PERIODIC_SIMULATION_TOOL,
parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"],
"execution_id": execution["execution_id"], "attempt": execution["attempt"]},
binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600,
)
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
except CalculationCapacityExceeded:
execution.update(calculation_job_id=None, status="failed", summary="计算队列已满,水力计算尚未启动,可按原输入重试。")
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
await repo.simulation_state(document["replay_id"], execution)
def project_job(execution: dict, job: dict | None) -> dict:
result = dict(execution)
if execution["status"] in {"completed", "failed"}:
return result
if not job:
result.update(status="failed", summary="本周期计算记录不可用,请重试。")
elif job["status"] in ACTIVE_JOB_STATES:
result.update(status="running" if job["status"] != "queued" else "pending")
result["steps"] = [dict(step) for step in execution["steps"]]
result["steps"][1].update(status="running" if job["status"] == "running" else "pending")
elif job["status"] == "succeeded" and job.get("result"):
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
else:
code = (job.get("error") or {}).get("code", job["status"])
result.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。")
return result
async def project_periodic_tasks(repo, replay_id, offset, limit):
items = await repo.periodic_tasks(replay_id, offset, limit)
queue = CalculationJobRepository(repo.connection, repo.project_id)
result = []
for execution in items:
job = (await queue.get(execution["calculation_job_id"], include_result=True)
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
result.append(project_job(execution, job) if execution.get("calculation_job_id") else execution)
return result
async def project_periodic_state(repo, document: dict) -> dict:
state = document.get("last_scheduled_simulation")
if not state or state["status"] not in {"pending", "running"}:
return document
batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"])
if execution:
queue = CalculationJobRepository(repo.connection, repo.project_id)
execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"],
"status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}}
return document
async def retry_periodic_task(repo, context, replay_id, execution_id, request_id):
require_execution_permissions(context)
async with repo.control():
document = await repo.get(replay_id)
# A retry is scoped to one frozen execution, including older periods.
execution = await repo.periodic_task(replay_id, execution_id)
if not execution or execution["task_key"] != "network-simulation":
raise HTTPException(404, "当前回放不存在此水力计算任务")
batch = await repo.batch(replay_id, execution["batch_sequence"])
receipts = batch.get("periodic_retry_receipts", {})
prior = receipts.get(str(request_id))
if prior:
if prior != str(execution_id):
raise HTTPException(409, "重试请求号已用于另一条执行记录")
return execution
queue = CalculationJobRepository(repo.connection, repo.project_id)
current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
if current["status"] != "failed":
raise HTTPException(409, "仅失败的水力计算可重试")
execution.update(attempt=execution["attempt"] + 1, status="pending", result=None,
updated_at=datetime.now(timezone.utc).isoformat())
for step in execution["steps"]:
step.update(status="pending", detail="")
await enqueue_simulation(repo, context, document, execution)
await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id)
return execution
async def execute_periodic_calculation(scope, row):
"""Internal worker job; admission and execution both recheck project rights."""
from app.mcp.models import QueryResult
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
from app.infra.epanet.model_snapshot import unpack_model_input
from app.mcp.analysis_service import bounded_calculation
require_execution_permissions(scope.project)
parameters = row["parameters"]
if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}:
raise HTTPException(403, "周期任务所属项目不匹配")
repo = ScadaReplayRepository(scope.connection, scope.project.project_id)
document = await repo.get(UUID(parameters["replay_id"]))
batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"])
if (not execution or execution["calculation_job_id"] != str(row["job_id"])
or str(document["created_by"]) != str(row["owner_user_id"])
or execution["attempt"] != parameters["attempt"]):
raise HTTPException(409, "周期计算执行身份已失效")
observations = await repo.detected_outputs(
document["replay_id"], document["config"]["device_ids"],
datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]),
execution["batch_sequence"], limit=50_001,
)
if len(observations) > 50_000:
raise HTTPException(413, "本周期观测超过水力复核预算")
previous = await repo.previous_simulations(document["replay_id"], execution["period_end"])
reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)]
references = await repo.reference_simulations(document["replay_id"], reference_times)
model_snapshot = await repo.batch(document["replay_id"], 0)
await scope.connection.commit()
if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot"
or model_snapshot["model_digest"] != document["scheduled_model_digest"]):
raise HTTPException(409, "冻结模型缺失或摘要不一致")
model_input = unpack_model_input(model_snapshot["model_input_gzip"])
result = await bounded_calculation(run_periodic_hydraulics, model_input,
document["devices"], observations,
execution["period_start"], execution["period_end"])
result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous]
current_metrics = {item["id"]: item for item in result["metrics"]}
contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"])
prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {}
worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics
and float(metric["value"]) > float(prior_metrics[key]["value"])
and metric["status"] != "normal"]
if worsening:
result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。")
result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references]
observed_references = {datetime.fromisoformat(item["period_end"]) for item in references}
missing_references = [time.isoformat() for time in reference_times if time not in observed_references]
if missing_references:
result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references))
execution.update(status="completed", conclusion=result["conclusion"], result=result,
metrics=result["metrics"], data_gaps=result["data_gaps"],
summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention"
else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data"
else "本周期水力计算与实测偏差复核已完成。"),
updated_at=datetime.now(timezone.utc).isoformat(),
evidence=[f"模型快照 {document['scheduled_model_digest']}",
f"冻结观测批次不超过 {execution['batch_sequence']}",
f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"])
for index, step in enumerate(execution["steps"]):
step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed")
execution["steps"][3]["detail"] = ";".join(result.get("cause_review", []))
return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"],
queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution)
async def persist_periodic_terminal(connection, row, terminal):
"""Called inside the supervisor's fenced terminal transaction."""
repo = ScadaReplayRepository(connection, row["project_id"])
parameters = row["parameters"]
execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"])
if not execution or execution["calculation_job_id"] != str(row["job_id"]):
return
if terminal["status"] == "succeeded":
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
else:
code = (terminal.get("error") or {}).get("code", terminal["status"])
execution.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。",
updated_at=datetime.now(timezone.utc).isoformat())
execution["steps"][1].update(status="failed", detail=execution["summary"])
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
await repo.simulation_state(parameters["replay_id"], execution)
+10 -2
View File
@@ -42,6 +42,9 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
"virtual_now", "created_at", "updated_at",
)},
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
scheduled_tasks_enabled=document.get("config", {}).get("scheduled_tasks_enabled", False),
periodic_pending=(document.get("last_scheduled_simulation") or {}).get("status") in {"pending", "running"},
periodic_error=(document.get("last_scheduled_simulation") or {}).get("error"),
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
devices=document["devices"], rules=document["config"]["rules"],
latest=list(document.get("latest", {}).values()),
@@ -132,6 +135,11 @@ class ScadaReplayService:
return snapshot(document)
if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
if document["config"].get("scheduled_tasks_enabled") and command.action in {"step", "monitor"}:
from app.services.periodic_tasks import project_periodic_state
document = await project_periodic_state(self.repo, document)
if snapshot(document).periodic_pending:
raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放")
action, status = command.action, document["status"]
if (action in {"step", "monitor", "retry"} and document.get("warmed")
and weighted_diagnosis(document)
@@ -332,7 +340,7 @@ class ScadaReplayService:
monitoring["anomaly_signals"] = list(signals.values())
if signals and monitoring["status"] == "normal":
monitoring["status"] = "attention"
monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断"
monitoring["detail"] = "本周期曾检出测点异常,进入运行异常与潜在事件诊断"
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]:
result.pop("step_target", None)
@@ -565,7 +573,7 @@ class ScadaReplayService:
trigger=TaskTrigger(
kind="event",
occurred_at=monitoring["period_end"],
detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管",
detail="SCADA 任一指标异常启动调查,核验爆管或污染等潜在事件;异常不等于现场确诊",
),
)
task = await tasks.create(command)