257 lines
16 KiB
Python
257 lines
16 KiB
Python
"""Business-clock executions reuse replay batches and the durable calculation queue."""
|
||
from datetime import datetime, timedelta, timezone
|
||
from uuid import UUID, NAMESPACE_URL, uuid5
|
||
|
||
from fastapi import HTTPException
|
||
|
||
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context
|
||
from app.domain.schemas.periodic_task import PeriodicTaskExecution
|
||
from app.mcp.errors import CalculationCapacityExceeded
|
||
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
|
||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
|
||
|
||
|
||
PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation"
|
||
ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"}
|
||
|
||
|
||
def require_execution_permissions(context) -> None:
|
||
if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context):
|
||
raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限")
|
||
|
||
|
||
async def initialize_periodic_tasks(repo, context, replay_id) -> None:
|
||
document = await repo.get(replay_id)
|
||
if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"):
|
||
return
|
||
require_execution_permissions(context)
|
||
if not await repo.queue_available():
|
||
raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构")
|
||
from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded
|
||
from app.services.content_digest import digest
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
try:
|
||
model_input = await bounded_calculation(capture_model_input, context.project_code)
|
||
packed = await bounded_calculation(pack_model_input, model_input)
|
||
except ModelSnapshotBudgetExceeded as error:
|
||
raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error
|
||
async with repo.control():
|
||
document = await repo.get(replay_id)
|
||
if not document.get("scheduled_model_ready"):
|
||
document["scheduled_model_digest"] = digest({"inp": model_input})
|
||
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
|
||
"model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]})
|
||
document["scheduled_model_ready"] = True
|
||
await repo.save(document)
|
||
|
||
|
||
async def ensure_periodic_tasks(repo, context, replay_id) -> None:
|
||
document = await repo.get(replay_id)
|
||
if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"):
|
||
return
|
||
require_execution_permissions(context)
|
||
await initialize_periodic_tasks(repo, context, replay_id)
|
||
async with repo.control():
|
||
document = await repo.get(replay_id)
|
||
monitoring = document["last_monitoring"]
|
||
batch = await repo.batch(replay_id, monitoring["batch_sequence"])
|
||
if not batch:
|
||
raise HTTPException(409, "本周期冻结数据批次尚未提交")
|
||
diagnosis = execution_record(document, monitoring, "scada-diagnosis")
|
||
if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}):
|
||
events = await repo.periodic_events(
|
||
replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"],
|
||
)
|
||
diagnosis = diagnosis_execution(document, monitoring, events)
|
||
await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis)
|
||
simulation = execution_record(document, monitoring, "network-simulation")
|
||
if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}):
|
||
await enqueue_simulation(repo, context, document, simulation)
|
||
|
||
|
||
async def enqueue_simulation(repo, context, document, execution) -> None:
|
||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||
identity = f"periodic:{execution['execution_id']}:{execution['attempt']}"
|
||
try:
|
||
row = await queue.enqueue(
|
||
owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)),
|
||
tool=PERIODIC_SIMULATION_TOOL,
|
||
parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"],
|
||
"execution_id": execution["execution_id"], "attempt": execution["attempt"]},
|
||
binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600,
|
||
)
|
||
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
|
||
except CalculationCapacityExceeded:
|
||
execution.update(failed_execution(execution, "计算队列已满,水力计算尚未启动,可按原输入重试。"), calculation_job_id=None)
|
||
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
|
||
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
|
||
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
|
||
await repo.simulation_state(document["replay_id"], execution)
|
||
|
||
|
||
def failed_execution(execution: dict, summary: str) -> dict:
|
||
result = {**execution, "status": "failed", "summary": summary,
|
||
"steps": [dict(step) for step in execution["steps"]]}
|
||
result["steps"][1].update(status="failed", detail=summary)
|
||
for step in result["steps"][2:]:
|
||
step.update(status="skipped", detail="本周期计算未完成,此步骤尚未执行。")
|
||
return result
|
||
|
||
|
||
def project_job(execution: dict, job: dict | None) -> dict:
|
||
result = dict(execution)
|
||
if execution["status"] == "completed":
|
||
return result
|
||
if execution["status"] == "failed":
|
||
return failed_execution(execution, execution["summary"])
|
||
if not job:
|
||
return failed_execution(execution, "本周期计算记录不可用,请重试。")
|
||
elif job["status"] in ACTIVE_JOB_STATES:
|
||
result.update(status="running" if job["status"] != "queued" else "pending")
|
||
result["steps"] = [dict(step) for step in execution["steps"]]
|
||
result["steps"][1].update(status="running" if job["status"] == "running" else "pending")
|
||
elif job["status"] == "succeeded" and job.get("result"):
|
||
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
|
||
else:
|
||
code = (job.get("error") or {}).get("code", job["status"])
|
||
return failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
||
return result
|
||
|
||
|
||
async def project_periodic_tasks(repo, replay_id, offset, limit):
|
||
items = await repo.periodic_tasks(replay_id, offset, limit)
|
||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||
result = []
|
||
for execution in items:
|
||
job = (await queue.get(execution["calculation_job_id"], include_result=True)
|
||
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
|
||
result.append(project_job(execution, job))
|
||
return result
|
||
|
||
|
||
async def project_periodic_state(repo, document: dict) -> dict:
|
||
state = document.get("last_scheduled_simulation")
|
||
if not state or state["status"] not in {"pending", "running"}:
|
||
return document
|
||
batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"])
|
||
execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"])
|
||
if execution:
|
||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||
execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
|
||
return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"],
|
||
"status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}}
|
||
return document
|
||
|
||
|
||
async def retry_periodic_task(repo, context, replay_id, execution_id, request_id):
|
||
require_execution_permissions(context)
|
||
async with repo.control():
|
||
document = await repo.get(replay_id)
|
||
# A retry is scoped to one frozen execution, including older periods.
|
||
execution = await repo.periodic_task(replay_id, execution_id)
|
||
if not execution or execution["task_key"] != "network-simulation":
|
||
raise HTTPException(404, "当前回放不存在此水力计算任务")
|
||
batch = await repo.batch(replay_id, execution["batch_sequence"])
|
||
receipts = batch.get("periodic_retry_receipts", {})
|
||
prior = receipts.get(str(request_id))
|
||
if prior:
|
||
if prior != str(execution_id):
|
||
raise HTTPException(409, "重试请求号已用于另一条执行记录")
|
||
return execution
|
||
queue = CalculationJobRepository(repo.connection, repo.project_id)
|
||
current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
|
||
if current["status"] != "failed":
|
||
raise HTTPException(409, "仅失败的水力计算可重试")
|
||
execution.update(attempt=execution["attempt"] + 1, status="pending", result=None,
|
||
updated_at=datetime.now(timezone.utc).isoformat())
|
||
for step in execution["steps"]:
|
||
step.update(status="pending", detail="")
|
||
await enqueue_simulation(repo, context, document, execution)
|
||
await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id)
|
||
return execution
|
||
|
||
|
||
async def execute_periodic_calculation(scope, row):
|
||
"""Internal worker job; admission and execution both recheck project rights."""
|
||
from app.mcp.models import QueryResult
|
||
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
|
||
from app.infra.epanet.model_snapshot import unpack_model_input
|
||
from app.mcp.analysis_service import bounded_calculation
|
||
require_execution_permissions(scope.project)
|
||
parameters = row["parameters"]
|
||
if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}:
|
||
raise HTTPException(403, "周期任务所属项目不匹配")
|
||
repo = ScadaReplayRepository(scope.connection, scope.project.project_id)
|
||
document = await repo.get(UUID(parameters["replay_id"]))
|
||
batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"])
|
||
execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"])
|
||
if (not execution or execution["calculation_job_id"] != str(row["job_id"])
|
||
or str(document["created_by"]) != str(row["owner_user_id"])
|
||
or execution["attempt"] != parameters["attempt"]):
|
||
raise HTTPException(409, "周期计算执行身份已失效")
|
||
observations = await repo.detected_outputs(
|
||
document["replay_id"], document["config"]["device_ids"],
|
||
datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]),
|
||
execution["batch_sequence"], limit=50_001,
|
||
)
|
||
if len(observations) > 50_000:
|
||
raise HTTPException(413, "本周期观测超过水力复核预算")
|
||
previous = await repo.previous_simulations(document["replay_id"], execution["period_end"])
|
||
reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)]
|
||
references = await repo.reference_simulations(document["replay_id"], reference_times)
|
||
model_snapshot = await repo.batch(document["replay_id"], 0)
|
||
await scope.connection.commit()
|
||
if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot"
|
||
or model_snapshot["model_digest"] != document["scheduled_model_digest"]):
|
||
raise HTTPException(409, "冻结模型缺失或摘要不一致")
|
||
model_input = unpack_model_input(model_snapshot["model_input_gzip"])
|
||
result = await bounded_calculation(run_periodic_hydraulics, model_input,
|
||
document["devices"], observations,
|
||
execution["period_start"], execution["period_end"])
|
||
result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous]
|
||
current_metrics = {item["id"]: item for item in result["metrics"]}
|
||
contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"])
|
||
prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {}
|
||
worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics
|
||
and float(metric["value"]) > float(prior_metrics[key]["value"])
|
||
and metric["status"] != "normal"]
|
||
if worsening:
|
||
result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。")
|
||
result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references]
|
||
observed_references = {datetime.fromisoformat(item["period_end"]) for item in references}
|
||
missing_references = [time.isoformat() for time in reference_times if time not in observed_references]
|
||
if missing_references:
|
||
result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references))
|
||
execution.update(status="completed", conclusion=result["conclusion"], result=result,
|
||
metrics=result["metrics"], data_gaps=result["data_gaps"],
|
||
summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention"
|
||
else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data"
|
||
else "本周期水力计算与实测偏差复核已完成。"),
|
||
updated_at=datetime.now(timezone.utc).isoformat(),
|
||
evidence=[f"模型快照 {document['scheduled_model_digest']}",
|
||
f"冻结观测批次不超过 {execution['batch_sequence']}",
|
||
f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"])
|
||
for index, step in enumerate(execution["steps"]):
|
||
step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed")
|
||
execution["steps"][3]["detail"] = ";".join(result.get("cause_review", []))
|
||
return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"],
|
||
queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution)
|
||
|
||
|
||
async def persist_periodic_terminal(connection, row, terminal):
|
||
"""Called inside the supervisor's fenced terminal transaction."""
|
||
repo = ScadaReplayRepository(connection, row["project_id"])
|
||
parameters = row["parameters"]
|
||
execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"])
|
||
if not execution or execution["calculation_job_id"] != str(row["job_id"]):
|
||
return
|
||
if terminal["status"] == "succeeded":
|
||
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
|
||
else:
|
||
code = (terminal.get("error") or {}).get("code", terminal["status"])
|
||
execution = failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
||
execution["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
|
||
await repo.simulation_state(parameters["replay_id"], execution)
|