Files
next-tjwater-server/app/services/periodic_tasks.py
T

247 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Business-clock executions reuse replay batches and the durable calculation queue."""
from datetime import datetime, timedelta, timezone
from uuid import UUID, NAMESPACE_URL, uuid5
from fastapi import HTTPException
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN, permissions_for_context
from app.domain.schemas.periodic_task import PeriodicTaskExecution
from app.mcp.errors import CalculationCapacityExceeded
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
PERIODIC_SIMULATION_TOOL = "scheduled_network_simulation"
ACTIVE_JOB_STATES = {"queued", "running", "cancel_requested"}
def require_execution_permissions(context) -> None:
if not {SCADA_VIEW, SCADA_CLEAN, SIMULATION_RUN} <= permissions_for_context(context):
raise HTTPException(403, "周期任务需要 SCADA 读取、清洗和水力计算权限")
async def initialize_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or document.get("scheduled_model_ready"):
return
require_execution_permissions(context)
if not await repo.queue_available():
raise HTTPException(503, "周期水力计算队列未配置,请先部署计算队列存储结构")
from app.infra.epanet.model_snapshot import capture_model_input, pack_model_input, ModelSnapshotBudgetExceeded
from app.services.content_digest import digest
from app.mcp.analysis_service import bounded_calculation
try:
model_input = await bounded_calculation(capture_model_input, context.project_code)
packed = await bounded_calculation(pack_model_input, model_input)
except ModelSnapshotBudgetExceeded as error:
raise HTTPException(413, "完整模型超过快照预算:原文最多 64 MiB,压缩后最多 8 MiB") from error
async with repo.control():
document = await repo.get(replay_id)
if not document.get("scheduled_model_ready"):
document["scheduled_model_digest"] = digest({"inp": model_input})
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
"model_input_gzip": packed, "model_digest": document["scheduled_model_digest"]})
document["scheduled_model_ready"] = True
await repo.save(document)
async def ensure_periodic_tasks(repo, context, replay_id) -> None:
document = await repo.get(replay_id)
if not document["config"].get("scheduled_tasks_enabled") or not document.get("last_monitoring"):
return
require_execution_permissions(context)
await initialize_periodic_tasks(repo, context, replay_id)
async with repo.control():
document = await repo.get(replay_id)
monitoring = document["last_monitoring"]
batch = await repo.batch(replay_id, monitoring["batch_sequence"])
if not batch:
raise HTTPException(409, "本周期冻结数据批次尚未提交")
diagnosis = execution_record(document, monitoring, "scada-diagnosis")
if str(diagnosis["execution_id"]) not in batch.get("scheduled_tasks", {}):
events = await repo.periodic_events(
replay_id, monitoring["period_start"], monitoring["period_end"], monitoring["batch_sequence"],
)
diagnosis = diagnosis_execution(document, monitoring, events)
await repo.save_periodic_task(replay_id, batch["sequence"], diagnosis)
simulation = execution_record(document, monitoring, "network-simulation")
if str(simulation["execution_id"]) not in batch.get("scheduled_tasks", {}):
await enqueue_simulation(repo, context, document, simulation)
async def enqueue_simulation(repo, context, document, execution) -> None:
queue = CalculationJobRepository(repo.connection, repo.project_id)
identity = f"periodic:{execution['execution_id']}:{execution['attempt']}"
try:
row = await queue.enqueue(
owner_user_id=document["created_by"], request_key=str(uuid5(NAMESPACE_URL, identity)),
tool=PERIODIC_SIMULATION_TOOL,
parameters={"replay_id": document["replay_id"], "batch_sequence": execution["batch_sequence"],
"execution_id": execution["execution_id"], "attempt": execution["attempt"]},
binding={"kind": "scheduled_task", "project_id": str(context.project_id)}, timeout_seconds=3600,
)
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
except CalculationCapacityExceeded:
execution.update(calculation_job_id=None, status="failed", summary="计算队列已满,水力计算尚未启动,可按原输入重试。")
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
await repo.simulation_state(document["replay_id"], execution)
def project_job(execution: dict, job: dict | None) -> dict:
result = dict(execution)
if execution["status"] in {"completed", "failed"}:
return result
if not job:
result.update(status="failed", summary="本周期计算记录不可用,请重试。")
elif job["status"] in ACTIVE_JOB_STATES:
result.update(status="running" if job["status"] != "queued" else "pending")
result["steps"] = [dict(step) for step in execution["steps"]]
result["steps"][1].update(status="running" if job["status"] == "running" else "pending")
elif job["status"] == "succeeded" and job.get("result"):
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
else:
code = (job.get("error") or {}).get("code", job["status"])
result.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。")
return result
async def project_periodic_tasks(repo, replay_id, offset, limit):
items = await repo.periodic_tasks(replay_id, offset, limit)
queue = CalculationJobRepository(repo.connection, repo.project_id)
result = []
for execution in items:
job = (await queue.get(execution["calculation_job_id"], include_result=True)
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
result.append(project_job(execution, job) if execution.get("calculation_job_id") else execution)
return result
async def project_periodic_state(repo, document: dict) -> dict:
state = document.get("last_scheduled_simulation")
if not state or state["status"] not in {"pending", "running"}:
return document
batch = await repo.batch(document["replay_id"], document["last_monitoring"]["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(state["execution_id"])
if execution:
queue = CalculationJobRepository(repo.connection, repo.project_id)
execution = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
return {**document, "last_scheduled_simulation": {"execution_id": state["execution_id"],
"status": execution["status"], "error": execution["summary"] if execution["status"] == "failed" else None}}
return document
async def retry_periodic_task(repo, context, replay_id, execution_id, request_id):
require_execution_permissions(context)
async with repo.control():
document = await repo.get(replay_id)
# A retry is scoped to one frozen execution, including older periods.
execution = await repo.periodic_task(replay_id, execution_id)
if not execution or execution["task_key"] != "network-simulation":
raise HTTPException(404, "当前回放不存在此水力计算任务")
batch = await repo.batch(replay_id, execution["batch_sequence"])
receipts = batch.get("periodic_retry_receipts", {})
prior = receipts.get(str(request_id))
if prior:
if prior != str(execution_id):
raise HTTPException(409, "重试请求号已用于另一条执行记录")
return execution
queue = CalculationJobRepository(repo.connection, repo.project_id)
current = project_job(execution, await queue.get(execution["calculation_job_id"], include_result=True))
if current["status"] != "failed":
raise HTTPException(409, "仅失败的水力计算可重试")
execution.update(attempt=execution["attempt"] + 1, status="pending", result=None,
updated_at=datetime.now(timezone.utc).isoformat())
for step in execution["steps"]:
step.update(status="pending", detail="")
await enqueue_simulation(repo, context, document, execution)
await repo.save_retry_receipt(replay_id, execution["batch_sequence"], request_id, execution_id)
return execution
async def execute_periodic_calculation(scope, row):
"""Internal worker job; admission and execution both recheck project rights."""
from app.mcp.models import QueryResult
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
from app.infra.epanet.model_snapshot import unpack_model_input
from app.mcp.analysis_service import bounded_calculation
require_execution_permissions(scope.project)
parameters = row["parameters"]
if row["binding"] != {"kind": "scheduled_task", "project_id": str(scope.project.project_id)}:
raise HTTPException(403, "周期任务所属项目不匹配")
repo = ScadaReplayRepository(scope.connection, scope.project.project_id)
document = await repo.get(UUID(parameters["replay_id"]))
batch = await repo.batch(parameters["replay_id"], parameters["batch_sequence"])
execution = (batch or {}).get("scheduled_tasks", {}).get(parameters["execution_id"])
if (not execution or execution["calculation_job_id"] != str(row["job_id"])
or str(document["created_by"]) != str(row["owner_user_id"])
or execution["attempt"] != parameters["attempt"]):
raise HTTPException(409, "周期计算执行身份已失效")
observations = await repo.detected_outputs(
document["replay_id"], document["config"]["device_ids"],
datetime.fromisoformat(execution["period_start"]), datetime.fromisoformat(execution["period_end"]),
execution["batch_sequence"], limit=50_001,
)
if len(observations) > 50_000:
raise HTTPException(413, "本周期观测超过水力复核预算")
previous = await repo.previous_simulations(document["replay_id"], execution["period_end"])
reference_times = [datetime.fromisoformat(execution["period_end"]) - timedelta(days=day) for day in range(1, 8)]
references = await repo.reference_simulations(document["replay_id"], reference_times)
model_snapshot = await repo.batch(document["replay_id"], 0)
await scope.connection.commit()
if (not model_snapshot or model_snapshot.get("kind") != "model_snapshot"
or model_snapshot["model_digest"] != document["scheduled_model_digest"]):
raise HTTPException(409, "冻结模型缺失或摘要不一致")
model_input = unpack_model_input(model_snapshot["model_input_gzip"])
result = await bounded_calculation(run_periodic_hydraulics, model_input,
document["devices"], observations,
execution["period_start"], execution["period_end"])
result["previous_periods"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in previous]
current_metrics = {item["id"]: item for item in result["metrics"]}
contiguous = previous and datetime.fromisoformat(previous[0]["period_end"]) == datetime.fromisoformat(execution["period_start"])
prior_metrics = {item["id"]: item for item in previous[0]["metrics"]} if contiguous else {}
worsening = [metric["label"] for key, metric in current_metrics.items() if key in prior_metrics
and float(metric["value"]) > float(prior_metrics[key]["value"])
and metric["status"] != "normal"]
if worsening:
result.setdefault("cause_review", []).append("较上一轮偏差扩大:" + "、".join(worsening) + ";建议核对泵站出水边界和关键阀门状态。")
result["same_phase_references"] = [{"period_end": item["period_end"], "metrics": item["metrics"]} for item in references]
observed_references = {datetime.fromisoformat(item["period_end"]) for item in references}
missing_references = [time.isoformat() for time in reference_times if time not in observed_references]
if missing_references:
result["data_gaps"].append("本轮冻结模型下缺少已完成的七日同窗复核:" + "、".join(missing_references))
execution.update(status="completed", conclusion=result["conclusion"], result=result,
metrics=result["metrics"], data_gaps=result["data_gaps"],
summary=("本周期水力复核发现偏差,需核对模型与运行边界。" if result["conclusion"] == "attention"
else "本周期数据不足,已保存模型复核报告。" if result["conclusion"] == "insufficient_data"
else "本周期水力计算与实测偏差复核已完成。"),
updated_at=datetime.now(timezone.utc).isoformat(),
evidence=[f"模型快照 {document['scheduled_model_digest']}",
f"冻结观测批次不超过 {execution['batch_sequence']}",
f"比较窗口 {execution['period_start']} 至 {execution['period_end']}"])
for index, step in enumerate(execution["steps"]):
step.update(status="skipped" if not result.get("snapshots") and index in {1, 2, 3} else "completed")
execution["steps"][3]["detail"] = ";".join(result.get("cause_review", []))
return QueryResult(source=PERIODIC_SIMULATION_TOOL, source_ref="periodic:" + execution["execution_id"],
queried_at=datetime.now(timezone.utc), semantics="analysis", data=execution)
async def persist_periodic_terminal(connection, row, terminal):
"""Called inside the supervisor's fenced terminal transaction."""
repo = ScadaReplayRepository(connection, row["project_id"])
parameters = row["parameters"]
execution = await repo.periodic_task(parameters["replay_id"], parameters["execution_id"])
if not execution or execution["calculation_job_id"] != str(row["job_id"]):
return
if terminal["status"] == "succeeded":
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
else:
code = (terminal.get("error") or {}).get("code", terminal["status"])
execution.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。",
updated_at=datetime.now(timezone.utc).isoformat())
execution["steps"][1].update(status="failed", detail=execution["summary"])
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
await repo.simulation_state(parameters["replay_id"], execution)