fix(replay): preserve failed periodic execution boundaries
This commit is contained in:
@@ -83,19 +83,30 @@ async def enqueue_simulation(repo, context, document, execution) -> None:
|
|||||||
)
|
)
|
||||||
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
|
execution.update(calculation_job_id=str(row["job_id"]), status="pending", summary="等待本周期水力计算。")
|
||||||
except CalculationCapacityExceeded:
|
except CalculationCapacityExceeded:
|
||||||
execution.update(calculation_job_id=None, status="failed", summary="计算队列已满,水力计算尚未启动,可按原输入重试。")
|
execution.update(failed_execution(execution, "计算队列已满,水力计算尚未启动,可按原输入重试。"), calculation_job_id=None)
|
||||||
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
|
execution["steps"][1].update(status="failed", detail="计算队列容量不足。")
|
||||||
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
|
execution["steps"][0].update(status="completed", detail="已冻结本轮模型和周期观测。")
|
||||||
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
|
await repo.save_periodic_task(document["replay_id"], execution["batch_sequence"], execution)
|
||||||
await repo.simulation_state(document["replay_id"], execution)
|
await repo.simulation_state(document["replay_id"], execution)
|
||||||
|
|
||||||
|
|
||||||
|
def failed_execution(execution: dict, summary: str) -> dict:
|
||||||
|
result = {**execution, "status": "failed", "summary": summary,
|
||||||
|
"steps": [dict(step) for step in execution["steps"]]}
|
||||||
|
result["steps"][1].update(status="failed", detail=summary)
|
||||||
|
for step in result["steps"][2:]:
|
||||||
|
step.update(status="skipped", detail="本周期计算未完成,此步骤尚未执行。")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def project_job(execution: dict, job: dict | None) -> dict:
|
def project_job(execution: dict, job: dict | None) -> dict:
|
||||||
result = dict(execution)
|
result = dict(execution)
|
||||||
if execution["status"] in {"completed", "failed"}:
|
if execution["status"] == "completed":
|
||||||
return result
|
return result
|
||||||
|
if execution["status"] == "failed":
|
||||||
|
return failed_execution(execution, execution["summary"])
|
||||||
if not job:
|
if not job:
|
||||||
result.update(status="failed", summary="本周期计算记录不可用,请重试。")
|
return failed_execution(execution, "本周期计算记录不可用,请重试。")
|
||||||
elif job["status"] in ACTIVE_JOB_STATES:
|
elif job["status"] in ACTIVE_JOB_STATES:
|
||||||
result.update(status="running" if job["status"] != "queued" else "pending")
|
result.update(status="running" if job["status"] != "queued" else "pending")
|
||||||
result["steps"] = [dict(step) for step in execution["steps"]]
|
result["steps"] = [dict(step) for step in execution["steps"]]
|
||||||
@@ -104,7 +115,7 @@ def project_job(execution: dict, job: dict | None) -> dict:
|
|||||||
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
|
return PeriodicTaskExecution.model_validate(job["result"]["data"]).model_dump(mode="json")
|
||||||
else:
|
else:
|
||||||
code = (job.get("error") or {}).get("code", job["status"])
|
code = (job.get("error") or {}).get("code", job["status"])
|
||||||
result.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
return failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -115,7 +126,7 @@ async def project_periodic_tasks(repo, replay_id, offset, limit):
|
|||||||
for execution in items:
|
for execution in items:
|
||||||
job = (await queue.get(execution["calculation_job_id"], include_result=True)
|
job = (await queue.get(execution["calculation_job_id"], include_result=True)
|
||||||
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
|
if execution.get("calculation_job_id") and execution["status"] not in {"completed", "failed"} else None)
|
||||||
result.append(project_job(execution, job) if execution.get("calculation_job_id") else execution)
|
result.append(project_job(execution, job))
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -239,8 +250,7 @@ async def persist_periodic_terminal(connection, row, terminal):
|
|||||||
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
|
execution = PeriodicTaskExecution.model_validate(terminal["result"]["data"]).model_dump(mode="json")
|
||||||
else:
|
else:
|
||||||
code = (terminal.get("error") or {}).get("code", terminal["status"])
|
code = (terminal.get("error") or {}).get("code", terminal["status"])
|
||||||
execution.update(status="failed", summary=f"本周期水力计算未完成({code}),可按原冻结输入重试。",
|
execution = failed_execution(execution, f"本周期水力计算未完成({code}),可按原冻结输入重试。")
|
||||||
updated_at=datetime.now(timezone.utc).isoformat())
|
execution["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
execution["steps"][1].update(status="failed", detail=execution["summary"])
|
|
||||||
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
|
await repo.save_periodic_task(parameters["replay_id"], parameters["batch_sequence"], execution)
|
||||||
await repo.simulation_state(parameters["replay_id"], execution)
|
await repo.simulation_state(parameters["replay_id"], execution)
|
||||||
|
|||||||
@@ -140,6 +140,8 @@ class ScadaReplayService:
|
|||||||
document = await project_periodic_state(self.repo, document)
|
document = await project_periodic_state(self.repo, document)
|
||||||
if snapshot(document).periodic_pending:
|
if snapshot(document).periodic_pending:
|
||||||
raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放")
|
raise HTTPException(409, "本周期水力计算尚未完成,请等待后继续播放")
|
||||||
|
if (document.get("last_scheduled_simulation") or {}).get("status") == "failed":
|
||||||
|
raise HTTPException(409, "本周期水力计算未完成,请在工况任务中重试,或停止后重新开始本轮")
|
||||||
action, status = command.action, document["status"]
|
action, status = command.action, document["status"]
|
||||||
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
||||||
and weighted_diagnosis(document)
|
and weighted_diagnosis(document)
|
||||||
|
|||||||
@@ -32,13 +32,17 @@
|
|||||||
|
|
||||||
worker 沿用现有并发限制、子进程隔离、超时、租约及代次栅栏。成功结果和终态在同一栅栏事务内回写周期记录,计算队列结果过期不删除已保存的业务报告。失败或中断可按原模型、批次和时间边界重试,增加 attempt,保留同一执行身份。
|
worker 沿用现有并发限制、子进程隔离、超时、租约及代次栅栏。成功结果和终态在同一栅栏事务内回写周期记录,计算队列结果过期不删除已保存的业务报告。失败或中断可按原模型、批次和时间边界重试,增加 attempt,保留同一执行身份。
|
||||||
|
|
||||||
|
水力计算排队或运行期间保留当前周期;失败、超时、中断或计算记录丢失后,播放与单步均停止推进。报告保留已完成的冻结步骤,将计算步骤标为失败、后续比较和复核步骤标为跳过;队列容量不足的失败同样适用。可在工况任务中重试本周期计算,成功后再继续播放,或停止后重新开始一轮。重复发送已接受命令仍返回原回执,不重新执行。SCADA 诊断已提交的正常、异常或数据不足结论均保留。
|
||||||
|
|
||||||
|
管网智能调度与泵站能耗检查继续使用原有 mock 动作模板,尚未接入执行算法。
|
||||||
|
|
||||||
## 接口与前端
|
## 接口与前端
|
||||||
|
|
||||||
- `GET /api/v1/scada-replays/{replay_id}/scheduled-tasks?offset=0&limit=100`:分页查询周期记录,仅读取已提交结果;不创建任务。
|
- `GET /api/v1/scada-replays/{replay_id}/scheduled-tasks?offset=0&limit=100`:分页查询周期记录,仅读取已提交结果;不创建任务。
|
||||||
- `POST /api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry`:提交 `{ "request_id": "UUID" }`,重试失败的水力计算。
|
- `POST /api/v1/scada-replays/{replay_id}/scheduled-tasks/{execution_id}/retry`:提交 `{ "request_id": "UUID" }`,重试失败的水力计算。
|
||||||
- 回放快照增加 `scheduled_tasks_enabled`、`periodic_pending`、`periodic_error`;轮询实际队列状态,不把排队视为完成。
|
- 回放快照增加 `scheduled_tasks_enabled`、`periodic_pending`、`periodic_error`;轮询实际队列状态,不把排队视为完成。
|
||||||
|
|
||||||
前端以当前回放业务时间生成保留的调度与能耗任务,暂停后进度冻结。实际回放启用时,SCADA 和定时模拟由真实周期记录替代对应生成条目;旧客户端与未启用的回放仍可使用完整原模板。列表显示最近六小时,接口保留分页历史。执行状态和诊断结论分开,报告展示实际步骤、指标、证据、缺项、调查与重试入口;刷新保留当前选中报告。
|
前端以当前回放业务时间生成保留的调度与能耗任务,暂停后进度冻结。SCADA 和管网定时模拟仅使用后端持久化周期记录,不因未启动回放、未启用周期作业或接口失败而退回生成模板。无回放时提示启动监测,旧回放未启用时提示重新开始;读取失败显示错误并保留已读取的后端记录。列表显示最近六小时,接口保留分页历史。执行状态和诊断结论分开,报告展示实际步骤、指标、证据、缺项、调查与重试入口;刷新保留当前选中报告。
|
||||||
|
|
||||||
## 验证与启用
|
## 验证与启用
|
||||||
|
|
||||||
|
|||||||
@@ -127,3 +127,36 @@ def test_client_cannot_override_project_frequency_on_creation(replay_api):
|
|||||||
})
|
})
|
||||||
assert response.status_code == 422
|
assert response.status_code == 422
|
||||||
create.assert_not_awaited()
|
create.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
def test_periodic_endpoint_returns_only_persisted_diagnosis_and_simulation_without_creating_runs(replay_api, monkeypatch):
|
||||||
|
from app.services import periodic_tasks
|
||||||
|
from app.services.periodic_diagnosis import execution_record
|
||||||
|
|
||||||
|
client, _, _, context, _ = replay_api
|
||||||
|
replay_id = uuid4()
|
||||||
|
document = {"replay_id": str(replay_id)}
|
||||||
|
monitoring = {"period_start": "2026-07-16T07:45:00+08:00", "period_end": "2026-07-16T08:00:00+08:00",
|
||||||
|
"knowledge_cutoff": "2026-07-16T08:00:00+08:00", "batch_sequence": 2}
|
||||||
|
records = [execution_record(document, monitoring, key) for key in ("scada-diagnosis", "network-simulation")]
|
||||||
|
for record in records:
|
||||||
|
record.update(status="completed", conclusion="normal", summary="已完成本周期实际处理。")
|
||||||
|
repo = SimpleNamespace(connection=object(), project_id=context.project_id,
|
||||||
|
available=AsyncMock(return_value=True), get=AsyncMock(return_value=document),
|
||||||
|
periodic_tasks=AsyncMock(return_value=records))
|
||||||
|
ensure = AsyncMock()
|
||||||
|
monkeypatch.setattr(scada_replay, "ensure_periodic_tasks", ensure)
|
||||||
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace())
|
||||||
|
client.app.dependency_overrides[scada_replay.repository] = lambda: repo
|
||||||
|
|
||||||
|
response = client.get(f"/scada-replays/{replay_id}/scheduled-tasks")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
items = response.json()["items"]
|
||||||
|
assert {item["task_key"] for item in items} == {"scada-diagnosis", "network-simulation"}
|
||||||
|
assert [item["execution_id"] for item in items] == [item["execution_id"] for item in records]
|
||||||
|
assert all(item["status"] == "completed" for item in items)
|
||||||
|
assert response.json()["next_offset"] is None
|
||||||
|
repo.get.assert_awaited_once_with(replay_id)
|
||||||
|
repo.periodic_tasks.assert_awaited_once_with(replay_id, 0, 101)
|
||||||
|
ensure.assert_not_awaited()
|
||||||
|
|||||||
@@ -220,6 +220,64 @@ def test_worker_interruption_is_reported_as_retryable_failure():
|
|||||||
assert record["status"] == "pending"
|
assert record["status"] == "pending"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("job", [None, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}},
|
||||||
|
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}])
|
||||||
|
def test_failed_calculation_marks_its_execution_step_and_skips_dependent_steps(job):
|
||||||
|
record = execution_record(document(), monitoring(), "network-simulation")
|
||||||
|
record["steps"][0]["status"] = "completed"
|
||||||
|
original = deepcopy(record)
|
||||||
|
result = periodic_tasks.project_job(record, job)
|
||||||
|
assert result["status"] == "failed"
|
||||||
|
assert result["steps"][0]["status"] == "completed"
|
||||||
|
assert result["steps"][1]["status"] == "failed"
|
||||||
|
assert result["steps"][1]["detail"] == result["summary"]
|
||||||
|
assert all(step["status"] == "skipped" for step in result["steps"][2:])
|
||||||
|
assert record == original
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_failure_persists_failed_and_skipped_steps_in_the_business_report(monkeypatch):
|
||||||
|
repo = Repository()
|
||||||
|
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
|
||||||
|
record["steps"][0]["status"] = "completed"
|
||||||
|
record["calculation_job_id"] = str(uuid4())
|
||||||
|
repo.tasks[record["execution_id"]] = record
|
||||||
|
monkeypatch.setattr(periodic_tasks, "ScadaReplayRepository", lambda *_: repo)
|
||||||
|
row = {"project_id": repo.project_id, "job_id": record["calculation_job_id"], "parameters": {
|
||||||
|
"replay_id": record["replay_id"], "execution_id": record["execution_id"], "batch_sequence": record["batch_sequence"]}}
|
||||||
|
asyncio.run(periodic_tasks.persist_periodic_terminal(repo.connection, row,
|
||||||
|
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}))
|
||||||
|
saved = repo.tasks[record["execution_id"]]
|
||||||
|
assert saved["status"] == repo.state["status"] == "failed"
|
||||||
|
assert [step["status"] for step in saved["steps"]] == ["completed", "failed", "skipped", "skipped", "skipped"]
|
||||||
|
assert saved["execution_id"] == record["execution_id"]
|
||||||
|
assert saved["batch_sequence"] == record["batch_sequence"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("action", ["step", "monitor"])
|
||||||
|
def test_failed_period_must_be_retried_before_advancing_but_can_be_stopped(monkeypatch, action):
|
||||||
|
from tests.unit.test_scada_replay_service import setup
|
||||||
|
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
||||||
|
repo, service, _, _, create = setup(monkeypatch)
|
||||||
|
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
|
||||||
|
|
||||||
|
async def run():
|
||||||
|
replay = await service.create(create, monitoring_interval_seconds=900)
|
||||||
|
repo.document["config"]["scheduled_tasks_enabled"] = True
|
||||||
|
repo.document["last_scheduled_simulation"] = {"status": "failed", "error": "EXECUTION_TIMEOUT"}
|
||||||
|
original = deepcopy(repo.document)
|
||||||
|
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action=action,
|
||||||
|
**({"at_time": replay.start_time} if action == "monitor" else {}))
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
await service.command(replay.replay_id, command)
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
assert "重试" in error.value.detail
|
||||||
|
assert repo.document == original
|
||||||
|
stopped = await service.command(replay.replay_id, ScadaReplayCommand(
|
||||||
|
request_id=uuid4(), expected_revision=replay.revision, action="stop"))
|
||||||
|
assert stopped.status == "stopped"
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
|
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
|
||||||
repo = Repository()
|
repo = Repository()
|
||||||
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
||||||
@@ -262,7 +320,8 @@ def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
|
|||||||
assert error.value.status_code == 403
|
assert error.value.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
|
@pytest.mark.parametrize("calculation_status", ["pending", "failed"])
|
||||||
|
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch, calculation_status):
|
||||||
from tests.unit.test_scada_replay_service import setup
|
from tests.unit.test_scada_replay_service import setup
|
||||||
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
||||||
repo, service, _, _, create = setup(monkeypatch)
|
repo, service, _, _, create = setup(monkeypatch)
|
||||||
@@ -273,7 +332,7 @@ def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
|
|||||||
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
|
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
|
||||||
accepted = await service.command(replay.replay_id, command)
|
accepted = await service.command(replay.replay_id, command)
|
||||||
repo.document["config"]["scheduled_tasks_enabled"] = True
|
repo.document["config"]["scheduled_tasks_enabled"] = True
|
||||||
repo.document["last_scheduled_simulation"] = {"status": "pending"}
|
repo.document["last_scheduled_simulation"] = {"status": calculation_status}
|
||||||
# A transport retry returns the original accepted period, even while its
|
# A transport retry returns the original accepted period, even while its
|
||||||
# calculation is queued; a different step must wait.
|
# calculation is queued; a different step must wait.
|
||||||
repeated = await service.command(replay.replay_id, command)
|
repeated = await service.command(replay.replay_id, command)
|
||||||
|
|||||||
Reference in New Issue
Block a user