fix(replay): preserve failed periodic execution boundaries
Generic Container CI/CD / test-build-publish (push) Successful in 4m23s
Server CI/CD v2 / build-test-publish-and-deploy (push) Successful in 4m23s

This commit is contained in:
2026-11-16 16:21:58 +08:00
parent 389892cb3f
commit 9cd1400723
5 changed files with 119 additions and 11 deletions
+33
View File
@@ -127,3 +127,36 @@ def test_client_cannot_override_project_frequency_on_creation(replay_api):
})
assert response.status_code == 422
create.assert_not_awaited()
def test_periodic_endpoint_returns_only_persisted_diagnosis_and_simulation_without_creating_runs(replay_api, monkeypatch):
from app.services import periodic_tasks
from app.services.periodic_diagnosis import execution_record
client, _, _, context, _ = replay_api
replay_id = uuid4()
document = {"replay_id": str(replay_id)}
monitoring = {"period_start": "2026-07-16T07:45:00+08:00", "period_end": "2026-07-16T08:00:00+08:00",
"knowledge_cutoff": "2026-07-16T08:00:00+08:00", "batch_sequence": 2}
records = [execution_record(document, monitoring, key) for key in ("scada-diagnosis", "network-simulation")]
for record in records:
record.update(status="completed", conclusion="normal", summary="已完成本周期实际处理。")
repo = SimpleNamespace(connection=object(), project_id=context.project_id,
available=AsyncMock(return_value=True), get=AsyncMock(return_value=document),
periodic_tasks=AsyncMock(return_value=records))
ensure = AsyncMock()
monkeypatch.setattr(scada_replay, "ensure_periodic_tasks", ensure)
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace())
client.app.dependency_overrides[scada_replay.repository] = lambda: repo
response = client.get(f"/scada-replays/{replay_id}/scheduled-tasks")
assert response.status_code == 200
items = response.json()["items"]
assert {item["task_key"] for item in items} == {"scada-diagnosis", "network-simulation"}
assert [item["execution_id"] for item in items] == [item["execution_id"] for item in records]
assert all(item["status"] == "completed" for item in items)
assert response.json()["next_offset"] is None
repo.get.assert_awaited_once_with(replay_id)
repo.periodic_tasks.assert_awaited_once_with(replay_id, 0, 101)
ensure.assert_not_awaited()
+61 -2
View File
@@ -220,6 +220,64 @@ def test_worker_interruption_is_reported_as_retryable_failure():
assert record["status"] == "pending"
@pytest.mark.parametrize("job", [None, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}},
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}])
def test_failed_calculation_marks_its_execution_step_and_skips_dependent_steps(job):
record = execution_record(document(), monitoring(), "network-simulation")
record["steps"][0]["status"] = "completed"
original = deepcopy(record)
result = periodic_tasks.project_job(record, job)
assert result["status"] == "failed"
assert result["steps"][0]["status"] == "completed"
assert result["steps"][1]["status"] == "failed"
assert result["steps"][1]["detail"] == result["summary"]
assert all(step["status"] == "skipped" for step in result["steps"][2:])
assert record == original
def test_worker_failure_persists_failed_and_skipped_steps_in_the_business_report(monkeypatch):
repo = Repository()
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
record["steps"][0]["status"] = "completed"
record["calculation_job_id"] = str(uuid4())
repo.tasks[record["execution_id"]] = record
monkeypatch.setattr(periodic_tasks, "ScadaReplayRepository", lambda *_: repo)
row = {"project_id": repo.project_id, "job_id": record["calculation_job_id"], "parameters": {
"replay_id": record["replay_id"], "execution_id": record["execution_id"], "batch_sequence": record["batch_sequence"]}}
asyncio.run(periodic_tasks.persist_periodic_terminal(repo.connection, row,
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}))
saved = repo.tasks[record["execution_id"]]
assert saved["status"] == repo.state["status"] == "failed"
assert [step["status"] for step in saved["steps"]] == ["completed", "failed", "skipped", "skipped", "skipped"]
assert saved["execution_id"] == record["execution_id"]
assert saved["batch_sequence"] == record["batch_sequence"]
@pytest.mark.parametrize("action", ["step", "monitor"])
def test_failed_period_must_be_retried_before_advancing_but_can_be_stopped(monkeypatch, action):
from tests.unit.test_scada_replay_service import setup
from app.domain.schemas.scada_replay import ScadaReplayCommand
repo, service, _, _, create = setup(monkeypatch)
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
async def run():
replay = await service.create(create, monitoring_interval_seconds=900)
repo.document["config"]["scheduled_tasks_enabled"] = True
repo.document["last_scheduled_simulation"] = {"status": "failed", "error": "EXECUTION_TIMEOUT"}
original = deepcopy(repo.document)
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action=action,
**({"at_time": replay.start_time} if action == "monitor" else {}))
with pytest.raises(HTTPException) as error:
await service.command(replay.replay_id, command)
assert error.value.status_code == 409
assert "重试" in error.value.detail
assert repo.document == original
stopped = await service.command(replay.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=replay.revision, action="stop"))
assert stopped.status == "stopped"
asyncio.run(run())
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
repo = Repository()
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
@@ -262,7 +320,8 @@ def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
assert error.value.status_code == 403
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
@pytest.mark.parametrize("calculation_status", ["pending", "failed"])
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch, calculation_status):
from tests.unit.test_scada_replay_service import setup
from app.domain.schemas.scada_replay import ScadaReplayCommand
repo, service, _, _, create = setup(monkeypatch)
@@ -273,7 +332,7 @@ def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
accepted = await service.command(replay.replay_id, command)
repo.document["config"]["scheduled_tasks_enabled"] = True
repo.document["last_scheduled_simulation"] = {"status": "pending"}
repo.document["last_scheduled_simulation"] = {"status": calculation_status}
# A transport retry returns the original accepted period, even while its
# calculation is queued; a different step must wait.
repeated = await service.command(replay.replay_id, command)