feat(replay): execute periodic diagnosis and hydraulic checks
This commit is contained in:
@@ -110,7 +110,7 @@ def test_monitoring_command_processes_pending_period_in_same_request(monkeypatch
|
||||
)
|
||||
|
||||
result = asyncio.run(scada_replay.replay_command(
|
||||
replay_id, command, repo="repo", ctx=context, timescale="timescale"
|
||||
replay_id, command, repo=SimpleNamespace(get=AsyncMock(return_value={"config": {}})), ctx=context, timescale="timescale"
|
||||
))
|
||||
|
||||
assert result == "completed-period"
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Native calculation and real SQL under an isolated project UUID, always rolled back."""
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
import pytest
|
||||
|
||||
from app.auth.project_dependencies import ProjectContext
|
||||
from app.infra.db.metadb.database import SessionLocal, engine
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
from app.infra.db.postgresql.calculation_jobs import CalculationJobRepository
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.services.periodic_tasks import ensure_periodic_tasks, execute_periodic_calculation, persist_periodic_terminal, project_periodic_tasks
|
||||
from app.infra.epanet.model_snapshot import pack_model_input
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(os.getenv("PERIODIC_TASK_STORAGE_TEST") != "1", reason="explicit rollback-only SQL test required")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend(): return "asyncio"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_frozen_period_queue_native_result_and_retry_storage():
|
||||
from tests.unit.test_periodic_hydraulics import MODEL
|
||||
async with SessionLocal() as session:
|
||||
metadata = MetadataRepository(session)
|
||||
routing = await metadata.get_project_db_routing(UUID("871d7538-589c-47de-a6d5-3185ccb9f0e1"), "biz_data")
|
||||
await engine.dispose()
|
||||
if not routing:
|
||||
pytest.skip("Project route unavailable")
|
||||
connection = await psycopg.AsyncConnection.connect(routing.dsn, row_factory=dict_row, connect_timeout=5)
|
||||
project_id, user_id, replay_id, request_id = uuid4(), uuid4(), uuid4(), uuid4()
|
||||
try:
|
||||
async with connection.transaction(force_rollback=True):
|
||||
repo = ScadaReplayRepository(connection, project_id)
|
||||
if not await repo.available() or not await repo.queue_available():
|
||||
pytest.skip("Replay or calculation storage is not provisioned")
|
||||
start, end = "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00"
|
||||
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m", "transmission_frequency": "0:15:00"},
|
||||
{"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h", "transmission_frequency": "0:15:00"}]
|
||||
rows = [{"device_id": device, "time": at, "quality": "valid", "cleaned_value": value}
|
||||
for at in (start, end) for device, value in (("P1", 39.8), ("F1", 3.6))]
|
||||
monitoring = {"monitoring_id": str(uuid4()), "period_start": start, "period_end": end, "knowledge_cutoff": end,
|
||||
"batch_sequence": 1, "status": "normal", "detail": "本周期正常", "task_id": None}
|
||||
document = {"project_id": str(project_id), "replay_id": str(replay_id), "created_by": str(user_id), "status": "paused",
|
||||
"config": {"request_id": str(request_id), "scheduled_tasks_enabled": True, "device_ids": ["P1", "F1"], "rules": []},
|
||||
"devices": devices, "latest": {row["device_id"]: row for row in rows},
|
||||
"monitoring_interval_seconds": 900, "last_monitoring": monitoring,
|
||||
"scheduled_model_ready": True, "scheduled_model_digest": "frozen-test-model",
|
||||
"updated_at": datetime.now(timezone.utc).isoformat()}
|
||||
await repo.insert(document)
|
||||
await repo.save_batch(replay_id, {"sequence": 0, "stage": "detected", "kind": "model_snapshot",
|
||||
"model_input_gzip": pack_model_input(MODEL), "model_digest": "frozen-test-model"})
|
||||
await repo.save_batch(replay_id, {"sequence": 1, "stage": "detected", "monitoring": monitoring, "outputs": rows})
|
||||
context = ProjectContext(project_id, "tjwater2_v2", user_id, "member")
|
||||
await ensure_periodic_tasks(repo, context, replay_id)
|
||||
await ensure_periodic_tasks(repo, context, replay_id)
|
||||
page = await project_periodic_tasks(repo, replay_id, 0, 100)
|
||||
assert len(page) == 2
|
||||
simulation = next(item for item in page if item["task_key"] == "network-simulation")
|
||||
assert simulation["status"] == "pending"
|
||||
queue = CalculationJobRepository(connection, project_id)
|
||||
owner = str(uuid4())
|
||||
job = await queue.claim(owner, 30)
|
||||
# Avoid committing the enclosing test rollback transaction: production
|
||||
# releases its read transaction before computation, tested separately.
|
||||
class ReadScopeConnection:
|
||||
def __getattr__(self, key): return getattr(connection, key)
|
||||
async def commit(self): pass
|
||||
result = await execute_periodic_calculation(SimpleNamespace(project=context, connection=ReadScopeConnection()), job)
|
||||
assert result.data["conclusion"] == "normal"
|
||||
assert len(result.data["result"]["comparisons"]) == 4
|
||||
terminal = {"status": "succeeded", "result": result.model_dump(mode="json")}
|
||||
assert await queue.finish(job, owner, **terminal)
|
||||
await persist_periodic_terminal(connection, job, terminal)
|
||||
# A stale worker must not republish another result after its fence closed.
|
||||
assert not await queue.finish(job, owner, **terminal)
|
||||
stored = await repo.periodic_task(replay_id, simulation["execution_id"])
|
||||
assert stored["status"] == "completed"
|
||||
assert len(stored["result"]["comparisons"]) == 4
|
||||
await repo.save_retry_receipt(replay_id, 1, request_id, simulation["execution_id"])
|
||||
assert (await repo.batch(replay_id, 1))["periodic_retry_receipts"][str(request_id)] == simulation["execution_id"]
|
||||
assert len(await repo.periodic_tasks(replay_id, 0, 1)) == 1
|
||||
assert len(await repo.periodic_tasks(replay_id, 1, 1)) == 1
|
||||
assert (await repo.get(replay_id))["last_scheduled_simulation"]["status"] == "completed"
|
||||
# Check rollback on the same connection, with no application rows retained.
|
||||
row = await (await connection.execute("SELECT count(*) AS count FROM peripheral_sample.scada_replays WHERE project_id=%s", (project_id,))).fetchone()
|
||||
assert row["count"] == 0
|
||||
finally:
|
||||
await connection.close()
|
||||
@@ -0,0 +1,27 @@
|
||||
import base64
|
||||
import gzip
|
||||
|
||||
import pytest
|
||||
|
||||
from app.infra.epanet import model_snapshot
|
||||
|
||||
|
||||
def test_full_model_larger_than_response_budget_round_trips_compressed():
|
||||
content = "[JUNCTIONS]\n" + "节点 12.345\n" * 900_000
|
||||
assert len(content.encode()) > 8 * 1024 * 1024
|
||||
packed = model_snapshot.pack_model_input(content)
|
||||
assert len(packed) < len(content)
|
||||
assert model_snapshot.unpack_model_input(packed) == content
|
||||
|
||||
|
||||
def test_decoding_enforces_expanded_model_budget(monkeypatch):
|
||||
packed = base64.b64encode(gzip.compress(b"A" * 1025)).decode()
|
||||
monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 1024)
|
||||
with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded):
|
||||
model_snapshot.unpack_model_input(packed)
|
||||
|
||||
|
||||
def test_oversized_model_is_rejected_without_compressing(monkeypatch):
|
||||
monkeypatch.setattr(model_snapshot, "MAX_MODEL_BYTES", 3)
|
||||
with pytest.raises(model_snapshot.ModelSnapshotBudgetExceeded):
|
||||
model_snapshot.pack_model_input("four")
|
||||
@@ -0,0 +1,67 @@
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.algorithms.periodic_hydraulics import run_periodic_hydraulics
|
||||
|
||||
|
||||
MODEL = """[JUNCTIONS]
|
||||
J1 10 1
|
||||
[RESERVOIRS]
|
||||
R1 50
|
||||
[PIPES]
|
||||
L1 R1 J1 500 150 100 0 OPEN
|
||||
[OPTIONS]
|
||||
UNITS LPS
|
||||
HEADLOSS H-W
|
||||
[TIMES]
|
||||
DURATION 0:15
|
||||
HYDRAULIC TIMESTEP 0:15
|
||||
REPORT TIMESTEP 0:15
|
||||
[END]
|
||||
"""
|
||||
|
||||
|
||||
def test_real_epanet_checks_all_model_elements_without_a_source_database():
|
||||
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"},
|
||||
{"device_id": "F1", "device_type": "pipe_flow", "link_id": "L1", "measurement_unit": "m3/h"}]
|
||||
rows = [{"time": at, "device_id": device, "quality": "valid", "cleaned_value": value}
|
||||
for at in ("2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00", "2026-07-16T08:15:00+08:00")
|
||||
for device, value in (("P1", 39.8), ("F1", 3.6))]
|
||||
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
|
||||
assert len(result["comparisons"]) == 4
|
||||
assert result["conclusion"] == "normal"
|
||||
assert result["compared_device_count"] == 2
|
||||
assert result["missing_device_ids"] == []
|
||||
assert all(snapshot["node_count"] == 2 and snapshot["link_count"] == 1 for snapshot in result["snapshots"])
|
||||
assert all(datetime.fromisoformat(item["time"]).hour <= 8 for item in result["comparisons"])
|
||||
assert {item["time"] for item in result["comparisons"]} == {row["time"] for row in rows[:4]}
|
||||
assert next(item for item in result["comparisons"] if item["device_id"] == "F1")["observed"] == .001
|
||||
|
||||
|
||||
def test_no_valid_measurement_is_a_data_gap_not_normal():
|
||||
result = run_periodic_hydraulics(MODEL, [], [], "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
|
||||
assert result["conclusion"] == "insufficient_data"
|
||||
assert result["comparisons"] == []
|
||||
|
||||
|
||||
def test_missing_mapping_and_zero_reference_are_not_silently_compared():
|
||||
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"},
|
||||
{"device_id": "F1", "device_type": "pipe_flow", "measurement_unit": "m3/h"}]
|
||||
rows = [{"time": "2026-07-16T08:00:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 0.}]
|
||||
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
|
||||
assert result["comparisons"][0]["relative_deviation_percent"] is None
|
||||
assert result["missing_device_ids"] == ["F1"]
|
||||
assert result["conclusion"] != "normal"
|
||||
|
||||
|
||||
def test_previous_sample_does_not_claim_current_period_is_normal():
|
||||
devices = [{"device_id": "P1", "device_type": "pressure", "node_id": "J1", "measurement_unit": "m"}]
|
||||
rows = [{"time": "2026-07-16T07:45:00+08:00", "device_id": "P1", "quality": "valid", "cleaned_value": 39.8}]
|
||||
result = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
|
||||
assert result["compared_device_count"] == 1
|
||||
assert result["current_device_count"] == 0
|
||||
assert result["conclusion"] == "insufficient_data"
|
||||
devices[0]["transmission_frequency"] = "0:30:00"
|
||||
slower = run_periodic_hydraulics(MODEL, devices, rows, "2026-07-16T07:45:00+08:00", "2026-07-16T08:00:00+08:00")
|
||||
assert slower["current_device_count"] == 1
|
||||
@@ -0,0 +1,306 @@
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.domain.schemas.periodic_task import PeriodicTaskExecution
|
||||
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
|
||||
from app.services import periodic_tasks
|
||||
|
||||
|
||||
def document():
|
||||
return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()),
|
||||
"monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest",
|
||||
"devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"},
|
||||
{"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}],
|
||||
"config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"],
|
||||
"rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]},
|
||||
"latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}}
|
||||
|
||||
|
||||
def monitoring():
|
||||
return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00",
|
||||
"period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00",
|
||||
"status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None}
|
||||
|
||||
|
||||
def test_water_risk_is_independent_of_incomplete_pressure_model():
|
||||
source, period = document(), monitoring()
|
||||
period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[
|
||||
{"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}])
|
||||
result = diagnosis_execution(source, period)
|
||||
assert result["conclusion"] == "suspected_pollution"
|
||||
assert result["investigation_task_id"] == period["task_id"]
|
||||
assert result["risk_signals"][0]["metric_name"] == "余氯"
|
||||
assert result["data_gaps"] == ["压力基线不足"]
|
||||
assert all(step["status"] == "completed" for step in result["steps"])
|
||||
|
||||
|
||||
def detected_water_event():
|
||||
from datetime import timedelta
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
||||
|
||||
source = document()
|
||||
at = datetime.fromisoformat(monitoring()["period_end"])
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15),
|
||||
device_ids=["Q1"], rules=[{
|
||||
"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"],
|
||||
"lower": 0.2, "unit": "mg/L", "metric_name": "余氯",
|
||||
}],
|
||||
)
|
||||
source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules]
|
||||
source, events = detect_frame(source, prepare_cleaning([
|
||||
{"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"}
|
||||
], config), config, at)
|
||||
event = next(event for event in events if event["rule_id"] == "water")
|
||||
task_id = str(uuid4())
|
||||
event["task_id"] = task_id
|
||||
source["rule_states"]["water"]["event"]["task_id"] = task_id
|
||||
return source, event
|
||||
|
||||
|
||||
def test_periodic_report_includes_water_event_excluded_from_burst_monitoring():
|
||||
from app.services.scada_normal_profile import diagnostic_signal
|
||||
|
||||
source, event = detected_water_event()
|
||||
period = monitoring()
|
||||
assert not diagnostic_signal(source, event)
|
||||
original = deepcopy(period)
|
||||
result = diagnosis_execution(source, period)
|
||||
|
||||
assert result["conclusion"] == "suspected_pollution"
|
||||
assert result["investigation_task_id"] == event["task_id"]
|
||||
water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution")
|
||||
assert water["device_ids"] == ["Q1"]
|
||||
assert water["task_id"] == event["task_id"]
|
||||
assert period == original
|
||||
|
||||
|
||||
def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence():
|
||||
source, event = detected_water_event()
|
||||
source["rule_states"]["water"]["event"] = None
|
||||
event.update(status="resolved", resolved_at=monitoring()["period_end"])
|
||||
period = monitoring()
|
||||
hydraulic_task_id = str(uuid4())
|
||||
hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()),
|
||||
"device_ids": ["P1"], "detail": "压力变化需要复核"}
|
||||
period.update(status="attention", task_id=hydraulic_task_id,
|
||||
weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal])
|
||||
original = deepcopy(period)
|
||||
result = diagnosis_execution(source, period, [event, deepcopy(event)])
|
||||
|
||||
assert result["conclusion"] == "suspected_pollution"
|
||||
assert result["investigation_task_id"] == hydraulic_task_id
|
||||
water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"]
|
||||
assert len(water) == 1
|
||||
assert water[0]["task_id"] == event["task_id"]
|
||||
assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"]
|
||||
for signal in result["risk_signals"])
|
||||
assert period == original
|
||||
assert result["result"]["monitoring"] == original
|
||||
|
||||
|
||||
def test_periodic_water_projection_excludes_telemetry_quality_events():
|
||||
source, event = detected_water_event()
|
||||
event["category"] = "data_quality"
|
||||
source["rule_states"]["water"]["event"] = deepcopy(event)
|
||||
result = diagnosis_execution(source, monitoring(), [event])
|
||||
|
||||
assert result["conclusion"] == "normal"
|
||||
assert result["risk_signals"] == []
|
||||
assert result["investigation_task_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("future", [
|
||||
{"occurred_at": "2026-07-16T08:15:00+08:00"},
|
||||
{"batch_sequence": 3},
|
||||
])
|
||||
def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future):
|
||||
source, event = detected_water_event()
|
||||
event.update(future)
|
||||
source["rule_states"]["water"]["event"] = event
|
||||
result = diagnosis_execution(source, monitoring(), [event])
|
||||
|
||||
assert result["conclusion"] == "normal"
|
||||
assert result["risk_signals"] == []
|
||||
assert result["investigation_task_id"] is None
|
||||
|
||||
|
||||
def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal():
|
||||
source, period = document(), monitoring()
|
||||
source["latest"]["P1"]["quality"] = "invalid"
|
||||
first = diagnosis_execution(source, period)
|
||||
assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"]
|
||||
assert first["conclusion"] == "attention"
|
||||
assert first["result"]["invalid_device_ids"] == ["P1"]
|
||||
source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00"
|
||||
assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"]
|
||||
other = dict(source, replay_id=str(uuid4()))
|
||||
assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"]
|
||||
|
||||
|
||||
class Repository:
|
||||
def __init__(self):
|
||||
self.document = document()
|
||||
self.document["last_monitoring"] = monitoring()
|
||||
self.project_id = self.document["project_id"]
|
||||
self.connection = object()
|
||||
self.tasks = {}
|
||||
self.receipts = {}
|
||||
self.state = None
|
||||
self.events = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def control(self): yield
|
||||
async def get(self, _): return deepcopy(self.document)
|
||||
async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)}
|
||||
async def periodic_events(self, *_): return deepcopy(self.events)
|
||||
async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution)
|
||||
async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution)
|
||||
async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id)))
|
||||
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id)
|
||||
|
||||
|
||||
def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch):
|
||||
repo = Repository()
|
||||
enqueue = AsyncMock(return_value={"job_id": uuid4()})
|
||||
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue))
|
||||
context = SimpleNamespace(project_id=repo.project_id)
|
||||
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
||||
|
||||
async def run():
|
||||
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
|
||||
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
|
||||
asyncio.run(run())
|
||||
assert len(repo.tasks) == 2
|
||||
assert enqueue.await_count == 1
|
||||
payload = enqueue.call_args.kwargs
|
||||
assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id}
|
||||
assert payload["owner_user_id"] == repo.document["created_by"]
|
||||
assert repo.state["status"] == "pending"
|
||||
for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task)
|
||||
|
||||
|
||||
def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch):
|
||||
source, event = detected_water_event()
|
||||
source["rule_states"]["water"]["event"] = None
|
||||
source["last_monitoring"] = monitoring()
|
||||
repo = Repository()
|
||||
repo.document = source
|
||||
repo.project_id = source["project_id"]
|
||||
repo.events = [event]
|
||||
read_events = AsyncMock(wraps=repo.periodic_events)
|
||||
repo.periodic_events = read_events
|
||||
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
||||
enqueue=AsyncMock(return_value={"job_id": uuid4()})))
|
||||
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
||||
|
||||
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"]))
|
||||
|
||||
read_events.assert_awaited_once()
|
||||
diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")
|
||||
assert diagnosis["conclusion"] == "suspected_pollution"
|
||||
assert diagnosis["investigation_task_id"] == event["task_id"]
|
||||
|
||||
|
||||
def test_worker_interruption_is_reported_as_retryable_failure():
|
||||
record = execution_record(document(), monitoring(), "network-simulation")
|
||||
result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}})
|
||||
assert result["status"] == "failed"
|
||||
assert "WORKER_INTERRUPTED" in result["summary"]
|
||||
assert record["status"] == "pending"
|
||||
|
||||
|
||||
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
|
||||
repo = Repository()
|
||||
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
||||
enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded())))
|
||||
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
||||
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"]))
|
||||
assert len(repo.tasks) == 2
|
||||
assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed"
|
||||
assert repo.state["status"] == "failed"
|
||||
assert repo.state["calculation_job_id"] is None
|
||||
assert "尚未启动" in repo.state["summary"]
|
||||
|
||||
|
||||
def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch):
|
||||
repo = Repository()
|
||||
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
|
||||
record.update(calculation_job_id=str(uuid4()), status="failed")
|
||||
repo.tasks[record["execution_id"]] = deepcopy(record)
|
||||
enqueue = AsyncMock(return_value={"job_id": uuid4()})
|
||||
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
||||
enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"})))
|
||||
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
||||
request = uuid4()
|
||||
async def run():
|
||||
for _ in range(2):
|
||||
await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id),
|
||||
repo.document["replay_id"], record["execution_id"], request)
|
||||
asyncio.run(run())
|
||||
retried = repo.tasks[record["execution_id"]]
|
||||
assert retried["attempt"] == 2
|
||||
assert retried["period_end"] == record["period_end"]
|
||||
assert retried["batch_sequence"] == record["batch_sequence"]
|
||||
assert enqueue.await_count == 1
|
||||
|
||||
|
||||
def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
|
||||
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"})
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {}))
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch):
|
||||
from tests.unit.test_scada_replay_service import setup
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
||||
repo, service, _, _, create = setup(monkeypatch)
|
||||
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
|
||||
|
||||
async def run():
|
||||
replay = await service.create(create, monitoring_interval_seconds=900)
|
||||
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
|
||||
accepted = await service.command(replay.replay_id, command)
|
||||
repo.document["config"]["scheduled_tasks_enabled"] = True
|
||||
repo.document["last_scheduled_simulation"] = {"status": "pending"}
|
||||
# A transport retry returns the original accepted period, even while its
|
||||
# calculation is queued; a different step must wait.
|
||||
repeated = await service.command(replay.replay_id, command)
|
||||
assert repeated.revision == accepted.revision
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await service.command(replay.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=accepted.revision, action="step"))
|
||||
assert error.value.status_code == 409
|
||||
assert "水力计算" in error.value.detail
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path):
|
||||
from app.services import calculation_worker
|
||||
from app.mcp.models import QueryResult
|
||||
@asynccontextmanager
|
||||
async def scope(_): yield SimpleNamespace(project="authorized-project")
|
||||
monkeypatch.setattr(calculation_worker, "execution_scope", scope)
|
||||
internal = AsyncMock(return_value=QueryResult(
|
||||
source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test",
|
||||
queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"}))
|
||||
monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal)
|
||||
ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding"))
|
||||
monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary)
|
||||
source, target = tmp_path / "job.json", tmp_path / "result.json"
|
||||
source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL}))
|
||||
asyncio.run(calculation_worker.execute_child(str(source), str(target)))
|
||||
assert json.loads(target.read_text())["status"] == "succeeded"
|
||||
internal.assert_awaited_once()
|
||||
ordinary.assert_not_awaited()
|
||||
Reference in New Issue
Block a user