366 lines
19 KiB
Python
366 lines
19 KiB
Python
import asyncio
|
|
import json
|
|
from contextlib import asynccontextmanager
|
|
from copy import deepcopy
|
|
from datetime import datetime, timezone
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
from app.domain.schemas.periodic_task import PeriodicTaskExecution
|
|
from app.services.periodic_diagnosis import diagnosis_execution, execution_record
|
|
from app.services import periodic_tasks
|
|
|
|
|
|
def document():
|
|
return {"replay_id": str(uuid4()), "project_id": str(uuid4()), "created_by": str(uuid4()),
|
|
"monitoring_interval_seconds": 900, "scheduled_model_ready": True, "scheduled_model_digest": "digest",
|
|
"devices": [{"device_id": "P1", "device_type": "pressure", "transmission_frequency": "0:15:00"},
|
|
{"device_id": "Q1", "device_type": "quality", "transmission_frequency": "0:15:00"}],
|
|
"config": {"scheduled_tasks_enabled": True, "device_ids": ["P1", "Q1"],
|
|
"rules": [{"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"], "metric_name": "余氯"}]},
|
|
"latest": {device: {"time": "2026-07-16T08:00:00+08:00", "quality": "valid"} for device in ("P1", "Q1")}}
|
|
|
|
|
|
def monitoring():
|
|
return {"monitoring_id": str(uuid4()), "batch_sequence": 2, "period_start": "2026-07-16T07:45:00+08:00",
|
|
"period_end": "2026-07-16T08:00:00+08:00", "knowledge_cutoff": "2026-07-16T08:00:00+08:00",
|
|
"status": "normal", "detail": "本周期正常", "pressure": {}, "task_id": None}
|
|
|
|
|
|
def test_water_risk_is_independent_of_incomplete_pressure_model():
|
|
source, period = document(), monitoring()
|
|
period.update(status="insufficient_data", detail="压力基线不足", task_id=str(uuid4()), anomaly_signals=[
|
|
{"rule_id": "water", "event_id": str(uuid4()), "device_ids": ["Q1"], "detail": "余氯低于配置下限"}])
|
|
result = diagnosis_execution(source, period)
|
|
assert result["conclusion"] == "suspected_pollution"
|
|
assert result["investigation_task_id"] == period["task_id"]
|
|
assert result["risk_signals"][0]["metric_name"] == "余氯"
|
|
assert result["data_gaps"] == ["压力基线不足"]
|
|
assert all(step["status"] == "completed" for step in result["steps"])
|
|
|
|
|
|
def detected_water_event():
|
|
from datetime import timedelta
|
|
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
|
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
|
|
|
source = document()
|
|
at = datetime.fromisoformat(monitoring()["period_end"])
|
|
config = ScadaReplayCreate(
|
|
request_id=uuid4(), start_time=at - timedelta(minutes=15), end_time=at + timedelta(minutes=15),
|
|
device_ids=["Q1"], rules=[{
|
|
"rule_id": "water", "kind": "bounds", "device_ids": ["Q1"],
|
|
"lower": 0.2, "unit": "mg/L", "metric_name": "余氯",
|
|
}],
|
|
)
|
|
source["config"]["rules"] = [rule.model_dump(mode="json") for rule in config.rules]
|
|
source, events = detect_frame(source, prepare_cleaning([
|
|
{"device_id": "Q1", "time": at, "monitored_value": 0.1, "source_hash": "water-frame"}
|
|
], config), config, at)
|
|
event = next(event for event in events if event["rule_id"] == "water")
|
|
task_id = str(uuid4())
|
|
event["task_id"] = task_id
|
|
source["rule_states"]["water"]["event"]["task_id"] = task_id
|
|
return source, event
|
|
|
|
|
|
def test_periodic_report_includes_water_event_excluded_from_burst_monitoring():
|
|
from app.services.scada_normal_profile import diagnostic_signal
|
|
|
|
source, event = detected_water_event()
|
|
period = monitoring()
|
|
assert not diagnostic_signal(source, event)
|
|
original = deepcopy(period)
|
|
result = diagnosis_execution(source, period)
|
|
|
|
assert result["conclusion"] == "suspected_pollution"
|
|
assert result["investigation_task_id"] == event["task_id"]
|
|
water = next(signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution")
|
|
assert water["device_ids"] == ["Q1"]
|
|
assert water["task_id"] == event["task_id"]
|
|
assert period == original
|
|
|
|
|
|
def test_recovered_period_water_event_keeps_its_separate_investigation_and_hydraulic_evidence():
|
|
source, event = detected_water_event()
|
|
source["rule_states"]["water"]["event"] = None
|
|
event.update(status="resolved", resolved_at=monitoring()["period_end"])
|
|
period = monitoring()
|
|
hydraulic_task_id = str(uuid4())
|
|
hydraulic_signal = {"rule_id": "pressure", "event_id": str(uuid4()),
|
|
"device_ids": ["P1"], "detail": "压力变化需要复核"}
|
|
period.update(status="attention", task_id=hydraulic_task_id,
|
|
weighted={"score": 0.12, "threshold": 0.8}, anomaly_signals=[hydraulic_signal])
|
|
original = deepcopy(period)
|
|
result = diagnosis_execution(source, period, [event, deepcopy(event)])
|
|
|
|
assert result["conclusion"] == "suspected_pollution"
|
|
assert result["investigation_task_id"] == hydraulic_task_id
|
|
water = [signal for signal in result["risk_signals"] if signal["kind"] == "suspected_pollution"]
|
|
assert len(water) == 1
|
|
assert water[0]["task_id"] == event["task_id"]
|
|
assert any(signal["kind"] == "hydraulic_anomaly" and signal["device_ids"] == ["P1"]
|
|
for signal in result["risk_signals"])
|
|
assert period == original
|
|
assert result["result"]["monitoring"] == original
|
|
|
|
|
|
def test_periodic_water_projection_excludes_telemetry_quality_events():
|
|
source, event = detected_water_event()
|
|
event["category"] = "data_quality"
|
|
source["rule_states"]["water"]["event"] = deepcopy(event)
|
|
result = diagnosis_execution(source, monitoring(), [event])
|
|
|
|
assert result["conclusion"] == "normal"
|
|
assert result["risk_signals"] == []
|
|
assert result["investigation_task_id"] is None
|
|
|
|
|
|
@pytest.mark.parametrize("future", [
|
|
{"occurred_at": "2026-07-16T08:15:00+08:00"},
|
|
{"batch_sequence": 3},
|
|
])
|
|
def test_periodic_report_rejects_water_state_after_its_frozen_cutoff(future):
|
|
source, event = detected_water_event()
|
|
event.update(future)
|
|
source["rule_states"]["water"]["event"] = event
|
|
result = diagnosis_execution(source, monitoring(), [event])
|
|
|
|
assert result["conclusion"] == "normal"
|
|
assert result["risk_signals"] == []
|
|
assert result["investigation_task_id"] is None
|
|
|
|
|
|
def test_same_period_is_stable_and_invalid_telemetry_cannot_be_normal():
|
|
source, period = document(), monitoring()
|
|
source["latest"]["P1"]["quality"] = "invalid"
|
|
first = diagnosis_execution(source, period)
|
|
assert first["execution_id"] == diagnosis_execution(source, period)["execution_id"]
|
|
assert first["conclusion"] == "attention"
|
|
assert first["result"]["invalid_device_ids"] == ["P1"]
|
|
source["latest"]["Q1"]["time"] = "2026-07-16T08:15:00+08:00"
|
|
assert diagnosis_execution(source, period)["result"]["missing_device_ids"] == ["Q1"]
|
|
other = dict(source, replay_id=str(uuid4()))
|
|
assert diagnosis_execution(other, period)["execution_id"] != first["execution_id"]
|
|
|
|
|
|
class Repository:
|
|
def __init__(self):
|
|
self.document = document()
|
|
self.document["last_monitoring"] = monitoring()
|
|
self.project_id = self.document["project_id"]
|
|
self.connection = object()
|
|
self.tasks = {}
|
|
self.receipts = {}
|
|
self.state = None
|
|
self.events = []
|
|
|
|
@asynccontextmanager
|
|
async def control(self): yield
|
|
async def get(self, _): return deepcopy(self.document)
|
|
async def batch(self, *_): return {"sequence": 2, "scheduled_tasks": deepcopy(self.tasks), "periodic_retry_receipts": dict(self.receipts)}
|
|
async def periodic_events(self, *_): return deepcopy(self.events)
|
|
async def save_periodic_task(self, replay_id, sequence, execution): self.tasks[execution["execution_id"]] = deepcopy(execution)
|
|
async def simulation_state(self, replay_id, execution): self.state = deepcopy(execution)
|
|
async def periodic_task(self, replay_id, execution_id): return deepcopy(self.tasks.get(str(execution_id)))
|
|
async def save_retry_receipt(self, replay_id, sequence, request_id, execution_id): self.receipts[str(request_id)] = str(execution_id)
|
|
|
|
|
|
def test_completed_period_enqueues_once_without_an_agent_session(monkeypatch):
|
|
repo = Repository()
|
|
enqueue = AsyncMock(return_value={"job_id": uuid4()})
|
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(enqueue=enqueue))
|
|
context = SimpleNamespace(project_id=repo.project_id)
|
|
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
|
|
|
async def run():
|
|
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
|
|
await periodic_tasks.ensure_periodic_tasks(repo, context, repo.document["replay_id"])
|
|
asyncio.run(run())
|
|
assert len(repo.tasks) == 2
|
|
assert enqueue.await_count == 1
|
|
payload = enqueue.call_args.kwargs
|
|
assert payload["binding"] == {"kind": "scheduled_task", "project_id": repo.project_id}
|
|
assert payload["owner_user_id"] == repo.document["created_by"]
|
|
assert repo.state["status"] == "pending"
|
|
for task in repo.tasks.values(): PeriodicTaskExecution.model_validate(task)
|
|
|
|
|
|
def test_periodic_execution_reads_frozen_water_events_even_after_recovery(monkeypatch):
|
|
source, event = detected_water_event()
|
|
source["rule_states"]["water"]["event"] = None
|
|
source["last_monitoring"] = monitoring()
|
|
repo = Repository()
|
|
repo.document = source
|
|
repo.project_id = source["project_id"]
|
|
repo.events = [event]
|
|
read_events = AsyncMock(wraps=repo.periodic_events)
|
|
repo.periodic_events = read_events
|
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
|
enqueue=AsyncMock(return_value={"job_id": uuid4()})))
|
|
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
|
|
|
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), source["replay_id"]))
|
|
|
|
read_events.assert_awaited_once()
|
|
diagnosis = next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")
|
|
assert diagnosis["conclusion"] == "suspected_pollution"
|
|
assert diagnosis["investigation_task_id"] == event["task_id"]
|
|
|
|
|
|
def test_worker_interruption_is_reported_as_retryable_failure():
|
|
record = execution_record(document(), monitoring(), "network-simulation")
|
|
result = periodic_tasks.project_job(record, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}})
|
|
assert result["status"] == "failed"
|
|
assert "WORKER_INTERRUPTED" in result["summary"]
|
|
assert record["status"] == "pending"
|
|
|
|
|
|
@pytest.mark.parametrize("job", [None, {"status": "interrupted", "error": {"code": "WORKER_INTERRUPTED"}},
|
|
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}])
|
|
def test_failed_calculation_marks_its_execution_step_and_skips_dependent_steps(job):
|
|
record = execution_record(document(), monitoring(), "network-simulation")
|
|
record["steps"][0]["status"] = "completed"
|
|
original = deepcopy(record)
|
|
result = periodic_tasks.project_job(record, job)
|
|
assert result["status"] == "failed"
|
|
assert result["steps"][0]["status"] == "completed"
|
|
assert result["steps"][1]["status"] == "failed"
|
|
assert result["steps"][1]["detail"] == result["summary"]
|
|
assert all(step["status"] == "skipped" for step in result["steps"][2:])
|
|
assert record == original
|
|
|
|
|
|
def test_worker_failure_persists_failed_and_skipped_steps_in_the_business_report(monkeypatch):
|
|
repo = Repository()
|
|
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
|
|
record["steps"][0]["status"] = "completed"
|
|
record["calculation_job_id"] = str(uuid4())
|
|
repo.tasks[record["execution_id"]] = record
|
|
monkeypatch.setattr(periodic_tasks, "ScadaReplayRepository", lambda *_: repo)
|
|
row = {"project_id": repo.project_id, "job_id": record["calculation_job_id"], "parameters": {
|
|
"replay_id": record["replay_id"], "execution_id": record["execution_id"], "batch_sequence": record["batch_sequence"]}}
|
|
asyncio.run(periodic_tasks.persist_periodic_terminal(repo.connection, row,
|
|
{"status": "timed_out", "error": {"code": "EXECUTION_TIMEOUT"}}))
|
|
saved = repo.tasks[record["execution_id"]]
|
|
assert saved["status"] == repo.state["status"] == "failed"
|
|
assert [step["status"] for step in saved["steps"]] == ["completed", "failed", "skipped", "skipped", "skipped"]
|
|
assert saved["execution_id"] == record["execution_id"]
|
|
assert saved["batch_sequence"] == record["batch_sequence"]
|
|
|
|
|
|
@pytest.mark.parametrize("action", ["step", "monitor"])
|
|
def test_failed_period_must_be_retried_before_advancing_but_can_be_stopped(monkeypatch, action):
|
|
from tests.unit.test_scada_replay_service import setup
|
|
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
|
repo, service, _, _, create = setup(monkeypatch)
|
|
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
|
|
|
|
async def run():
|
|
replay = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document["config"]["scheduled_tasks_enabled"] = True
|
|
repo.document["last_scheduled_simulation"] = {"status": "failed", "error": "EXECUTION_TIMEOUT"}
|
|
original = deepcopy(repo.document)
|
|
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action=action,
|
|
**({"at_time": replay.start_time} if action == "monitor" else {}))
|
|
with pytest.raises(HTTPException) as error:
|
|
await service.command(replay.replay_id, command)
|
|
assert error.value.status_code == 409
|
|
assert "重试" in error.value.detail
|
|
assert repo.document == original
|
|
stopped = await service.command(replay.replay_id, ScadaReplayCommand(
|
|
request_id=uuid4(), expected_revision=replay.revision, action="stop"))
|
|
assert stopped.status == "stopped"
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_full_calculation_queue_preserves_completed_scada_diagnosis(monkeypatch):
|
|
repo = Repository()
|
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
|
enqueue=AsyncMock(side_effect=periodic_tasks.CalculationCapacityExceeded())))
|
|
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
|
asyncio.run(periodic_tasks.ensure_periodic_tasks(repo, SimpleNamespace(project_id=repo.project_id), repo.document["replay_id"]))
|
|
assert len(repo.tasks) == 2
|
|
assert next(task for task in repo.tasks.values() if task["task_key"] == "scada-diagnosis")["status"] == "completed"
|
|
assert repo.state["status"] == "failed"
|
|
assert repo.state["calculation_job_id"] is None
|
|
assert "尚未启动" in repo.state["summary"]
|
|
|
|
|
|
def test_retry_keeps_inputs_and_period_identity_and_is_idempotent(monkeypatch):
|
|
repo = Repository()
|
|
record = execution_record(repo.document, repo.document["last_monitoring"], "network-simulation")
|
|
record.update(calculation_job_id=str(uuid4()), status="failed")
|
|
repo.tasks[record["execution_id"]] = deepcopy(record)
|
|
enqueue = AsyncMock(return_value={"job_id": uuid4()})
|
|
monkeypatch.setattr(periodic_tasks, "CalculationJobRepository", lambda *_: SimpleNamespace(
|
|
enqueue=enqueue, get=AsyncMock(return_value={"status": "failed"})))
|
|
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view", "scada.clean", "simulation.run"})
|
|
request = uuid4()
|
|
async def run():
|
|
for _ in range(2):
|
|
await periodic_tasks.retry_periodic_task(repo, SimpleNamespace(project_id=repo.project_id),
|
|
repo.document["replay_id"], record["execution_id"], request)
|
|
asyncio.run(run())
|
|
retried = repo.tasks[record["execution_id"]]
|
|
assert retried["attempt"] == 2
|
|
assert retried["period_end"] == record["period_end"]
|
|
assert retried["batch_sequence"] == record["batch_sequence"]
|
|
assert enqueue.await_count == 1
|
|
|
|
|
|
def test_worker_rechecks_permission_before_reading_frozen_inputs(monkeypatch):
|
|
monkeypatch.setattr(periodic_tasks, "permissions_for_context", lambda _: {"scada.view"})
|
|
with pytest.raises(HTTPException) as error:
|
|
asyncio.run(periodic_tasks.execute_periodic_calculation(SimpleNamespace(project=object()), {}))
|
|
assert error.value.status_code == 403
|
|
|
|
|
|
@pytest.mark.parametrize("calculation_status", ["pending", "failed"])
|
|
def test_pending_hydraulics_preserves_original_command_receipt(monkeypatch, calculation_status):
|
|
from tests.unit.test_scada_replay_service import setup
|
|
from app.domain.schemas.scada_replay import ScadaReplayCommand
|
|
repo, service, _, _, create = setup(monkeypatch)
|
|
monkeypatch.setattr(periodic_tasks, "project_periodic_state", AsyncMock(side_effect=lambda _, value: value))
|
|
|
|
async def run():
|
|
replay = await service.create(create, monitoring_interval_seconds=900)
|
|
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=replay.revision, action="step")
|
|
accepted = await service.command(replay.replay_id, command)
|
|
repo.document["config"]["scheduled_tasks_enabled"] = True
|
|
repo.document["last_scheduled_simulation"] = {"status": calculation_status}
|
|
# A transport retry returns the original accepted period, even while its
|
|
# calculation is queued; a different step must wait.
|
|
repeated = await service.command(replay.replay_id, command)
|
|
assert repeated.revision == accepted.revision
|
|
with pytest.raises(HTTPException) as error:
|
|
await service.command(replay.replay_id, ScadaReplayCommand(
|
|
request_id=uuid4(), expected_revision=accepted.revision, action="step"))
|
|
assert error.value.status_code == 409
|
|
assert "水力计算" in error.value.detail
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_worker_routes_server_owned_periodic_jobs_without_an_agent_binding(monkeypatch, tmp_path):
|
|
from app.services import calculation_worker
|
|
from app.mcp.models import QueryResult
|
|
@asynccontextmanager
|
|
async def scope(_): yield SimpleNamespace(project="authorized-project")
|
|
monkeypatch.setattr(calculation_worker, "execution_scope", scope)
|
|
internal = AsyncMock(return_value=QueryResult(
|
|
source=periodic_tasks.PERIODIC_SIMULATION_TOOL, source_ref="periodic:test",
|
|
queried_at=datetime.now(timezone.utc), semantics="analysis", data={"status": "completed"}))
|
|
monkeypatch.setattr(periodic_tasks, "execute_periodic_calculation", internal)
|
|
ordinary = AsyncMock(side_effect=AssertionError("Periodic jobs must not restore an Agent binding"))
|
|
monkeypatch.setattr("app.mcp.calculation_jobs.execute_calculation_job", ordinary)
|
|
source, target = tmp_path / "job.json", tmp_path / "result.json"
|
|
source.write_text(json.dumps({"tool": periodic_tasks.PERIODIC_SIMULATION_TOOL}))
|
|
asyncio.run(calculation_worker.execute_child(str(source), str(target)))
|
|
assert json.loads(target.read_text())["status"] == "succeeded"
|
|
internal.assert_awaited_once()
|
|
ordinary.assert_not_awaited()
|