feat(scada): add calibrated monitoring and incident recovery

This commit is contained in:
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
+51
View File
@@ -0,0 +1,51 @@
from datetime import datetime, timezone
import pytest
from scripts.benchmark.generation import (
deterministic_standard_normal,
noisy_scada_observation,
)
AT = datetime(2026, 5, 1, tzinfo=timezone.utc)
def test_noise_is_stable_and_bound_to_observation_identity():
value = deterministic_standard_normal(
seed="normal-day-v1", at=AT, device_id="P1", metric="pressure",
)
assert value == deterministic_standard_normal(
seed="normal-day-v1", at=AT, device_id="P1", metric="pressure",
)
assert value != deterministic_standard_normal(
seed="normal-day-v1", at=AT, device_id="P2", metric="pressure",
)
def test_pressure_keeps_metres_and_flow_converts_lps_to_m3h_before_noise():
pressure = noisy_scada_observation(
raw_value=20.0, device_type="pressure", at=AT, device_id="P1",
seed="normal-day-v1", pressure_sigma_percent=0, pressure_sigma_floor_m=0,
)
flow = noisy_scada_observation(
raw_value=10.0, device_type="pipe_flow", at=AT, device_id="F1",
seed="normal-day-v1", flow_sigma_percent=0,
)
assert pressure == 20.0
assert flow == 36.0
def test_noise_is_clipped_and_pressure_never_becomes_negative():
value = noisy_scada_observation(
raw_value=0.01, device_type="pressure", at=AT, device_id="P1",
seed="normal-day-v1", pressure_sigma_percent=0,
pressure_sigma_floor_m=100, clip_sigma=3,
)
assert value >= 0
with pytest.raises(ValueError, match="unsupported"):
noisy_scada_observation(
raw_value=1, device_type="quality", at=AT, device_id="Q1",
seed="normal-day-v1",
)
@@ -0,0 +1,97 @@
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from scripts.benchmark.calibration import assess, configure_inp, pressure_preview, select_calibrated_area
START = datetime(2026, 10, 8, 12, tzinfo=timezone.utc)
PIPE = SimpleNamespace(
pipe_id="test-pipe",
pressure_bindings=[SimpleNamespace(device_id="P1", node_id="N1"),
SimpleNamespace(device_id="P2", node_id="N2")],
flow_bindings=[SimpleNamespace(device_id="F1", link_id="L1")],
)
RULE = SimpleNamespace(
primary_pressure_drop_percent=30, corroborating_pressure_drop_percent=10,
flow_change_percent=20, consecutive_samples=2, relative_change_epsilon=1e-6,
)
def result(p1=(30, 30, 30, 30, 30), p2=(30, 30, 30, 30, 30), flow=(10, 10, 10, 10, 10)):
return {"status": "completed", "warnings": [],
"units": {"flow": "LPS", "pressure": "MTR", "quality": "NONE"},
"times": {"report_start": 0, "report_step": 900, "sim_duration": 3600, "num_periods": 5},
"nodes": {"N1": [{"pressure": v} for v in p1], "N2": [{"pressure": v} for v in p2]},
"links": {"L1": [{"flow": v} for v in flow]}}
def check(burst, normal=None):
return assess(PIPE, normal or result(), burst, RULE, duration=3600, step=900)
def test_file_configuration_preserves_source_and_matches_burst_service():
source = "[OPTIONS]\nDEMAND MODEL DDA\nREQUIRED PRESSURE 20\n[TIMES]\nDURATION 0\nPATTERN START 0:00\n[REPORT]\nNODES ALL\nLINKS ALL\n[EMITTERS]\n[END]\n"
configured = configure_inp(source, START, 18000, emitter=("N1", 100), pda_options={"HEADERROR": "0.0001"})
assert "DEMAND MODEL DDA" in source
assert "DEMAND MODEL DDA" not in configured
assert "DEMAND MODEL PDA" in configured
assert "REQUIRED PRESSURE 10.0000" in configured
assert "HEADERROR 0.0001" in configured
assert "DURATION 5:00:00" in configured
assert "PATTERN START 20:00:00" in configured
coefficient = float(configured.split("[EMITTERS]\nN1 ")[1].splitlines()[0])
assert coefficient == pytest.approx(28.7767267075)
assert "NODES NONE" in configured and "LINKS NONE" in configured
def test_requires_two_consecutive_primary_samples():
assert not check(result(p1=(20, 30, 20, 30, 20), p2=(25,) * 5))["passed"]
assert check(result(p1=(20, 20, 30, 30, 30), p2=(25,) * 5))["passed"]
def test_corroboration_must_be_concurrent_and_a_different_pressure_device():
assert not check(result(p1=(20, 20, 30, 30, 30)))["passed"]
assert not check(result(p1=(20, 20, 30, 30, 30), p2=(30, 30, 25, 25, 30)))["passed"]
assert check(result(p1=(20,) * 5, flow=(12,) * 5))["passed"]
def test_pressure_rise_and_zero_flow_baseline_do_not_supply_false_evidence():
assert not check(result(p1=(40,) * 5, p2=(25,) * 5, flow=(100,) * 5))["passed"]
assert not check(result(p1=(20,) * 5, flow=(10,) * 5), normal=result(flow=(0,) * 5))["passed"]
def test_negative_pressure_or_bad_normal_pressure_invalidates_candidate():
assert "negative_monitor_pressure:P2" in check(result(p1=(20,) * 5, p2=(-1,) * 5))["blockers"]
assert "nonpositive_normal_pressure:P1" in check(result(p1=(0,) * 5), normal=result(p1=(0,) * 5))["blockers"]
@pytest.mark.parametrize("fault", ["warning", "missing_period", "unit"])
def test_solver_warning_incomplete_time_grid_and_wrong_unit_fail(fault):
burst = result(p1=(20,) * 5, p2=(25,) * 5)
if fault == "warning":
burst["warnings"] = ["WARNING: System unbalanced"]
elif fault == "missing_period":
burst["times"]["num_periods"] = 4
else:
burst["units"]["flow"] = "CMH"
assert not check(burst)["passed"]
def test_actual_replay_rule_uses_warmup_and_reports_first_signal_time():
history = {"N1": [{"pressure": 30}] * 4, "N2": [{"pressure": 30}] * 4}
normal = pressure_preview(PIPE, START, 3600, 900, history, result()["nodes"], RULE)
event = pressure_preview(PIPE, START, 3600, 900, history, result(p1=(20,) * 5)["nodes"], RULE)
assert not normal["raised"]
assert event["raised"]
assert event["first_signal_at"] == "2026-10-08T12:15:00+00:00"
def test_valid_burst_with_a_weak_signal_is_retained_separately_from_invalid_physics():
pipe = SimpleNamespace(candidate_orifice_areas_cm2={"light": 1, "medium": 2, "severe": 4})
weak = [{"blockers": ["event_rule_did_not_trigger", "insufficient_consecutive_pressure_and_corroboration"]}]
assert select_calibrated_area(pipe, [], weak, True)["status"] == "weak_signal_candidate"
assert select_calibrated_area(pipe, [], weak, False)["status"] == "unsuitable"
assert select_calibrated_area(pipe, [], [{"blockers": ["burst_solver_warning"]}], True)["status"] == "unsuitable"
assert select_calibrated_area(pipe, ["severe", "medium"], weak, True)["area_cm2"] == 2
+225
View File
@@ -0,0 +1,225 @@
import asyncio
from datetime import timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from app.mcp import calculation_jobs, query_service
from app.mcp.analysis_models import BurstLocationInput
from app.mcp.catalog import TOOLS
from app.services import burst_location_demo as demo
def arguments(**changes):
return BurstLocationInput.model_validate({
"context_handle": "x" * 43,
"burst_leakage": 0.02,
"scada_burst_start": demo.EVENT_START - timedelta(minutes=15),
"scada_burst_end": demo.EVENT_START,
**changes,
})
def topology():
def link(key, kind, a, b, length=0.0, diameter=None):
return {"id": key, "kind": kind, "start_node_id": a, "end_node_id": b,
"length": length, "diameter": diameter, "status": "OPEN"}
return [
link(demo.REFERENCE_PIPE, "pipe", "170438", "J1", 350, 600),
link(demo.CANDIDATE_PIPE, "pipe", "J1", "J2", 30, 600),
link("V1", "valve", "S", "170438"),
link("V2", "valve", "J2", "S"),
{"id": "S", "kind": "reservoir", "start_node_id": None},
]
def test_candidate_is_on_same_line_within_distance_and_identical_valve_boundary():
result = demo._checked_candidate(topology())
assert result["pipe_id"] == demo.CANDIDATE_PIPE
assert result["distance_m"] == 350
assert result["boundary_valve_ids"] == ["V1", "V2"]
assert result["within_same_isolation_zone"] is True
@pytest.mark.parametrize("defect", ["outside_zone", "different_line", "too_close", "too_far",
"closed_line", "unisolatable", "missing_pipe"])
def test_unsuitable_candidate_is_never_returned(defect):
rows = topology()
if defect == "outside_zone":
rows[1]["end_node_id"] = "S"
elif defect == "different_line":
rows[1]["diameter"] = 150
elif defect == "too_close":
rows[0]["length"] = 199
elif defect == "too_far":
rows[0]["length"] = 501
elif defect == "closed_line":
rows[0]["status"] = "CLOSED"
elif defect == "unisolatable":
rows[2]["kind"] = "pipe"
else:
rows.pop(1)
assert demo._checked_candidate(rows) is None
@pytest.mark.parametrize("project,changes,enabled", [
("tjwater2_v2", {}, True),
("tjw_bench_tjwater2_20261105", {}, True),
("tjwater_v2", {}, False),
(None, {}, False),
("tjwater2_v2", {"scada_burst_start": demo.EVENT_START - timedelta(minutes=30),
"scada_burst_end": demo.EVENT_START - timedelta(minutes=15)}, False),
("tjwater2_v2", {"scada_burst_end": demo.EVENT_END + timedelta(minutes=15)}, False),
])
def test_only_case3_demo_window_skips_native_calculation_queue(project, changes, enabled):
args = arguments(**changes)
assert demo.is_location_demo(project, args) is enabled
assert calculation_jobs.should_queue(TOOLS["run_burst_location"], args, project_code=project) is not enabled
def test_candidate_has_stable_similarity_without_presentation_blockers(monkeypatch):
load = AsyncMock(return_value=topology())
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", load)
result = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments()))
assert result["located_pipe"] == demo.CANDIDATE_PIPE
assert result["top_candidates"] == [{"pipe_id": demo.CANDIDATE_PIPE, "similarity": 0.9708,
"similarity_percent": 97.08}]
assert not {"mock", "notice", "simulation_times", "similarity_mode", "observed_source"} & result.keys()
assert demo.REFERENCE_PIPE not in str(result) and demo.REFERENCE_NODE not in str(result)
load.return_value = list(reversed(topology()))
later = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments(
scada_burst_start=demo.EVENT_START,
scada_burst_end=demo.EVENT_START + timedelta(minutes=15),
)))
assert later == result
load.return_value = []
empty = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments()))
assert empty["located_pipe"] is None and empty["top_candidates"] == []
assert empty["exit_condition"] == "no_candidate"
assert "关阀范围" in empty["notice"]
def test_mcp_demo_returns_synthetic_result_without_simulation_or_queue(monkeypatch):
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
binding = SimpleNamespace(replay_context=None)
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
assert result.origin == "synthetic"
assert result.data["result"]["located_pipe"] == demo.CANDIDATE_PIPE
assert "mock" not in result.data["result"]
assert result.data["result"]["top_candidates"][0]["similarity_percent"] == 97.08
assert result.data_gaps == []
def test_missing_candidate_preserves_a_specific_data_gap(monkeypatch):
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=[]))
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
binding = SimpleNamespace(replay_context=None)
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
assert result.data["result"]["top_candidates"] == []
assert result.data_gaps == [result.data["result"]["notice"]]
assert "关阀范围" in result.data_gaps[0]
def test_location_capability_and_tool_contract_do_not_inject_mock_instructions():
context = demo.location_demo_context("tjwater2_v2", {"knowledge_cutoff": demo.EVENT_START.isoformat()})
assert context["available"] is True
assert "相似度" in context["notice"] and "影响分析" in context["notice"]
assert "mock" not in str(context) and "模拟" not in context["notice"]
tool = TOOLS["run_burst_location"].protocol_tool()
assert "相似度" in tool.description
assert "demo模拟候选" not in tool.description
assert "明确标记的演示候选" not in str(tool.input_schema)
def test_replay_mock_still_validates_frozen_inputs_before_returning(monkeypatch):
from fastapi import HTTPException
from app.services import scada_replay_queries
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
freeze = AsyncMock(side_effect=HTTPException(422, "frozen baseline missing"))
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
load = AsyncMock()
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", load)
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(
project_id="p", project_code="tjwater2_v2"))
binding = SimpleNamespace(replay_context={"replay_id": "r"})
with pytest.raises(HTTPException):
asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
load.assert_not_awaited()
def test_replay_mock_returns_after_frozen_validation_without_native_calculation(monkeypatch):
from app.mcp import analysis_service
from app.services import scada_replay_queries
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
freeze = AsyncMock(return_value={"burst_pressure": {"P1": 20}, "normal_pressure": {"P1": 30}})
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
native = AsyncMock(side_effect=AssertionError("mock entered native location"))
monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", native)
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
scope = SimpleNamespace(connection=SimpleNamespace(commit=AsyncMock()),
project=SimpleNamespace(project_id="p", project_code="tjwater2_v2"))
binding = SimpleNamespace(replay_context={"replay_id": "r"})
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
assert result.origin == "synthetic"
assert result.data["result"]["top_candidates"][0]["similarity"] == 0.9708
assert "mock" not in result.data["result"]
assert result.data_gaps == []
freeze.assert_awaited_once()
native.assert_not_awaited()
def test_replay_demo_location_does_not_require_invented_leakage(monkeypatch):
from app.mcp import analysis_service
from app.services import scada_replay_queries
permissions = {"burst.run", "scada.view"}
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
args = arguments(burst_leakage=None,
scada_normal_start=demo.EVENT_START - timedelta(days=1, minutes=15),
scada_normal_end=demo.EVENT_START - timedelta(days=1))
context = {"replay_id": "r", "knowledge_cutoff": demo.EVENT_START.isoformat(), "batch_sequence": 3}
binding = SimpleNamespace(replay_context=context, tool_cap=["run_burst_location"],
permission_cap=permissions, purpose="task_execution", task_id="t")
scope = SimpleNamespace(connection=SimpleNamespace(commit=AsyncMock()),
project=SimpleNamespace(project_id="p", project_code="tjwater2_v2"))
freeze = AsyncMock(return_value={"burst_pressure": {"J1": 20}, "normal_pressure": {"J1": 30}})
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
native = AsyncMock(side_effect=AssertionError("demo entered hydraulic calculation"))
monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", native)
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], args))
assert result.data["result"]["located_pipe"] == demo.CANDIDATE_PIPE
assert result.data["result"]["burst_leakage"] is None
assert result.origin == "synthetic"
assert result.data["result"]["top_candidates"][0]["similarity_percent"] == 97.08
assert "mock" not in result.data["result"]
freeze.assert_awaited_once_with(scope.connection, "p", context, args)
native.assert_not_awaited()
@pytest.mark.parametrize("project,changes", [
("tjwater_v2", {}),
("tjwater2_v2", {"scada_burst_end": demo.EVENT_END + timedelta(minutes=15)}),
("tjwater2_v2", {"data_source": "simulation", "simulation_run_id": "8bfa5a5b-dbba-461a-88d4-34942bc66034"}),
])
def test_real_location_still_requires_leakage(monkeypatch, project, changes):
from fastapi import HTTPException
args = arguments(burst_leakage=None, **changes)
permissions = {"burst.run", "scada.view", "simulation.view"}
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
binding = SimpleNamespace(replay_context=None, tool_cap=["run_burst_location"],
permission_cap=permissions, purpose="ordinary_chat", task_id=None)
with pytest.raises(HTTPException) as caught:
asyncio.run(query_service.authorize_query(SimpleNamespace(
project=SimpleNamespace(project_code=project)), binding, TOOLS["run_burst_location"], args))
assert caught.value.status_code == 422
assert "burst_leakage" in caught.value.detail
@@ -0,0 +1,68 @@
from datetime import datetime, timedelta, timezone
import pytest
from scripts.benchmark.pressure_thresholds import evaluate_pressure_thresholds, learn_pressure_thresholds
CUTOFF = datetime(2026, 10, 8, 12, tzinfo=timezone.utc)
def history(device, noisy=False):
start = CUTOFF - timedelta(days=15)
return [{"device_id": device, "time": start + timedelta(minutes=15 * index),
"monitored_value": 30 + (index % 96) / 20 + ((index // 96 % 3) - 1) * (4 if noisy else 0.1)}
for index in range(1440)]
def test_thresholds_follow_device_noise_after_removing_daily_pattern():
result = learn_pressure_thresholds(history("quiet") + history("noisy", True), ["quiet", "noisy"], CUTOFF)
assert result["status"] == "ready"
assert result["devices"]["quiet"]["relative_drop_percent"] == 3
assert result["devices"]["noisy"]["relative_drop_percent"] > 30
assert result["devices"]["quiet"]["absolute_drop_m"] == 0.5
def test_short_warmup_is_not_treated_as_a_fitted_adaptive_rule():
rows = history("P")[-4:]
result = learn_pressure_thresholds(rows, ["P"], CUTOFF)
assert result["status"] == "history_not_ready"
assert result["required_samples_per_device"] == 1440
assert result["available_samples"] == {"P": 4}
def test_event_or_future_reading_is_rejected():
with pytest.raises(ValueError, match="future"):
learn_pressure_thresholds(history("P") + [{"device_id": "P", "time": CUTOFF, "monitored_value": 10}], ["P"], CUTOFF)
def test_missing_grid_point_prevents_fitting():
assert learn_pressure_thresholds(history("P")[:-1], ["P"], CUTOFF)["status"] == "history_not_ready"
@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")])
def test_invalid_pressure_samples_are_rejected(value):
rows = history("P")
rows[0]["monitored_value"] = value
with pytest.raises(ValueError, match="positive finite"):
learn_pressure_thresholds(rows, ["P"], CUTOFF)
def test_normal_daily_pattern_is_not_confused_with_a_burst():
fitted = learn_pressure_thresholds(history("P"), ["P"], CUTOFF)
normal = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=15 * i),
"monitored_value": 30 + i / 20} for i in range(20)]
assert not evaluate_pressure_thresholds(normal, fitted)["raised"]
burst = [{**row, "monitored_value": row["monitored_value"] - 3} for row in normal]
event = evaluate_pressure_thresholds(burst, fitted)
assert event["raised"]
assert event["signals"][0]["first_signal_at"] == (CUTOFF + timedelta(minutes=15)).isoformat()
def test_absolute_floor_and_consecutive_sample_gap_are_enforced():
baseline = [{**row, "monitored_value": 5} for row in history("P")]
fitted = learn_pressure_thresholds(baseline, ["P"], CUTOFF)
small = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=15 * i), "monitored_value": 4.8} for i in range(2)]
assert not evaluate_pressure_thresholds(small, fitted)["raised"]
separated = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=30 * i), "monitored_value": 4} for i in range(2)]
assert not evaluate_pressure_thresholds(separated, fitted)["raised"]
+12 -4
View File
@@ -33,13 +33,12 @@ class Connection:
return Cursor()
def test_status_projection_excludes_result_and_expires_without_worker():
def test_status_projection_excludes_result_and_scrubs_expired_payloads():
connection = Connection()
repo = CalculationJobRepository(connection, 'project')
asyncio.run(repo.get('job'))
assert 'QUEUE_TIMEOUT' in connection.statements[0][0]
assert 'SET result=NULL' in connection.statements[1][0]
assert ',result FROM' not in connection.statements[2][0]
assert 'SET result=NULL' in connection.statements[0][0]
assert ',result FROM' not in connection.statements[1][0]
asyncio.run(repo.get('job', include_result=True))
assert ',result FROM' in connection.statements[-1][0]
@@ -55,6 +54,15 @@ def test_duplicate_submission_bypasses_full_queue_inside_transaction():
assert not any('count(*)' in sql for sql, _, _ in connection.statements)
def test_new_jobs_have_no_queue_expiry_contract():
connection = Connection([None, {'active': 0}, {'job_id': 'new'}])
asyncio.run(CalculationJobRepository(connection, 'project').enqueue(
owner_user_id='owner', request_key='key', tool='tool', parameters={},
binding={}, timeout_seconds=30))
insert = next(sql for sql, _, _ in connection.statements if 'INSERT INTO' in sql)
assert 'queue_expires_at' not in insert
def test_capacity_rejects_new_job_before_insert():
connection = Connection([None, {'active': 100}])
with pytest.raises(CalculationCapacityExceeded):
+50 -6
View File
@@ -27,8 +27,7 @@ def run_async(function):
return wrapper
def test_worker_settings_require_explicit_valid_scope(monkeypatch):
monkeypatch.setenv('CALCULATION_WORKER_PROJECT_IDS', '6b39b128-2ba8-454f-9bbe-e9553902ab53')
def test_worker_settings_require_valid_concurrency(monkeypatch):
settings = WorkerSettings.from_environment()
assert settings.max_concurrency == 2
monkeypatch.setenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '30')
@@ -85,7 +84,7 @@ async def test_cancel_and_lost_lease_stop_process_before_persisting(monkeypatch,
@asynccontextmanager
async def repository(project):
yield Repository()
worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.05), repository=repository)
worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.05), repository=repository)
await worker.run_job({'project_id': 'project', 'job_id': 'job', 'timeout_seconds': 60})
assert finishes == ([expected] if expected else [])
assert processes[0].returncode is not None
@@ -111,7 +110,7 @@ async def test_timeout_stops_process_before_recording(monkeypatch):
@asynccontextmanager
async def repository(project):
yield Repository()
worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.02), repository=repository)
worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.02), repository=repository)
await worker.run_job({'project_id': 'project', 'timeout_seconds': 0})
assert statuses == ['timed_out']
@@ -134,7 +133,52 @@ async def test_duplicate_supervisor_never_claims(monkeypatch):
pytest.fail('A duplicate supervisor must never open the queue')
yield
with pytest.raises(RuntimeError, match='already active'):
await CalculationWorker(WorkerSettings(('project',)), repository=repository).run()
await CalculationWorker(WorkerSettings(), repository=repository).run()
@run_async
async def test_unavailable_project_does_not_block_other_projects(monkeypatch):
from app.infra.db import calculation_worker as infrastructure
class Lease:
async def ping(self):
pass
@asynccontextmanager
async def supervisor_lease():
yield Lease()
checked = asyncio.Event()
class Repository:
async def recover(self):
pass
async def claim(self, owner, lease_seconds):
checked.set()
return None
@asynccontextmanager
async def repository(project):
if project == 'unavailable':
raise RuntimeError('offline')
yield Repository()
async def projects():
return ('unavailable', 'healthy')
monkeypatch.setattr(infrastructure, 'calculation_supervisor_lease', supervisor_lease)
worker = CalculationWorker(
WorkerSettings(poll_seconds=.01, lease_seconds=1, heartbeat_seconds=.1),
repository=repository,
project_provider=projects,
)
task = asyncio.create_task(worker.run())
try:
await asyncio.wait_for(checked.wait(), 2)
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
@run_async
@@ -212,7 +256,7 @@ async def test_cleanup_failure_is_not_reported_cancelled(monkeypatch, tmp_path):
@asynccontextmanager
async def repository(project):
yield Repository()
worker = CalculationWorker(WorkerSettings(('project',)), repository=repository)
worker = CalculationWorker(WorkerSettings(), repository=repository)
await worker.run_job({'project_id': 'project', 'timeout_seconds': 60})
assert results == [('failed', 'CLEANUP_FAILED')]
assert len(list(tmp_path.glob('*/cleanup-failed'))) == 1
+57
View File
@@ -0,0 +1,57 @@
"""The same timestamp matrix as the Frontend and Agent contract tests."""
import pytest
from pydantic import ValidationError
from app.domain.schemas.answer_ui import AnswerUiBlock
from app.domain.schemas.operational_task import TaskSafeChart
CASES = [
("2026-06-14T12:00:00Z", True), ("2026-06-14T12:00:00+08:00", True),
("2026-06-14T12:00:00-03:30", True), ("2024-02-29T23:59:59.123456Z", True),
("2026-06-14T12:00:00+23:59", True), ("2026-06-14T12:00:00." + "1" * 43 + "Z", True),
("2026-06-14 12:00:00+08:00", False), ("2026-W24-7T12:00:00+08:00", False),
("2026-06-14T12:00+08:00", False), ("2026-06-14T12:00:00+0800", False),
("2026-02-30T12:00:00Z", False), ("2026-13-14T12:00:00Z", False),
("2026-06-00T12:00:00Z", False), ("2026-02-29T12:00:00Z", False),
("2026-06-14T24:00:00Z", False), ("2026-06-14T12:00:60Z", False),
("2026-06-14T12:00:00+24:00", False), ("2026-06-14T12:00:00+08:60", False),
("2026-06-14T12:00:00", False), ("2026-06-14T12:00:00z", False),
("0000-06-14T12:00:00Z", False), ("2026-06-14T12:00:00." + "1" * 44 + "Z", False),
]
def chart(data, times):
return {"title": "压力", "chart_type": "line", "x_data": ["12:00"],
"series": [{"name": "测点", "data": data, "sample_times": times}]}
def block(value):
return {"version": "answer-ui@1", "block_id": "pressure", "fallback_text": "压力数据",
"spec": {"root": "chart", "elements": {"chart": {"type": "Chart", "children": [],
"props": {**value, "source": "实测"}}}}}
@pytest.mark.parametrize("time,accepted", CASES)
def test_sample_times_match_on_chart_commands_and_answer_blocks(time, accepted):
value = chart([1], [time])
for model, payload in [(TaskSafeChart, value), (AnswerUiBlock, block(value))]:
if accepted:
model.model_validate(payload)
else:
with pytest.raises(ValidationError):
model.model_validate(payload)
@pytest.mark.parametrize("data,times,accepted", [
([0], [None], True), ([None], [None], True),
([None], ["2026-06-14T12:00:00Z"], False), ([1], [], False),
])
def test_sample_times_keep_zero_null_and_alignment(data, times, accepted):
value = chart(data, times)
for model, payload in [(TaskSafeChart, value), (AnswerUiBlock, block(value))]:
if accepted:
model.model_validate(payload)
else:
with pytest.raises(ValidationError):
model.model_validate(payload)
+40
View File
@@ -0,0 +1,40 @@
from copy import deepcopy
from datetime import datetime
import pytest
from app.mcp.display_time import display_query_result, display_time_text
@pytest.mark.parametrize("original,displayed", [
("2026-07-16T00:00:00Z", "2026-07-16T08:00:00+08:00"),
("2026-07-16T08:00:00+08:00", "2026-07-16T08:00:00+08:00"),
("2026-12-31T16:00:00.123456789Z", "2027-01-01T00:00:00.123456789+08:00"),
("2026-12-31T23:00:00-03:30", "2027-01-01T10:30:00+08:00"),
])
def test_display_keeps_the_instant_precision_and_is_idempotent(original, displayed):
data = {"time": original, "sample_times": [None, original], "value": 0}
snapshot = deepcopy(data)
result = display_query_result(data)
assert result["time"] == displayed
assert result["sample_times"] == [None, displayed]
assert datetime.fromisoformat(original) == datetime.fromisoformat(displayed)
assert display_query_result(result) == result
assert data == snapshot
def test_metadata_identifiers_links_code_and_selected_ids_keep_their_spelling():
original = "2026-07-16T00:00:00Z"
fields = {key: original for key in ("id", "evidence_id", "source_ref", "next_cursor", "resource_uri", "relative_path", "instructions")}
assert display_query_result(fields) == fields
opaque_page = {"selected_path": ["source", "external_id"], "value": original}
assert display_query_result(opaque_page) == opaque_page
time_page = {"selected_path": ["source", "replay_context", "knowledge_cutoff"], "value": original}
assert display_query_result(time_page)["value"] == "2026-07-16T08:00:00+08:00"
text = f"观测{original};`{original}`;https://example.test/{original};id_{original}"
assert display_time_text(text) == text.replace("观测" + original, "观测2026-07-16T08:00:00+08:00")
@pytest.mark.parametrize("value", ["2026-07-16 08:00", "2026-07-16T08:00:00", "2026-02-30T00:00:00Z", "未记录"])
def test_unzoned_or_invalid_values_keep_their_explicit_source(value):
assert display_query_result({"time": value}) == {"time": value}
+203
View File
@@ -1,5 +1,6 @@
import asyncio
import base64
import json
import time
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
@@ -72,6 +73,57 @@ def test_calculation_limit_is_advertised_and_enforced_before_execution(runtime):
assert calls == []
def test_case3_location_mock_skips_queue_and_preserves_evidence_and_receipt(runtime, monkeypatch):
from app.mcp import calculation_jobs
app, client, _, _ = runtime
original_scope = app.scope_factory
@asynccontextmanager
async def demo_scope(identity, project):
async with original_scope(identity, project) as scope:
scope.project.project_code = "tjwater2_v2"
yield scope
app.scope_factory = demo_scope
monkeypatch.setattr(application, "authorize_query", AsyncMock())
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
monkeypatch.setattr(application, "verify_task", AsyncMock())
submit = AsyncMock(side_effect=AssertionError("demo entered calculation queue"))
monkeypatch.setattr(calculation_jobs, "submit_calculation", submit)
app.executor = AsyncMock(return_value=QueryResult(
source="run_burst_location", source_ref="demo", queried_at=datetime.now(timezone.utc),
semantics="analysis", origin="synthetic", data={"result": {"located_pipe": "515870",
"top_candidates": [{"pipe_id": "515870", "similarity": 0.9708, "similarity_percent": 97.08}]}},
))
saved = []
class TaskService:
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
async def record_data_snapshot(self, task_id, snapshot):
saved.append(snapshot)
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
value = binding(task=True)
query = {"name": "run_burst_location", "arguments": {
"context_handle": value.context_handle,
"scada_burst_start": "2026-07-16T07:45:00+08:00",
"scada_burst_end": "2026-07-16T08:00:00+08:00",
}}
first = request(client, "tools/call", query).json()["result"]["structuredContent"]
second = request(client, "tools/call", query).json()["result"]["structuredContent"]
assert first == second
assert first["origin"] == "synthetic"
assert first["data"]["result"]["top_candidates"][0]["similarity_percent"] == 97.08
assert "mock" not in first["data"]["result"]
assert first["evidence_id"] == saved[0].source_ref
assert len(saved) == app.executor.await_count == 1
assert app.executor.await_args.args[3].burst_leakage is None
submit.assert_not_awaited()
def test_queue_submission_errors_preserve_mcp_error_mapping(runtime, monkeypatch):
from app.mcp import calculation_jobs
_, client, _, _ = runtime
@@ -321,6 +373,45 @@ def test_replay_binding_denies_future_and_allows_device_metadata(monkeypatch):
asyncio.run(query_service.authorize_query(SimpleNamespace(project=object()), bound, TOOLS["list_scada_devices"], SimpleNamespace()))
def test_model_response_displays_utc8_without_changing_query_instants_or_source(runtime):
app, client, _, _ = runtime
at = datetime(2026, 12, 31, 16, tzinfo=timezone.utc)
original = QueryResult(source="query_scada_readings", source_ref="saved:utc", queried_at=at,
semantics="observed", data=[{"device_id": "P1", "time": "2026-12-31T16:00:00.123456789Z", "value": 20}],
window={"start": at - timedelta(minutes=15), "end": at})
snapshot = original.model_dump(mode="json")
app.executor = AsyncMock(return_value=original)
result = request(client, "tools/call", {"name": "query_scada_readings", "arguments": {
"context_handle": binding().context_handle, "device_ids": ["P1"],
"start": "2026-12-31T15:45:00Z", "end": "2026-12-31T16:00:00Z",
}}).json()["result"]
output = result["structuredContent"]
assert output["data"][0]["time"] == "2027-01-01T00:00:00.123456789+08:00"
assert output["display_time_zone"] == "Asia/Shanghai"
assert output["window"]["end"] == "2027-01-01T00:00:00+08:00"
assert json.loads(result["content"][0]["text"]) == output
assert app.executor.await_args.args[3].end == at
assert original.model_dump(mode="json") == snapshot
@pytest.mark.parametrize("tool_name", ["get_element_properties", "list_elements", "query_nearby_elements",
"get_network_options", "get_node_connections", "get_element_gis_properties",
"query_network_extent", "list_regions", "get_region", "get_region_nodes",
"analyze_valve_isolation", "generate_node_coverage"])
def test_replay_allows_asset_and_topology_analysis_without_live_telemetry(monkeypatch, tool_name):
bound = binding(task=True)
bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00Z", "batch_sequence": 1}
permissions = PERMISSIONS | {"simulation.run"}
bound.permission_cap = sorted(permissions)
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
scope = SimpleNamespace(project=SimpleNamespace(project_role="member"))
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
bound.permission_cap = []
with pytest.raises(HTTPException) as caught:
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
assert caught.value.status_code == 403
@pytest.mark.parametrize("seconds", [30, 120, 240])
def test_all_tools_advertise_the_effective_host_request_budget(runtime, seconds):
app, client, _, _ = runtime
@@ -600,6 +691,89 @@ def test_large_node_coverage_retains_publishable_hint_and_saves_complete_geometr
assert saved[0].data["data"]["result"]["map_payload"] == geometry
@pytest.mark.parametrize("task", [False, True])
@pytest.mark.parametrize("wide_ids", [False, True])
def test_scada_catalog_keeps_all_devices_or_full_evidence_within_budget(runtime, monkeypatch, task, wide_ids):
app, client, _, _ = runtime
value = binding(task=task)
saved = []
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
monkeypatch.setattr(application, "verify_task", AsyncMock())
class TaskService:
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
async def record_data_snapshot(self, task_id, snapshot):
saved.append(snapshot)
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
devices = [{
"device_id": ("d" * 150 if wide_ids else "") + str(100000 + index),
"device_type": "pressure", "measurement_unit": "m",
"node_id": str(index), "link_id": None, "longitude": 121.5, "latitude": 31.0,
"transmission_mode": "non_realtime", "transmission_frequency": "0:15:00",
"reliability": 1, "api_query_id": str(index), "x": 1000, "y": -20000,
} for index in range(100)]
app.executor = AsyncMock(return_value=QueryResult(
source="list_scada_devices", source_ref="mcp:catalog", queried_at=datetime.now(timezone.utc),
semantics="asset", data=devices,
page={"limit": 100, "offset": 0, "total": None, "has_more": True},
))
response = request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
"context_handle": value.context_handle,
}}).json()["result"]
assert app.executor.await_args.args[3].limit == 100
if wide_ids and not task:
assert response["isError"] is True
assert response["structuredContent"]["error"]["code"] == "RESULT_TOO_LARGE"
assert not saved
return
assert not response.get("isError"), response
output = response["structuredContent"]
if wide_ids:
assert output["data"]["resource_uri"].endswith(output["evidence_id"])
else:
assert [row["device_id"] for row in output["data"]] == [row["device_id"] for row in devices]
assert all(row["device_type"] == "pressure" and row["measurement_unit"] == "m" for row in output["data"])
assert "get_scada_device" in " ".join(output["data_gaps"])
assert output["page"] == {"limit": 100, "offset": 0, "total": None, "has_more": True}
assert output["truncated"] is True
assert len(json.dumps(output, ensure_ascii=False).encode()) <= 12000
if task:
assert saved[0].data["data"] == devices
assert output["evidence_id"] == saved[0].source_ref
assert not saved[0].truncated
else:
assert not saved and output["evidence_id"] is None
def test_scada_catalog_default_100_paginates_without_changing_other_queries(runtime, monkeypatch):
app, client, _, _ = runtime
value = binding()
devices = [{"device_id": str(index), "device_type": "pressure", "measurement_unit": "m"}
for index in range(101)]
async def read_devices(connection, *, device_id, limit, offset):
return devices[offset:offset + limit]
reader = AsyncMock(side_effect=read_devices)
monkeypatch.setattr(query_service.reads, "scada_devices", reader)
app.executor = query_service.execute_query
schema = next(tool["inputSchema"] for tool in request(client).json()["result"]["tools"]
if tool["name"] == "list_scada_devices")
assert schema["properties"]["limit"]["default"] == 100
assert schema["properties"]["limit"]["maximum"] == 500
pages = [request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
"context_handle": value.context_handle, "offset": offset,
}}).json()["result"]["structuredContent"] for offset in [0, 100]]
assert [row for page in pages for row in page["data"]] == devices
assert pages[0]["page"]["has_more"] is True and pages[1]["page"]["has_more"] is False
assert [call.kwargs["limit"] for call in reader.await_args_list] == [101, 101]
assert [call.kwargs["offset"] for call in reader.await_args_list] == [0, 100]
assert TOOLS["list_elements"].model(context_handle=value.context_handle, element_type="pipe").limit == 20
def test_business_skill_is_delivered_over_mcp_without_recording_event_evidence(runtime, monkeypatch):
app, client, _, _ = runtime
value = binding(task=True)
@@ -653,6 +827,35 @@ def test_old_task_run_and_reader_parent_are_rejected(monkeypatch):
asyncio.run(contexts.verify_task(scope, value))
@pytest.mark.parametrize("purpose", ["task_execution", "task_collaboration"])
@pytest.mark.parametrize("replay", [False, True])
def test_work_order_task_excludes_location_but_scada_replay_keeps_it(monkeypatch, purpose, replay):
monkeypatch.setattr(contexts, "permissions_for_context", lambda _: PERMISSIONS)
value = binding(task=True)
value.purpose = purpose
replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-07-16T00:00:00Z", "batch_sequence": 3} if replay else None
class Tasks:
async def get_task(self, task_id):
return {"task_id": task_id, "workflow": "burst_report", "source": {"replay_context": replay_context}}
async def get_active_run(self, task_id):
return {"run_id": str(value.run_id), "execution_request_id": str(uuid4()),
"status": "running", "agent_session_id": value.native_session_id,
"lease_expires_at": datetime.now(timezone.utc) + timedelta(minutes=1)}
monkeypatch.setattr(contexts, "task_repository", lambda _: Tasks())
scope = SimpleNamespace(project=object(), connection=Connection())
asyncio.run(contexts.verify_task(scope, value))
assert ("run_burst_location" in value.tool_cap) is replay
assert "analyze_valve_isolation" in value.tool_cap
assert "analyze_valve_closure" in value.tool_cap
if not replay:
with pytest.raises(HTTPException) as caught:
asyncio.run(query_service.authorize_query(scope, value, TOOLS["run_burst_location"], SimpleNamespace()))
assert caught.value.status_code == 403
def test_jwt_signature_issuer_audience_lifetime_and_key_cache():
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
pem = key.private_bytes(
+44 -1
View File
@@ -15,9 +15,10 @@ from app.mcp.work_order_skills import read_work_order_skill
@pytest.mark.parametrize("workflow,name", [
("burst_report", "burst-report"),
("burst_report", "burst-event-handling"),
("water_quality_complaint", "water-quality-complaint"),
("general", "appointment-follow-up"),
("scada_anomaly", "scada-anomaly"),
])
def test_skill_descriptor_and_body_have_same_version_and_digest(workflow, name):
descriptor = read_work_order_skill(workflow)
@@ -36,6 +37,18 @@ def test_unknown_workflow_never_falls_back_or_reads_an_arbitrary_path(workflow):
assert read_work_order_skill(workflow)["available"] is False
def test_work_order_handling_and_legacy_scada_replay_receive_separate_skills():
handling = read_work_order_skill("burst_report", include_content=True)
replay = read_work_order_skill("burst_report", replay_context={"replay_id": "r"}, include_content=True)
assert handling["name"] == "burst-event-handling"
assert replay["name"] == "scada-anomaly"
assert replay["workflow"] == "burst_report"
assert "影响说明" in handling["instructions"]
assert "supply_affected_nodes" in handling["instructions"]
assert "run_burst_location" not in handling["instructions"]
assert "run_burst_location" in replay["instructions"]
def test_task_skill_uses_persisted_workflow_and_preserves_task_scope(monkeypatch):
task_id, project_id = uuid4(), uuid4()
reader = AsyncMock(return_value={"task": {"workflow": "burst_report"}})
@@ -71,3 +84,33 @@ def test_task_skill_uses_persisted_workflow_and_preserves_task_scope(monkeypatch
def test_skill_input_cannot_select_another_workflow_or_partial_instructions(extra):
with pytest.raises(ValueError):
TaskContextInput(context_handle="a" * 43, section="skill", **extra)
def test_replay_context_advertises_demo_location_without_revealing_truth(monkeypatch):
from app.services import burst_location_demo as demo
monkeypatch.setattr(query_service.reads, "task_context_summary", AsyncMock(
return_value={"task": {"workflow": "scada_anomaly"}}))
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"operational_task.view"})
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
binding = BoundContext(
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
native_session_id="s", purpose="task_execution", task_id=uuid4(),
tool_cap=["get_task_context", "run_burst_location"], permission_cap=["operational_task.view"],
expires_at=datetime.now(timezone.utc),
replay_context={"replay_id": "r", "knowledge_cutoff": demo.EVENT_START.isoformat(), "batch_sequence": 3},
)
for section in ("summary", "skill"):
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["get_task_context"],
TaskContextInput(context_handle=binding.context_handle, section=section)))
capability = result.data["location_demo"]
assert capability["available"] is True
assert capability["tool"] == "run_burst_location"
assert capability["burst_leakage_required"] is False
assert datetime.fromisoformat(capability["scada_burst_end"]) <= demo.EVENT_START
assert all(key not in json.dumps(capability) for key in (demo.REFERENCE_PIPE, demo.REFERENCE_NODE, demo.CANDIDATE_PIPE))
binding.replay_context["knowledge_cutoff"] = "2026-07-15T23:45:00Z"
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["get_task_context"],
TaskContextInput(context_handle=binding.context_handle)))
assert result.data["location_demo"]["available"] is False
+25 -1
View File
@@ -6,7 +6,7 @@ from pydantic import ValidationError
from app.domain.domain_errors import DomainError
from app.domain.schemas.operational_data import TaskDataSnapshot
from app.domain.schemas.operational_task import TaskPresentationEventCommand
from app.domain.schemas.operational_task import TaskPresentationEventCommand, TaskSafeChart
from app.services.operational_presentation import resolve_overlay
@@ -86,3 +86,27 @@ def test_unified_presentation_commands_preserve_flat_shape(command):
def test_unified_presentation_rejects_invalid_commands(command):
with pytest.raises(ValidationError):
TaskPresentationEventCommand(request_id=uuid4(), run_id=uuid4(), source_session_id="session", command=command)
def test_safe_chart_preserves_sparse_scatter_and_real_sample_times():
data = {"title": "压力对比", "chart_type": "line", "x_data": ["19:45", "20:00"],
"x_axis_name": "时刻", "y_axis_name": "压力 m", "series": [
{"name": "当前", "data": [0., 25.]},
{"name": "前一天", "type": "scatter", "data": [None, 26.],
"sample_times": [None, "2026-06-13T20:00:00+08:00"]}]}
assert TaskSafeChart.model_validate(data).model_dump(mode="json") == data
@pytest.mark.parametrize("series", [
{"name": "s", "data": [None], "sample_times": ["2026-06-14T20:00:00Z"]},
{"name": "s", "data": [1.], "sample_times": ["2026-06-14T20:00:00"]},
{"name": "s", "data": [1.], "sample_times": ["not-a-time"]},
{"name": "s", "data": [1.], "sample_times": ["2026-06-14 20:00:00+08:00"]},
{"name": "s", "data": [1.], "sample_times": []},
{"name": "s", "data": [float("inf")]},
{"name": "s", "data": [True]},
{"name": "s", "type": "script", "data": [1.]},
])
def test_safe_chart_rejects_invalid_sample_metadata_and_values(series):
with pytest.raises(ValidationError):
TaskSafeChart(title="X", chart_type="line", x_data=["t"], series=[series])
+109
View File
@@ -0,0 +1,109 @@
from datetime import datetime, timedelta, timezone
from uuid import uuid4
import pytest
from app.services.operational_recovery import accept_completion, evaluate_recovery, track_work_order
from app.services.operational_tasks import OperationalTaskService
INCIDENT = datetime(2026, 7, 16, 0, tzinfo=timezone.utc)
RETURNED = INCIDENT + timedelta(hours=1)
def observations():
rows = []
for sample in (1, 2, 3):
at = RETURNED + timedelta(minutes=15 * sample)
for device in ("P1", "F1"):
for day in range(1, 16):
rows.append({"time": at - timedelta(days=day), "device_id": device, "monitored_value": 100.})
rows.append({"time": at, "device_id": device, "monitored_value": 100.})
return rows
def evaluate(rows):
return evaluate_recovery(rows, device_ids=["P1", "F1"], returned_at=RETURNED,
incident_at=INCIDENT, interval_seconds=900)
def test_recovery_uses_post_receipt_samples_and_pre_incident_same_phase_reference():
result = evaluate(observations())
assert result["status"] == "passed"
assert [item["at"] for item in result["samples"]] == [
(RETURNED + timedelta(minutes=30)).isoformat(), (RETURNED + timedelta(minutes=45)).isoformat()]
assert all(check["normal_day_count"] == 15 for sample in result["samples"] for check in sample["devices"])
@pytest.mark.parametrize("problem", ["missing_device", "low_pressure", "high_flow", "nan", "bad_quality", "missing_baseline", "gap"])
def test_incomplete_or_abnormal_data_never_proves_recovery(problem):
rows = observations()
current = [row for row in rows if row["time"] > RETURNED]
for row in current:
if problem == "missing_device" and row["device_id"] == "P1":
rows.remove(row)
elif problem in {"low_pressure", "high_flow", "nan"}:
row["monitored_value"] = {"low_pressure": 60., "high_flow": 130., "nan": float("nan")}[problem]
elif problem == "bad_quality":
row["quality"] = "invalid"
elif problem == "gap":
row["time"] += timedelta(minutes=15 * current.index(row))
if problem == "missing_baseline":
rows = [row for row in rows if row["time"] >= INCIDENT - timedelta(days=14)]
assert evaluate(rows)["status"] == "waiting"
def test_samples_at_or_before_return_time_cannot_close_incident():
rows = [row for row in observations() if row["time"] < INCIDENT]
rows.extend({"time": RETURNED, "device_id": device, "monitored_value": 100.} for device in ("P1", "F1"))
assert evaluate(rows)["status"] == "waiting"
def test_faster_sampling_counts_monitoring_periods_and_rejects_later_relapse():
rows = []
for minute in range(5, 36, 5):
at = RETURNED + timedelta(minutes=minute)
for device in ("P1", "F1"):
rows.extend({"time": at - timedelta(days=day), "device_id": device, "monitored_value": 100.}
for day in range(1, 16))
rows.append({"time": at, "device_id": device, "monitored_value": 100.})
assert evaluate(rows)["status"] == "passed"
next(row for row in rows if row["time"] == RETURNED + timedelta(minutes=25) and row["device_id"] == "P1")["monitored_value"] = 50.
assert evaluate(rows)["status"] == "waiting"
def test_dispatch_waits_for_all_orders_and_accepts_corrected_field_completion_time():
task = {"workflow": "burst_report"}
for order_id in ("one", "two"):
track_work_order(task, {"kind": "create_work_order", "action_id": str(uuid4()), "execution_status": "succeeded",
"receipt": {"adapter": "peripheral_sample", "receipt_id": order_id}})
assert OperationalTaskService._investigation_status([], task) == "waiting_human"
assert not accept_completion(task, {"source_event_id": "peripheral_sample:unrelated"})
feedback = {"source_event_id": "peripheral_sample:one", "status": "completed", "source_revision": 2,
"occurred_at": RETURNED.isoformat()}
assert accept_completion(task, feedback)
assert task["handling_context"]["phase"] == "waiting_work_order"
accept_completion(task, {**feedback, "source_event_id": "peripheral_sample:two"})
assert task["handling_context"]["phase"] == "waiting_scada"
task["handling_context"]["phase"] = "verification_ready"
task["handling_context"].update(verification_evidence_id="previous-proof", last_check={"status": "passed"})
accept_completion(task, {**feedback, "source_revision": 3})
assert task["handling_context"]["phase"] == "verification_ready"
assert OperationalTaskService._investigation_status([], task) == "completed"
corrected = RETURNED + timedelta(hours=1)
accept_completion(task, {**feedback, "source_revision": 4, "occurred_at": corrected.isoformat()})
assert task["handling_context"]["returned_at"] == corrected.isoformat()
assert task["handling_context"]["phase"] == "waiting_scada"
assert "verification_evidence_id" not in task["handling_context"] and "last_check" not in task["handling_context"]
def test_field_feedback_time_requires_a_real_note_and_explicit_timezone():
from pydantic import ValidationError
from samples.peripheral.schemas import UpdateWorkOrder
body = {"request_id": uuid4(), "expected_revision": 1, "status": "completed",
"progress_note": "演示抢修完成并恢复供水", "occurred_at": RETURNED.isoformat()}
assert UpdateWorkOrder.model_validate(body).occurred_at == RETURNED
with pytest.raises(ValidationError):
UpdateWorkOrder.model_validate({**body, "progress_note": None})
with pytest.raises(ValidationError):
UpdateWorkOrder.model_validate({**body, "occurred_at": "2026-07-16T09:00:00"})
+195
View File
@@ -576,6 +576,65 @@ async def test_presentation_cursor_and_event_are_read_only_and_idempotent():
await service.presentation_cursor(uuid4())
@pytest.mark.anyio
async def test_replay_investigation_can_publish_evidenced_composite_chart_but_cannot_propose_real_action():
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task["source"]["replay_context"] = {
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
}
repository = RunTaskRepository(task, _active_run(task, now))
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
evidence, _ = await service.record_data_snapshot(task["task_id"], TaskDataSnapshot(
source="query_scada_readings", source_ref="mcp:frozen-scada", queried_at=now,
origin="synthetic", data={"items": []}, truncated=False,
))
result = await service.publish_presentation_event(task["task_id"], TaskPresentationEventCommand(
request_id=uuid4(), run_id=repository.run["run_id"], source_session_id="session-1",
command={"action": "open_chart", "chart_id": "pressure", "evidence_refs": [evidence.evidence_id],
"chart": {"title": "压力", "chart_type": "line", "x_data": ["19:45", "20:00"],
"y_axis_name": "压力 m", "series": [
{"name": "当前", "type": "line", "data": [27., 26.]},
{"name": "前一天", "type": "scatter", "data": [None, 28.],
"sample_times": [None, "2026-10-09T20:00:00+08:00"]},
]}},
))
assert result.recorded
assert repository.events[-1]["metadata"]["command"]["chart"]["series"][1]["data"] == [None, 28.]
with pytest.raises(DomainError) as caught:
await service.propose_action(task["task_id"], ActionProposalCommand(
request_id=uuid4(), kind="create_work_order", reason="演示不得创建真实工单",
payload={"title": "现场核查", "description": "核查", "work_order_type": "general"},
))
assert caught.value.code == "REPLAY_ACTION_DISABLED"
@pytest.mark.anyio
@pytest.mark.parametrize("operation", ["draft", "approve", "submit"])
async def test_replay_cannot_edit_approve_or_submit_preexisting_real_actions(operation):
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task["source"]["replay_context"] = {
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
}
repository = RunTaskRepository(task, _active_run(task, now))
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
dispatcher = SimpleNamespace(dispatch=AsyncMock())
args = {"request_id": uuid4(), "expected_version": 1}
with pytest.raises(DomainError) as caught:
if operation == "draft":
await service.update_work_order_draft(task["task_id"], uuid4(), WorkOrderDraftUpdateCommand(
**args, payload={"title": "现场核查", "description": "核查", "work_order_type": "general"}))
elif operation == "approve":
await service.decide_action(task["task_id"], uuid4(), ActionDecisionCommand(**args, decision="approved"))
else:
await service.submit_action(task["task_id"], uuid4(), ActionExecuteCommand(**args), dispatcher, None)
assert caught.value.code == "REPLAY_ACTION_DISABLED"
dispatcher.dispatch.assert_not_awaited()
@pytest.mark.anyio
@pytest.mark.parametrize("payload", [
{"action": "locate_features", "layer": "scada", "feature_ids": ["D-1"]},
@@ -1025,6 +1084,93 @@ def _seed_concluded_report(repository, now):
})
@pytest.mark.anyio
async def test_burst_waits_for_completion_then_scada_evidence_then_agent_report(monkeypatch):
from copy import deepcopy
from app.domain.schemas.project_settings import ProjectSettings
from app.services.operational_recovery import reconcile_recoveries, track_work_order
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
incident = now - timedelta(hours=4)
returned = incident + timedelta(hours=1)
context = _context()
task = _task_document(context, now, lease=now + timedelta(minutes=1))
task.update(workflow="burst_report", trigger={"kind": "event", "occurred_at": incident.isoformat(), "detail": "爆管"})
repository = RunTaskRepository(task, _active_run(task, now))
repository.connection = object()
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
monkeypatch.setattr(service, "_resolve_investigation_alerts", AsyncMock())
_seed_concluded_report(repository, now)
action = await service.propose_action(task["task_id"], ActionProposalCommand(
request_id=uuid4(), kind="create_work_order", reason="隔离抢修与恢复供水",
payload={"title": "抢修恢复", "description": "回传完工时间和恢复供水情况", "work_order_type": "general"}))
child_id = str(uuid4())
repository.action = action.model_copy(update={"approval_status": "approved", "execution_status": "succeeded"}).model_dump(mode="json")
repository.action["receipt"] = {"adapter": "peripheral_sample", "receipt_id": child_id,
"submitted_at": now.isoformat(), "completed_at": now.isoformat(), "external_status": "open", "detail": "仅创建成功"}
track_work_order(repository.task, repository.action)
finished = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert finished.status == "waiting_human"
service._resolve_investigation_alerts.assert_not_awaited()
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=2,
status="completed", occurred_at=returned, detail="抢修完成,已恢复供水"))
assert repository.task["handling_context"]["phase"] == "waiting_scada"
assert datetime.fromisoformat(repository.task["handling_context"]["returned_at"]) == returned
settings = ProjectSettings(scada_monitoring_interval_seconds=900, scada_monitoring={"pressure_device_ids": ["P1"]})
repository.recovery_candidates = AsyncMock(side_effect=lambda *_: [deepcopy(repository.task)]
if repository.task["handling_context"]["phase"] == "waiting_scada" else [])
timescale = SimpleNamespace(transaction=repository.transaction)
rows = []
for sample in (1, 2):
at = returned + timedelta(minutes=15 * sample)
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
reader = AsyncMock(return_value=rows[:-1])
monkeypatch.setattr("app.infra.db.timescaledb.repositories.scada.ScadaRepository.get_scada_by_ids_time_range", reader)
assert await reconcile_recoveries(service, timescale, settings) == 0
assert repository.task["handling_context"]["phase"] == "waiting_scada"
reader.return_value = rows
assert await reconcile_recoveries(service, timescale, settings) == 1
assert await reconcile_recoveries(service, timescale, settings) == 0
recovery = repository.task["handling_context"]
evidence_id = recovery["verification_evidence_id"]
assert repository.evidence[evidence_id]["data_snapshot"]["data"]["status"] == "passed"
service._resolve_investigation_alerts.assert_not_awaited()
# Correcting the appointment's field return time invalidates the old proof
# and requires two actual observations after the new business time.
corrected_time = returned + timedelta(minutes=15)
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=3,
status="completed", occurred_at=corrected_time, detail="修订现场实际完工时间"))
assert repository.task["handling_context"]["phase"] == "waiting_scada"
assert "verification_evidence_id" not in repository.task["handling_context"]
assert await reconcile_recoveries(service, timescale, settings) == 0
at = returned + timedelta(minutes=45)
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
assert await reconcile_recoveries(service, timescale, settings) == 1
revised_id = repository.task["handling_context"]["verification_evidence_id"]
assert revised_id != evidence_id and evidence_id in repository.evidence
evidence_id = revised_id
repository.task.update(status="running", agent_session_id="session-1")
repository.run = _active_run(repository.task, now)
_seed_concluded_report(repository, now)
with pytest.raises(DomainError) as rejected:
await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert rejected.value.code == "RECOVERY_EVIDENCE_REQUIRED"
repository.reports[-1]["report"]["evidence_refs"] = [evidence_id]
result = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
assert result.status == "completed" and result.handling_context["phase"] == "completed"
service._resolve_investigation_alerts.assert_awaited_once_with(task["task_id"], evidence_id)
@pytest.mark.anyio
async def test_plan_only_keeps_lease_until_same_generation_report_closes_run():
now = datetime(2026, 9, 23, 8, tzinfo=timezone.utc)
@@ -2087,3 +2233,52 @@ async def test_missing_calculation_migration_fails_finalization_closed(monkeypat
await repo.pending_calculation_jobs(str(uuid4()), str(uuid4()), str(uuid4()))
assert error.value.code == 'DEPENDENCY_UNAVAILABLE'
assert error.value.status == 503
@pytest.mark.anyio
async def test_recovery_alert_query_scopes_active_alerts_to_project_and_task():
alerts = [{"alert_id": "first"}, {"alert_id": "second"}]
cursor = SimpleNamespace(fetchall=AsyncMock(return_value=[{"document": item} for item in alerts]))
connection = SimpleNamespace(execute=AsyncMock(return_value=cursor))
project_id, task_id = uuid4(), str(uuid4())
repository = OperationalTaskRepository(connection, project_id)
assert await repository.list_investigation_alerts(task_id) == alerts
statement, parameters = connection.execute.await_args.args
assert parameters == (str(project_id), f"operational-task:{task_id}")
assert "status IN ('active','acknowledged')" in statement
@pytest.mark.anyio
async def test_recovery_resolves_alerts_with_original_facts_and_stable_event_id(monkeypatch):
from uuid import NAMESPACE_URL, uuid5
from app.services.task_alerts import TaskAlertService
task_id, project_id = str(uuid4()), str(uuid4())
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
alert = {
"alert_id": "incident-1", "producer_id": "producer-1", "revision": 2,
"condition_id": f"operational-task:{task_id}", "status": "acknowledged",
"source": {"task_type": "scheduled_task", "task_id": task_id, "run_id": "run-1"},
"title": "压力异常", "description": "等待恢复", "severity": "warning",
"occurred_at": now.isoformat(), "updated_at": now.isoformat(),
}
connection = object()
repository = SimpleNamespace(connection=connection, project_id=project_id,
list_investigation_alerts=AsyncMock(return_value=[alert]))
calls = []
async def push(service, command):
calls.append((service, command))
monkeypatch.setattr(TaskAlertService, "push", push)
service = OperationalTaskService(repository, _context(), set())
for _ in range(2):
await service._resolve_investigation_alerts(task_id, "recovery:evidence-1")
assert len(calls) == 2
for publisher, command in calls:
assert publisher.repo.connection is connection and publisher.repo.project_id == project_id
assert publisher.internal_producer_id == alert["producer_id"]
assert command.event_type == "resolved"
assert command.event_id == uuid5(NAMESPACE_URL, f"recovery:{task_id}:recovery:evidence-1:incident-1")
assert command.source.model_dump(mode="json") == alert["source"]
assert command.title == alert["title"] and command.description == alert["description"]
+9 -8
View File
@@ -48,9 +48,10 @@ async def test_all_worker_operations_share_investigation_creation_scope():
assert request.url.params["task_type"] == "event_investigation"
assert request.url.params["task_created_from"] == boundary
return httpx.Response(200, json={"items": []})
assert request.url.path.endswith("leases/recover")
assert request.url.path.endswith(("leases/recover", "recovery/reconcile"))
assert json.loads(request.content) == {
"limit": 100, "task_type": "event_investigation", "task_created_from": boundary,
"limit": 20 if request.url.path.endswith("recovery/reconcile") else 100,
"task_type": "event_investigation", "task_created_from": boundary,
}
return httpx.Response(200, json={"task_ids": []})
@@ -58,7 +59,7 @@ async def test_all_worker_operations_share_investigation_creation_scope():
bearer="TEST_ONLY", task_created_from=boundary)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
assert await OperationalWorker(settings, client).tick() == 0
assert len(requests) == 5
assert len(requests) == 6
@pytest.mark.anyio
@@ -68,7 +69,7 @@ async def test_worker_reconciles_messages_at_capacity_and_preserves_transport_re
def handler(request):
path = request.url.path
if path.endswith("leases/recover"):
if path.endswith(("leases/recover", "recovery/reconcile")):
return httpx.Response(200, json={"items": []})
if path.endswith("agent-messages/pending"):
return httpx.Response(200, json={"items": pending})
@@ -129,7 +130,7 @@ async def test_persisted_queue_retries_with_backend_execution_identity():
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
assert request.headers["Authorization"] == "Bearer TEST_ONLY"
assert not request.url.path.endswith("schedules/dispatch-due")
if request.url.path.endswith("leases/recover"):
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
return httpx.Response(200, json={"items": []})
if request.method == "GET":
if request.url.params.get("status") != "queued":
@@ -191,7 +192,7 @@ async def test_bounded_queue_continues_after_one_agent_handoff_fails():
limits.append(int(request.url.params["limit"]))
return httpx.Response(200, json={"items": runs})
if request.url.host == "server.test":
if request.url.path.endswith("leases/recover"):
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
return httpx.Response(200, json={"items": []})
failures.append(json.loads(request.content))
return httpx.Response(200, json={})
@@ -269,7 +270,7 @@ async def test_worker_refreshes_running_agent_credentials_after_restart():
def handler(request):
if request.url.path == "/health":
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
if request.url.path.endswith("leases/recover"):
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
return httpx.Response(200, json={"items": []})
if request.method == "GET":
items = [{
@@ -300,7 +301,7 @@ async def test_worker_only_restores_sessions_owned_by_target_agent_instance():
def handler(request):
if request.url.path == "/health":
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
if request.url.path.endswith("leases/recover"):
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
return httpx.Response(200, json={"items": []})
if request.method == "GET":
items = [{
+3 -1
View File
@@ -255,7 +255,7 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
"title": "采样工单", "priority": "normal",
"work_order_type": "general", "created_at": "2026-09-23T08:00:00Z",
"scheduled_at": None,
"activity": ([{"sequence": 2, "occurred_at": "2026-09-23T09:00:00Z", "actor": "现场人员",
"activity": ([{"sequence": 2, "occurred_at": "2026-09-23T08:45:00Z", "actor": "现场人员",
"kind": "feedback", "title": "现场反馈", "detail": "复测余氯0.2mg/L"}]
if order_revision > 1 else []),
}
@@ -271,6 +271,8 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
assert ("work_order_snapshot" in payload) == request.url.path.endswith(f"{projection_id}/external-feedback")
if order_revision > 1:
assert "复测余氯0.2mg/L" in payload["evidence"]["summary"]
assert payload["occurred_at"] == "2026-09-23T08:45:00Z"
assert payload["evidence"]["observed_at"] == "2026-09-23T08:45:00Z"
return httpx.Response(200, json={})
if request.url.path.endswith(investigation_id):
return httpx.Response(200, json={"actions": [{"kind": "create_work_order", "receipt": {
@@ -0,0 +1,46 @@
import pytest
from scripts.prepare_regional_burst_design import require_isolatable_pipe
from scripts.screen_burst_benchmark_pipes import nearest_pressure_distances
def test_region_without_valves_is_rejected():
with pytest.raises(ValueError, match="cannot be isolated"):
require_isolatable_pipe(["supply:pipe:R:A", "target:pipe:A:B"], "target", ["R"], [])
def test_nearby_valve_does_not_qualify_an_uncontrolled_source_path():
links = ["v:valve:R:A", "bypass:pipe:R:B", "target:pipe:A:B"]
with pytest.raises(ValueError, match="cannot be isolated"):
require_isolatable_pipe(links, "target", ["R"], [])
def test_single_boundary_valve_can_qualify_a_branch():
links = ["v:valve:R:A", "target:pipe:A:B"]
result = require_isolatable_pipe(links, "target", ["R"], [])
assert result["must_close_valves"] == ["v"]
assert set(result["affected_nodes"]) == {"A", "B"}
assert result["source_check_performed"]
def test_source_metadata_is_mandatory():
with pytest.raises(ValueError, match="cannot be isolated"):
require_isolatable_pipe(["v:valve:R:A", "target:pipe:A:B"], "target", [], [])
def test_distance_is_to_the_nearest_of_all_meters():
graph = {"A": [("P", 1500)], "B": [("P", 100)], "P": [("A", 1500), ("B", 100)]}
devices = [{"device_id": "far", "node_id": "A"}, {"device_id": "near", "node_id": "B"}]
assert nearest_pressure_distances(graph, devices, 3000)["P"] == (100, "near")
def test_network_path_length_and_cutoff_are_used():
graph = {"A": [("B", 600)], "B": [("A", 600), ("C", 700)], "C": [("B", 700)]}
devices = [{"device_id": "meter", "node_id": "A"}]
assert "C" not in nearest_pressure_distances(graph, devices, 1200)
assert nearest_pressure_distances(graph, devices, 1500)["C"] == (1300, "meter")
def test_unreachable_pipe_is_not_a_zero_distance_candidate():
graph = {"A": [], "P": []}
assert "P" not in nearest_pressure_distances(graph, [{"device_id": "meter", "node_id": "A"}], 3000)
+11
View File
@@ -40,6 +40,17 @@ def test_report_ui_round_trips_with_a_version_and_keeps_legacy_reports_readable(
assert InvestigationReportRevision.model_validate(old).report.ui_blocks == []
def test_report_ui_preserves_sparse_scada_series_and_sample_times():
data = rich_report()
chart = data["ui_blocks"][0]["block"]["spec"]["elements"]["chart"]["props"]
chart["y_axis_name"] = "压力 MPa"
chart["series"].append({"name": "前一天", "type": "scatter", "data": [None, 0.28],
"sample_times": [None, "2026-10-19T09:00:00+08:00"]})
restored = InvestigationReportInput.model_validate_json(
InvestigationReportInput.model_validate(data).model_dump_json())
assert restored.ui_blocks[0].block.spec.elements["chart"].props == chart
@pytest.mark.parametrize("phase", [None, "final"])
def test_report_ui_accepts_final_and_legacy_phases(phase):
data = rich_report()
@@ -0,0 +1,84 @@
from datetime import datetime, timedelta
from math import sin
import pytest
from pydantic import ValidationError
from app.domain.schemas.project_settings import ScadaMonitoringSettings, ScadaMonitoringWeights
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder, adaptive_score_threshold
from app.services import scada_replay_monitoring as monitoring
from test_scada_replay_monitoring import TARGET, PRESSURE_IDS, FLOW_IDS, _document
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY, normal_profile_bounds
def setup():
document = _document()
document["config"]["start_time"] = TARGET.isoformat()
document["monitoring_config"] = ScadaMonitoringSettings(
pressure_device_ids=PRESSURE_IDS, outlet_flow_device_ids=FLOW_IDS,
).model_dump(mode="json")
config = ScadaReplayCreate.model_validate(document["config"])
start = TARGET - timedelta(days=15)
rows = [{"device_id": device, "time": (start + timedelta(minutes=15 * i)).isoformat(),
"cleaned_value": 100 + 5 * sin(i / 96 * 6.283) + 0.1 * sin(i * 1.7), "quality": "valid"}
for i in range(15 * 96) for device in PRESSURE_IDS + FLOW_IDS]
return document, config, rows
def test_streamed_thresholds_are_frozen_from_complete_pre_replay_history(monkeypatch):
document, config, rows = setup()
calls = []
def detect(**kwargs):
frame = kwargs["observed_pressure_data"]
assert frame.index.max().to_pydatetime() < config.start_time
assert frame.shape == (1440, 5)
calls.append(frame)
return {"rows": [{"Score": value} for value in [-0.03, -0.01, 0.01, 0.02, 0.03] * 3]}
monkeypatch.setattr(monitoring, "run_burst_detection", detect)
builder = MonitoringCalibrationBuilder(document, config)
for offset in range(0, len(rows), 997):
builder.add(rows[offset:offset + 997])
result = builder.fit()
assert result["status"] == "ready"
assert result["version"] == CALIBRATION_VERSION
assert result["diagnostic_policy"] == DIAGNOSTIC_POLICY
assert result["knowledge_cutoff"] == config.start_time.isoformat()
assert result["burst_detection"]["threshold"] < -0.04
assert result["pressure"][PRESSURE_IDS[0]]["threshold_percent"] == 3.0
assert result["outlet_flow"][FLOW_IDS[0]]["threshold_percent"] == 5.0
assert len(calls) == 1
assert sum(values.nbytes for values in builder.values.values()) == 1440 * 6 * 8
for row in rows:
bounds = normal_profile_bounds(result["diagnostic"][row["device_id"]], datetime.fromisoformat(row["time"]))
assert bounds[0] <= row["cleaned_value"] <= bounds[1]
def test_history_gap_and_future_samples_cannot_be_hidden(monkeypatch):
document, config, rows = setup()
monkeypatch.setattr(monitoring, "run_burst_detection", lambda **_: pytest.fail("incomplete history entered detection"))
builder = MonitoringCalibrationBuilder(document, config)
builder.add(rows[1:])
assert builder.fit()["status"] == "insufficient_data"
with pytest.raises(ValueError, match="之后"):
builder.add([{**rows[0], "time": config.start_time.isoformat()}])
def test_score_cutoff_follows_normal_noise_instead_of_target_scores():
quiet = adaptive_score_threshold([0.01, 0.02, 0.03, 0.02, 0.01])
noisy = adaptive_score_threshold([-0.07, -0.03, 0.01, 0.04, 0.02])
assert noisy["threshold"] < quiet["threshold"]
assert -0.2 < noisy["threshold"]
assert adaptive_score_threshold([0.01])["status"] == "insufficient_data"
@pytest.mark.parametrize("values", [
{"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.5},
{"burst_detection": -0.1, "pressure": 1, "outlet_flow": 0.1},
{"outlet_flow": float("nan")},
])
def test_invalid_weights_are_rejected(values):
with pytest.raises(ValidationError):
ScadaMonitoringWeights(**values)
+202
View File
@@ -1,8 +1,11 @@
from datetime import datetime, timedelta, timezone
from uuid import uuid4
import pytest
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.services import scada_replay_monitoring
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
TARGET = datetime(2026, 10, 16, tzinfo=timezone.utc)
@@ -68,6 +71,14 @@ def test_multiple_indicators_create_suspected_burst_result(monkeypatch):
assert result["pressure"]["hit_device_ids"] == PRESSURE_IDS
assert result["outlet_flow"]["hit_device_ids"] == FLOW_IDS
assert result["burst_detection"]["hit"] is True
pressure = result["pressure"]["devices"][0]
assert pressure["value_change"] == -30
assert pressure["unit"] == "m"
assert pressure["baseline_method"] == "recent_samples_median"
assert pressure["baseline_sample_count"] == 4
assert pressure["baseline_end"] == result["period_start"]
assert result["outlet_flow"]["devices"][0]["value_change"] == 50
assert result["burst_detection"]["score_interpretation"] == "lower_is_more_anomalous_not_probability"
def test_single_indicator_stays_attention(monkeypatch):
@@ -106,3 +117,194 @@ def test_missing_project_roles_is_data_gap_not_normal():
assert result["status"] == "insufficient_data"
assert result["burst_detection"]["status"] == "not_run"
def _weighted_document():
document = _document()
document["config"]["start_time"] = (TARGET - timedelta(minutes=30)).isoformat()
document["monitoring_config"] = ScadaMonitoringSettings(
pressure_device_ids=PRESSURE_IDS, outlet_flow_device_ids=FLOW_IDS,
).model_dump(mode="json")
document["monitoring_calibration"] = {
"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
"status": "ready", "knowledge_cutoff": (TARGET - timedelta(minutes=30)).isoformat(),
"pressure": {device: {"threshold_percent": 10, "method": "frozen_normal_history"} for device in PRESSURE_IDS},
"outlet_flow": {"F1": {"threshold_percent": 5, "method": "frozen_normal_history"}},
"burst_detection": {"threshold": -0.08},
}
profile = {"phase_origin": document["config"]["start_time"], "interval_seconds": 900,
"lower": [99.], "upper": [101.]}
calibration = document["monitoring_calibration"]
calibration["diagnostic"] = {device: profile for device in PRESSURE_IDS + FLOW_IDS}
for kind in ("pressure", "outlet_flow"):
for item in calibration[kind].values():
item["normal_profile"] = profile
return document
@pytest.mark.parametrize("flow,expected_score", [("increase", 1.0), ("flat", 0.9), ("missing", 0.9)])
def test_pressure_and_detection_can_trigger_with_low_or_unavailable_flow_weight(monkeypatch, flow, expected_score):
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
"rows": [{"Score": -0.2}], "summary": {},
})
rows = _rows()
for row in rows:
if flow == "flat" and row["device_id"] == "F1":
row["cleaned_value"] = 100
if flow == "missing":
rows = [row for row in rows if not (row["device_id"] == "F1"
and datetime.fromisoformat(row["time"]) >= TARGET - timedelta(minutes=15))]
result = scada_replay_monitoring.evaluate_monitoring_period(
_weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
period_end=TARGET, batch_sequence=1440,
)
assert result["status"] == "suspected_burst"
assert result["weighted"]["score"] == expected_score
assert result["weighted"]["weights"]["outlet_flow"] == 0.1
assert result["pressure"]["devices"][0]["threshold_percent"] == 10
assert result["burst_detection"]["score_threshold"] == -0.08
if flow == "missing":
assert result["weighted"]["unavailable_indicators"] == ["outlet_flow"]
@pytest.mark.parametrize("indicator", ["pressure", "outlet_flow", "burst_detection"])
def test_a_single_indicator_cannot_create_a_weighted_burst_alert(monkeypatch, indicator):
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
"rows": [{"Score": -0.2 if indicator == "burst_detection" else 0.02}], "summary": {},
})
rows = _rows()
for row in rows:
if (row["device_id"] in PRESSURE_IDS and indicator != "pressure"
or row["device_id"] == "F1" and indicator != "outlet_flow"):
row["cleaned_value"] = 100
result = scada_replay_monitoring.evaluate_monitoring_period(
_weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
period_end=TARGET, batch_sequence=1440,
)
assert result["status"] == "attention"
assert result["weighted"]["score"] < 0.8
def test_weighted_policy_requires_a_persisted_causal_calibration():
document = _weighted_document()
document.pop("monitoring_calibration")
result = scada_replay_monitoring.evaluate_monitoring_period(
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
)
assert result["status"] == "insufficient_data"
def test_calibration_from_another_or_future_cutoff_is_rejected():
document = _weighted_document()
document["monitoring_calibration"]["knowledge_cutoff"] = (TARGET + timedelta(minutes=15)).isoformat()
result = scada_replay_monitoring.evaluate_monitoring_period(
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
)
assert result["status"] == "insufficient_data"
def test_ready_flow_anomaly_survives_missing_pressure_calibration(monkeypatch):
document = _weighted_document()
document["monitoring_calibration"].update(status="insufficient_data", pressure={})
document["monitoring_calibration"].pop("burst_detection")
def unexpected_detector(**_):
raise AssertionError("Unavailable pressure calibration must not run the detector")
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", unexpected_detector)
result = scada_replay_monitoring.evaluate_monitoring_period(
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
)
assert result["status"] == "insufficient_data"
assert result["outlet_flow"]["hit"] is True
assert result["weighted"]["score"] == 0.1
assert result["burst_detection"]["status"] == "baseline_not_ready"
def test_relative_baseline_matches_calibration_immediately_previous_samples():
config = ScadaReplayCreate.model_validate(_document()["config"])
rows = [{"device_id": "P1", "time": (TARGET - timedelta(minutes=15 * (5 - i))).isoformat(),
"cleaned_value": value, "quality": "valid"}
for i, value in enumerate((120., 115., 110., 105., 100., 95.))]
result = scada_replay_monitoring._relative_indicator(
rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
)
assert result["devices"][0]["baseline"] == 107.5
assert result["devices"][0]["baseline_end"] == (TARGET - timedelta(minutes=15)).isoformat()
def test_period_start_reading_is_not_a_new_observation():
config = ScadaReplayCreate.model_validate(_document()["config"])
rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
result = scada_replay_monitoring._relative_indicator(
rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
)
assert result["status"] == "insufficient_data"
assert result["gaps"][0]["reason"] == "period_reading_missing"
def test_detector_cannot_reuse_the_previous_period_score(monkeypatch):
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection",
lambda **_: pytest.fail("stale target reached detector"))
rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
result = scada_replay_monitoring._burst_detection(rows, PRESSURE_IDS, 900, TARGET)
assert result["status"] == "current_reading_missing"
assert result["hit"] is False
def test_old_calibration_version_requires_a_new_replay():
document = _weighted_document()
document["monitoring_calibration"]["version"] = "scada-monitor-calibration@1"
result = scada_replay_monitoring.evaluate_monitoring_period(
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
)
assert result["status"] == "insufficient_data"
assert "新建" in result["detail"]
assert not any(result[key]["hit"] for key in ("pressure", "outlet_flow", "burst_detection"))
def test_normal_diurnal_change_then_t_and_t_plus_fifteen_share_one_incident(monkeypatch):
from app.services.scada_replay_processing import detect_frame
from app.services.scada_monitoring_incidents import update_incident
document = _weighted_document()
config = ScadaReplayCreate.model_validate(document["config"])
profile = {"phase_origin": config.start_time.isoformat(), "interval_seconds": 900,
"lower": [99., 199., 199., 199.], "upper": [101., 201., 201., 201.]}
document["monitoring_calibration"]["diagnostic"]["F1"] = profile
document["monitoring_calibration"]["outlet_flow"]["F1"]["normal_profile"] = profile
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **kwargs: {
"rows": [{"Score": -0.2 if kwargs["observed_pressure_data"].iloc[-1].min() < 90 else 0.02}],
"summary": {},
})
rows = _rows()
rows += [{"time": (TARGET - timedelta(days=15)).isoformat(), "device_id": device,
"cleaned_value": 100., "quality": "valid"} for device in PRESSURE_IDS + FLOW_IDS]
for row in rows:
if row["device_id"] == "F1" and _time_for_test(row) >= TARGET - timedelta(minutes=15):
row["cleaned_value"] = 200.
rows += [{"time": (TARGET + timedelta(minutes=15)).isoformat(), "device_id": device,
"cleaned_value": 200. if device == "F1" else 70., "quality": "valid"}
for device in PRESSURE_IDS + FLOW_IDS]
document["histories"] = {device: [100.] * 4 for device in PRESSURE_IDS + FLOW_IDS}
actions, statuses = [], []
for sequence, offset in enumerate((-15, 0, 15), start=1):
at = TARGET + timedelta(minutes=offset)
frame = [row for row in rows if _time_for_test(row) == at]
document, changes = detect_frame(document, frame, config, at)
result = scada_replay_monitoring.evaluate_monitoring_period(
document, [row for row in rows if _time_for_test(row) <= at],
period_start=at - timedelta(minutes=15), period_end=at, batch_sequence=sequence,
)
result["anomaly_signals"] = [state["event"] for state in document["rule_states"].values()
if state.get("event")]
statuses.append(result["status"])
incident, action = update_incident(document, result, sequence)
actions.append(action)
if incident and not incident.get("task_id"):
incident["task_id"] = str(uuid4())
if offset == -15:
assert changes == [] # Normal doubling of flow does not launch diagnosis.
assert statuses == ["normal", "suspected_burst", "suspected_burst"]
assert actions == [None, "raised", "ongoing"]
def _time_for_test(row):
return datetime.fromisoformat(row["time"])
@@ -1,8 +1,12 @@
from datetime import datetime, timedelta, timezone
from uuid import uuid4
import pytest
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
from app.services.scada_normal_profile import (
CALIBRATION_VERSION, DIAGNOSTIC_POLICY, diagnostic_signal,
)
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
@@ -64,3 +68,60 @@ def test_empty_clock_ticks_do_not_break_consecutive_observations():
doc, _ = detect_frame(doc, [], cfg, START + timedelta(minutes=1))
_, events = detect_frame(doc, prepare_cleaning(rows(20, 15), cfg), cfg, START + timedelta(minutes=15))
assert events[0]["change"] == "raised"
def weighted_document(low=97., high=103.):
return {"replay_id": "run", "histories": {"P1": [100.] * 4},
"monitoring_config": {"decision_policy": "weighted"},
"config": config().model_dump(mode="json"),
"monitoring_calibration": {
"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
"knowledge_cutoff": START.isoformat(),
"diagnostic": {"P1": {"phase_origin": START.isoformat(), "interval_seconds": 900,
"lower": [low], "upper": [high]}},
}}
def test_weighted_diagnosis_ignores_normal_diurnal_large_relative_changes():
cfg = config(rules=[{"rule_id": "flow", "device_ids": ["P1"], "kind": "relative_change",
"direction": "both", "threshold": 30, "unit": "m3/h"}])
document = weighted_document(25., 35.)
document["histories"]["P1"] = [10.] * 4
_, events = detect_frame(document, prepare_cleaning(rows(30), cfg), cfg, START)
assert events == [] # +200% is normal at this daily sampling phase.
_, events = detect_frame(document, prepare_cleaning(rows(36), cfg), cfg, START)
assert events[0]["change"] == "raised"
@pytest.mark.parametrize("baseline,value,low,high", [
(100., 95., 97., 103.), (0., -5., -1., 1.),
])
def test_weighted_single_indicator_uses_history_envelope_without_thirty_percent_gate(baseline, value, low, high):
cfg = config()
document = weighted_document(low, high)
document["histories"]["P1"] = [baseline] * 4
_, events = detect_frame(document, prepare_cleaning(rows(value), cfg), cfg, START)
assert len(events) == 1
assert events[0]["diagnostic_policy"] == DIAGNOSTIC_POLICY
assert diagnostic_signal(document, events[0])
assert "阈值 30" not in events[0]["detail"]
@pytest.mark.parametrize("missing", ["version", "diagnostic", "knowledge_cutoff"])
def test_weighted_missing_calibration_does_not_fall_back_to_thirty_percent(missing):
cfg = config()
document = weighted_document()
document["monitoring_calibration"].pop(missing)
_, events = detect_frame(document, prepare_cleaning(rows(1), cfg), cfg, START)
assert events == []
def test_old_active_relative_state_cannot_block_or_supply_new_diagnosis():
cfg = config()
document = weighted_document()
old = {"event_id": "old", "rule_id": "pressure", "device_ids": ["P1"], "status": "active"}
document["rule_states"] = {"pressure": {"event": old, "hits": 1, "recoveries": 0}}
result, events = detect_frame(document, prepare_cleaning(rows(95), cfg), cfg, START)
assert not diagnostic_signal(document, old)
assert result["rule_states"]["pressure"]["event"] == old
assert events[0]["diagnostic_policy"] == DIAGNOSTIC_POLICY
+55 -1
View File
@@ -12,7 +12,7 @@ from app.mcp.analysis_models import BurstLocationInput
from app.mcp.catalog import TOOLS
from app.mcp.models import BoundContext
from app.mcp import query_service
from app.services.scada_replay_queries import replay_burst_location_inputs
from app.services.scada_replay_queries import replay_burst_location_inputs, replay_snapshot_rows
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
@@ -85,6 +85,60 @@ def test_replay_location_builds_series_only_from_frozen_outputs():
assert connection.statements[1][1][2] == 9
def test_replay_queries_frozen_history_with_project_replay_and_batch_bounds():
project_id, replay_id = uuid4(), str(uuid4())
at = START - timedelta(days=7)
run = {"start_time": START.isoformat(), "config": {"device_ids": ["P1"]},
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()}}
reading = {"time": at.isoformat(), "device_id": "P1", "monitored_value": 23.,
"cleaned_value": 23., "quality": "valid", "reason": "冻结历史基线"}
connection = Connection([[{"document": run}], [{"reading": reading}]])
context = {"replay_id": replay_id, "knowledge_cutoff": START.isoformat(), "batch_sequence": 9}
result = asyncio.run(replay_snapshot_rows(connection, project_id, context, ["P1"], at, at, 10))
assert result == [reading]
statement, params = connection.statements[1]
assert "b.stage='detected'" in statement and "b.sequence<=%s" in statement
assert params[:4] == (str(project_id), replay_id, 9, ["P1"])
assert params[4:6] == (at, at)
@pytest.mark.parametrize("defect,status", [("old_replay", 422), ("too_early", 422), ("future", 422), ("foreign_device", 403)])
def test_replay_history_rejects_missing_freeze_or_scope_escape(defect, status):
at = START - timedelta(days=7)
run = {"start_time": START.isoformat(), "config": {"device_ids": ["P1"]},
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()}}
if defect == "old_replay": run.pop("baseline")
if defect == "too_early": at = START - timedelta(days=16)
if defect == "future": at = START + timedelta(minutes=1)
connection = Connection([[{"document": run}]])
context = {"replay_id": str(uuid4()), "knowledge_cutoff": START.isoformat(), "batch_sequence": 9}
with pytest.raises(HTTPException) as caught:
asyncio.run(replay_snapshot_rows(connection, uuid4(), context,
["P2" if defect == "foreign_device" else "P1"], at, at, 10))
assert caught.value.status_code == status
assert len(connection.statements) <= 1
def test_location_accepts_frozen_pre_replay_normal_window():
replay_id = str(uuid4())
normal = START - timedelta(days=7)
run = {"start_time": START.isoformat(),
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()},
"devices": [{"device_id": "P1", "node_id": "J1"}, {"device_id": "F1", "link_id": "L1"}],
"monitoring_config": {"pressure_device_ids": ["P1"], "outlet_flow_device_ids": ["F1"]}}
readings = [{"reading": {"time": at.isoformat(), "device_id": device,
"cleaned_value": value, "quality": "valid"}}
for at, device, value in [(normal, "P1", 100), (normal, "F1", 50),
(START + timedelta(minutes=15), "P1", 70),
(START + timedelta(minutes=15), "F1", 80)]]
connection = Connection([[{"document": run}], readings])
context = {"replay_id": replay_id, "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(), "batch_sequence": 9}
result = asyncio.run(replay_burst_location_inputs(connection, uuid4(), context,
_args(scada_normal_start=normal, scada_normal_end=normal + timedelta(minutes=14))))
assert result["normal_pressure"] == {"J1": 100}
assert result["burst_flow"] == {"L1": 80}
def test_replay_location_authorization_requires_explicit_bounded_normal_window(monkeypatch):
binding = BoundContext(
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
+731 -7
View File
@@ -10,23 +10,38 @@ import pytest
from fastapi import HTTPException
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.infra.db.postgresql.scada import ScadaInfoRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.scada_replay import ScadaReplayService
from app.services.scada_replay import BaselineBudgetExceeded, ScadaReplayService, snapshot
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
class Connection:
def __init__(self, owner=None):
self.statements, self.depth, self.owner = [], 0, owner
async def execute(self, statement): self.statements.append(statement)
@asynccontextmanager
async def transaction(self):
yield
prior = (deepcopy(self.owner.document), deepcopy(self.owner.batches)) if self.owner else None
self.depth += 1
try:
yield
except BaseException:
if prior:
self.owner.document, self.owner.batches = prior
raise
finally:
self.depth -= 1
class Repository:
def __init__(self):
self.project_id = str(uuid4())
self.connection = Connection()
self.connection = Connection(self)
self.document = None
self.batches = {}
@@ -43,9 +58,20 @@ class Repository:
async def get(self, _): return deepcopy(self.document)
async def current(self): return deepcopy(self.document)
async def pending_batch(self, _):
return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None)
return next((deepcopy(batch) for batch in self.batches.values()
if batch["sequence"] > 0 and batch["stage"] != "detected"), None)
async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch)
async def monitoring_outputs(self, *_args): return []
async def clear_unpublished_baseline(self, _):
self.batches = {key: value for key, value in self.batches.items()
if not (key < 0 and value.get("kind") == "baseline" and value["stage"] == "clean_prepared")}
async def publish_baseline(self, _):
for key, value in self.batches.items():
if key < 0 and value.get("kind") == "baseline" and value["stage"] == "clean_prepared":
value["stage"] = "detected"
async def monitoring_outputs(self, _replay_id, device_ids, start, end, before_sequence):
return [row for batch in self.batches.values() if batch["sequence"] < before_sequence
and batch["stage"] == "detected" for row in batch.get("outputs", [])
if row["device_id"] in device_ids and start <= datetime.fromisoformat(row["time"]) <= end]
def setup(monkeypatch):
@@ -59,6 +85,8 @@ def setup(monkeypatch):
"device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00"
}]))
async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_):
if _end <= START:
return []
return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}]
if after is None else [])
query = AsyncMock(side_effect=raw_page)
@@ -66,11 +94,269 @@ def setup(monkeypatch):
monkeypatch.setattr(ScadaRepository, "get_raw_page", query)
monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning)
monkeypatch.setattr(service, "_create_investigations", AsyncMock())
monkeypatch.setattr(service, "_publish_monitoring_alert", AsyncMock())
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService.create",
AsyncMock(return_value=SimpleNamespace(task_id=uuid4())))
create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}])
return repo, service, query, cleaning, create
def test_baseline_freezes_all_pages_silently_without_cleaning_the_source(monkeypatch):
repo, service, query, cleaning, create = setup(monkeypatch)
before = START - timedelta(minutes=15)
query.side_effect = None
query.side_effect = [
[{"time": before - timedelta(minutes=15), "device_id": "P1", "monitored_value": 100., "source_hash": "a"}],
[{"time": before, "device_id": "P1", "monitored_value": 200., "source_hash": "b"}],
[],
]
async def run():
await service.create(create, monitoring_interval_seconds=900)
document = await repo.get(None)
await service._freeze_baseline(document, create)
assert datetime.fromisoformat(repo.document["virtual_now"]) == create.start_time
assert repo.document["batch_sequence"] == repo.document["processed_count"] == 0
assert repo.document["anomaly_count"] == repo.document["monitoring_count"] == 0
assert repo.document["rule_states"] == repo.document["latest"] == {}
assert repo.document["baseline"]["row_count"] == 2
assert repo.document["histories"]["P1"] == [100., 200.]
assert set(repo.batches) == {-1, -2}
assert all(batch["kind"] == "baseline" and batch["stage"] == "detected"
and batch["events"] == [] for batch in repo.batches.values())
assert repo.batches[-2]["outputs"][0]["cleaned_value"] == 200.
asyncio.run(run())
assert service.timescale.statements == ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY"]
assert query.await_args_list[0].args[2:4] == (START - timedelta(days=15), START)
assert query.await_args_list[1].args[6] == (before - timedelta(minutes=15), "P1")
cleaning.assert_not_awaited()
service._create_investigations.assert_not_awaited()
def test_snapshot_returns_frozen_monitoring_rules_and_actual_calibration(monkeypatch):
repo, service, _, _, create = setup(monkeypatch)
settings = ScadaMonitoringSettings(pressure_device_ids=["P1"])
async def run():
await service.create(create, monitoring_interval_seconds=900, monitoring_config=settings)
repo.document["monitoring_calibration"] = {
"status": "ready", "burst_detection": {"threshold": -0.0843},
"pressure": {"P1": {"threshold_percent": 6.5}},
}
# Project settings may change later; the replay still reports its frozen configuration.
settings.weights.pressure = 0.3
settings.weights.burst_detection = 0.6
result = snapshot(repo.document)
assert result.monitoring_config.weights.pressure == 0.4
assert result.monitoring_config.weights.burst_detection == 0.5
assert result.monitoring_calibration["burst_detection"]["threshold"] == -0.0843
repo.document["monitoring_config"].pop("decision_policy")
assert snapshot(repo.document).monitoring_config.decision_policy == "all_signals"
asyncio.run(run())
def test_baseline_stages_each_page_before_reading_more_without_publishing_or_holding_business_transaction(monkeypatch):
repo, service, query, _, create = setup(monkeypatch)
calls = 0
async def raw_page(*args, **kwargs):
nonlocal calls
assert service.timescale.depth == 1
assert repo.connection.depth == 0
if calls:
assert repo.batches[-calls]["stage"] == "clean_prepared"
assert not repo.document.get("warmed") and "baseline" not in repo.document
assert await repo.pending_batch(None) is None
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
calls += 1
if calls > 2:
return []
return [{"time": START - timedelta(minutes=45 - calls * 15), "device_id": "P1",
"monitored_value": 30., "source_hash": str(calls)}]
query.side_effect = raw_page
async def run():
await service.create(create, monitoring_interval_seconds=900)
await service._freeze_baseline(await repo.get(None), create)
assert repo.document["warmed"] and repo.document["baseline"]["row_count"] == 2
assert all(batch["stage"] == "detected" for batch in repo.batches.values())
asyncio.run(run())
def test_failed_baseline_read_never_publishes_partial_history(monkeypatch):
repo, service, query, cleaning, create = setup(monkeypatch)
query.side_effect = [
[{"time": START - timedelta(minutes=15), "device_id": "P1", "monitored_value": 30., "source_hash": "a"}],
RuntimeError("source unavailable"),
]
async def run():
await service.create(create, monitoring_interval_seconds=900)
with pytest.raises(RuntimeError, match="source unavailable"):
await service._freeze_baseline(await repo.get(None), create)
assert "baseline" not in repo.document and not repo.document.get("warmed")
assert repo.batches[-1]["stage"] == "clean_prepared"
assert await repo.pending_batch(None) is None
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
asyncio.run(run())
cleaning.assert_not_awaited()
def test_baseline_retry_discards_unpublished_pages_and_refreezes_one_source_snapshot(monkeypatch):
repo, service, query, _, create = setup(monkeypatch)
def row(minutes, value):
return {"time": START - timedelta(minutes=minutes), "device_id": "P1",
"monitored_value": value, "source_hash": str(value)}
query.side_effect = [[row(30, 100.)], [row(15, 200.)], RuntimeError("source unavailable")]
async def run():
first = await service.create(create, monitoring_interval_seconds=900)
await service.command(first.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="step"))
with pytest.raises(RuntimeError, match="source unavailable"):
await service.process_pending_step(first.replay_id)
assert repo.document["status"] == "failed" and set(repo.batches) == {-1, -2}
assert all(batch["stage"] == "clean_prepared" for batch in repo.batches.values())
query.side_effect = [[row(15, 400.)], [], []]
await service.command(first.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=2, action="retry"))
result = await service.process_pending_step(first.replay_id)
assert not result.step_pending and result.status == "paused"
assert repo.document["baseline"]["row_count"] == 1
assert repo.document["histories"]["P1"] == [400.]
assert -2 not in repo.batches and repo.batches[-1]["stage"] == "detected"
assert repo.batches[-1]["outputs"][0]["cleaned_value"] == 400.
assert result.anomaly_count == result.processed_count == 0
asyncio.run(run())
@pytest.mark.parametrize("budget", ["rows", "bytes"])
def test_baseline_budget_stops_before_saving_excess_rows_and_exposes_actionable_error(monkeypatch, budget):
repo, service, query, cleaning, create = setup(monkeypatch)
if budget == "rows":
monkeypatch.setattr("app.services.scada_replay.BASELINE_MAX_ROWS", 2)
else:
monkeypatch.setattr("app.services.scada_replay.BASELINE_MAX_BYTES", 1)
query.side_effect = [
[{"time": START - timedelta(minutes=minute), "device_id": "P1",
"monitored_value": 30., "source_hash": str(minute)} for minute in [45, 30]],
[{"time": START - timedelta(minutes=15), "device_id": "P1", "monitored_value": 30., "source_hash": "15"}],
]
async def run():
first = await service.create(create, monitoring_interval_seconds=900)
await service.command(first.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="step"))
with pytest.raises(BaselineBudgetExceeded):
await service.process_pending_step(first.replay_id)
result = snapshot(repo.document)
assert result.status == "failed" and not result.step_pending
assert "BASELINE_BUDGET_EXCEEDED" in result.error and "历史基线" in result.error and "重试" in result.error
assert "baseline" not in repo.document and not repo.document.get("warmed")
assert result.processed_count == result.anomaly_count == result.monitoring_count == 0
assert result.virtual_now == START
assert set(repo.batches) == ({-1} if budget == "rows" else set())
assert all(batch["stage"] == "clean_prepared" for batch in repo.batches.values())
asyncio.run(run())
cleaning.assert_not_awaited()
service._create_investigations.assert_not_awaited()
assert query.await_count == (2 if budget == "rows" else 1)
if budget == "rows":
assert query.await_args_list[1].args[5] == 1
def test_failed_baseline_publication_rolls_back_visibility_and_initialized_marker(monkeypatch):
repo, service, query, _, create = setup(monkeypatch)
query.side_effect = [[{"time": START - timedelta(minutes=15), "device_id": "P1",
"monitored_value": 30., "source_hash": "a"}], []]
async def run():
await service.create(create, monitoring_interval_seconds=900)
monkeypatch.setattr(repo, "save", AsyncMock(side_effect=RuntimeError("publish unavailable")))
with pytest.raises(RuntimeError, match="publish unavailable"):
await service._freeze_baseline(await repo.get(None), create)
assert repo.batches[-1]["stage"] == "clean_prepared"
assert "baseline" not in repo.document and not repo.document.get("warmed")
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
asyncio.run(run())
def test_baseline_already_published_is_not_refrozen_or_replaced(monkeypatch):
repo, service, query, _, create = setup(monkeypatch)
async def run():
await service.create(create, monitoring_interval_seconds=900)
repo.document.update(warmed=True, baseline={"row_count": 1})
repo.batches[-1] = {"sequence": -1, "stage": "detected", "kind": "baseline", "outputs": []}
before = deepcopy((repo.document, repo.batches))
await service._freeze_baseline(await repo.get(None), create)
assert (repo.document, repo.batches) == before
asyncio.run(run())
query.assert_not_awaited()
def test_first_step_uses_frozen_fifteen_days_without_counting_historical_events(monkeypatch):
repo, service, query, cleaning, _ = setup(monkeypatch)
pressure_ids, flow_id = [f"P{i}" for i in range(5)], "F1"
device_ids = pressure_ids + [flow_id]
config = ScadaReplayCreate(
request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1), device_ids=device_ids,
rules=[{"rule_id": "pressure", "kind": "relative_change", "device_ids": pressure_ids,
"direction": "decrease", "threshold": 20, "unit": "m"},
{"rule_id": "flow", "kind": "relative_change", "device_ids": [flow_id],
"direction": "increase", "threshold": 20, "unit": "m3/h"}],
)
monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
"device_id": device, "device_type": "pressure" if device in pressure_ids else "pipe_flow",
"measurement_unit": "m" if device in pressure_ids else "m3/h",
"transmission_frequency": "0:15:00",
} for device in device_ids]))
source = [{"time": START + timedelta(minutes=15 * i), "device_id": device,
"monitored_value": (70. if device in pressure_ids else 150.) if i == 1 else 100.,
"source_hash": f"{i}-{device}"}
for i in range(-15 * 96, 2) for device in sorted(device_ids)]
async def raw_page(_connection, _devices, start, end, through, limit, after=None, **options):
rows = [row for row in source if start <= row["time"] < end and row["time"] <= through
and (after is None or (row["time"], row["device_id"]) > after)]
if options.get("frame") and rows:
rows = [row for row in rows if row["time"] == rows[0]["time"]]
return rows[:limit]
query.side_effect = raw_page
def detection(**kwargs):
target = kwargs["observed_pressure_data"].iloc[-1].min()
return {"rows": [{"Score": 0.02}] * 14 + [{"Score": -0.2 if target < 90 else 0.02}], "summary": {}}
monkeypatch.setattr("app.services.scada_replay_monitoring.run_burst_detection", detection)
monkeypatch.setattr(service, "_create_monitoring_task", AsyncMock())
async def run():
first = await service.create(config, monitoring_interval_seconds=900,
monitoring_config=ScadaMonitoringSettings(
pressure_device_ids=pressure_ids, outlet_flow_device_ids=[flow_id]))
await service.command(first.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="step"))
result = await service.process_pending_step(first.replay_id)
assert result.last_monitoring.status == "suspected_burst"
assert result.last_monitoring.weighted["weights"]["outlet_flow"] == 0.1
assert repo.document["monitoring_calibration"]["status"] == "ready"
assert result.processed_count == 12 and result.monitoring_count == 1
assert repo.document["baseline"]["row_count"] == 8640
assert result.virtual_now == START + timedelta(minutes=15)
assert all(not batch["events"] for key, batch in repo.batches.items() if key < 0)
asyncio.run(run())
assert cleaning.await_count == 2
assert all(row["time"] >= START.isoformat()
for call in cleaning.await_args_list for row in call.args[1])
def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch):
repo, service, query, cleaning, create = setup(monkeypatch)
async def run():
@@ -89,6 +375,79 @@ def test_step_command_is_revisioned_idempotent_and_does_not_process_until_reques
query.assert_not_awaited(); cleaning.assert_not_awaited()
@pytest.mark.parametrize("minutes", [0, 10])
def test_monitoring_stops_exactly_at_selected_time_and_reuses_same_period(monkeypatch, minutes):
repo, service, query, cleaning, create = setup(monkeypatch)
target = START + timedelta(minutes=minutes)
async def run():
initial = await service.create(create, monitoring_interval_seconds=900)
pending = await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="monitor", at_time=target))
assert pending.step_pending
assert datetime.fromisoformat(repo.document["step_target"]) == target
result = await service.process_pending_step(initial.replay_id)
assert result.virtual_now == target
assert result.last_monitoring.period_end == target
assert result.last_monitoring.knowledge_cutoff == target
assert result.monitoring_count == 1
reads, writes, batches = query.await_count, cleaning.await_count, len(repo.batches)
repeated = await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=result.revision, action="monitor", at_time=target))
assert not repeated.step_pending
assert repeated.last_monitoring == result.last_monitoring
assert repeated.monitoring_count == 1
assert (query.await_count, cleaning.await_count, len(repo.batches)) == (reads, writes, batches)
await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=repeated.revision, action="step"))
assert datetime.fromisoformat(repo.document["step_target"]) == START + timedelta(minutes=15)
asyncio.run(run())
assert all(call.args[4] <= target for call in query.await_args_list)
@pytest.mark.parametrize("minutes,status_code", [(-5, 422), (65, 422), (15, 409)])
def test_selected_monitoring_rejects_out_of_range_or_backward_targets_without_processing(monkeypatch, minutes, status_code):
repo, service, query, cleaning, create = setup(monkeypatch)
async def run():
initial = await service.create(create, monitoring_interval_seconds=900)
repo.document["virtual_now"] = (START + timedelta(minutes=30)).isoformat()
before = deepcopy(repo.document)
with pytest.raises(HTTPException) as failure:
await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="monitor", at_time=START + timedelta(minutes=minutes)))
assert failure.value.status_code == status_code
assert repo.document == before
asyncio.run(run())
query.assert_not_awaited()
cleaning.assert_not_awaited()
def test_selected_monitoring_cannot_replace_a_pending_period(monkeypatch):
repo, service, query, cleaning, create = setup(monkeypatch)
async def run():
initial = await service.create(create, monitoring_interval_seconds=900)
await service.command(initial.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
before = deepcopy(repo.document)
with pytest.raises(HTTPException, match="当前时间步正在处理"):
await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=2, action="monitor", at_time=START))
assert repo.document == before
asyncio.run(run())
query.assert_not_awaited()
cleaning.assert_not_awaited()
@pytest.mark.parametrize("action,at_time", [("monitor", None), ("monitor", "2026-07-16T08:00:00"), ("step", "2026-07-16T00:00:00Z")])
def test_selected_monitoring_requires_an_explicit_zoned_time_only_for_monitor(action, at_time):
with pytest.raises(ValueError):
ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action=action, at_time=at_time)
def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch):
repo, service, query, cleaning, create = setup(monkeypatch)
async def run():
@@ -212,7 +571,8 @@ def test_partial_final_step_checks_empty_interval_and_completes_without_fabricat
assert query.await_args.args[4] == create.end_time
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch):
@pytest.mark.parametrize("weighted", [False, True])
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch, weighted):
repo, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
@@ -231,12 +591,376 @@ def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(mo
"burst_detection": {"target_score": -0.05},
"task_id": None,
}
if weighted:
monitoring["burst_detection"]["score_threshold"] = -0.0843
monitoring["weighted"] = {
"score": 0.9, "threshold": 0.8,
"weights": {"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.1},
}
document = {"replay_id": str(uuid4())}
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8}))
command = create_task.await_args.args[0]
assert command.workflow == "burst_report"
assert command.workflow == "scada_anomaly"
assert command.source.replay_context.batch_sequence == 8
assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
assert monitoring["task_id"] == str(create_task.return_value.task_id)
assert document["monitoring_incident"]["alert"]["condition_id"] == f"operational-task:{monitoring['task_id']}"
assert document["monitoring_incident"]["alert"]["description"] == "已启动调查,详细诊断见调查报告。"
if weighted:
assert "-0.0843" in command.summary
assert "综合评分 0.90" in command.summary
assert "流量 10%" in command.summary
def test_monitoring_summary_counts_signals_without_copying_reading_details(monkeypatch):
_, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
monitoring = incident_monitoring(0)
monitoring["anomaly_signals"] = [
{"device_ids": [f"P{i}"], "detail": f"P{i}=28.14402136077437 m,超出冻结正常同时段范围 [28.3253, 28.8685] m"}
for i in range(100)
]
asyncio.run(service._create_monitoring_task({"replay_id": str(uuid4())}, monitoring, {"sequence": 1}))
summary = create_task.await_args.args[0].summary
assert len(summary) <= 300
assert "100" in summary
assert "28.14402136077437" not in summary
assert monitoring["anomaly_signals"][0]["detail"].startswith("P0=28.14402136077437")
def test_continuing_burst_reuses_task_across_persisted_periods(monkeypatch):
repo, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
advance_task = AsyncMock()
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
lambda *_: SimpleNamespace(create=create_task, advance_replay_investigation=advance_task))
document = {"replay_id": str(uuid4())}
async def run():
nonlocal document
for step in range(3):
monitoring = incident_monitoring(step, status="attention" if step == 0 else "suspected_burst")
await service._create_monitoring_task(document, monitoring, {"sequence": step + 1})
document = deepcopy(document) # Simulate loading persisted state after restart.
assert monitoring["task_id"] == str(create_task.return_value.task_id)
assert create_task.await_count == 1
service._publish_monitoring_alert.assert_awaited_once()
assert document["monitoring_incident"]["pressure_anchors"]["P1"]["baseline"] == 100.
advance_task.assert_awaited_once()
assert advance_task.await_args.args[0] == create_task.return_value.task_id
assert advance_task.await_args.args[1]["knowledge_cutoff"] == (START + timedelta(minutes=30)).isoformat()
asyncio.run(run())
def incident_monitoring(step, *, status="suspected_burst", value=60.):
return {
"monitoring_id": str(uuid4()), "status": status,
"period_start": (START + timedelta(minutes=15 * step)).isoformat(),
"period_end": (START + timedelta(minutes=15 * (step + 1))).isoformat(),
"knowledge_cutoff": (START + timedelta(minutes=15 * (step + 1))).isoformat(),
"rule_version": "burst-monitor-v2", "detail": "监测结论", "task_id": None,
"pressure": {"status": "ready", "hit": status == "suspected_burst", "hit_device_ids": ["P1"],
"devices": [{"device_id": "P1", "baseline": 100., "value": value,
"threshold_percent": 10., "hit": status == "suspected_burst",
"observed_at": (START + timedelta(minutes=15 * (step + 1))).isoformat()}]},
"outlet_flow": {"status": "ready", "hit": False, "hit_device_ids": []},
"burst_detection": {"status": "ready", "hit": status == "suspected_burst", "target_score": -0.2},
}
def test_incident_recovers_once_then_allows_new_accident(monkeypatch):
_, service, *_ = setup(monkeypatch)
create_task = AsyncMock(side_effect=[SimpleNamespace(task_id=uuid4()), SimpleNamespace(task_id=uuid4())])
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
lambda *_: SimpleNamespace(create=create_task))
document = {"replay_id": str(uuid4())}
async def run():
first = incident_monitoring(0)
await service._create_monitoring_task(document, first, {"sequence": 1})
for step in (1, 2):
result = incident_monitoring(step, status="normal", value=100.)
await service._create_monitoring_task(document, result, {"sequence": step + 1})
assert result["incident"]["action"] == "resolved"
assert result["task_id"] == first["task_id"]
recurrence = incident_monitoring(3)
await service._create_monitoring_task(document, recurrence, {"sequence": 4})
assert recurrence["incident"]["incident_id"] != first["incident"]["incident_id"]
assert recurrence["task_id"] != first["task_id"]
assert create_task.await_count == 2
assert [call.args[2] for call in service._publish_monitoring_alert.await_args_list] == ["raised", "resolved", "raised"]
commands = [call.args[0] for call in create_task.await_args_list]
assert commands[0].source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
asyncio.run(run())
@pytest.mark.parametrize("interruption", ["insufficient_data", "attention", "normal_low_pressure", "missing_sensor", "stale_reading", "time_gap"])
def test_incident_needs_two_fresh_complete_pressure_recoveries(interruption):
from app.services.scada_monitoring_incidents import update_incident
document = {"replay_id": str(uuid4())}
incident, _ = update_incident(document, incident_monitoring(0), 1)
incident["task_id"] = str(uuid4())
update_incident(document, incident_monitoring(1, status="normal", value=100.), 2)
interrupted = incident_monitoring(2, status="normal", value=100.)
if interruption in {"insufficient_data", "attention"}:
interrupted["status"] = interruption
elif interruption == "normal_low_pressure":
interrupted["pressure"]["devices"][0]["value"] = 60.
elif interruption == "missing_sensor":
interrupted["pressure"]["devices"] = []
elif interruption == "stale_reading":
interrupted["pressure"]["devices"][0]["observed_at"] = incident["last_observed_at"]
else:
interrupted["period_start"] = START.isoformat()
incident, action = update_incident(document, interrupted, 3)
assert action == "ongoing"
assert incident["recovery_count"] == (1 if interruption == "time_gap" else 0)
if interruption != "time_gap":
_, action = update_incident(document, incident_monitoring(3, status="normal", value=100.), 4)
assert action == "ongoing"
def test_same_period_retry_does_not_count_as_second_recovery():
from app.services.scada_monitoring_incidents import update_incident
document = {"replay_id": str(uuid4())}
update_incident(document, incident_monitoring(0), 1)
recovery = incident_monitoring(1, status="normal", value=100.)
update_incident(document, recovery, 2)
incident, action = update_incident(deepcopy(document), recovery, 2)
assert action == "ongoing" and incident["recovery_count"] == 1
def test_older_replay_adopts_existing_task_without_creating_another(monkeypatch):
_, service, *_ = setup(monkeypatch)
previous = incident_monitoring(0)
previous["task_id"] = str(uuid4())
document = {"replay_id": str(uuid4()), "last_monitoring": previous}
current = incident_monitoring(1)
asyncio.run(service._create_monitoring_task(document, current, {"sequence": 2}))
assert current["task_id"] == previous["task_id"]
assert current["incident"]["action"] == "ongoing"
service._publish_monitoring_alert.assert_not_awaited()
def test_combined_monitoring_alert_uses_same_facts_for_recovery(monkeypatch):
repo, service, *_ = setup(monkeypatch)
push = AsyncMock()
monkeypatch.setattr("app.infra.db.postgresql.task_alerts.TaskAlertRepository", lambda *_: object())
monkeypatch.setattr("app.services.task_alerts.TaskAlertService", lambda *_args, **_kwargs: SimpleNamespace(push=push))
incident = {"incident_id": str(uuid4()), "alert": {
"alert_id": "test-accident", "producer_id": "scada-replay:demo",
"occurred_at": START.isoformat(), "condition_id": None,
"source": {"kind": "scada_replay", "replay_id": "demo", "rule_id": "burst_monitor", "batch_sequence": 1},
"title": "SCADA 监测疑似爆管", "description": "同一事故", "severity": "warning",
}}
async def run():
for change in ("raised", "resolved"):
await ScadaReplayService._publish_monitoring_alert(service, {"replay_id": "demo"}, incident, change)
asyncio.run(run())
first, second = [call.args[0].model_dump(mode="json") for call in push.await_args_list]
assert first.pop("event_id") != second.pop("event_id")
assert first.pop("event_type") == "raised"
assert second.pop("event_type") == "resolved"
assert first == second
def test_combined_monitoring_keeps_rule_evidence_and_quality_notifications(monkeypatch):
_, service, *_ = setup(monkeypatch)
push = AsyncMock()
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.task_alerts.TaskAlertRepository", lambda *_: object())
monkeypatch.setattr("app.services.task_alerts.TaskAlertService", lambda *_args, **_kwargs: SimpleNamespace(push=push))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=AsyncMock()))
def event(rule, category=None):
return {"event_id": str(uuid4()), "rule_id": rule, "change": "raised", "occurred_at": START.isoformat(),
"detail": "检测信号", "category": category, "task_id": None}
pressure, water, quality = event("p"), event("water"), event("quality-P1", "data_quality")
document = {"replay_id": str(uuid4()), "monitoring_config": {"decision_policy": "weighted"},
"config": {"rules": [{"rule_id": "p", "kind": "relative_change"}, {"rule_id": "water", "kind": "bounds"}]},
"rule_states": {item["rule_id"]: {"event": deepcopy(item)} for item in (pressure, water, quality)}}
async def run():
await ScadaReplayService._create_investigations(service, document, [pressure, water, quality], {"sequence": 1}, create_tasks=False)
assert push.await_count == 1 # Measurement signals share diagnosis; telemetry quality remains independent.
assert document["rule_states"]["p"]["event"]["notification_suppressed"]
assert document["rule_states"]["water"]["event"]["notification_suppressed"]
resolved = {**document["rule_states"]["p"]["event"], "change": "resolved"}
await ScadaReplayService._create_investigations(service, document, [resolved], {"sequence": 2}, create_tasks=False)
assert push.await_count == 1
resolved.pop("notification_suppressed") # An old alert raised before this change still resolves.
await ScadaReplayService._create_investigations(service, document, [resolved], {"sequence": 2}, create_tasks=False)
assert push.await_count == 2
asyncio.run(run())
def test_failed_incident_publication_rolls_back_state_and_retries_same_task_request(monkeypatch):
repo, service, _, _, create = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
return {**incident_monitoring(0), "period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
service._publish_monitoring_alert.side_effect = [RuntimeError("alert unavailable"), None]
async def run():
first = await service.create(create, monitoring_interval_seconds=900)
repo.document["warmed"] = True
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
with pytest.raises(RuntimeError, match="alert unavailable"):
await service.process_pending_step(first.replay_id)
assert repo.document["status"] == "failed"
assert "monitoring_incident" not in repo.document
assert repo.document["monitoring_count"] == 0
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry"))
result = await service.process_pending_step(first.replay_id)
assert result.last_monitoring.incident.action == "raised"
assert result.monitoring_count == 1
first_command, retried_command = [call.args[0] for call in create_task.await_args_list]
assert first_command == retried_command
assert result.last_monitoring.task_id == create_task.return_value.task_id
asyncio.run(run())
@pytest.mark.parametrize("indicator", ["pressure", "outlet_flow", "burst_detection", "quality", "partial_data"])
def test_any_single_measurement_anomaly_enters_diagnosis_below_burst_threshold(monkeypatch, indicator):
_, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
monitoring = incident_monitoring(0, status="attention")
for key in ("pressure", "outlet_flow", "burst_detection"):
monitoring[key]["hit"] = key == indicator
monitoring["weighted"] = {"score": 0.1, "threshold": 0.8,
"weights": {"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.1}}
if indicator == "quality":
monitoring["status"] = "insufficient_data"
monitoring["anomaly_signals"] = [{"event_id": str(uuid4()), "rule_id": "quality_bounds",
"device_ids": ["Q1"], "detail": "水质测点超出已配置上限"}]
monitoring["burst_detection"] = {"status": "not_run", "hit": False}
if indicator == "partial_data":
monitoring["status"] = "insufficient_data"
monitoring["pressure"]["hit"] = True
document = {"replay_id": str(uuid4())}
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 1}))
command = create_task.await_args.args[0]
assert command.workflow == "scada_anomaly"
assert command.source.event_kind == "scada_anomaly"
assert "任一指标异常即启动调查" in command.summary
assert "爆管嫌疑门槛 0.80" in command.summary
assert monitoring["task_id"] == str(create_task.return_value.task_id)
assert monitoring["incident"]["action"] == "raised"
def test_flow_only_incident_does_not_recover_while_absolute_flow_remains_high():
from app.services.scada_monitoring_incidents import update_incident
document = {"replay_id": str(uuid4())}
first = incident_monitoring(0, status="attention", value=100.)
first["outlet_flow"] = {"status": "ready", "hit": True, "devices": [
{"device_id": "F1", "hit": True, "baseline": 100., "threshold_percent": 10., "value": 150.}
]}
incident, _ = update_incident(document, first, 1)
incident["task_id"] = str(uuid4())
for step in (1, 2):
normal = incident_monitoring(step, status="normal", value=100.)
normal["outlet_flow"] = {"status": "ready", "hit": False, "devices": [
{"device_id": "F1", "value": 150., "observed_at": normal["period_end"]}
]}
_, action = update_incident(document, normal, step + 1)
assert action == "ongoing"
for step in (3, 4):
normal = incident_monitoring(step, status="normal", value=100.)
normal["outlet_flow"] = {"status": "ready", "hit": False, "devices": [
{"device_id": "F1", "value": 100., "observed_at": normal["period_end"]}
]}
_, action = update_incident(document, normal, step + 1)
assert action == "resolved"
def test_transient_measurement_anomaly_still_enters_diagnosis_at_period_end(monkeypatch):
repo, service, query, _, create = setup(monkeypatch)
readings = [(START, 20.), (START + timedelta(minutes=5), 5.), (START + timedelta(minutes=10), 5.)]
async def raw_page(_conn, _devices, _start, _end, through, _limit, after=None, **_):
return [{"time": at, "device_id": "P1", "monitored_value": value, "source_hash": str(index)}
for index, (at, value) in enumerate(readings) if at <= through and (after is None or at > after[0])][:1]
query.side_effect = raw_page
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
return {**incident_monitoring(0, status="normal", value=5.),
"period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
async def run():
initial = await service.create(create, monitoring_interval_seconds=900)
repo.document["warmed"] = True
await service.command(initial.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
result = await service.process_pending_step(initial.replay_id)
assert result.active_events == []
assert result.last_monitoring.status == "attention"
assert result.last_monitoring.anomaly_signals[0]["rule_id"] == "p"
assert result.last_monitoring.task_id is not None
assert result.last_monitoring.incident.action == "raised"
assert "step_anomaly_signals" not in repo.document
asyncio.run(run())
@pytest.mark.parametrize("action,status", [("step", "paused"), ("retry", "failed")])
def test_warmed_weighted_replay_with_old_calibration_requires_rebuild(monkeypatch, action, status):
repo, service, _, _, create = setup(monkeypatch)
async def run():
initial = await service.create(create, monitoring_interval_seconds=900,
monitoring_config=ScadaMonitoringSettings())
repo.document.update(warmed=True, status=status,
monitoring_calibration={"version": "scada-monitor-calibration@1"})
with pytest.raises(HTTPException) as error:
await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action=action))
assert error.value.status_code == 409
assert "新建" in error.value.detail
asyncio.run(run())
def test_old_active_percentage_event_does_not_enter_new_period_diagnosis(monkeypatch):
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
repo, service, _, _, _ = setup(monkeypatch)
config = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "relative_change",
"direction": "both", "threshold": 30, "unit": "m"}])
monkeypatch.setattr(service, "_create_monitoring_task", AsyncMock())
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
return {**incident_monitoring(0, status="normal", value=20.),
"period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
async def run():
initial = await service.create(config, monitoring_interval_seconds=900,
monitoring_config=ScadaMonitoringSettings())
old = {"event_id": str(uuid4()), "rule_id": "p", "device_ids": ["P1"], "status": "active",
"occurred_at": START.isoformat(), "last_observed_at": START.isoformat(),
"detail": "旧30%事件", "task_id": None}
repo.document.update(warmed=True, rule_states={"p": {"event": old, "hits": 1, "recoveries": 0}},
monitoring_calibration={"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
"knowledge_cutoff": START.isoformat(), "diagnostic": {"P1": {
"phase_origin": START.isoformat(), "interval_seconds": 900, "lower": [10.], "upper": [30.]}}})
await service.command(initial.replay_id, ScadaReplayCommand(
request_id=uuid4(), expected_revision=1, action="step"))
result = await service.process_pending_step(initial.replay_id)
assert result.last_monitoring.status == "normal"
assert result.last_monitoring.anomaly_signals == []
assert repo.document["rule_states"]["p"]["event"] == old
asyncio.run(run())
+319
View File
@@ -0,0 +1,319 @@
"""Real spawned workers share a listener while established streams drain."""
from contextlib import contextmanager
import json
import multiprocessing
from pathlib import Path
from threading import Event, Thread
import time
from types import SimpleNamespace
import httpx
import pytest
import uvicorn
from scripts.server_handoff import HandoffSupervisor
def wait_for(predicate, *, timeout=5.):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.02)
raise AssertionError("test-owned worker did not reach the expected state")
def next_event(lines, *, after_ns=0):
for line in lines:
if line.startswith("data: "):
event = json.loads(line[len("data: "):])
if event.get("time_ns", 0) >= after_ns:
return event
raise AssertionError("SSE ended before its fixture release marker")
def owned_workers(supervisor):
return [worker for worker in [supervisor.current, *supervisor.retiring] if worker is not None]
def close_owned_supervisor(supervisor, directory, original_children):
"""Release every fixture stream before waiting; bound cleanup even on failure."""
(directory / "release-all").touch()
(directory / "allow-startup").touch()
workers = owned_workers(supervisor)
errors = []
def close():
try:
supervisor.close()
except BaseException as error:
errors.append(error)
closing = Thread(target=close, daemon=True)
closing.start()
closing.join(timeout=8)
leaked = [process for process in multiprocessing.active_children()
if process.pid not in original_children and process.is_alive()]
if closing.is_alive() or leaked:
# This is emergency teardown of processes created by this test, never
# production reload policy or an operation on the developer's service.
for process in leaked:
process.terminate()
for process in leaked:
process.join(timeout=3)
closing.join(timeout=3)
pytest.fail("handoff cleanup left a test-owned worker running")
for worker in workers:
worker.process.join(timeout=1)
assert not worker.process.is_alive()
if errors:
raise errors[0]
@pytest.fixture
def handoff(monkeypatch, tmp_path):
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "fixtures"))
monkeypatch.setenv("SCADA_HANDOFF_FIXTURE_DIR", str(tmp_path))
(tmp_path / "version").write_text("v1")
(tmp_path / "startup-mode").write_text("ready")
(tmp_path / "readiness").write_text("true")
original_children = {process.pid for process in multiprocessing.active_children()}
@contextmanager
def running(*, startup_timeout=5., readiness_check=None):
config = uvicorn.Config(
"server_handoff_app:app", host="127.0.0.1", port=0,
lifespan="on", log_level="error", access_log=False,
timeout_graceful_shutdown=None,
)
supervisor = HandoffSupervisor(config, startup_timeout=startup_timeout, poll_interval=.02,
readiness_check=readiness_check)
try:
yield supervisor, tmp_path
finally:
close_owned_supervisor(supervisor, tmp_path, original_children)
return running
def base_url(supervisor):
return f"http://127.0.0.1:{supervisor.socket.getsockname()[1]}"
def health(supervisor, *, timeout=3.):
# A fresh TCP connection makes each assertion exercise current acceptance.
with httpx.Client(timeout=timeout, trust_env=False) as client:
response = client.get(base_url(supervisor) + "/health")
response.raise_for_status()
return response.json()
def test_first_worker_is_spawned_and_ready_before_start_returns(handoff):
with handoff() as (supervisor, _):
assert supervisor.start() is True
worker = supervisor.current
assert worker.process._start_method == "spawn"
assert worker.process.is_alive()
assert health(supervisor) == {"pid": worker.process.pid, "version": "v1"}
assert supervisor.retiring == []
def test_reload_keeps_old_sse_until_release_and_new_requests_use_new_worker(handoff):
with handoff() as (supervisor, directory):
assert supervisor.start() is True
old = supervisor.current
with httpx.Client(timeout=3., trust_env=False) as client:
with client.stream("GET", base_url(supervisor) + "/events") as stream:
stream.raise_for_status()
lines = stream.iter_lines()
assert next_event(lines)["pid"] == old.process.pid
(directory / "version").write_text("v2")
(directory / "startup-mode").write_text("delay")
stop_probe = Event()
samples, errors = [], []
def probe_health():
while not stop_probe.is_set():
try:
samples.append(health(supervisor, timeout=.3))
except Exception as error:
errors.append(repr(error))
stop_probe.wait(.01)
probing = Thread(target=probe_health, daemon=True)
probing.start()
try:
wait_for(lambda: samples)
assert supervisor.reload() is True
wait_for(lambda: any(sample["version"] == "v2" for sample in samples))
finally:
stop_probe.set()
probing.join(timeout=2)
assert not probing.is_alive()
assert errors == [], f"HTTP availability gap during reload: {errors}"
assert {sample["version"] for sample in samples} == {"v1", "v2"}
new = supervisor.current
assert new.process.pid != old.process.pid
assert old in supervisor.retiring
assert old.process.is_alive()
for _ in range(3):
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
after_handoff = time.time_ns()
event = next_event(lines, after_ns=after_handoff)
assert (event["pid"], event["version"]) == (old.process.pid, "v1")
assert not (directory / f"stopped-{old.process.pid}").exists()
(directory / f"release-{old.process.pid}").touch()
assert any(line == "event: complete" for line in lines)
old.process.join(timeout=5)
assert old.process.exitcode == 0
assert (directory / f"stopped-{old.process.pid}").read_text() == "v1"
supervisor.reap()
assert old not in supervisor.retiring
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
@pytest.mark.parametrize("mode", ["fail", "wait"])
def test_candidate_startup_failure_keeps_previous_worker_serving(handoff, mode):
with handoff() as (supervisor, directory):
assert supervisor.start() is True
old = supervisor.current
(directory / "version").write_text("broken-v2")
(directory / "startup-mode").write_text(mode)
if mode == "wait":
supervisor.startup_timeout = .75
assert supervisor.reload() is False
assert supervisor.current is old
assert old.process.is_alive()
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
assert supervisor.retiring == []
assert len(list(directory.glob("started-*"))) == 2
def test_candidate_process_start_error_keeps_previous_worker_healthy(handoff, monkeypatch):
def fail_start():
raise OSError("test process spawn rejected")
with handoff() as (supervisor, _):
assert supervisor.start() is True
old = supervisor.current
candidate = SimpleNamespace(pid=None, start=fail_start)
monkeypatch.setattr("scripts.server_handoff.get_subprocess", lambda **_: candidate)
assert supervisor.reload() is False
assert supervisor.current is old
assert supervisor.retiring == []
assert old.process.is_alive()
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
def test_readiness_check_rejects_unusable_candidate_before_handoff(handoff):
with handoff(readiness_check="server_handoff_app:readiness_check") as (supervisor, directory):
assert supervisor.start() is True
old = supervisor.current
assert (directory / f"checked-{old.process.pid}").read_text() == "true"
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
(directory / "version").write_text("not-ready-v2")
(directory / "readiness").write_text("false")
stop_probe = Event()
samples, errors = [], []
def probe_health():
while not stop_probe.is_set():
try:
samples.append(health(supervisor, timeout=.3))
except Exception as error:
errors.append(repr(error))
stop_probe.wait(.01)
probing = Thread(target=probe_health, daemon=True)
probing.start()
try:
wait_for(lambda: samples)
assert supervisor.reload() is False
finally:
stop_probe.set()
probing.join(timeout=2)
assert not probing.is_alive()
assert errors == []
assert all(sample == {"pid": old.process.pid, "version": "v1"} for sample in samples)
rejected = [path for path in directory.glob("checked-*") if path.name != f"checked-{old.process.pid}"]
assert len(rejected) == 1
assert rejected[0].read_text() == "false"
assert supervisor.current is old
assert old.process.is_alive()
assert supervisor.retiring == []
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
(directory / "version").write_text("ready-v3")
(directory / "readiness").write_text("true")
assert supervisor.reload() is True
new = supervisor.current
assert new.process.pid != old.process.pid
assert (directory / f"checked-{new.process.pid}").read_text() == "true"
assert health(supervisor) == {"pid": new.process.pid, "version": "ready-v3"}
def test_close_waits_for_existing_stream_to_finish_without_forcing_worker(handoff):
with handoff() as (supervisor, directory):
assert supervisor.start() is True
worker = supervisor.current
with httpx.Client(timeout=3., trust_env=False) as client:
with client.stream("GET", base_url(supervisor) + "/events") as stream:
lines = stream.iter_lines()
assert next_event(lines)["pid"] == worker.process.pid
errors = []
def close():
try:
supervisor.close()
except BaseException as error:
errors.append(error)
closing = Thread(target=close, daemon=True)
closing.start()
try:
after_close = time.time_ns()
assert next_event(lines, after_ns=after_close)["pid"] == worker.process.pid
assert closing.is_alive()
assert worker.process.is_alive()
(directory / f"release-{worker.process.pid}").touch()
assert any(line == "event: complete" for line in lines)
finally:
(directory / "release-all").touch()
closing.join(timeout=5)
assert not closing.is_alive()
assert errors == []
assert worker.process.exitcode == 0
assert (directory / f"stopped-{worker.process.pid}").exists()
@pytest.mark.parametrize("active_alive,edited,expected_reloads", [(False, False, 1), (True, True, 0)])
def test_retiring_limit_coalesces_edits_but_never_blocks_crash_recovery(
monkeypatch, active_alive, edited, expected_reloads,
):
class OneWatchIteration:
calls = 0
def wait(self, _timeout):
self.calls += 1
return self.calls > 1
def set(self):
self.calls = 2
supervisor = HandoffSupervisor.__new__(HandoffSupervisor)
supervisor.current = SimpleNamespace(process=SimpleNamespace(is_alive=lambda: active_alive))
supervisor.retiring = [object(), object()]
supervisor.should_exit = OneWatchIteration()
supervisor.poll_interval = .001
supervisor._files = {}
calls = []
monkeypatch.setattr(supervisor, "start", lambda: True)
monkeypatch.setattr(supervisor, "reap", lambda: None)
monkeypatch.setattr(supervisor, "_snapshot", lambda: {"edited.py": (1, 1)} if edited else {})
monkeypatch.setattr(supervisor, "reload", lambda: calls.append("reload"))
monkeypatch.setattr(supervisor, "close", lambda: calls.append("close"))
supervisor.run()
assert calls.count("reload") == expected_reloads
assert calls[-1] == "close"
+24
View File
@@ -281,6 +281,30 @@ def test_temporary_project_database_cleans_up_after_failure(monkeypatch) -> None
]
def test_temporary_project_database_copies_snapshot_without_template(monkeypatch) -> None:
calls: list[tuple[str, ...]] = []
@contextmanager
def temporary_template(project, purpose):
calls.append(("schema", project, purpose))
yield "isolated_run"
monkeypatch.setattr(projects, "have_project", lambda name: False)
monkeypatch.setattr(projects, "temporary_template_database", temporary_template)
monkeypatch.setattr(
"app.native.wndb.core.model_replace.copy_network_tables",
lambda source, target: calls.append(("model", source, target)),
)
with projects.temporary_project_database("project_a", "burst") as name:
assert name == "isolated_run"
assert calls == [
("schema", "project_a", "burst"),
("model", "project_a", "isolated_run"),
]
def test_temporary_template_database_does_not_clone_a_project(monkeypatch) -> None:
calls: list[tuple[str, ...]] = []
monkeypatch.setattr(