feat(scada): add calibrated monitoring and incident recovery
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.benchmark.generation import (
|
||||
deterministic_standard_normal,
|
||||
noisy_scada_observation,
|
||||
)
|
||||
|
||||
|
||||
AT = datetime(2026, 5, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_noise_is_stable_and_bound_to_observation_identity():
|
||||
value = deterministic_standard_normal(
|
||||
seed="normal-day-v1", at=AT, device_id="P1", metric="pressure",
|
||||
)
|
||||
assert value == deterministic_standard_normal(
|
||||
seed="normal-day-v1", at=AT, device_id="P1", metric="pressure",
|
||||
)
|
||||
assert value != deterministic_standard_normal(
|
||||
seed="normal-day-v1", at=AT, device_id="P2", metric="pressure",
|
||||
)
|
||||
|
||||
|
||||
def test_pressure_keeps_metres_and_flow_converts_lps_to_m3h_before_noise():
|
||||
pressure = noisy_scada_observation(
|
||||
raw_value=20.0, device_type="pressure", at=AT, device_id="P1",
|
||||
seed="normal-day-v1", pressure_sigma_percent=0, pressure_sigma_floor_m=0,
|
||||
)
|
||||
flow = noisy_scada_observation(
|
||||
raw_value=10.0, device_type="pipe_flow", at=AT, device_id="F1",
|
||||
seed="normal-day-v1", flow_sigma_percent=0,
|
||||
)
|
||||
assert pressure == 20.0
|
||||
assert flow == 36.0
|
||||
|
||||
|
||||
def test_noise_is_clipped_and_pressure_never_becomes_negative():
|
||||
value = noisy_scada_observation(
|
||||
raw_value=0.01, device_type="pressure", at=AT, device_id="P1",
|
||||
seed="normal-day-v1", pressure_sigma_percent=0,
|
||||
pressure_sigma_floor_m=100, clip_sigma=3,
|
||||
)
|
||||
assert value >= 0
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported"):
|
||||
noisy_scada_observation(
|
||||
raw_value=1, device_type="quality", at=AT, device_id="Q1",
|
||||
seed="normal-day-v1",
|
||||
)
|
||||
@@ -0,0 +1,97 @@
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.benchmark.calibration import assess, configure_inp, pressure_preview, select_calibrated_area
|
||||
|
||||
|
||||
START = datetime(2026, 10, 8, 12, tzinfo=timezone.utc)
|
||||
PIPE = SimpleNamespace(
|
||||
pipe_id="test-pipe",
|
||||
pressure_bindings=[SimpleNamespace(device_id="P1", node_id="N1"),
|
||||
SimpleNamespace(device_id="P2", node_id="N2")],
|
||||
flow_bindings=[SimpleNamespace(device_id="F1", link_id="L1")],
|
||||
)
|
||||
RULE = SimpleNamespace(
|
||||
primary_pressure_drop_percent=30, corroborating_pressure_drop_percent=10,
|
||||
flow_change_percent=20, consecutive_samples=2, relative_change_epsilon=1e-6,
|
||||
)
|
||||
|
||||
|
||||
def result(p1=(30, 30, 30, 30, 30), p2=(30, 30, 30, 30, 30), flow=(10, 10, 10, 10, 10)):
|
||||
return {"status": "completed", "warnings": [],
|
||||
"units": {"flow": "LPS", "pressure": "MTR", "quality": "NONE"},
|
||||
"times": {"report_start": 0, "report_step": 900, "sim_duration": 3600, "num_periods": 5},
|
||||
"nodes": {"N1": [{"pressure": v} for v in p1], "N2": [{"pressure": v} for v in p2]},
|
||||
"links": {"L1": [{"flow": v} for v in flow]}}
|
||||
|
||||
|
||||
def check(burst, normal=None):
|
||||
return assess(PIPE, normal or result(), burst, RULE, duration=3600, step=900)
|
||||
|
||||
|
||||
def test_file_configuration_preserves_source_and_matches_burst_service():
|
||||
source = "[OPTIONS]\nDEMAND MODEL DDA\nREQUIRED PRESSURE 20\n[TIMES]\nDURATION 0\nPATTERN START 0:00\n[REPORT]\nNODES ALL\nLINKS ALL\n[EMITTERS]\n[END]\n"
|
||||
configured = configure_inp(source, START, 18000, emitter=("N1", 100), pda_options={"HEADERROR": "0.0001"})
|
||||
assert "DEMAND MODEL DDA" in source
|
||||
assert "DEMAND MODEL DDA" not in configured
|
||||
assert "DEMAND MODEL PDA" in configured
|
||||
assert "REQUIRED PRESSURE 10.0000" in configured
|
||||
assert "HEADERROR 0.0001" in configured
|
||||
assert "DURATION 5:00:00" in configured
|
||||
assert "PATTERN START 20:00:00" in configured
|
||||
coefficient = float(configured.split("[EMITTERS]\nN1 ")[1].splitlines()[0])
|
||||
assert coefficient == pytest.approx(28.7767267075)
|
||||
assert "NODES NONE" in configured and "LINKS NONE" in configured
|
||||
|
||||
|
||||
def test_requires_two_consecutive_primary_samples():
|
||||
assert not check(result(p1=(20, 30, 20, 30, 20), p2=(25,) * 5))["passed"]
|
||||
assert check(result(p1=(20, 20, 30, 30, 30), p2=(25,) * 5))["passed"]
|
||||
|
||||
|
||||
def test_corroboration_must_be_concurrent_and_a_different_pressure_device():
|
||||
assert not check(result(p1=(20, 20, 30, 30, 30)))["passed"]
|
||||
assert not check(result(p1=(20, 20, 30, 30, 30), p2=(30, 30, 25, 25, 30)))["passed"]
|
||||
assert check(result(p1=(20,) * 5, flow=(12,) * 5))["passed"]
|
||||
|
||||
|
||||
def test_pressure_rise_and_zero_flow_baseline_do_not_supply_false_evidence():
|
||||
assert not check(result(p1=(40,) * 5, p2=(25,) * 5, flow=(100,) * 5))["passed"]
|
||||
assert not check(result(p1=(20,) * 5, flow=(10,) * 5), normal=result(flow=(0,) * 5))["passed"]
|
||||
|
||||
|
||||
def test_negative_pressure_or_bad_normal_pressure_invalidates_candidate():
|
||||
assert "negative_monitor_pressure:P2" in check(result(p1=(20,) * 5, p2=(-1,) * 5))["blockers"]
|
||||
assert "nonpositive_normal_pressure:P1" in check(result(p1=(0,) * 5), normal=result(p1=(0,) * 5))["blockers"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fault", ["warning", "missing_period", "unit"])
|
||||
def test_solver_warning_incomplete_time_grid_and_wrong_unit_fail(fault):
|
||||
burst = result(p1=(20,) * 5, p2=(25,) * 5)
|
||||
if fault == "warning":
|
||||
burst["warnings"] = ["WARNING: System unbalanced"]
|
||||
elif fault == "missing_period":
|
||||
burst["times"]["num_periods"] = 4
|
||||
else:
|
||||
burst["units"]["flow"] = "CMH"
|
||||
assert not check(burst)["passed"]
|
||||
|
||||
|
||||
def test_actual_replay_rule_uses_warmup_and_reports_first_signal_time():
|
||||
history = {"N1": [{"pressure": 30}] * 4, "N2": [{"pressure": 30}] * 4}
|
||||
normal = pressure_preview(PIPE, START, 3600, 900, history, result()["nodes"], RULE)
|
||||
event = pressure_preview(PIPE, START, 3600, 900, history, result(p1=(20,) * 5)["nodes"], RULE)
|
||||
assert not normal["raised"]
|
||||
assert event["raised"]
|
||||
assert event["first_signal_at"] == "2026-10-08T12:15:00+00:00"
|
||||
|
||||
|
||||
def test_valid_burst_with_a_weak_signal_is_retained_separately_from_invalid_physics():
|
||||
pipe = SimpleNamespace(candidate_orifice_areas_cm2={"light": 1, "medium": 2, "severe": 4})
|
||||
weak = [{"blockers": ["event_rule_did_not_trigger", "insufficient_consecutive_pressure_and_corroboration"]}]
|
||||
assert select_calibrated_area(pipe, [], weak, True)["status"] == "weak_signal_candidate"
|
||||
assert select_calibrated_area(pipe, [], weak, False)["status"] == "unsuitable"
|
||||
assert select_calibrated_area(pipe, [], [{"blockers": ["burst_solver_warning"]}], True)["status"] == "unsuitable"
|
||||
assert select_calibrated_area(pipe, ["severe", "medium"], weak, True)["area_cm2"] == 2
|
||||
@@ -0,0 +1,225 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.mcp import calculation_jobs, query_service
|
||||
from app.mcp.analysis_models import BurstLocationInput
|
||||
from app.mcp.catalog import TOOLS
|
||||
from app.services import burst_location_demo as demo
|
||||
|
||||
|
||||
def arguments(**changes):
|
||||
return BurstLocationInput.model_validate({
|
||||
"context_handle": "x" * 43,
|
||||
"burst_leakage": 0.02,
|
||||
"scada_burst_start": demo.EVENT_START - timedelta(minutes=15),
|
||||
"scada_burst_end": demo.EVENT_START,
|
||||
**changes,
|
||||
})
|
||||
|
||||
|
||||
def topology():
|
||||
def link(key, kind, a, b, length=0.0, diameter=None):
|
||||
return {"id": key, "kind": kind, "start_node_id": a, "end_node_id": b,
|
||||
"length": length, "diameter": diameter, "status": "OPEN"}
|
||||
return [
|
||||
link(demo.REFERENCE_PIPE, "pipe", "170438", "J1", 350, 600),
|
||||
link(demo.CANDIDATE_PIPE, "pipe", "J1", "J2", 30, 600),
|
||||
link("V1", "valve", "S", "170438"),
|
||||
link("V2", "valve", "J2", "S"),
|
||||
{"id": "S", "kind": "reservoir", "start_node_id": None},
|
||||
]
|
||||
|
||||
|
||||
def test_candidate_is_on_same_line_within_distance_and_identical_valve_boundary():
|
||||
result = demo._checked_candidate(topology())
|
||||
assert result["pipe_id"] == demo.CANDIDATE_PIPE
|
||||
assert result["distance_m"] == 350
|
||||
assert result["boundary_valve_ids"] == ["V1", "V2"]
|
||||
assert result["within_same_isolation_zone"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defect", ["outside_zone", "different_line", "too_close", "too_far",
|
||||
"closed_line", "unisolatable", "missing_pipe"])
|
||||
def test_unsuitable_candidate_is_never_returned(defect):
|
||||
rows = topology()
|
||||
if defect == "outside_zone":
|
||||
rows[1]["end_node_id"] = "S"
|
||||
elif defect == "different_line":
|
||||
rows[1]["diameter"] = 150
|
||||
elif defect == "too_close":
|
||||
rows[0]["length"] = 199
|
||||
elif defect == "too_far":
|
||||
rows[0]["length"] = 501
|
||||
elif defect == "closed_line":
|
||||
rows[0]["status"] = "CLOSED"
|
||||
elif defect == "unisolatable":
|
||||
rows[2]["kind"] = "pipe"
|
||||
else:
|
||||
rows.pop(1)
|
||||
assert demo._checked_candidate(rows) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("project,changes,enabled", [
|
||||
("tjwater2_v2", {}, True),
|
||||
("tjw_bench_tjwater2_20261105", {}, True),
|
||||
("tjwater_v2", {}, False),
|
||||
(None, {}, False),
|
||||
("tjwater2_v2", {"scada_burst_start": demo.EVENT_START - timedelta(minutes=30),
|
||||
"scada_burst_end": demo.EVENT_START - timedelta(minutes=15)}, False),
|
||||
("tjwater2_v2", {"scada_burst_end": demo.EVENT_END + timedelta(minutes=15)}, False),
|
||||
])
|
||||
def test_only_case3_demo_window_skips_native_calculation_queue(project, changes, enabled):
|
||||
args = arguments(**changes)
|
||||
assert demo.is_location_demo(project, args) is enabled
|
||||
assert calculation_jobs.should_queue(TOOLS["run_burst_location"], args, project_code=project) is not enabled
|
||||
|
||||
|
||||
def test_candidate_has_stable_similarity_without_presentation_blockers(monkeypatch):
|
||||
load = AsyncMock(return_value=topology())
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", load)
|
||||
result = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments()))
|
||||
assert result["located_pipe"] == demo.CANDIDATE_PIPE
|
||||
assert result["top_candidates"] == [{"pipe_id": demo.CANDIDATE_PIPE, "similarity": 0.9708,
|
||||
"similarity_percent": 97.08}]
|
||||
assert not {"mock", "notice", "simulation_times", "similarity_mode", "observed_source"} & result.keys()
|
||||
assert demo.REFERENCE_PIPE not in str(result) and demo.REFERENCE_NODE not in str(result)
|
||||
load.return_value = list(reversed(topology()))
|
||||
later = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments(
|
||||
scada_burst_start=demo.EVENT_START,
|
||||
scada_burst_end=demo.EVENT_START + timedelta(minutes=15),
|
||||
)))
|
||||
assert later == result
|
||||
load.return_value = []
|
||||
empty = asyncio.run(demo.location_demo_result(object(), "tjwater2_v2", arguments()))
|
||||
assert empty["located_pipe"] is None and empty["top_candidates"] == []
|
||||
assert empty["exit_condition"] == "no_candidate"
|
||||
assert "关阀范围" in empty["notice"]
|
||||
|
||||
|
||||
def test_mcp_demo_returns_synthetic_result_without_simulation_or_queue(monkeypatch):
|
||||
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
|
||||
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
|
||||
binding = SimpleNamespace(replay_context=None)
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
|
||||
assert result.origin == "synthetic"
|
||||
assert result.data["result"]["located_pipe"] == demo.CANDIDATE_PIPE
|
||||
assert "mock" not in result.data["result"]
|
||||
assert result.data["result"]["top_candidates"][0]["similarity_percent"] == 97.08
|
||||
assert result.data_gaps == []
|
||||
|
||||
|
||||
def test_missing_candidate_preserves_a_specific_data_gap(monkeypatch):
|
||||
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=[]))
|
||||
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
|
||||
binding = SimpleNamespace(replay_context=None)
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
|
||||
assert result.data["result"]["top_candidates"] == []
|
||||
assert result.data_gaps == [result.data["result"]["notice"]]
|
||||
assert "关阀范围" in result.data_gaps[0]
|
||||
|
||||
|
||||
def test_location_capability_and_tool_contract_do_not_inject_mock_instructions():
|
||||
context = demo.location_demo_context("tjwater2_v2", {"knowledge_cutoff": demo.EVENT_START.isoformat()})
|
||||
assert context["available"] is True
|
||||
assert "相似度" in context["notice"] and "影响分析" in context["notice"]
|
||||
assert "mock" not in str(context) and "模拟" not in context["notice"]
|
||||
tool = TOOLS["run_burst_location"].protocol_tool()
|
||||
assert "相似度" in tool.description
|
||||
assert "demo模拟候选" not in tool.description
|
||||
assert "明确标记的演示候选" not in str(tool.input_schema)
|
||||
|
||||
|
||||
def test_replay_mock_still_validates_frozen_inputs_before_returning(monkeypatch):
|
||||
from fastapi import HTTPException
|
||||
from app.services import scada_replay_queries
|
||||
|
||||
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
|
||||
freeze = AsyncMock(side_effect=HTTPException(422, "frozen baseline missing"))
|
||||
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
|
||||
load = AsyncMock()
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", load)
|
||||
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(
|
||||
project_id="p", project_code="tjwater2_v2"))
|
||||
binding = SimpleNamespace(replay_context={"replay_id": "r"})
|
||||
with pytest.raises(HTTPException):
|
||||
asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
|
||||
load.assert_not_awaited()
|
||||
|
||||
|
||||
def test_replay_mock_returns_after_frozen_validation_without_native_calculation(monkeypatch):
|
||||
from app.mcp import analysis_service
|
||||
from app.services import scada_replay_queries
|
||||
|
||||
monkeypatch.setattr(query_service, "authorize_query", AsyncMock())
|
||||
freeze = AsyncMock(return_value={"burst_pressure": {"P1": 20}, "normal_pressure": {"P1": 30}})
|
||||
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
|
||||
native = AsyncMock(side_effect=AssertionError("mock entered native location"))
|
||||
monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", native)
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
|
||||
scope = SimpleNamespace(connection=SimpleNamespace(commit=AsyncMock()),
|
||||
project=SimpleNamespace(project_id="p", project_code="tjwater2_v2"))
|
||||
binding = SimpleNamespace(replay_context={"replay_id": "r"})
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], arguments()))
|
||||
assert result.origin == "synthetic"
|
||||
assert result.data["result"]["top_candidates"][0]["similarity"] == 0.9708
|
||||
assert "mock" not in result.data["result"]
|
||||
assert result.data_gaps == []
|
||||
freeze.assert_awaited_once()
|
||||
native.assert_not_awaited()
|
||||
|
||||
|
||||
def test_replay_demo_location_does_not_require_invented_leakage(monkeypatch):
|
||||
from app.mcp import analysis_service
|
||||
from app.services import scada_replay_queries
|
||||
|
||||
permissions = {"burst.run", "scada.view"}
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
|
||||
args = arguments(burst_leakage=None,
|
||||
scada_normal_start=demo.EVENT_START - timedelta(days=1, minutes=15),
|
||||
scada_normal_end=demo.EVENT_START - timedelta(days=1))
|
||||
context = {"replay_id": "r", "knowledge_cutoff": demo.EVENT_START.isoformat(), "batch_sequence": 3}
|
||||
binding = SimpleNamespace(replay_context=context, tool_cap=["run_burst_location"],
|
||||
permission_cap=permissions, purpose="task_execution", task_id="t")
|
||||
scope = SimpleNamespace(connection=SimpleNamespace(commit=AsyncMock()),
|
||||
project=SimpleNamespace(project_id="p", project_code="tjwater2_v2"))
|
||||
freeze = AsyncMock(return_value={"burst_pressure": {"J1": 20}, "normal_pressure": {"J1": 30}})
|
||||
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", freeze)
|
||||
monkeypatch.setattr(demo.NetworkAssetRepository, "get_isolation_model", AsyncMock(return_value=topology()))
|
||||
native = AsyncMock(side_effect=AssertionError("demo entered hydraulic calculation"))
|
||||
monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", native)
|
||||
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["run_burst_location"], args))
|
||||
|
||||
assert result.data["result"]["located_pipe"] == demo.CANDIDATE_PIPE
|
||||
assert result.data["result"]["burst_leakage"] is None
|
||||
assert result.origin == "synthetic"
|
||||
assert result.data["result"]["top_candidates"][0]["similarity_percent"] == 97.08
|
||||
assert "mock" not in result.data["result"]
|
||||
freeze.assert_awaited_once_with(scope.connection, "p", context, args)
|
||||
native.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("project,changes", [
|
||||
("tjwater_v2", {}),
|
||||
("tjwater2_v2", {"scada_burst_end": demo.EVENT_END + timedelta(minutes=15)}),
|
||||
("tjwater2_v2", {"data_source": "simulation", "simulation_run_id": "8bfa5a5b-dbba-461a-88d4-34942bc66034"}),
|
||||
])
|
||||
def test_real_location_still_requires_leakage(monkeypatch, project, changes):
|
||||
from fastapi import HTTPException
|
||||
|
||||
args = arguments(burst_leakage=None, **changes)
|
||||
permissions = {"burst.run", "scada.view", "simulation.view"}
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
|
||||
binding = SimpleNamespace(replay_context=None, tool_cap=["run_burst_location"],
|
||||
permission_cap=permissions, purpose="ordinary_chat", task_id=None)
|
||||
with pytest.raises(HTTPException) as caught:
|
||||
asyncio.run(query_service.authorize_query(SimpleNamespace(
|
||||
project=SimpleNamespace(project_code=project)), binding, TOOLS["run_burst_location"], args))
|
||||
assert caught.value.status_code == 422
|
||||
assert "burst_leakage" in caught.value.detail
|
||||
@@ -0,0 +1,68 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.benchmark.pressure_thresholds import evaluate_pressure_thresholds, learn_pressure_thresholds
|
||||
|
||||
|
||||
CUTOFF = datetime(2026, 10, 8, 12, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def history(device, noisy=False):
|
||||
start = CUTOFF - timedelta(days=15)
|
||||
return [{"device_id": device, "time": start + timedelta(minutes=15 * index),
|
||||
"monitored_value": 30 + (index % 96) / 20 + ((index // 96 % 3) - 1) * (4 if noisy else 0.1)}
|
||||
for index in range(1440)]
|
||||
|
||||
|
||||
def test_thresholds_follow_device_noise_after_removing_daily_pattern():
|
||||
result = learn_pressure_thresholds(history("quiet") + history("noisy", True), ["quiet", "noisy"], CUTOFF)
|
||||
assert result["status"] == "ready"
|
||||
assert result["devices"]["quiet"]["relative_drop_percent"] == 3
|
||||
assert result["devices"]["noisy"]["relative_drop_percent"] > 30
|
||||
assert result["devices"]["quiet"]["absolute_drop_m"] == 0.5
|
||||
|
||||
|
||||
def test_short_warmup_is_not_treated_as_a_fitted_adaptive_rule():
|
||||
rows = history("P")[-4:]
|
||||
result = learn_pressure_thresholds(rows, ["P"], CUTOFF)
|
||||
assert result["status"] == "history_not_ready"
|
||||
assert result["required_samples_per_device"] == 1440
|
||||
assert result["available_samples"] == {"P": 4}
|
||||
|
||||
|
||||
def test_event_or_future_reading_is_rejected():
|
||||
with pytest.raises(ValueError, match="future"):
|
||||
learn_pressure_thresholds(history("P") + [{"device_id": "P", "time": CUTOFF, "monitored_value": 10}], ["P"], CUTOFF)
|
||||
|
||||
|
||||
def test_missing_grid_point_prevents_fitting():
|
||||
assert learn_pressure_thresholds(history("P")[:-1], ["P"], CUTOFF)["status"] == "history_not_ready"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")])
|
||||
def test_invalid_pressure_samples_are_rejected(value):
|
||||
rows = history("P")
|
||||
rows[0]["monitored_value"] = value
|
||||
with pytest.raises(ValueError, match="positive finite"):
|
||||
learn_pressure_thresholds(rows, ["P"], CUTOFF)
|
||||
|
||||
|
||||
def test_normal_daily_pattern_is_not_confused_with_a_burst():
|
||||
fitted = learn_pressure_thresholds(history("P"), ["P"], CUTOFF)
|
||||
normal = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=15 * i),
|
||||
"monitored_value": 30 + i / 20} for i in range(20)]
|
||||
assert not evaluate_pressure_thresholds(normal, fitted)["raised"]
|
||||
burst = [{**row, "monitored_value": row["monitored_value"] - 3} for row in normal]
|
||||
event = evaluate_pressure_thresholds(burst, fitted)
|
||||
assert event["raised"]
|
||||
assert event["signals"][0]["first_signal_at"] == (CUTOFF + timedelta(minutes=15)).isoformat()
|
||||
|
||||
|
||||
def test_absolute_floor_and_consecutive_sample_gap_are_enforced():
|
||||
baseline = [{**row, "monitored_value": 5} for row in history("P")]
|
||||
fitted = learn_pressure_thresholds(baseline, ["P"], CUTOFF)
|
||||
small = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=15 * i), "monitored_value": 4.8} for i in range(2)]
|
||||
assert not evaluate_pressure_thresholds(small, fitted)["raised"]
|
||||
separated = [{"device_id": "P", "time": CUTOFF + timedelta(minutes=30 * i), "monitored_value": 4} for i in range(2)]
|
||||
assert not evaluate_pressure_thresholds(separated, fitted)["raised"]
|
||||
@@ -33,13 +33,12 @@ class Connection:
|
||||
return Cursor()
|
||||
|
||||
|
||||
def test_status_projection_excludes_result_and_expires_without_worker():
|
||||
def test_status_projection_excludes_result_and_scrubs_expired_payloads():
|
||||
connection = Connection()
|
||||
repo = CalculationJobRepository(connection, 'project')
|
||||
asyncio.run(repo.get('job'))
|
||||
assert 'QUEUE_TIMEOUT' in connection.statements[0][0]
|
||||
assert 'SET result=NULL' in connection.statements[1][0]
|
||||
assert ',result FROM' not in connection.statements[2][0]
|
||||
assert 'SET result=NULL' in connection.statements[0][0]
|
||||
assert ',result FROM' not in connection.statements[1][0]
|
||||
asyncio.run(repo.get('job', include_result=True))
|
||||
assert ',result FROM' in connection.statements[-1][0]
|
||||
|
||||
@@ -55,6 +54,15 @@ def test_duplicate_submission_bypasses_full_queue_inside_transaction():
|
||||
assert not any('count(*)' in sql for sql, _, _ in connection.statements)
|
||||
|
||||
|
||||
def test_new_jobs_have_no_queue_expiry_contract():
|
||||
connection = Connection([None, {'active': 0}, {'job_id': 'new'}])
|
||||
asyncio.run(CalculationJobRepository(connection, 'project').enqueue(
|
||||
owner_user_id='owner', request_key='key', tool='tool', parameters={},
|
||||
binding={}, timeout_seconds=30))
|
||||
insert = next(sql for sql, _, _ in connection.statements if 'INSERT INTO' in sql)
|
||||
assert 'queue_expires_at' not in insert
|
||||
|
||||
|
||||
def test_capacity_rejects_new_job_before_insert():
|
||||
connection = Connection([None, {'active': 100}])
|
||||
with pytest.raises(CalculationCapacityExceeded):
|
||||
|
||||
@@ -27,8 +27,7 @@ def run_async(function):
|
||||
return wrapper
|
||||
|
||||
|
||||
def test_worker_settings_require_explicit_valid_scope(monkeypatch):
|
||||
monkeypatch.setenv('CALCULATION_WORKER_PROJECT_IDS', '6b39b128-2ba8-454f-9bbe-e9553902ab53')
|
||||
def test_worker_settings_require_valid_concurrency(monkeypatch):
|
||||
settings = WorkerSettings.from_environment()
|
||||
assert settings.max_concurrency == 2
|
||||
monkeypatch.setenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '30')
|
||||
@@ -85,7 +84,7 @@ async def test_cancel_and_lost_lease_stop_process_before_persisting(monkeypatch,
|
||||
@asynccontextmanager
|
||||
async def repository(project):
|
||||
yield Repository()
|
||||
worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.05), repository=repository)
|
||||
worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.05), repository=repository)
|
||||
await worker.run_job({'project_id': 'project', 'job_id': 'job', 'timeout_seconds': 60})
|
||||
assert finishes == ([expected] if expected else [])
|
||||
assert processes[0].returncode is not None
|
||||
@@ -111,7 +110,7 @@ async def test_timeout_stops_process_before_recording(monkeypatch):
|
||||
@asynccontextmanager
|
||||
async def repository(project):
|
||||
yield Repository()
|
||||
worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.02), repository=repository)
|
||||
worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.02), repository=repository)
|
||||
await worker.run_job({'project_id': 'project', 'timeout_seconds': 0})
|
||||
assert statuses == ['timed_out']
|
||||
|
||||
@@ -134,7 +133,52 @@ async def test_duplicate_supervisor_never_claims(monkeypatch):
|
||||
pytest.fail('A duplicate supervisor must never open the queue')
|
||||
yield
|
||||
with pytest.raises(RuntimeError, match='already active'):
|
||||
await CalculationWorker(WorkerSettings(('project',)), repository=repository).run()
|
||||
await CalculationWorker(WorkerSettings(), repository=repository).run()
|
||||
|
||||
|
||||
@run_async
|
||||
async def test_unavailable_project_does_not_block_other_projects(monkeypatch):
|
||||
from app.infra.db import calculation_worker as infrastructure
|
||||
|
||||
class Lease:
|
||||
async def ping(self):
|
||||
pass
|
||||
|
||||
@asynccontextmanager
|
||||
async def supervisor_lease():
|
||||
yield Lease()
|
||||
|
||||
checked = asyncio.Event()
|
||||
|
||||
class Repository:
|
||||
async def recover(self):
|
||||
pass
|
||||
|
||||
async def claim(self, owner, lease_seconds):
|
||||
checked.set()
|
||||
return None
|
||||
|
||||
@asynccontextmanager
|
||||
async def repository(project):
|
||||
if project == 'unavailable':
|
||||
raise RuntimeError('offline')
|
||||
yield Repository()
|
||||
|
||||
async def projects():
|
||||
return ('unavailable', 'healthy')
|
||||
|
||||
monkeypatch.setattr(infrastructure, 'calculation_supervisor_lease', supervisor_lease)
|
||||
worker = CalculationWorker(
|
||||
WorkerSettings(poll_seconds=.01, lease_seconds=1, heartbeat_seconds=.1),
|
||||
repository=repository,
|
||||
project_provider=projects,
|
||||
)
|
||||
task = asyncio.create_task(worker.run())
|
||||
try:
|
||||
await asyncio.wait_for(checked.wait(), 2)
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
|
||||
@run_async
|
||||
@@ -212,7 +256,7 @@ async def test_cleanup_failure_is_not_reported_cancelled(monkeypatch, tmp_path):
|
||||
@asynccontextmanager
|
||||
async def repository(project):
|
||||
yield Repository()
|
||||
worker = CalculationWorker(WorkerSettings(('project',)), repository=repository)
|
||||
worker = CalculationWorker(WorkerSettings(), repository=repository)
|
||||
await worker.run_job({'project_id': 'project', 'timeout_seconds': 60})
|
||||
assert results == [('failed', 'CLEANUP_FAILED')]
|
||||
assert len(list(tmp_path.glob('*/cleanup-failed'))) == 1
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""The same timestamp matrix as the Frontend and Agent contract tests."""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.domain.schemas.answer_ui import AnswerUiBlock
|
||||
from app.domain.schemas.operational_task import TaskSafeChart
|
||||
|
||||
|
||||
CASES = [
|
||||
("2026-06-14T12:00:00Z", True), ("2026-06-14T12:00:00+08:00", True),
|
||||
("2026-06-14T12:00:00-03:30", True), ("2024-02-29T23:59:59.123456Z", True),
|
||||
("2026-06-14T12:00:00+23:59", True), ("2026-06-14T12:00:00." + "1" * 43 + "Z", True),
|
||||
("2026-06-14 12:00:00+08:00", False), ("2026-W24-7T12:00:00+08:00", False),
|
||||
("2026-06-14T12:00+08:00", False), ("2026-06-14T12:00:00+0800", False),
|
||||
("2026-02-30T12:00:00Z", False), ("2026-13-14T12:00:00Z", False),
|
||||
("2026-06-00T12:00:00Z", False), ("2026-02-29T12:00:00Z", False),
|
||||
("2026-06-14T24:00:00Z", False), ("2026-06-14T12:00:60Z", False),
|
||||
("2026-06-14T12:00:00+24:00", False), ("2026-06-14T12:00:00+08:60", False),
|
||||
("2026-06-14T12:00:00", False), ("2026-06-14T12:00:00z", False),
|
||||
("0000-06-14T12:00:00Z", False), ("2026-06-14T12:00:00." + "1" * 44 + "Z", False),
|
||||
]
|
||||
|
||||
|
||||
def chart(data, times):
|
||||
return {"title": "压力", "chart_type": "line", "x_data": ["12:00"],
|
||||
"series": [{"name": "测点", "data": data, "sample_times": times}]}
|
||||
|
||||
|
||||
def block(value):
|
||||
return {"version": "answer-ui@1", "block_id": "pressure", "fallback_text": "压力数据",
|
||||
"spec": {"root": "chart", "elements": {"chart": {"type": "Chart", "children": [],
|
||||
"props": {**value, "source": "实测"}}}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("time,accepted", CASES)
|
||||
def test_sample_times_match_on_chart_commands_and_answer_blocks(time, accepted):
|
||||
value = chart([1], [time])
|
||||
for model, payload in [(TaskSafeChart, value), (AnswerUiBlock, block(value))]:
|
||||
if accepted:
|
||||
model.model_validate(payload)
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
model.model_validate(payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data,times,accepted", [
|
||||
([0], [None], True), ([None], [None], True),
|
||||
([None], ["2026-06-14T12:00:00Z"], False), ([1], [], False),
|
||||
])
|
||||
def test_sample_times_keep_zero_null_and_alignment(data, times, accepted):
|
||||
value = chart(data, times)
|
||||
for model, payload in [(TaskSafeChart, value), (AnswerUiBlock, block(value))]:
|
||||
if accepted:
|
||||
model.model_validate(payload)
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
model.model_validate(payload)
|
||||
@@ -0,0 +1,40 @@
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.mcp.display_time import display_query_result, display_time_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("original,displayed", [
|
||||
("2026-07-16T00:00:00Z", "2026-07-16T08:00:00+08:00"),
|
||||
("2026-07-16T08:00:00+08:00", "2026-07-16T08:00:00+08:00"),
|
||||
("2026-12-31T16:00:00.123456789Z", "2027-01-01T00:00:00.123456789+08:00"),
|
||||
("2026-12-31T23:00:00-03:30", "2027-01-01T10:30:00+08:00"),
|
||||
])
|
||||
def test_display_keeps_the_instant_precision_and_is_idempotent(original, displayed):
|
||||
data = {"time": original, "sample_times": [None, original], "value": 0}
|
||||
snapshot = deepcopy(data)
|
||||
result = display_query_result(data)
|
||||
assert result["time"] == displayed
|
||||
assert result["sample_times"] == [None, displayed]
|
||||
assert datetime.fromisoformat(original) == datetime.fromisoformat(displayed)
|
||||
assert display_query_result(result) == result
|
||||
assert data == snapshot
|
||||
|
||||
|
||||
def test_metadata_identifiers_links_code_and_selected_ids_keep_their_spelling():
|
||||
original = "2026-07-16T00:00:00Z"
|
||||
fields = {key: original for key in ("id", "evidence_id", "source_ref", "next_cursor", "resource_uri", "relative_path", "instructions")}
|
||||
assert display_query_result(fields) == fields
|
||||
opaque_page = {"selected_path": ["source", "external_id"], "value": original}
|
||||
assert display_query_result(opaque_page) == opaque_page
|
||||
time_page = {"selected_path": ["source", "replay_context", "knowledge_cutoff"], "value": original}
|
||||
assert display_query_result(time_page)["value"] == "2026-07-16T08:00:00+08:00"
|
||||
text = f"观测{original};`{original}`;https://example.test/{original};id_{original}"
|
||||
assert display_time_text(text) == text.replace("观测" + original, "观测2026-07-16T08:00:00+08:00")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["2026-07-16 08:00", "2026-07-16T08:00:00", "2026-02-30T00:00:00Z", "未记录"])
|
||||
def test_unzoned_or_invalid_values_keep_their_explicit_source(value):
|
||||
assert display_query_result({"time": value}) == {"time": value}
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -72,6 +73,57 @@ def test_calculation_limit_is_advertised_and_enforced_before_execution(runtime):
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_case3_location_mock_skips_queue_and_preserves_evidence_and_receipt(runtime, monkeypatch):
|
||||
from app.mcp import calculation_jobs
|
||||
|
||||
app, client, _, _ = runtime
|
||||
original_scope = app.scope_factory
|
||||
|
||||
@asynccontextmanager
|
||||
async def demo_scope(identity, project):
|
||||
async with original_scope(identity, project) as scope:
|
||||
scope.project.project_code = "tjwater2_v2"
|
||||
yield scope
|
||||
|
||||
app.scope_factory = demo_scope
|
||||
monkeypatch.setattr(application, "authorize_query", AsyncMock())
|
||||
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
|
||||
monkeypatch.setattr(application, "verify_task", AsyncMock())
|
||||
submit = AsyncMock(side_effect=AssertionError("demo entered calculation queue"))
|
||||
monkeypatch.setattr(calculation_jobs, "submit_calculation", submit)
|
||||
app.executor = AsyncMock(return_value=QueryResult(
|
||||
source="run_burst_location", source_ref="demo", queried_at=datetime.now(timezone.utc),
|
||||
semantics="analysis", origin="synthetic", data={"result": {"located_pipe": "515870",
|
||||
"top_candidates": [{"pipe_id": "515870", "similarity": 0.9708, "similarity_percent": 97.08}]}},
|
||||
))
|
||||
saved = []
|
||||
|
||||
class TaskService:
|
||||
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
|
||||
|
||||
async def record_data_snapshot(self, task_id, snapshot):
|
||||
saved.append(snapshot)
|
||||
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
|
||||
|
||||
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
|
||||
value = binding(task=True)
|
||||
query = {"name": "run_burst_location", "arguments": {
|
||||
"context_handle": value.context_handle,
|
||||
"scada_burst_start": "2026-07-16T07:45:00+08:00",
|
||||
"scada_burst_end": "2026-07-16T08:00:00+08:00",
|
||||
}}
|
||||
first = request(client, "tools/call", query).json()["result"]["structuredContent"]
|
||||
second = request(client, "tools/call", query).json()["result"]["structuredContent"]
|
||||
assert first == second
|
||||
assert first["origin"] == "synthetic"
|
||||
assert first["data"]["result"]["top_candidates"][0]["similarity_percent"] == 97.08
|
||||
assert "mock" not in first["data"]["result"]
|
||||
assert first["evidence_id"] == saved[0].source_ref
|
||||
assert len(saved) == app.executor.await_count == 1
|
||||
assert app.executor.await_args.args[3].burst_leakage is None
|
||||
submit.assert_not_awaited()
|
||||
|
||||
|
||||
def test_queue_submission_errors_preserve_mcp_error_mapping(runtime, monkeypatch):
|
||||
from app.mcp import calculation_jobs
|
||||
_, client, _, _ = runtime
|
||||
@@ -321,6 +373,45 @@ def test_replay_binding_denies_future_and_allows_device_metadata(monkeypatch):
|
||||
asyncio.run(query_service.authorize_query(SimpleNamespace(project=object()), bound, TOOLS["list_scada_devices"], SimpleNamespace()))
|
||||
|
||||
|
||||
def test_model_response_displays_utc8_without_changing_query_instants_or_source(runtime):
|
||||
app, client, _, _ = runtime
|
||||
at = datetime(2026, 12, 31, 16, tzinfo=timezone.utc)
|
||||
original = QueryResult(source="query_scada_readings", source_ref="saved:utc", queried_at=at,
|
||||
semantics="observed", data=[{"device_id": "P1", "time": "2026-12-31T16:00:00.123456789Z", "value": 20}],
|
||||
window={"start": at - timedelta(minutes=15), "end": at})
|
||||
snapshot = original.model_dump(mode="json")
|
||||
app.executor = AsyncMock(return_value=original)
|
||||
result = request(client, "tools/call", {"name": "query_scada_readings", "arguments": {
|
||||
"context_handle": binding().context_handle, "device_ids": ["P1"],
|
||||
"start": "2026-12-31T15:45:00Z", "end": "2026-12-31T16:00:00Z",
|
||||
}}).json()["result"]
|
||||
output = result["structuredContent"]
|
||||
assert output["data"][0]["time"] == "2027-01-01T00:00:00.123456789+08:00"
|
||||
assert output["display_time_zone"] == "Asia/Shanghai"
|
||||
assert output["window"]["end"] == "2027-01-01T00:00:00+08:00"
|
||||
assert json.loads(result["content"][0]["text"]) == output
|
||||
assert app.executor.await_args.args[3].end == at
|
||||
assert original.model_dump(mode="json") == snapshot
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_name", ["get_element_properties", "list_elements", "query_nearby_elements",
|
||||
"get_network_options", "get_node_connections", "get_element_gis_properties",
|
||||
"query_network_extent", "list_regions", "get_region", "get_region_nodes",
|
||||
"analyze_valve_isolation", "generate_node_coverage"])
|
||||
def test_replay_allows_asset_and_topology_analysis_without_live_telemetry(monkeypatch, tool_name):
|
||||
bound = binding(task=True)
|
||||
bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00Z", "batch_sequence": 1}
|
||||
permissions = PERMISSIONS | {"simulation.run"}
|
||||
bound.permission_cap = sorted(permissions)
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
|
||||
scope = SimpleNamespace(project=SimpleNamespace(project_role="member"))
|
||||
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
|
||||
bound.permission_cap = []
|
||||
with pytest.raises(HTTPException) as caught:
|
||||
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seconds", [30, 120, 240])
|
||||
def test_all_tools_advertise_the_effective_host_request_budget(runtime, seconds):
|
||||
app, client, _, _ = runtime
|
||||
@@ -600,6 +691,89 @@ def test_large_node_coverage_retains_publishable_hint_and_saves_complete_geometr
|
||||
assert saved[0].data["data"]["result"]["map_payload"] == geometry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", [False, True])
|
||||
@pytest.mark.parametrize("wide_ids", [False, True])
|
||||
def test_scada_catalog_keeps_all_devices_or_full_evidence_within_budget(runtime, monkeypatch, task, wide_ids):
|
||||
app, client, _, _ = runtime
|
||||
value = binding(task=task)
|
||||
saved = []
|
||||
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
|
||||
monkeypatch.setattr(application, "verify_task", AsyncMock())
|
||||
|
||||
class TaskService:
|
||||
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
|
||||
|
||||
async def record_data_snapshot(self, task_id, snapshot):
|
||||
saved.append(snapshot)
|
||||
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
|
||||
|
||||
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
|
||||
devices = [{
|
||||
"device_id": ("d" * 150 if wide_ids else "") + str(100000 + index),
|
||||
"device_type": "pressure", "measurement_unit": "m",
|
||||
"node_id": str(index), "link_id": None, "longitude": 121.5, "latitude": 31.0,
|
||||
"transmission_mode": "non_realtime", "transmission_frequency": "0:15:00",
|
||||
"reliability": 1, "api_query_id": str(index), "x": 1000, "y": -20000,
|
||||
} for index in range(100)]
|
||||
app.executor = AsyncMock(return_value=QueryResult(
|
||||
source="list_scada_devices", source_ref="mcp:catalog", queried_at=datetime.now(timezone.utc),
|
||||
semantics="asset", data=devices,
|
||||
page={"limit": 100, "offset": 0, "total": None, "has_more": True},
|
||||
))
|
||||
response = request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
|
||||
"context_handle": value.context_handle,
|
||||
}}).json()["result"]
|
||||
assert app.executor.await_args.args[3].limit == 100
|
||||
if wide_ids and not task:
|
||||
assert response["isError"] is True
|
||||
assert response["structuredContent"]["error"]["code"] == "RESULT_TOO_LARGE"
|
||||
assert not saved
|
||||
return
|
||||
assert not response.get("isError"), response
|
||||
output = response["structuredContent"]
|
||||
if wide_ids:
|
||||
assert output["data"]["resource_uri"].endswith(output["evidence_id"])
|
||||
else:
|
||||
assert [row["device_id"] for row in output["data"]] == [row["device_id"] for row in devices]
|
||||
assert all(row["device_type"] == "pressure" and row["measurement_unit"] == "m" for row in output["data"])
|
||||
assert "get_scada_device" in " ".join(output["data_gaps"])
|
||||
assert output["page"] == {"limit": 100, "offset": 0, "total": None, "has_more": True}
|
||||
assert output["truncated"] is True
|
||||
assert len(json.dumps(output, ensure_ascii=False).encode()) <= 12000
|
||||
if task:
|
||||
assert saved[0].data["data"] == devices
|
||||
assert output["evidence_id"] == saved[0].source_ref
|
||||
assert not saved[0].truncated
|
||||
else:
|
||||
assert not saved and output["evidence_id"] is None
|
||||
|
||||
|
||||
def test_scada_catalog_default_100_paginates_without_changing_other_queries(runtime, monkeypatch):
|
||||
app, client, _, _ = runtime
|
||||
value = binding()
|
||||
devices = [{"device_id": str(index), "device_type": "pressure", "measurement_unit": "m"}
|
||||
for index in range(101)]
|
||||
|
||||
async def read_devices(connection, *, device_id, limit, offset):
|
||||
return devices[offset:offset + limit]
|
||||
|
||||
reader = AsyncMock(side_effect=read_devices)
|
||||
monkeypatch.setattr(query_service.reads, "scada_devices", reader)
|
||||
app.executor = query_service.execute_query
|
||||
schema = next(tool["inputSchema"] for tool in request(client).json()["result"]["tools"]
|
||||
if tool["name"] == "list_scada_devices")
|
||||
assert schema["properties"]["limit"]["default"] == 100
|
||||
assert schema["properties"]["limit"]["maximum"] == 500
|
||||
pages = [request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
|
||||
"context_handle": value.context_handle, "offset": offset,
|
||||
}}).json()["result"]["structuredContent"] for offset in [0, 100]]
|
||||
assert [row for page in pages for row in page["data"]] == devices
|
||||
assert pages[0]["page"]["has_more"] is True and pages[1]["page"]["has_more"] is False
|
||||
assert [call.kwargs["limit"] for call in reader.await_args_list] == [101, 101]
|
||||
assert [call.kwargs["offset"] for call in reader.await_args_list] == [0, 100]
|
||||
assert TOOLS["list_elements"].model(context_handle=value.context_handle, element_type="pipe").limit == 20
|
||||
|
||||
|
||||
def test_business_skill_is_delivered_over_mcp_without_recording_event_evidence(runtime, monkeypatch):
|
||||
app, client, _, _ = runtime
|
||||
value = binding(task=True)
|
||||
@@ -653,6 +827,35 @@ def test_old_task_run_and_reader_parent_are_rejected(monkeypatch):
|
||||
asyncio.run(contexts.verify_task(scope, value))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["task_execution", "task_collaboration"])
|
||||
@pytest.mark.parametrize("replay", [False, True])
|
||||
def test_work_order_task_excludes_location_but_scada_replay_keeps_it(monkeypatch, purpose, replay):
|
||||
monkeypatch.setattr(contexts, "permissions_for_context", lambda _: PERMISSIONS)
|
||||
value = binding(task=True)
|
||||
value.purpose = purpose
|
||||
replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-07-16T00:00:00Z", "batch_sequence": 3} if replay else None
|
||||
|
||||
class Tasks:
|
||||
async def get_task(self, task_id):
|
||||
return {"task_id": task_id, "workflow": "burst_report", "source": {"replay_context": replay_context}}
|
||||
|
||||
async def get_active_run(self, task_id):
|
||||
return {"run_id": str(value.run_id), "execution_request_id": str(uuid4()),
|
||||
"status": "running", "agent_session_id": value.native_session_id,
|
||||
"lease_expires_at": datetime.now(timezone.utc) + timedelta(minutes=1)}
|
||||
|
||||
monkeypatch.setattr(contexts, "task_repository", lambda _: Tasks())
|
||||
scope = SimpleNamespace(project=object(), connection=Connection())
|
||||
asyncio.run(contexts.verify_task(scope, value))
|
||||
assert ("run_burst_location" in value.tool_cap) is replay
|
||||
assert "analyze_valve_isolation" in value.tool_cap
|
||||
assert "analyze_valve_closure" in value.tool_cap
|
||||
if not replay:
|
||||
with pytest.raises(HTTPException) as caught:
|
||||
asyncio.run(query_service.authorize_query(scope, value, TOOLS["run_burst_location"], SimpleNamespace()))
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
def test_jwt_signature_issuer_audience_lifetime_and_key_cache():
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
pem = key.private_bytes(
|
||||
|
||||
@@ -15,9 +15,10 @@ from app.mcp.work_order_skills import read_work_order_skill
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workflow,name", [
|
||||
("burst_report", "burst-report"),
|
||||
("burst_report", "burst-event-handling"),
|
||||
("water_quality_complaint", "water-quality-complaint"),
|
||||
("general", "appointment-follow-up"),
|
||||
("scada_anomaly", "scada-anomaly"),
|
||||
])
|
||||
def test_skill_descriptor_and_body_have_same_version_and_digest(workflow, name):
|
||||
descriptor = read_work_order_skill(workflow)
|
||||
@@ -36,6 +37,18 @@ def test_unknown_workflow_never_falls_back_or_reads_an_arbitrary_path(workflow):
|
||||
assert read_work_order_skill(workflow)["available"] is False
|
||||
|
||||
|
||||
def test_work_order_handling_and_legacy_scada_replay_receive_separate_skills():
|
||||
handling = read_work_order_skill("burst_report", include_content=True)
|
||||
replay = read_work_order_skill("burst_report", replay_context={"replay_id": "r"}, include_content=True)
|
||||
assert handling["name"] == "burst-event-handling"
|
||||
assert replay["name"] == "scada-anomaly"
|
||||
assert replay["workflow"] == "burst_report"
|
||||
assert "影响说明" in handling["instructions"]
|
||||
assert "supply_affected_nodes" in handling["instructions"]
|
||||
assert "run_burst_location" not in handling["instructions"]
|
||||
assert "run_burst_location" in replay["instructions"]
|
||||
|
||||
|
||||
def test_task_skill_uses_persisted_workflow_and_preserves_task_scope(monkeypatch):
|
||||
task_id, project_id = uuid4(), uuid4()
|
||||
reader = AsyncMock(return_value={"task": {"workflow": "burst_report"}})
|
||||
@@ -71,3 +84,33 @@ def test_task_skill_uses_persisted_workflow_and_preserves_task_scope(monkeypatch
|
||||
def test_skill_input_cannot_select_another_workflow_or_partial_instructions(extra):
|
||||
with pytest.raises(ValueError):
|
||||
TaskContextInput(context_handle="a" * 43, section="skill", **extra)
|
||||
|
||||
|
||||
def test_replay_context_advertises_demo_location_without_revealing_truth(monkeypatch):
|
||||
from app.services import burst_location_demo as demo
|
||||
|
||||
monkeypatch.setattr(query_service.reads, "task_context_summary", AsyncMock(
|
||||
return_value={"task": {"workflow": "scada_anomaly"}}))
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"operational_task.view"})
|
||||
scope = SimpleNamespace(connection=object(), project=SimpleNamespace(project_code="tjwater2_v2"))
|
||||
binding = BoundContext(
|
||||
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
|
||||
native_session_id="s", purpose="task_execution", task_id=uuid4(),
|
||||
tool_cap=["get_task_context", "run_burst_location"], permission_cap=["operational_task.view"],
|
||||
expires_at=datetime.now(timezone.utc),
|
||||
replay_context={"replay_id": "r", "knowledge_cutoff": demo.EVENT_START.isoformat(), "batch_sequence": 3},
|
||||
)
|
||||
for section in ("summary", "skill"):
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["get_task_context"],
|
||||
TaskContextInput(context_handle=binding.context_handle, section=section)))
|
||||
capability = result.data["location_demo"]
|
||||
assert capability["available"] is True
|
||||
assert capability["tool"] == "run_burst_location"
|
||||
assert capability["burst_leakage_required"] is False
|
||||
assert datetime.fromisoformat(capability["scada_burst_end"]) <= demo.EVENT_START
|
||||
assert all(key not in json.dumps(capability) for key in (demo.REFERENCE_PIPE, demo.REFERENCE_NODE, demo.CANDIDATE_PIPE))
|
||||
|
||||
binding.replay_context["knowledge_cutoff"] = "2026-07-15T23:45:00Z"
|
||||
result = asyncio.run(query_service.execute_query(scope, binding, TOOLS["get_task_context"],
|
||||
TaskContextInput(context_handle=binding.context_handle)))
|
||||
assert result.data["location_demo"]["available"] is False
|
||||
|
||||
@@ -6,7 +6,7 @@ from pydantic import ValidationError
|
||||
|
||||
from app.domain.domain_errors import DomainError
|
||||
from app.domain.schemas.operational_data import TaskDataSnapshot
|
||||
from app.domain.schemas.operational_task import TaskPresentationEventCommand
|
||||
from app.domain.schemas.operational_task import TaskPresentationEventCommand, TaskSafeChart
|
||||
from app.services.operational_presentation import resolve_overlay
|
||||
|
||||
|
||||
@@ -86,3 +86,27 @@ def test_unified_presentation_commands_preserve_flat_shape(command):
|
||||
def test_unified_presentation_rejects_invalid_commands(command):
|
||||
with pytest.raises(ValidationError):
|
||||
TaskPresentationEventCommand(request_id=uuid4(), run_id=uuid4(), source_session_id="session", command=command)
|
||||
|
||||
|
||||
def test_safe_chart_preserves_sparse_scatter_and_real_sample_times():
|
||||
data = {"title": "压力对比", "chart_type": "line", "x_data": ["19:45", "20:00"],
|
||||
"x_axis_name": "时刻", "y_axis_name": "压力 m", "series": [
|
||||
{"name": "当前", "data": [0., 25.]},
|
||||
{"name": "前一天", "type": "scatter", "data": [None, 26.],
|
||||
"sample_times": [None, "2026-06-13T20:00:00+08:00"]}]}
|
||||
assert TaskSafeChart.model_validate(data).model_dump(mode="json") == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("series", [
|
||||
{"name": "s", "data": [None], "sample_times": ["2026-06-14T20:00:00Z"]},
|
||||
{"name": "s", "data": [1.], "sample_times": ["2026-06-14T20:00:00"]},
|
||||
{"name": "s", "data": [1.], "sample_times": ["not-a-time"]},
|
||||
{"name": "s", "data": [1.], "sample_times": ["2026-06-14 20:00:00+08:00"]},
|
||||
{"name": "s", "data": [1.], "sample_times": []},
|
||||
{"name": "s", "data": [float("inf")]},
|
||||
{"name": "s", "data": [True]},
|
||||
{"name": "s", "type": "script", "data": [1.]},
|
||||
])
|
||||
def test_safe_chart_rejects_invalid_sample_metadata_and_values(series):
|
||||
with pytest.raises(ValidationError):
|
||||
TaskSafeChart(title="X", chart_type="line", x_data=["t"], series=[series])
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.operational_recovery import accept_completion, evaluate_recovery, track_work_order
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
|
||||
|
||||
INCIDENT = datetime(2026, 7, 16, 0, tzinfo=timezone.utc)
|
||||
RETURNED = INCIDENT + timedelta(hours=1)
|
||||
|
||||
|
||||
def observations():
|
||||
rows = []
|
||||
for sample in (1, 2, 3):
|
||||
at = RETURNED + timedelta(minutes=15 * sample)
|
||||
for device in ("P1", "F1"):
|
||||
for day in range(1, 16):
|
||||
rows.append({"time": at - timedelta(days=day), "device_id": device, "monitored_value": 100.})
|
||||
rows.append({"time": at, "device_id": device, "monitored_value": 100.})
|
||||
return rows
|
||||
|
||||
|
||||
def evaluate(rows):
|
||||
return evaluate_recovery(rows, device_ids=["P1", "F1"], returned_at=RETURNED,
|
||||
incident_at=INCIDENT, interval_seconds=900)
|
||||
|
||||
|
||||
def test_recovery_uses_post_receipt_samples_and_pre_incident_same_phase_reference():
|
||||
result = evaluate(observations())
|
||||
assert result["status"] == "passed"
|
||||
assert [item["at"] for item in result["samples"]] == [
|
||||
(RETURNED + timedelta(minutes=30)).isoformat(), (RETURNED + timedelta(minutes=45)).isoformat()]
|
||||
assert all(check["normal_day_count"] == 15 for sample in result["samples"] for check in sample["devices"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("problem", ["missing_device", "low_pressure", "high_flow", "nan", "bad_quality", "missing_baseline", "gap"])
|
||||
def test_incomplete_or_abnormal_data_never_proves_recovery(problem):
|
||||
rows = observations()
|
||||
current = [row for row in rows if row["time"] > RETURNED]
|
||||
for row in current:
|
||||
if problem == "missing_device" and row["device_id"] == "P1":
|
||||
rows.remove(row)
|
||||
elif problem in {"low_pressure", "high_flow", "nan"}:
|
||||
row["monitored_value"] = {"low_pressure": 60., "high_flow": 130., "nan": float("nan")}[problem]
|
||||
elif problem == "bad_quality":
|
||||
row["quality"] = "invalid"
|
||||
elif problem == "gap":
|
||||
row["time"] += timedelta(minutes=15 * current.index(row))
|
||||
if problem == "missing_baseline":
|
||||
rows = [row for row in rows if row["time"] >= INCIDENT - timedelta(days=14)]
|
||||
assert evaluate(rows)["status"] == "waiting"
|
||||
|
||||
|
||||
def test_samples_at_or_before_return_time_cannot_close_incident():
|
||||
rows = [row for row in observations() if row["time"] < INCIDENT]
|
||||
rows.extend({"time": RETURNED, "device_id": device, "monitored_value": 100.} for device in ("P1", "F1"))
|
||||
assert evaluate(rows)["status"] == "waiting"
|
||||
|
||||
|
||||
def test_faster_sampling_counts_monitoring_periods_and_rejects_later_relapse():
|
||||
rows = []
|
||||
for minute in range(5, 36, 5):
|
||||
at = RETURNED + timedelta(minutes=minute)
|
||||
for device in ("P1", "F1"):
|
||||
rows.extend({"time": at - timedelta(days=day), "device_id": device, "monitored_value": 100.}
|
||||
for day in range(1, 16))
|
||||
rows.append({"time": at, "device_id": device, "monitored_value": 100.})
|
||||
assert evaluate(rows)["status"] == "passed"
|
||||
next(row for row in rows if row["time"] == RETURNED + timedelta(minutes=25) and row["device_id"] == "P1")["monitored_value"] = 50.
|
||||
assert evaluate(rows)["status"] == "waiting"
|
||||
|
||||
|
||||
def test_dispatch_waits_for_all_orders_and_accepts_corrected_field_completion_time():
|
||||
task = {"workflow": "burst_report"}
|
||||
for order_id in ("one", "two"):
|
||||
track_work_order(task, {"kind": "create_work_order", "action_id": str(uuid4()), "execution_status": "succeeded",
|
||||
"receipt": {"adapter": "peripheral_sample", "receipt_id": order_id}})
|
||||
assert OperationalTaskService._investigation_status([], task) == "waiting_human"
|
||||
assert not accept_completion(task, {"source_event_id": "peripheral_sample:unrelated"})
|
||||
feedback = {"source_event_id": "peripheral_sample:one", "status": "completed", "source_revision": 2,
|
||||
"occurred_at": RETURNED.isoformat()}
|
||||
assert accept_completion(task, feedback)
|
||||
assert task["handling_context"]["phase"] == "waiting_work_order"
|
||||
accept_completion(task, {**feedback, "source_event_id": "peripheral_sample:two"})
|
||||
assert task["handling_context"]["phase"] == "waiting_scada"
|
||||
task["handling_context"]["phase"] = "verification_ready"
|
||||
task["handling_context"].update(verification_evidence_id="previous-proof", last_check={"status": "passed"})
|
||||
accept_completion(task, {**feedback, "source_revision": 3})
|
||||
assert task["handling_context"]["phase"] == "verification_ready"
|
||||
assert OperationalTaskService._investigation_status([], task) == "completed"
|
||||
corrected = RETURNED + timedelta(hours=1)
|
||||
accept_completion(task, {**feedback, "source_revision": 4, "occurred_at": corrected.isoformat()})
|
||||
assert task["handling_context"]["returned_at"] == corrected.isoformat()
|
||||
assert task["handling_context"]["phase"] == "waiting_scada"
|
||||
assert "verification_evidence_id" not in task["handling_context"] and "last_check" not in task["handling_context"]
|
||||
|
||||
|
||||
def test_field_feedback_time_requires_a_real_note_and_explicit_timezone():
|
||||
from pydantic import ValidationError
|
||||
from samples.peripheral.schemas import UpdateWorkOrder
|
||||
body = {"request_id": uuid4(), "expected_revision": 1, "status": "completed",
|
||||
"progress_note": "演示抢修完成并恢复供水", "occurred_at": RETURNED.isoformat()}
|
||||
assert UpdateWorkOrder.model_validate(body).occurred_at == RETURNED
|
||||
with pytest.raises(ValidationError):
|
||||
UpdateWorkOrder.model_validate({**body, "progress_note": None})
|
||||
with pytest.raises(ValidationError):
|
||||
UpdateWorkOrder.model_validate({**body, "occurred_at": "2026-07-16T09:00:00"})
|
||||
@@ -576,6 +576,65 @@ async def test_presentation_cursor_and_event_are_read_only_and_idempotent():
|
||||
await service.presentation_cursor(uuid4())
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_replay_investigation_can_publish_evidenced_composite_chart_but_cannot_propose_real_action():
|
||||
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
|
||||
context = _context()
|
||||
task = _task_document(context, now, lease=now + timedelta(minutes=1))
|
||||
task["source"]["replay_context"] = {
|
||||
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
|
||||
}
|
||||
repository = RunTaskRepository(task, _active_run(task, now))
|
||||
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
|
||||
evidence, _ = await service.record_data_snapshot(task["task_id"], TaskDataSnapshot(
|
||||
source="query_scada_readings", source_ref="mcp:frozen-scada", queried_at=now,
|
||||
origin="synthetic", data={"items": []}, truncated=False,
|
||||
))
|
||||
result = await service.publish_presentation_event(task["task_id"], TaskPresentationEventCommand(
|
||||
request_id=uuid4(), run_id=repository.run["run_id"], source_session_id="session-1",
|
||||
command={"action": "open_chart", "chart_id": "pressure", "evidence_refs": [evidence.evidence_id],
|
||||
"chart": {"title": "压力", "chart_type": "line", "x_data": ["19:45", "20:00"],
|
||||
"y_axis_name": "压力 m", "series": [
|
||||
{"name": "当前", "type": "line", "data": [27., 26.]},
|
||||
{"name": "前一天", "type": "scatter", "data": [None, 28.],
|
||||
"sample_times": [None, "2026-10-09T20:00:00+08:00"]},
|
||||
]}},
|
||||
))
|
||||
assert result.recorded
|
||||
assert repository.events[-1]["metadata"]["command"]["chart"]["series"][1]["data"] == [None, 28.]
|
||||
with pytest.raises(DomainError) as caught:
|
||||
await service.propose_action(task["task_id"], ActionProposalCommand(
|
||||
request_id=uuid4(), kind="create_work_order", reason="演示不得创建真实工单",
|
||||
payload={"title": "现场核查", "description": "核查", "work_order_type": "general"},
|
||||
))
|
||||
assert caught.value.code == "REPLAY_ACTION_DISABLED"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("operation", ["draft", "approve", "submit"])
|
||||
async def test_replay_cannot_edit_approve_or_submit_preexisting_real_actions(operation):
|
||||
now = datetime(2026, 10, 10, tzinfo=timezone.utc)
|
||||
context = _context()
|
||||
task = _task_document(context, now, lease=now + timedelta(minutes=1))
|
||||
task["source"]["replay_context"] = {
|
||||
"replay_id": str(uuid4()), "knowledge_cutoff": now.isoformat(), "batch_sequence": 1,
|
||||
}
|
||||
repository = RunTaskRepository(task, _active_run(task, now))
|
||||
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
|
||||
dispatcher = SimpleNamespace(dispatch=AsyncMock())
|
||||
args = {"request_id": uuid4(), "expected_version": 1}
|
||||
with pytest.raises(DomainError) as caught:
|
||||
if operation == "draft":
|
||||
await service.update_work_order_draft(task["task_id"], uuid4(), WorkOrderDraftUpdateCommand(
|
||||
**args, payload={"title": "现场核查", "description": "核查", "work_order_type": "general"}))
|
||||
elif operation == "approve":
|
||||
await service.decide_action(task["task_id"], uuid4(), ActionDecisionCommand(**args, decision="approved"))
|
||||
else:
|
||||
await service.submit_action(task["task_id"], uuid4(), ActionExecuteCommand(**args), dispatcher, None)
|
||||
assert caught.value.code == "REPLAY_ACTION_DISABLED"
|
||||
dispatcher.dispatch.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("payload", [
|
||||
{"action": "locate_features", "layer": "scada", "feature_ids": ["D-1"]},
|
||||
@@ -1025,6 +1084,93 @@ def _seed_concluded_report(repository, now):
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_burst_waits_for_completion_then_scada_evidence_then_agent_report(monkeypatch):
|
||||
from copy import deepcopy
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
from app.services.operational_recovery import reconcile_recoveries, track_work_order
|
||||
|
||||
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
|
||||
incident = now - timedelta(hours=4)
|
||||
returned = incident + timedelta(hours=1)
|
||||
context = _context()
|
||||
task = _task_document(context, now, lease=now + timedelta(minutes=1))
|
||||
task.update(workflow="burst_report", trigger={"kind": "event", "occurred_at": incident.isoformat(), "detail": "爆管"})
|
||||
repository = RunTaskRepository(task, _active_run(task, now))
|
||||
repository.connection = object()
|
||||
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
|
||||
monkeypatch.setattr(service, "_resolve_investigation_alerts", AsyncMock())
|
||||
_seed_concluded_report(repository, now)
|
||||
action = await service.propose_action(task["task_id"], ActionProposalCommand(
|
||||
request_id=uuid4(), kind="create_work_order", reason="隔离抢修与恢复供水",
|
||||
payload={"title": "抢修恢复", "description": "回传完工时间和恢复供水情况", "work_order_type": "general"}))
|
||||
child_id = str(uuid4())
|
||||
repository.action = action.model_copy(update={"approval_status": "approved", "execution_status": "succeeded"}).model_dump(mode="json")
|
||||
repository.action["receipt"] = {"adapter": "peripheral_sample", "receipt_id": child_id,
|
||||
"submitted_at": now.isoformat(), "completed_at": now.isoformat(), "external_status": "open", "detail": "仅创建成功"}
|
||||
track_work_order(repository.task, repository.action)
|
||||
finished = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
|
||||
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
|
||||
assert finished.status == "waiting_human"
|
||||
service._resolve_investigation_alerts.assert_not_awaited()
|
||||
|
||||
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
|
||||
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=2,
|
||||
status="completed", occurred_at=returned, detail="抢修完成,已恢复供水"))
|
||||
assert repository.task["handling_context"]["phase"] == "waiting_scada"
|
||||
assert datetime.fromisoformat(repository.task["handling_context"]["returned_at"]) == returned
|
||||
settings = ProjectSettings(scada_monitoring_interval_seconds=900, scada_monitoring={"pressure_device_ids": ["P1"]})
|
||||
repository.recovery_candidates = AsyncMock(side_effect=lambda *_: [deepcopy(repository.task)]
|
||||
if repository.task["handling_context"]["phase"] == "waiting_scada" else [])
|
||||
timescale = SimpleNamespace(transaction=repository.transaction)
|
||||
rows = []
|
||||
for sample in (1, 2):
|
||||
at = returned + timedelta(minutes=15 * sample)
|
||||
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
|
||||
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
|
||||
reader = AsyncMock(return_value=rows[:-1])
|
||||
monkeypatch.setattr("app.infra.db.timescaledb.repositories.scada.ScadaRepository.get_scada_by_ids_time_range", reader)
|
||||
assert await reconcile_recoveries(service, timescale, settings) == 0
|
||||
assert repository.task["handling_context"]["phase"] == "waiting_scada"
|
||||
reader.return_value = rows
|
||||
assert await reconcile_recoveries(service, timescale, settings) == 1
|
||||
assert await reconcile_recoveries(service, timescale, settings) == 0
|
||||
recovery = repository.task["handling_context"]
|
||||
evidence_id = recovery["verification_evidence_id"]
|
||||
assert repository.evidence[evidence_id]["data_snapshot"]["data"]["status"] == "passed"
|
||||
service._resolve_investigation_alerts.assert_not_awaited()
|
||||
|
||||
# Correcting the appointment's field return time invalidates the old proof
|
||||
# and requires two actual observations after the new business time.
|
||||
corrected_time = returned + timedelta(minutes=15)
|
||||
await service.external_feedback(task["task_id"], ExternalFeedbackCommand(
|
||||
request_id=uuid4(), source_event_id=f"peripheral_sample:{child_id}", source_revision=3,
|
||||
status="completed", occurred_at=corrected_time, detail="修订现场实际完工时间"))
|
||||
assert repository.task["handling_context"]["phase"] == "waiting_scada"
|
||||
assert "verification_evidence_id" not in repository.task["handling_context"]
|
||||
assert await reconcile_recoveries(service, timescale, settings) == 0
|
||||
at = returned + timedelta(minutes=45)
|
||||
rows.extend({"time": at - timedelta(days=day), "device_id": "P1", "monitored_value": 100.} for day in range(1, 16))
|
||||
rows.append({"time": at, "device_id": "P1", "monitored_value": 100.})
|
||||
assert await reconcile_recoveries(service, timescale, settings) == 1
|
||||
revised_id = repository.task["handling_context"]["verification_evidence_id"]
|
||||
assert revised_id != evidence_id and evidence_id in repository.evidence
|
||||
evidence_id = revised_id
|
||||
|
||||
repository.task.update(status="running", agent_session_id="session-1")
|
||||
repository.run = _active_run(repository.task, now)
|
||||
_seed_concluded_report(repository, now)
|
||||
with pytest.raises(DomainError) as rejected:
|
||||
await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
|
||||
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
|
||||
assert rejected.value.code == "RECOVERY_EVIDENCE_REQUIRED"
|
||||
repository.reports[-1]["report"]["evidence_refs"] = [evidence_id]
|
||||
result = await service.complete_run(task["task_id"], repository.run["run_id"], AgentRunCommand(
|
||||
request_id=uuid4(), expected_revision=repository.run["revision"], source_session_id="session-1"))
|
||||
assert result.status == "completed" and result.handling_context["phase"] == "completed"
|
||||
service._resolve_investigation_alerts.assert_awaited_once_with(task["task_id"], evidence_id)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_plan_only_keeps_lease_until_same_generation_report_closes_run():
|
||||
now = datetime(2026, 9, 23, 8, tzinfo=timezone.utc)
|
||||
@@ -2087,3 +2233,52 @@ async def test_missing_calculation_migration_fails_finalization_closed(monkeypat
|
||||
await repo.pending_calculation_jobs(str(uuid4()), str(uuid4()), str(uuid4()))
|
||||
assert error.value.code == 'DEPENDENCY_UNAVAILABLE'
|
||||
assert error.value.status == 503
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_recovery_alert_query_scopes_active_alerts_to_project_and_task():
|
||||
alerts = [{"alert_id": "first"}, {"alert_id": "second"}]
|
||||
cursor = SimpleNamespace(fetchall=AsyncMock(return_value=[{"document": item} for item in alerts]))
|
||||
connection = SimpleNamespace(execute=AsyncMock(return_value=cursor))
|
||||
project_id, task_id = uuid4(), str(uuid4())
|
||||
repository = OperationalTaskRepository(connection, project_id)
|
||||
assert await repository.list_investigation_alerts(task_id) == alerts
|
||||
statement, parameters = connection.execute.await_args.args
|
||||
assert parameters == (str(project_id), f"operational-task:{task_id}")
|
||||
assert "status IN ('active','acknowledged')" in statement
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_recovery_resolves_alerts_with_original_facts_and_stable_event_id(monkeypatch):
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
from app.services.task_alerts import TaskAlertService
|
||||
|
||||
task_id, project_id = str(uuid4()), str(uuid4())
|
||||
now = datetime(2026, 7, 16, 4, tzinfo=timezone.utc)
|
||||
alert = {
|
||||
"alert_id": "incident-1", "producer_id": "producer-1", "revision": 2,
|
||||
"condition_id": f"operational-task:{task_id}", "status": "acknowledged",
|
||||
"source": {"task_type": "scheduled_task", "task_id": task_id, "run_id": "run-1"},
|
||||
"title": "压力异常", "description": "等待恢复", "severity": "warning",
|
||||
"occurred_at": now.isoformat(), "updated_at": now.isoformat(),
|
||||
}
|
||||
connection = object()
|
||||
repository = SimpleNamespace(connection=connection, project_id=project_id,
|
||||
list_investigation_alerts=AsyncMock(return_value=[alert]))
|
||||
calls = []
|
||||
|
||||
async def push(service, command):
|
||||
calls.append((service, command))
|
||||
|
||||
monkeypatch.setattr(TaskAlertService, "push", push)
|
||||
service = OperationalTaskService(repository, _context(), set())
|
||||
for _ in range(2):
|
||||
await service._resolve_investigation_alerts(task_id, "recovery:evidence-1")
|
||||
assert len(calls) == 2
|
||||
for publisher, command in calls:
|
||||
assert publisher.repo.connection is connection and publisher.repo.project_id == project_id
|
||||
assert publisher.internal_producer_id == alert["producer_id"]
|
||||
assert command.event_type == "resolved"
|
||||
assert command.event_id == uuid5(NAMESPACE_URL, f"recovery:{task_id}:recovery:evidence-1:incident-1")
|
||||
assert command.source.model_dump(mode="json") == alert["source"]
|
||||
assert command.title == alert["title"] and command.description == alert["description"]
|
||||
|
||||
@@ -48,9 +48,10 @@ async def test_all_worker_operations_share_investigation_creation_scope():
|
||||
assert request.url.params["task_type"] == "event_investigation"
|
||||
assert request.url.params["task_created_from"] == boundary
|
||||
return httpx.Response(200, json={"items": []})
|
||||
assert request.url.path.endswith("leases/recover")
|
||||
assert request.url.path.endswith(("leases/recover", "recovery/reconcile"))
|
||||
assert json.loads(request.content) == {
|
||||
"limit": 100, "task_type": "event_investigation", "task_created_from": boundary,
|
||||
"limit": 20 if request.url.path.endswith("recovery/reconcile") else 100,
|
||||
"task_type": "event_investigation", "task_created_from": boundary,
|
||||
}
|
||||
return httpx.Response(200, json={"task_ids": []})
|
||||
|
||||
@@ -58,7 +59,7 @@ async def test_all_worker_operations_share_investigation_creation_scope():
|
||||
bearer="TEST_ONLY", task_created_from=boundary)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
assert await OperationalWorker(settings, client).tick() == 0
|
||||
assert len(requests) == 5
|
||||
assert len(requests) == 6
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@@ -68,7 +69,7 @@ async def test_worker_reconciles_messages_at_capacity_and_preserves_transport_re
|
||||
|
||||
def handler(request):
|
||||
path = request.url.path
|
||||
if path.endswith("leases/recover"):
|
||||
if path.endswith(("leases/recover", "recovery/reconcile")):
|
||||
return httpx.Response(200, json={"items": []})
|
||||
if path.endswith("agent-messages/pending"):
|
||||
return httpx.Response(200, json={"items": pending})
|
||||
@@ -129,7 +130,7 @@ async def test_persisted_queue_retries_with_backend_execution_identity():
|
||||
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
|
||||
assert request.headers["Authorization"] == "Bearer TEST_ONLY"
|
||||
assert not request.url.path.endswith("schedules/dispatch-due")
|
||||
if request.url.path.endswith("leases/recover"):
|
||||
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
|
||||
return httpx.Response(200, json={"items": []})
|
||||
if request.method == "GET":
|
||||
if request.url.params.get("status") != "queued":
|
||||
@@ -191,7 +192,7 @@ async def test_bounded_queue_continues_after_one_agent_handoff_fails():
|
||||
limits.append(int(request.url.params["limit"]))
|
||||
return httpx.Response(200, json={"items": runs})
|
||||
if request.url.host == "server.test":
|
||||
if request.url.path.endswith("leases/recover"):
|
||||
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
|
||||
return httpx.Response(200, json={"items": []})
|
||||
failures.append(json.loads(request.content))
|
||||
return httpx.Response(200, json={})
|
||||
@@ -269,7 +270,7 @@ async def test_worker_refreshes_running_agent_credentials_after_restart():
|
||||
def handler(request):
|
||||
if request.url.path == "/health":
|
||||
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
|
||||
if request.url.path.endswith("leases/recover"):
|
||||
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
|
||||
return httpx.Response(200, json={"items": []})
|
||||
if request.method == "GET":
|
||||
items = [{
|
||||
@@ -300,7 +301,7 @@ async def test_worker_only_restores_sessions_owned_by_target_agent_instance():
|
||||
def handler(request):
|
||||
if request.url.path == "/health":
|
||||
return httpx.Response(200, json={"ok": True, "agent_instance_id": "agent-a"})
|
||||
if request.url.path.endswith("leases/recover"):
|
||||
if request.url.path.endswith(("leases/recover", "recovery/reconcile")):
|
||||
return httpx.Response(200, json={"items": []})
|
||||
if request.method == "GET":
|
||||
items = [{
|
||||
|
||||
@@ -255,7 +255,7 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
|
||||
"title": "采样工单", "priority": "normal",
|
||||
"work_order_type": "general", "created_at": "2026-09-23T08:00:00Z",
|
||||
"scheduled_at": None,
|
||||
"activity": ([{"sequence": 2, "occurred_at": "2026-09-23T09:00:00Z", "actor": "现场人员",
|
||||
"activity": ([{"sequence": 2, "occurred_at": "2026-09-23T08:45:00Z", "actor": "现场人员",
|
||||
"kind": "feedback", "title": "现场反馈", "detail": "复测余氯0.2mg/L"}]
|
||||
if order_revision > 1 else []),
|
||||
}
|
||||
@@ -271,6 +271,8 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
|
||||
assert ("work_order_snapshot" in payload) == request.url.path.endswith(f"{projection_id}/external-feedback")
|
||||
if order_revision > 1:
|
||||
assert "复测余氯0.2mg/L" in payload["evidence"]["summary"]
|
||||
assert payload["occurred_at"] == "2026-09-23T08:45:00Z"
|
||||
assert payload["evidence"]["observed_at"] == "2026-09-23T08:45:00Z"
|
||||
return httpx.Response(200, json={})
|
||||
if request.url.path.endswith(investigation_id):
|
||||
return httpx.Response(200, json={"actions": [{"kind": "create_work_order", "receipt": {
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import pytest
|
||||
|
||||
from scripts.prepare_regional_burst_design import require_isolatable_pipe
|
||||
from scripts.screen_burst_benchmark_pipes import nearest_pressure_distances
|
||||
|
||||
|
||||
def test_region_without_valves_is_rejected():
|
||||
with pytest.raises(ValueError, match="cannot be isolated"):
|
||||
require_isolatable_pipe(["supply:pipe:R:A", "target:pipe:A:B"], "target", ["R"], [])
|
||||
|
||||
|
||||
def test_nearby_valve_does_not_qualify_an_uncontrolled_source_path():
|
||||
links = ["v:valve:R:A", "bypass:pipe:R:B", "target:pipe:A:B"]
|
||||
with pytest.raises(ValueError, match="cannot be isolated"):
|
||||
require_isolatable_pipe(links, "target", ["R"], [])
|
||||
|
||||
|
||||
def test_single_boundary_valve_can_qualify_a_branch():
|
||||
links = ["v:valve:R:A", "target:pipe:A:B"]
|
||||
result = require_isolatable_pipe(links, "target", ["R"], [])
|
||||
assert result["must_close_valves"] == ["v"]
|
||||
assert set(result["affected_nodes"]) == {"A", "B"}
|
||||
assert result["source_check_performed"]
|
||||
|
||||
|
||||
def test_source_metadata_is_mandatory():
|
||||
with pytest.raises(ValueError, match="cannot be isolated"):
|
||||
require_isolatable_pipe(["v:valve:R:A", "target:pipe:A:B"], "target", [], [])
|
||||
|
||||
|
||||
def test_distance_is_to_the_nearest_of_all_meters():
|
||||
graph = {"A": [("P", 1500)], "B": [("P", 100)], "P": [("A", 1500), ("B", 100)]}
|
||||
devices = [{"device_id": "far", "node_id": "A"}, {"device_id": "near", "node_id": "B"}]
|
||||
assert nearest_pressure_distances(graph, devices, 3000)["P"] == (100, "near")
|
||||
|
||||
|
||||
def test_network_path_length_and_cutoff_are_used():
|
||||
graph = {"A": [("B", 600)], "B": [("A", 600), ("C", 700)], "C": [("B", 700)]}
|
||||
devices = [{"device_id": "meter", "node_id": "A"}]
|
||||
assert "C" not in nearest_pressure_distances(graph, devices, 1200)
|
||||
assert nearest_pressure_distances(graph, devices, 1500)["C"] == (1300, "meter")
|
||||
|
||||
|
||||
def test_unreachable_pipe_is_not_a_zero_distance_candidate():
|
||||
graph = {"A": [], "P": []}
|
||||
assert "P" not in nearest_pressure_distances(graph, [{"device_id": "meter", "node_id": "A"}], 3000)
|
||||
@@ -40,6 +40,17 @@ def test_report_ui_round_trips_with_a_version_and_keeps_legacy_reports_readable(
|
||||
assert InvestigationReportRevision.model_validate(old).report.ui_blocks == []
|
||||
|
||||
|
||||
def test_report_ui_preserves_sparse_scada_series_and_sample_times():
|
||||
data = rich_report()
|
||||
chart = data["ui_blocks"][0]["block"]["spec"]["elements"]["chart"]["props"]
|
||||
chart["y_axis_name"] = "压力 MPa"
|
||||
chart["series"].append({"name": "前一天", "type": "scatter", "data": [None, 0.28],
|
||||
"sample_times": [None, "2026-10-19T09:00:00+08:00"]})
|
||||
restored = InvestigationReportInput.model_validate_json(
|
||||
InvestigationReportInput.model_validate(data).model_dump_json())
|
||||
assert restored.ui_blocks[0].block.spec.elements["chart"].props == chart
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", [None, "final"])
|
||||
def test_report_ui_accepts_final_and_legacy_phases(phase):
|
||||
data = rich_report()
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
from datetime import datetime, timedelta
|
||||
from math import sin
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings, ScadaMonitoringWeights
|
||||
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder, adaptive_score_threshold
|
||||
from app.services import scada_replay_monitoring as monitoring
|
||||
from test_scada_replay_monitoring import TARGET, PRESSURE_IDS, FLOW_IDS, _document
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY, normal_profile_bounds
|
||||
|
||||
|
||||
def setup():
|
||||
document = _document()
|
||||
document["config"]["start_time"] = TARGET.isoformat()
|
||||
document["monitoring_config"] = ScadaMonitoringSettings(
|
||||
pressure_device_ids=PRESSURE_IDS, outlet_flow_device_ids=FLOW_IDS,
|
||||
).model_dump(mode="json")
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
start = TARGET - timedelta(days=15)
|
||||
rows = [{"device_id": device, "time": (start + timedelta(minutes=15 * i)).isoformat(),
|
||||
"cleaned_value": 100 + 5 * sin(i / 96 * 6.283) + 0.1 * sin(i * 1.7), "quality": "valid"}
|
||||
for i in range(15 * 96) for device in PRESSURE_IDS + FLOW_IDS]
|
||||
return document, config, rows
|
||||
|
||||
|
||||
def test_streamed_thresholds_are_frozen_from_complete_pre_replay_history(monkeypatch):
|
||||
document, config, rows = setup()
|
||||
calls = []
|
||||
|
||||
def detect(**kwargs):
|
||||
frame = kwargs["observed_pressure_data"]
|
||||
assert frame.index.max().to_pydatetime() < config.start_time
|
||||
assert frame.shape == (1440, 5)
|
||||
calls.append(frame)
|
||||
return {"rows": [{"Score": value} for value in [-0.03, -0.01, 0.01, 0.02, 0.03] * 3]}
|
||||
|
||||
monkeypatch.setattr(monitoring, "run_burst_detection", detect)
|
||||
builder = MonitoringCalibrationBuilder(document, config)
|
||||
for offset in range(0, len(rows), 997):
|
||||
builder.add(rows[offset:offset + 997])
|
||||
result = builder.fit()
|
||||
assert result["status"] == "ready"
|
||||
assert result["version"] == CALIBRATION_VERSION
|
||||
assert result["diagnostic_policy"] == DIAGNOSTIC_POLICY
|
||||
assert result["knowledge_cutoff"] == config.start_time.isoformat()
|
||||
assert result["burst_detection"]["threshold"] < -0.04
|
||||
assert result["pressure"][PRESSURE_IDS[0]]["threshold_percent"] == 3.0
|
||||
assert result["outlet_flow"][FLOW_IDS[0]]["threshold_percent"] == 5.0
|
||||
assert len(calls) == 1
|
||||
assert sum(values.nbytes for values in builder.values.values()) == 1440 * 6 * 8
|
||||
for row in rows:
|
||||
bounds = normal_profile_bounds(result["diagnostic"][row["device_id"]], datetime.fromisoformat(row["time"]))
|
||||
assert bounds[0] <= row["cleaned_value"] <= bounds[1]
|
||||
|
||||
|
||||
def test_history_gap_and_future_samples_cannot_be_hidden(monkeypatch):
|
||||
document, config, rows = setup()
|
||||
monkeypatch.setattr(monitoring, "run_burst_detection", lambda **_: pytest.fail("incomplete history entered detection"))
|
||||
builder = MonitoringCalibrationBuilder(document, config)
|
||||
builder.add(rows[1:])
|
||||
assert builder.fit()["status"] == "insufficient_data"
|
||||
with pytest.raises(ValueError, match="之后"):
|
||||
builder.add([{**rows[0], "time": config.start_time.isoformat()}])
|
||||
|
||||
|
||||
def test_score_cutoff_follows_normal_noise_instead_of_target_scores():
|
||||
quiet = adaptive_score_threshold([0.01, 0.02, 0.03, 0.02, 0.01])
|
||||
noisy = adaptive_score_threshold([-0.07, -0.03, 0.01, 0.04, 0.02])
|
||||
assert noisy["threshold"] < quiet["threshold"]
|
||||
assert -0.2 < noisy["threshold"]
|
||||
assert adaptive_score_threshold([0.01])["status"] == "insufficient_data"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [
|
||||
{"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.5},
|
||||
{"burst_detection": -0.1, "pressure": 1, "outlet_flow": 0.1},
|
||||
{"outlet_flow": float("nan")},
|
||||
])
|
||||
def test_invalid_weights_are_rejected(values):
|
||||
with pytest.raises(ValidationError):
|
||||
ScadaMonitoringWeights(**values)
|
||||
@@ -1,8 +1,11 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
import pytest
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.services import scada_replay_monitoring
|
||||
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
|
||||
|
||||
|
||||
TARGET = datetime(2026, 10, 16, tzinfo=timezone.utc)
|
||||
@@ -68,6 +71,14 @@ def test_multiple_indicators_create_suspected_burst_result(monkeypatch):
|
||||
assert result["pressure"]["hit_device_ids"] == PRESSURE_IDS
|
||||
assert result["outlet_flow"]["hit_device_ids"] == FLOW_IDS
|
||||
assert result["burst_detection"]["hit"] is True
|
||||
pressure = result["pressure"]["devices"][0]
|
||||
assert pressure["value_change"] == -30
|
||||
assert pressure["unit"] == "m"
|
||||
assert pressure["baseline_method"] == "recent_samples_median"
|
||||
assert pressure["baseline_sample_count"] == 4
|
||||
assert pressure["baseline_end"] == result["period_start"]
|
||||
assert result["outlet_flow"]["devices"][0]["value_change"] == 50
|
||||
assert result["burst_detection"]["score_interpretation"] == "lower_is_more_anomalous_not_probability"
|
||||
|
||||
|
||||
def test_single_indicator_stays_attention(monkeypatch):
|
||||
@@ -106,3 +117,194 @@ def test_missing_project_roles_is_data_gap_not_normal():
|
||||
|
||||
assert result["status"] == "insufficient_data"
|
||||
assert result["burst_detection"]["status"] == "not_run"
|
||||
|
||||
|
||||
def _weighted_document():
|
||||
document = _document()
|
||||
document["config"]["start_time"] = (TARGET - timedelta(minutes=30)).isoformat()
|
||||
document["monitoring_config"] = ScadaMonitoringSettings(
|
||||
pressure_device_ids=PRESSURE_IDS, outlet_flow_device_ids=FLOW_IDS,
|
||||
).model_dump(mode="json")
|
||||
document["monitoring_calibration"] = {
|
||||
"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
|
||||
"status": "ready", "knowledge_cutoff": (TARGET - timedelta(minutes=30)).isoformat(),
|
||||
"pressure": {device: {"threshold_percent": 10, "method": "frozen_normal_history"} for device in PRESSURE_IDS},
|
||||
"outlet_flow": {"F1": {"threshold_percent": 5, "method": "frozen_normal_history"}},
|
||||
"burst_detection": {"threshold": -0.08},
|
||||
}
|
||||
profile = {"phase_origin": document["config"]["start_time"], "interval_seconds": 900,
|
||||
"lower": [99.], "upper": [101.]}
|
||||
calibration = document["monitoring_calibration"]
|
||||
calibration["diagnostic"] = {device: profile for device in PRESSURE_IDS + FLOW_IDS}
|
||||
for kind in ("pressure", "outlet_flow"):
|
||||
for item in calibration[kind].values():
|
||||
item["normal_profile"] = profile
|
||||
return document
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flow,expected_score", [("increase", 1.0), ("flat", 0.9), ("missing", 0.9)])
|
||||
def test_pressure_and_detection_can_trigger_with_low_or_unavailable_flow_weight(monkeypatch, flow, expected_score):
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
|
||||
"rows": [{"Score": -0.2}], "summary": {},
|
||||
})
|
||||
rows = _rows()
|
||||
for row in rows:
|
||||
if flow == "flat" and row["device_id"] == "F1":
|
||||
row["cleaned_value"] = 100
|
||||
if flow == "missing":
|
||||
rows = [row for row in rows if not (row["device_id"] == "F1"
|
||||
and datetime.fromisoformat(row["time"]) >= TARGET - timedelta(minutes=15))]
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
_weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
|
||||
period_end=TARGET, batch_sequence=1440,
|
||||
)
|
||||
assert result["status"] == "suspected_burst"
|
||||
assert result["weighted"]["score"] == expected_score
|
||||
assert result["weighted"]["weights"]["outlet_flow"] == 0.1
|
||||
assert result["pressure"]["devices"][0]["threshold_percent"] == 10
|
||||
assert result["burst_detection"]["score_threshold"] == -0.08
|
||||
if flow == "missing":
|
||||
assert result["weighted"]["unavailable_indicators"] == ["outlet_flow"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("indicator", ["pressure", "outlet_flow", "burst_detection"])
|
||||
def test_a_single_indicator_cannot_create_a_weighted_burst_alert(monkeypatch, indicator):
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
|
||||
"rows": [{"Score": -0.2 if indicator == "burst_detection" else 0.02}], "summary": {},
|
||||
})
|
||||
rows = _rows()
|
||||
for row in rows:
|
||||
if (row["device_id"] in PRESSURE_IDS and indicator != "pressure"
|
||||
or row["device_id"] == "F1" and indicator != "outlet_flow"):
|
||||
row["cleaned_value"] = 100
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
_weighted_document(), rows, period_start=TARGET - timedelta(minutes=15),
|
||||
period_end=TARGET, batch_sequence=1440,
|
||||
)
|
||||
assert result["status"] == "attention"
|
||||
assert result["weighted"]["score"] < 0.8
|
||||
|
||||
|
||||
def test_weighted_policy_requires_a_persisted_causal_calibration():
|
||||
document = _weighted_document()
|
||||
document.pop("monitoring_calibration")
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
|
||||
)
|
||||
assert result["status"] == "insufficient_data"
|
||||
|
||||
|
||||
def test_calibration_from_another_or_future_cutoff_is_rejected():
|
||||
document = _weighted_document()
|
||||
document["monitoring_calibration"]["knowledge_cutoff"] = (TARGET + timedelta(minutes=15)).isoformat()
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
|
||||
)
|
||||
assert result["status"] == "insufficient_data"
|
||||
|
||||
|
||||
def test_ready_flow_anomaly_survives_missing_pressure_calibration(monkeypatch):
|
||||
document = _weighted_document()
|
||||
document["monitoring_calibration"].update(status="insufficient_data", pressure={})
|
||||
document["monitoring_calibration"].pop("burst_detection")
|
||||
def unexpected_detector(**_):
|
||||
raise AssertionError("Unavailable pressure calibration must not run the detector")
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", unexpected_detector)
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
|
||||
)
|
||||
assert result["status"] == "insufficient_data"
|
||||
assert result["outlet_flow"]["hit"] is True
|
||||
assert result["weighted"]["score"] == 0.1
|
||||
assert result["burst_detection"]["status"] == "baseline_not_ready"
|
||||
|
||||
|
||||
def test_relative_baseline_matches_calibration_immediately_previous_samples():
|
||||
config = ScadaReplayCreate.model_validate(_document()["config"])
|
||||
rows = [{"device_id": "P1", "time": (TARGET - timedelta(minutes=15 * (5 - i))).isoformat(),
|
||||
"cleaned_value": value, "quality": "valid"}
|
||||
for i, value in enumerate((120., 115., 110., 105., 100., 95.))]
|
||||
result = scada_replay_monitoring._relative_indicator(
|
||||
rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
|
||||
)
|
||||
assert result["devices"][0]["baseline"] == 107.5
|
||||
assert result["devices"][0]["baseline_end"] == (TARGET - timedelta(minutes=15)).isoformat()
|
||||
|
||||
|
||||
def test_period_start_reading_is_not_a_new_observation():
|
||||
config = ScadaReplayCreate.model_validate(_document()["config"])
|
||||
rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
|
||||
result = scada_replay_monitoring._relative_indicator(
|
||||
rows, config, ["P1"], TARGET - timedelta(minutes=15), TARGET, direction="decrease",
|
||||
)
|
||||
assert result["status"] == "insufficient_data"
|
||||
assert result["gaps"][0]["reason"] == "period_reading_missing"
|
||||
|
||||
|
||||
def test_detector_cannot_reuse_the_previous_period_score(monkeypatch):
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection",
|
||||
lambda **_: pytest.fail("stale target reached detector"))
|
||||
rows = [row for row in _rows() if row["time"] < TARGET.isoformat()]
|
||||
result = scada_replay_monitoring._burst_detection(rows, PRESSURE_IDS, 900, TARGET)
|
||||
assert result["status"] == "current_reading_missing"
|
||||
assert result["hit"] is False
|
||||
|
||||
|
||||
def test_old_calibration_version_requires_a_new_replay():
|
||||
document = _weighted_document()
|
||||
document["monitoring_calibration"]["version"] = "scada-monitor-calibration@1"
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, _rows(), period_start=TARGET - timedelta(minutes=15), period_end=TARGET, batch_sequence=1,
|
||||
)
|
||||
assert result["status"] == "insufficient_data"
|
||||
assert "新建" in result["detail"]
|
||||
assert not any(result[key]["hit"] for key in ("pressure", "outlet_flow", "burst_detection"))
|
||||
|
||||
|
||||
def test_normal_diurnal_change_then_t_and_t_plus_fifteen_share_one_incident(monkeypatch):
|
||||
from app.services.scada_replay_processing import detect_frame
|
||||
from app.services.scada_monitoring_incidents import update_incident
|
||||
document = _weighted_document()
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
profile = {"phase_origin": config.start_time.isoformat(), "interval_seconds": 900,
|
||||
"lower": [99., 199., 199., 199.], "upper": [101., 201., 201., 201.]}
|
||||
document["monitoring_calibration"]["diagnostic"]["F1"] = profile
|
||||
document["monitoring_calibration"]["outlet_flow"]["F1"]["normal_profile"] = profile
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **kwargs: {
|
||||
"rows": [{"Score": -0.2 if kwargs["observed_pressure_data"].iloc[-1].min() < 90 else 0.02}],
|
||||
"summary": {},
|
||||
})
|
||||
rows = _rows()
|
||||
rows += [{"time": (TARGET - timedelta(days=15)).isoformat(), "device_id": device,
|
||||
"cleaned_value": 100., "quality": "valid"} for device in PRESSURE_IDS + FLOW_IDS]
|
||||
for row in rows:
|
||||
if row["device_id"] == "F1" and _time_for_test(row) >= TARGET - timedelta(minutes=15):
|
||||
row["cleaned_value"] = 200.
|
||||
rows += [{"time": (TARGET + timedelta(minutes=15)).isoformat(), "device_id": device,
|
||||
"cleaned_value": 200. if device == "F1" else 70., "quality": "valid"}
|
||||
for device in PRESSURE_IDS + FLOW_IDS]
|
||||
document["histories"] = {device: [100.] * 4 for device in PRESSURE_IDS + FLOW_IDS}
|
||||
actions, statuses = [], []
|
||||
for sequence, offset in enumerate((-15, 0, 15), start=1):
|
||||
at = TARGET + timedelta(minutes=offset)
|
||||
frame = [row for row in rows if _time_for_test(row) == at]
|
||||
document, changes = detect_frame(document, frame, config, at)
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, [row for row in rows if _time_for_test(row) <= at],
|
||||
period_start=at - timedelta(minutes=15), period_end=at, batch_sequence=sequence,
|
||||
)
|
||||
result["anomaly_signals"] = [state["event"] for state in document["rule_states"].values()
|
||||
if state.get("event")]
|
||||
statuses.append(result["status"])
|
||||
incident, action = update_incident(document, result, sequence)
|
||||
actions.append(action)
|
||||
if incident and not incident.get("task_id"):
|
||||
incident["task_id"] = str(uuid4())
|
||||
if offset == -15:
|
||||
assert changes == [] # Normal doubling of flow does not launch diagnosis.
|
||||
assert statuses == ["normal", "suspected_burst", "suspected_burst"]
|
||||
assert actions == [None, "raised", "ongoing"]
|
||||
|
||||
|
||||
def _time_for_test(row):
|
||||
return datetime.fromisoformat(row["time"])
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
import pytest
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||||
from app.services.scada_normal_profile import (
|
||||
CALIBRATION_VERSION, DIAGNOSTIC_POLICY, diagnostic_signal,
|
||||
)
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
@@ -64,3 +68,60 @@ def test_empty_clock_ticks_do_not_break_consecutive_observations():
|
||||
doc, _ = detect_frame(doc, [], cfg, START + timedelta(minutes=1))
|
||||
_, events = detect_frame(doc, prepare_cleaning(rows(20, 15), cfg), cfg, START + timedelta(minutes=15))
|
||||
assert events[0]["change"] == "raised"
|
||||
|
||||
|
||||
def weighted_document(low=97., high=103.):
|
||||
return {"replay_id": "run", "histories": {"P1": [100.] * 4},
|
||||
"monitoring_config": {"decision_policy": "weighted"},
|
||||
"config": config().model_dump(mode="json"),
|
||||
"monitoring_calibration": {
|
||||
"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
|
||||
"knowledge_cutoff": START.isoformat(),
|
||||
"diagnostic": {"P1": {"phase_origin": START.isoformat(), "interval_seconds": 900,
|
||||
"lower": [low], "upper": [high]}},
|
||||
}}
|
||||
|
||||
|
||||
def test_weighted_diagnosis_ignores_normal_diurnal_large_relative_changes():
|
||||
cfg = config(rules=[{"rule_id": "flow", "device_ids": ["P1"], "kind": "relative_change",
|
||||
"direction": "both", "threshold": 30, "unit": "m3/h"}])
|
||||
document = weighted_document(25., 35.)
|
||||
document["histories"]["P1"] = [10.] * 4
|
||||
_, events = detect_frame(document, prepare_cleaning(rows(30), cfg), cfg, START)
|
||||
assert events == [] # +200% is normal at this daily sampling phase.
|
||||
_, events = detect_frame(document, prepare_cleaning(rows(36), cfg), cfg, START)
|
||||
assert events[0]["change"] == "raised"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("baseline,value,low,high", [
|
||||
(100., 95., 97., 103.), (0., -5., -1., 1.),
|
||||
])
|
||||
def test_weighted_single_indicator_uses_history_envelope_without_thirty_percent_gate(baseline, value, low, high):
|
||||
cfg = config()
|
||||
document = weighted_document(low, high)
|
||||
document["histories"]["P1"] = [baseline] * 4
|
||||
_, events = detect_frame(document, prepare_cleaning(rows(value), cfg), cfg, START)
|
||||
assert len(events) == 1
|
||||
assert events[0]["diagnostic_policy"] == DIAGNOSTIC_POLICY
|
||||
assert diagnostic_signal(document, events[0])
|
||||
assert "阈值 30" not in events[0]["detail"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", ["version", "diagnostic", "knowledge_cutoff"])
|
||||
def test_weighted_missing_calibration_does_not_fall_back_to_thirty_percent(missing):
|
||||
cfg = config()
|
||||
document = weighted_document()
|
||||
document["monitoring_calibration"].pop(missing)
|
||||
_, events = detect_frame(document, prepare_cleaning(rows(1), cfg), cfg, START)
|
||||
assert events == []
|
||||
|
||||
|
||||
def test_old_active_relative_state_cannot_block_or_supply_new_diagnosis():
|
||||
cfg = config()
|
||||
document = weighted_document()
|
||||
old = {"event_id": "old", "rule_id": "pressure", "device_ids": ["P1"], "status": "active"}
|
||||
document["rule_states"] = {"pressure": {"event": old, "hits": 1, "recoveries": 0}}
|
||||
result, events = detect_frame(document, prepare_cleaning(rows(95), cfg), cfg, START)
|
||||
assert not diagnostic_signal(document, old)
|
||||
assert result["rule_states"]["pressure"]["event"] == old
|
||||
assert events[0]["diagnostic_policy"] == DIAGNOSTIC_POLICY
|
||||
|
||||
@@ -12,7 +12,7 @@ from app.mcp.analysis_models import BurstLocationInput
|
||||
from app.mcp.catalog import TOOLS
|
||||
from app.mcp.models import BoundContext
|
||||
from app.mcp import query_service
|
||||
from app.services.scada_replay_queries import replay_burst_location_inputs
|
||||
from app.services.scada_replay_queries import replay_burst_location_inputs, replay_snapshot_rows
|
||||
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
@@ -85,6 +85,60 @@ def test_replay_location_builds_series_only_from_frozen_outputs():
|
||||
assert connection.statements[1][1][2] == 9
|
||||
|
||||
|
||||
def test_replay_queries_frozen_history_with_project_replay_and_batch_bounds():
|
||||
project_id, replay_id = uuid4(), str(uuid4())
|
||||
at = START - timedelta(days=7)
|
||||
run = {"start_time": START.isoformat(), "config": {"device_ids": ["P1"]},
|
||||
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()}}
|
||||
reading = {"time": at.isoformat(), "device_id": "P1", "monitored_value": 23.,
|
||||
"cleaned_value": 23., "quality": "valid", "reason": "冻结历史基线"}
|
||||
connection = Connection([[{"document": run}], [{"reading": reading}]])
|
||||
context = {"replay_id": replay_id, "knowledge_cutoff": START.isoformat(), "batch_sequence": 9}
|
||||
result = asyncio.run(replay_snapshot_rows(connection, project_id, context, ["P1"], at, at, 10))
|
||||
assert result == [reading]
|
||||
statement, params = connection.statements[1]
|
||||
assert "b.stage='detected'" in statement and "b.sequence<=%s" in statement
|
||||
assert params[:4] == (str(project_id), replay_id, 9, ["P1"])
|
||||
assert params[4:6] == (at, at)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defect,status", [("old_replay", 422), ("too_early", 422), ("future", 422), ("foreign_device", 403)])
|
||||
def test_replay_history_rejects_missing_freeze_or_scope_escape(defect, status):
|
||||
at = START - timedelta(days=7)
|
||||
run = {"start_time": START.isoformat(), "config": {"device_ids": ["P1"]},
|
||||
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()}}
|
||||
if defect == "old_replay": run.pop("baseline")
|
||||
if defect == "too_early": at = START - timedelta(days=16)
|
||||
if defect == "future": at = START + timedelta(minutes=1)
|
||||
connection = Connection([[{"document": run}]])
|
||||
context = {"replay_id": str(uuid4()), "knowledge_cutoff": START.isoformat(), "batch_sequence": 9}
|
||||
with pytest.raises(HTTPException) as caught:
|
||||
asyncio.run(replay_snapshot_rows(connection, uuid4(), context,
|
||||
["P2" if defect == "foreign_device" else "P1"], at, at, 10))
|
||||
assert caught.value.status_code == status
|
||||
assert len(connection.statements) <= 1
|
||||
|
||||
|
||||
def test_location_accepts_frozen_pre_replay_normal_window():
|
||||
replay_id = str(uuid4())
|
||||
normal = START - timedelta(days=7)
|
||||
run = {"start_time": START.isoformat(),
|
||||
"baseline": {"start_time": (START - timedelta(days=15)).isoformat()},
|
||||
"devices": [{"device_id": "P1", "node_id": "J1"}, {"device_id": "F1", "link_id": "L1"}],
|
||||
"monitoring_config": {"pressure_device_ids": ["P1"], "outlet_flow_device_ids": ["F1"]}}
|
||||
readings = [{"reading": {"time": at.isoformat(), "device_id": device,
|
||||
"cleaned_value": value, "quality": "valid"}}
|
||||
for at, device, value in [(normal, "P1", 100), (normal, "F1", 50),
|
||||
(START + timedelta(minutes=15), "P1", 70),
|
||||
(START + timedelta(minutes=15), "F1", 80)]]
|
||||
connection = Connection([[{"document": run}], readings])
|
||||
context = {"replay_id": replay_id, "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(), "batch_sequence": 9}
|
||||
result = asyncio.run(replay_burst_location_inputs(connection, uuid4(), context,
|
||||
_args(scada_normal_start=normal, scada_normal_end=normal + timedelta(minutes=14))))
|
||||
assert result["normal_pressure"] == {"J1": 100}
|
||||
assert result["burst_flow"] == {"L1": 80}
|
||||
|
||||
|
||||
def test_replay_location_authorization_requires_explicit_bounded_normal_window(monkeypatch):
|
||||
binding = BoundContext(
|
||||
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
|
||||
|
||||
@@ -10,23 +10,38 @@ import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.scada_replay import ScadaReplayService
|
||||
from app.services.scada_replay import BaselineBudgetExceeded, ScadaReplayService, snapshot
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class Connection:
|
||||
def __init__(self, owner=None):
|
||||
self.statements, self.depth, self.owner = [], 0, owner
|
||||
|
||||
async def execute(self, statement): self.statements.append(statement)
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
yield
|
||||
prior = (deepcopy(self.owner.document), deepcopy(self.owner.batches)) if self.owner else None
|
||||
self.depth += 1
|
||||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
if prior:
|
||||
self.owner.document, self.owner.batches = prior
|
||||
raise
|
||||
finally:
|
||||
self.depth -= 1
|
||||
|
||||
|
||||
class Repository:
|
||||
def __init__(self):
|
||||
self.project_id = str(uuid4())
|
||||
self.connection = Connection()
|
||||
self.connection = Connection(self)
|
||||
self.document = None
|
||||
self.batches = {}
|
||||
|
||||
@@ -43,9 +58,20 @@ class Repository:
|
||||
async def get(self, _): return deepcopy(self.document)
|
||||
async def current(self): return deepcopy(self.document)
|
||||
async def pending_batch(self, _):
|
||||
return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None)
|
||||
return next((deepcopy(batch) for batch in self.batches.values()
|
||||
if batch["sequence"] > 0 and batch["stage"] != "detected"), None)
|
||||
async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch)
|
||||
async def monitoring_outputs(self, *_args): return []
|
||||
async def clear_unpublished_baseline(self, _):
|
||||
self.batches = {key: value for key, value in self.batches.items()
|
||||
if not (key < 0 and value.get("kind") == "baseline" and value["stage"] == "clean_prepared")}
|
||||
async def publish_baseline(self, _):
|
||||
for key, value in self.batches.items():
|
||||
if key < 0 and value.get("kind") == "baseline" and value["stage"] == "clean_prepared":
|
||||
value["stage"] = "detected"
|
||||
async def monitoring_outputs(self, _replay_id, device_ids, start, end, before_sequence):
|
||||
return [row for batch in self.batches.values() if batch["sequence"] < before_sequence
|
||||
and batch["stage"] == "detected" for row in batch.get("outputs", [])
|
||||
if row["device_id"] in device_ids and start <= datetime.fromisoformat(row["time"]) <= end]
|
||||
|
||||
|
||||
def setup(monkeypatch):
|
||||
@@ -59,6 +85,8 @@ def setup(monkeypatch):
|
||||
"device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00"
|
||||
}]))
|
||||
async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_):
|
||||
if _end <= START:
|
||||
return []
|
||||
return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}]
|
||||
if after is None else [])
|
||||
query = AsyncMock(side_effect=raw_page)
|
||||
@@ -66,11 +94,269 @@ def setup(monkeypatch):
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", query)
|
||||
monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning)
|
||||
monkeypatch.setattr(service, "_create_investigations", AsyncMock())
|
||||
monkeypatch.setattr(service, "_publish_monitoring_alert", AsyncMock())
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService.create",
|
||||
AsyncMock(return_value=SimpleNamespace(task_id=uuid4())))
|
||||
create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
|
||||
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}])
|
||||
return repo, service, query, cleaning, create
|
||||
|
||||
|
||||
def test_baseline_freezes_all_pages_silently_without_cleaning_the_source(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
before = START - timedelta(minutes=15)
|
||||
query.side_effect = None
|
||||
query.side_effect = [
|
||||
[{"time": before - timedelta(minutes=15), "device_id": "P1", "monitored_value": 100., "source_hash": "a"}],
|
||||
[{"time": before, "device_id": "P1", "monitored_value": 200., "source_hash": "b"}],
|
||||
[],
|
||||
]
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900)
|
||||
document = await repo.get(None)
|
||||
await service._freeze_baseline(document, create)
|
||||
assert datetime.fromisoformat(repo.document["virtual_now"]) == create.start_time
|
||||
assert repo.document["batch_sequence"] == repo.document["processed_count"] == 0
|
||||
assert repo.document["anomaly_count"] == repo.document["monitoring_count"] == 0
|
||||
assert repo.document["rule_states"] == repo.document["latest"] == {}
|
||||
assert repo.document["baseline"]["row_count"] == 2
|
||||
assert repo.document["histories"]["P1"] == [100., 200.]
|
||||
assert set(repo.batches) == {-1, -2}
|
||||
assert all(batch["kind"] == "baseline" and batch["stage"] == "detected"
|
||||
and batch["events"] == [] for batch in repo.batches.values())
|
||||
assert repo.batches[-2]["outputs"][0]["cleaned_value"] == 200.
|
||||
|
||||
asyncio.run(run())
|
||||
assert service.timescale.statements == ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY"]
|
||||
assert query.await_args_list[0].args[2:4] == (START - timedelta(days=15), START)
|
||||
assert query.await_args_list[1].args[6] == (before - timedelta(minutes=15), "P1")
|
||||
cleaning.assert_not_awaited()
|
||||
service._create_investigations.assert_not_awaited()
|
||||
|
||||
|
||||
def test_snapshot_returns_frozen_monitoring_rules_and_actual_calibration(monkeypatch):
|
||||
repo, service, _, _, create = setup(monkeypatch)
|
||||
settings = ScadaMonitoringSettings(pressure_device_ids=["P1"])
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900, monitoring_config=settings)
|
||||
repo.document["monitoring_calibration"] = {
|
||||
"status": "ready", "burst_detection": {"threshold": -0.0843},
|
||||
"pressure": {"P1": {"threshold_percent": 6.5}},
|
||||
}
|
||||
# Project settings may change later; the replay still reports its frozen configuration.
|
||||
settings.weights.pressure = 0.3
|
||||
settings.weights.burst_detection = 0.6
|
||||
result = snapshot(repo.document)
|
||||
assert result.monitoring_config.weights.pressure == 0.4
|
||||
assert result.monitoring_config.weights.burst_detection == 0.5
|
||||
assert result.monitoring_calibration["burst_detection"]["threshold"] == -0.0843
|
||||
repo.document["monitoring_config"].pop("decision_policy")
|
||||
assert snapshot(repo.document).monitoring_config.decision_policy == "all_signals"
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_baseline_stages_each_page_before_reading_more_without_publishing_or_holding_business_transaction(monkeypatch):
|
||||
repo, service, query, _, create = setup(monkeypatch)
|
||||
calls = 0
|
||||
|
||||
async def raw_page(*args, **kwargs):
|
||||
nonlocal calls
|
||||
assert service.timescale.depth == 1
|
||||
assert repo.connection.depth == 0
|
||||
if calls:
|
||||
assert repo.batches[-calls]["stage"] == "clean_prepared"
|
||||
assert not repo.document.get("warmed") and "baseline" not in repo.document
|
||||
assert await repo.pending_batch(None) is None
|
||||
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
|
||||
calls += 1
|
||||
if calls > 2:
|
||||
return []
|
||||
return [{"time": START - timedelta(minutes=45 - calls * 15), "device_id": "P1",
|
||||
"monitored_value": 30., "source_hash": str(calls)}]
|
||||
|
||||
query.side_effect = raw_page
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900)
|
||||
await service._freeze_baseline(await repo.get(None), create)
|
||||
assert repo.document["warmed"] and repo.document["baseline"]["row_count"] == 2
|
||||
assert all(batch["stage"] == "detected" for batch in repo.batches.values())
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_failed_baseline_read_never_publishes_partial_history(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
query.side_effect = [
|
||||
[{"time": START - timedelta(minutes=15), "device_id": "P1", "monitored_value": 30., "source_hash": "a"}],
|
||||
RuntimeError("source unavailable"),
|
||||
]
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900)
|
||||
with pytest.raises(RuntimeError, match="source unavailable"):
|
||||
await service._freeze_baseline(await repo.get(None), create)
|
||||
assert "baseline" not in repo.document and not repo.document.get("warmed")
|
||||
assert repo.batches[-1]["stage"] == "clean_prepared"
|
||||
assert await repo.pending_batch(None) is None
|
||||
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
|
||||
|
||||
asyncio.run(run())
|
||||
cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
def test_baseline_retry_discards_unpublished_pages_and_refreezes_one_source_snapshot(monkeypatch):
|
||||
repo, service, query, _, create = setup(monkeypatch)
|
||||
def row(minutes, value):
|
||||
return {"time": START - timedelta(minutes=minutes), "device_id": "P1",
|
||||
"monitored_value": value, "source_hash": str(value)}
|
||||
query.side_effect = [[row(30, 100.)], [row(15, 200.)], RuntimeError("source unavailable")]
|
||||
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"))
|
||||
with pytest.raises(RuntimeError, match="source unavailable"):
|
||||
await service.process_pending_step(first.replay_id)
|
||||
assert repo.document["status"] == "failed" and set(repo.batches) == {-1, -2}
|
||||
assert all(batch["stage"] == "clean_prepared" for batch in repo.batches.values())
|
||||
query.side_effect = [[row(15, 400.)], [], []]
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=2, action="retry"))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert not result.step_pending and result.status == "paused"
|
||||
assert repo.document["baseline"]["row_count"] == 1
|
||||
assert repo.document["histories"]["P1"] == [400.]
|
||||
assert -2 not in repo.batches and repo.batches[-1]["stage"] == "detected"
|
||||
assert repo.batches[-1]["outputs"][0]["cleaned_value"] == 400.
|
||||
assert result.anomaly_count == result.processed_count == 0
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("budget", ["rows", "bytes"])
|
||||
def test_baseline_budget_stops_before_saving_excess_rows_and_exposes_actionable_error(monkeypatch, budget):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
if budget == "rows":
|
||||
monkeypatch.setattr("app.services.scada_replay.BASELINE_MAX_ROWS", 2)
|
||||
else:
|
||||
monkeypatch.setattr("app.services.scada_replay.BASELINE_MAX_BYTES", 1)
|
||||
query.side_effect = [
|
||||
[{"time": START - timedelta(minutes=minute), "device_id": "P1",
|
||||
"monitored_value": 30., "source_hash": str(minute)} for minute in [45, 30]],
|
||||
[{"time": START - timedelta(minutes=15), "device_id": "P1", "monitored_value": 30., "source_hash": "15"}],
|
||||
]
|
||||
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"))
|
||||
with pytest.raises(BaselineBudgetExceeded):
|
||||
await service.process_pending_step(first.replay_id)
|
||||
result = snapshot(repo.document)
|
||||
assert result.status == "failed" and not result.step_pending
|
||||
assert "BASELINE_BUDGET_EXCEEDED" in result.error and "历史基线" in result.error and "重试" in result.error
|
||||
assert "baseline" not in repo.document and not repo.document.get("warmed")
|
||||
assert result.processed_count == result.anomaly_count == result.monitoring_count == 0
|
||||
assert result.virtual_now == START
|
||||
assert set(repo.batches) == ({-1} if budget == "rows" else set())
|
||||
assert all(batch["stage"] == "clean_prepared" for batch in repo.batches.values())
|
||||
asyncio.run(run())
|
||||
cleaning.assert_not_awaited()
|
||||
service._create_investigations.assert_not_awaited()
|
||||
assert query.await_count == (2 if budget == "rows" else 1)
|
||||
if budget == "rows":
|
||||
assert query.await_args_list[1].args[5] == 1
|
||||
|
||||
|
||||
def test_failed_baseline_publication_rolls_back_visibility_and_initialized_marker(monkeypatch):
|
||||
repo, service, query, _, create = setup(monkeypatch)
|
||||
query.side_effect = [[{"time": START - timedelta(minutes=15), "device_id": "P1",
|
||||
"monitored_value": 30., "source_hash": "a"}], []]
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900)
|
||||
monkeypatch.setattr(repo, "save", AsyncMock(side_effect=RuntimeError("publish unavailable")))
|
||||
with pytest.raises(RuntimeError, match="publish unavailable"):
|
||||
await service._freeze_baseline(await repo.get(None), create)
|
||||
assert repo.batches[-1]["stage"] == "clean_prepared"
|
||||
assert "baseline" not in repo.document and not repo.document.get("warmed")
|
||||
assert await repo.monitoring_outputs(None, ["P1"], START - timedelta(days=15), START, 1) == []
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_baseline_already_published_is_not_refrozen_or_replaced(monkeypatch):
|
||||
repo, service, query, _, create = setup(monkeypatch)
|
||||
|
||||
async def run():
|
||||
await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document.update(warmed=True, baseline={"row_count": 1})
|
||||
repo.batches[-1] = {"sequence": -1, "stage": "detected", "kind": "baseline", "outputs": []}
|
||||
before = deepcopy((repo.document, repo.batches))
|
||||
await service._freeze_baseline(await repo.get(None), create)
|
||||
assert (repo.document, repo.batches) == before
|
||||
asyncio.run(run())
|
||||
query.assert_not_awaited()
|
||||
|
||||
|
||||
def test_first_step_uses_frozen_fifteen_days_without_counting_historical_events(monkeypatch):
|
||||
repo, service, query, cleaning, _ = setup(monkeypatch)
|
||||
pressure_ids, flow_id = [f"P{i}" for i in range(5)], "F1"
|
||||
device_ids = pressure_ids + [flow_id]
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1), device_ids=device_ids,
|
||||
rules=[{"rule_id": "pressure", "kind": "relative_change", "device_ids": pressure_ids,
|
||||
"direction": "decrease", "threshold": 20, "unit": "m"},
|
||||
{"rule_id": "flow", "kind": "relative_change", "device_ids": [flow_id],
|
||||
"direction": "increase", "threshold": 20, "unit": "m3/h"}],
|
||||
)
|
||||
monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
|
||||
"device_id": device, "device_type": "pressure" if device in pressure_ids else "pipe_flow",
|
||||
"measurement_unit": "m" if device in pressure_ids else "m3/h",
|
||||
"transmission_frequency": "0:15:00",
|
||||
} for device in device_ids]))
|
||||
source = [{"time": START + timedelta(minutes=15 * i), "device_id": device,
|
||||
"monitored_value": (70. if device in pressure_ids else 150.) if i == 1 else 100.,
|
||||
"source_hash": f"{i}-{device}"}
|
||||
for i in range(-15 * 96, 2) for device in sorted(device_ids)]
|
||||
|
||||
async def raw_page(_connection, _devices, start, end, through, limit, after=None, **options):
|
||||
rows = [row for row in source if start <= row["time"] < end and row["time"] <= through
|
||||
and (after is None or (row["time"], row["device_id"]) > after)]
|
||||
if options.get("frame") and rows:
|
||||
rows = [row for row in rows if row["time"] == rows[0]["time"]]
|
||||
return rows[:limit]
|
||||
|
||||
query.side_effect = raw_page
|
||||
def detection(**kwargs):
|
||||
target = kwargs["observed_pressure_data"].iloc[-1].min()
|
||||
return {"rows": [{"Score": 0.02}] * 14 + [{"Score": -0.2 if target < 90 else 0.02}], "summary": {}}
|
||||
monkeypatch.setattr("app.services.scada_replay_monitoring.run_burst_detection", detection)
|
||||
monkeypatch.setattr(service, "_create_monitoring_task", AsyncMock())
|
||||
|
||||
async def run():
|
||||
first = await service.create(config, monitoring_interval_seconds=900,
|
||||
monitoring_config=ScadaMonitoringSettings(
|
||||
pressure_device_ids=pressure_ids, outlet_flow_device_ids=[flow_id]))
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.last_monitoring.status == "suspected_burst"
|
||||
assert result.last_monitoring.weighted["weights"]["outlet_flow"] == 0.1
|
||||
assert repo.document["monitoring_calibration"]["status"] == "ready"
|
||||
assert result.processed_count == 12 and result.monitoring_count == 1
|
||||
assert repo.document["baseline"]["row_count"] == 8640
|
||||
assert result.virtual_now == START + timedelta(minutes=15)
|
||||
assert all(not batch["events"] for key, batch in repo.batches.items() if key < 0)
|
||||
|
||||
asyncio.run(run())
|
||||
assert cleaning.await_count == 2
|
||||
assert all(row["time"] >= START.isoformat()
|
||||
for call in cleaning.await_args_list for row in call.args[1])
|
||||
|
||||
|
||||
def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
async def run():
|
||||
@@ -89,6 +375,79 @@ def test_step_command_is_revisioned_idempotent_and_does_not_process_until_reques
|
||||
query.assert_not_awaited(); cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("minutes", [0, 10])
|
||||
def test_monitoring_stops_exactly_at_selected_time_and_reuses_same_period(monkeypatch, minutes):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
target = START + timedelta(minutes=minutes)
|
||||
|
||||
async def run():
|
||||
initial = await service.create(create, monitoring_interval_seconds=900)
|
||||
pending = await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="monitor", at_time=target))
|
||||
assert pending.step_pending
|
||||
assert datetime.fromisoformat(repo.document["step_target"]) == target
|
||||
result = await service.process_pending_step(initial.replay_id)
|
||||
assert result.virtual_now == target
|
||||
assert result.last_monitoring.period_end == target
|
||||
assert result.last_monitoring.knowledge_cutoff == target
|
||||
assert result.monitoring_count == 1
|
||||
reads, writes, batches = query.await_count, cleaning.await_count, len(repo.batches)
|
||||
repeated = await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=result.revision, action="monitor", at_time=target))
|
||||
assert not repeated.step_pending
|
||||
assert repeated.last_monitoring == result.last_monitoring
|
||||
assert repeated.monitoring_count == 1
|
||||
assert (query.await_count, cleaning.await_count, len(repo.batches)) == (reads, writes, batches)
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=repeated.revision, action="step"))
|
||||
assert datetime.fromisoformat(repo.document["step_target"]) == START + timedelta(minutes=15)
|
||||
|
||||
asyncio.run(run())
|
||||
assert all(call.args[4] <= target for call in query.await_args_list)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("minutes,status_code", [(-5, 422), (65, 422), (15, 409)])
|
||||
def test_selected_monitoring_rejects_out_of_range_or_backward_targets_without_processing(monkeypatch, minutes, status_code):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
|
||||
async def run():
|
||||
initial = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["virtual_now"] = (START + timedelta(minutes=30)).isoformat()
|
||||
before = deepcopy(repo.document)
|
||||
with pytest.raises(HTTPException) as failure:
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="monitor", at_time=START + timedelta(minutes=minutes)))
|
||||
assert failure.value.status_code == status_code
|
||||
assert repo.document == before
|
||||
|
||||
asyncio.run(run())
|
||||
query.assert_not_awaited()
|
||||
cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
def test_selected_monitoring_cannot_replace_a_pending_period(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
|
||||
async def run():
|
||||
initial = await service.create(create, monitoring_interval_seconds=900)
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
before = deepcopy(repo.document)
|
||||
with pytest.raises(HTTPException, match="当前时间步正在处理"):
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=2, action="monitor", at_time=START))
|
||||
assert repo.document == before
|
||||
|
||||
asyncio.run(run())
|
||||
query.assert_not_awaited()
|
||||
cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action,at_time", [("monitor", None), ("monitor", "2026-07-16T08:00:00"), ("step", "2026-07-16T00:00:00Z")])
|
||||
def test_selected_monitoring_requires_an_explicit_zoned_time_only_for_monitor(action, at_time):
|
||||
with pytest.raises(ValueError):
|
||||
ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action=action, at_time=at_time)
|
||||
|
||||
|
||||
def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
async def run():
|
||||
@@ -212,7 +571,8 @@ def test_partial_final_step_checks_empty_interval_and_completes_without_fabricat
|
||||
assert query.await_args.args[4] == create.end_time
|
||||
|
||||
|
||||
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch):
|
||||
@pytest.mark.parametrize("weighted", [False, True])
|
||||
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch, weighted):
|
||||
repo, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
@@ -231,12 +591,376 @@ def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(mo
|
||||
"burst_detection": {"target_score": -0.05},
|
||||
"task_id": None,
|
||||
}
|
||||
if weighted:
|
||||
monitoring["burst_detection"]["score_threshold"] = -0.0843
|
||||
monitoring["weighted"] = {
|
||||
"score": 0.9, "threshold": 0.8,
|
||||
"weights": {"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.1},
|
||||
}
|
||||
document = {"replay_id": str(uuid4())}
|
||||
|
||||
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8}))
|
||||
|
||||
command = create_task.await_args.args[0]
|
||||
assert command.workflow == "burst_report"
|
||||
assert command.workflow == "scada_anomaly"
|
||||
assert command.source.replay_context.batch_sequence == 8
|
||||
assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
|
||||
assert monitoring["task_id"] == str(create_task.return_value.task_id)
|
||||
assert document["monitoring_incident"]["alert"]["condition_id"] == f"operational-task:{monitoring['task_id']}"
|
||||
assert document["monitoring_incident"]["alert"]["description"] == "已启动调查,详细诊断见调查报告。"
|
||||
if weighted:
|
||||
assert "-0.0843" in command.summary
|
||||
assert "综合评分 0.90" in command.summary
|
||||
assert "流量 10%" in command.summary
|
||||
|
||||
|
||||
def test_monitoring_summary_counts_signals_without_copying_reading_details(monkeypatch):
|
||||
_, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
|
||||
monitoring = incident_monitoring(0)
|
||||
monitoring["anomaly_signals"] = [
|
||||
{"device_ids": [f"P{i}"], "detail": f"P{i}=28.14402136077437 m,超出冻结正常同时段范围 [28.3253, 28.8685] m"}
|
||||
for i in range(100)
|
||||
]
|
||||
|
||||
asyncio.run(service._create_monitoring_task({"replay_id": str(uuid4())}, monitoring, {"sequence": 1}))
|
||||
|
||||
summary = create_task.await_args.args[0].summary
|
||||
assert len(summary) <= 300
|
||||
assert "100" in summary
|
||||
assert "28.14402136077437" not in summary
|
||||
assert monitoring["anomaly_signals"][0]["detail"].startswith("P0=28.14402136077437")
|
||||
|
||||
|
||||
def test_continuing_burst_reuses_task_across_persisted_periods(monkeypatch):
|
||||
repo, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
advance_task = AsyncMock()
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
|
||||
lambda *_: SimpleNamespace(create=create_task, advance_replay_investigation=advance_task))
|
||||
document = {"replay_id": str(uuid4())}
|
||||
|
||||
async def run():
|
||||
nonlocal document
|
||||
for step in range(3):
|
||||
monitoring = incident_monitoring(step, status="attention" if step == 0 else "suspected_burst")
|
||||
await service._create_monitoring_task(document, monitoring, {"sequence": step + 1})
|
||||
document = deepcopy(document) # Simulate loading persisted state after restart.
|
||||
assert monitoring["task_id"] == str(create_task.return_value.task_id)
|
||||
assert create_task.await_count == 1
|
||||
service._publish_monitoring_alert.assert_awaited_once()
|
||||
assert document["monitoring_incident"]["pressure_anchors"]["P1"]["baseline"] == 100.
|
||||
advance_task.assert_awaited_once()
|
||||
assert advance_task.await_args.args[0] == create_task.return_value.task_id
|
||||
assert advance_task.await_args.args[1]["knowledge_cutoff"] == (START + timedelta(minutes=30)).isoformat()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def incident_monitoring(step, *, status="suspected_burst", value=60.):
|
||||
return {
|
||||
"monitoring_id": str(uuid4()), "status": status,
|
||||
"period_start": (START + timedelta(minutes=15 * step)).isoformat(),
|
||||
"period_end": (START + timedelta(minutes=15 * (step + 1))).isoformat(),
|
||||
"knowledge_cutoff": (START + timedelta(minutes=15 * (step + 1))).isoformat(),
|
||||
"rule_version": "burst-monitor-v2", "detail": "监测结论", "task_id": None,
|
||||
"pressure": {"status": "ready", "hit": status == "suspected_burst", "hit_device_ids": ["P1"],
|
||||
"devices": [{"device_id": "P1", "baseline": 100., "value": value,
|
||||
"threshold_percent": 10., "hit": status == "suspected_burst",
|
||||
"observed_at": (START + timedelta(minutes=15 * (step + 1))).isoformat()}]},
|
||||
"outlet_flow": {"status": "ready", "hit": False, "hit_device_ids": []},
|
||||
"burst_detection": {"status": "ready", "hit": status == "suspected_burst", "target_score": -0.2},
|
||||
}
|
||||
|
||||
|
||||
def test_incident_recovers_once_then_allows_new_accident(monkeypatch):
|
||||
_, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(side_effect=[SimpleNamespace(task_id=uuid4()), SimpleNamespace(task_id=uuid4())])
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
|
||||
lambda *_: SimpleNamespace(create=create_task))
|
||||
document = {"replay_id": str(uuid4())}
|
||||
|
||||
async def run():
|
||||
first = incident_monitoring(0)
|
||||
await service._create_monitoring_task(document, first, {"sequence": 1})
|
||||
for step in (1, 2):
|
||||
result = incident_monitoring(step, status="normal", value=100.)
|
||||
await service._create_monitoring_task(document, result, {"sequence": step + 1})
|
||||
assert result["incident"]["action"] == "resolved"
|
||||
assert result["task_id"] == first["task_id"]
|
||||
recurrence = incident_monitoring(3)
|
||||
await service._create_monitoring_task(document, recurrence, {"sequence": 4})
|
||||
assert recurrence["incident"]["incident_id"] != first["incident"]["incident_id"]
|
||||
assert recurrence["task_id"] != first["task_id"]
|
||||
assert create_task.await_count == 2
|
||||
assert [call.args[2] for call in service._publish_monitoring_alert.await_args_list] == ["raised", "resolved", "raised"]
|
||||
commands = [call.args[0] for call in create_task.await_args_list]
|
||||
assert commands[0].source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interruption", ["insufficient_data", "attention", "normal_low_pressure", "missing_sensor", "stale_reading", "time_gap"])
|
||||
def test_incident_needs_two_fresh_complete_pressure_recoveries(interruption):
|
||||
from app.services.scada_monitoring_incidents import update_incident
|
||||
document = {"replay_id": str(uuid4())}
|
||||
incident, _ = update_incident(document, incident_monitoring(0), 1)
|
||||
incident["task_id"] = str(uuid4())
|
||||
update_incident(document, incident_monitoring(1, status="normal", value=100.), 2)
|
||||
interrupted = incident_monitoring(2, status="normal", value=100.)
|
||||
if interruption in {"insufficient_data", "attention"}:
|
||||
interrupted["status"] = interruption
|
||||
elif interruption == "normal_low_pressure":
|
||||
interrupted["pressure"]["devices"][0]["value"] = 60.
|
||||
elif interruption == "missing_sensor":
|
||||
interrupted["pressure"]["devices"] = []
|
||||
elif interruption == "stale_reading":
|
||||
interrupted["pressure"]["devices"][0]["observed_at"] = incident["last_observed_at"]
|
||||
else:
|
||||
interrupted["period_start"] = START.isoformat()
|
||||
incident, action = update_incident(document, interrupted, 3)
|
||||
assert action == "ongoing"
|
||||
assert incident["recovery_count"] == (1 if interruption == "time_gap" else 0)
|
||||
if interruption != "time_gap":
|
||||
_, action = update_incident(document, incident_monitoring(3, status="normal", value=100.), 4)
|
||||
assert action == "ongoing"
|
||||
|
||||
|
||||
def test_same_period_retry_does_not_count_as_second_recovery():
|
||||
from app.services.scada_monitoring_incidents import update_incident
|
||||
document = {"replay_id": str(uuid4())}
|
||||
update_incident(document, incident_monitoring(0), 1)
|
||||
recovery = incident_monitoring(1, status="normal", value=100.)
|
||||
update_incident(document, recovery, 2)
|
||||
incident, action = update_incident(deepcopy(document), recovery, 2)
|
||||
assert action == "ongoing" and incident["recovery_count"] == 1
|
||||
|
||||
|
||||
def test_older_replay_adopts_existing_task_without_creating_another(monkeypatch):
|
||||
_, service, *_ = setup(monkeypatch)
|
||||
previous = incident_monitoring(0)
|
||||
previous["task_id"] = str(uuid4())
|
||||
document = {"replay_id": str(uuid4()), "last_monitoring": previous}
|
||||
current = incident_monitoring(1)
|
||||
asyncio.run(service._create_monitoring_task(document, current, {"sequence": 2}))
|
||||
assert current["task_id"] == previous["task_id"]
|
||||
assert current["incident"]["action"] == "ongoing"
|
||||
service._publish_monitoring_alert.assert_not_awaited()
|
||||
|
||||
|
||||
def test_combined_monitoring_alert_uses_same_facts_for_recovery(monkeypatch):
|
||||
repo, service, *_ = setup(monkeypatch)
|
||||
push = AsyncMock()
|
||||
monkeypatch.setattr("app.infra.db.postgresql.task_alerts.TaskAlertRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.task_alerts.TaskAlertService", lambda *_args, **_kwargs: SimpleNamespace(push=push))
|
||||
incident = {"incident_id": str(uuid4()), "alert": {
|
||||
"alert_id": "test-accident", "producer_id": "scada-replay:demo",
|
||||
"occurred_at": START.isoformat(), "condition_id": None,
|
||||
"source": {"kind": "scada_replay", "replay_id": "demo", "rule_id": "burst_monitor", "batch_sequence": 1},
|
||||
"title": "SCADA 监测疑似爆管", "description": "同一事故", "severity": "warning",
|
||||
}}
|
||||
|
||||
async def run():
|
||||
for change in ("raised", "resolved"):
|
||||
await ScadaReplayService._publish_monitoring_alert(service, {"replay_id": "demo"}, incident, change)
|
||||
asyncio.run(run())
|
||||
first, second = [call.args[0].model_dump(mode="json") for call in push.await_args_list]
|
||||
assert first.pop("event_id") != second.pop("event_id")
|
||||
assert first.pop("event_type") == "raised"
|
||||
assert second.pop("event_type") == "resolved"
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_combined_monitoring_keeps_rule_evidence_and_quality_notifications(monkeypatch):
|
||||
_, service, *_ = setup(monkeypatch)
|
||||
push = AsyncMock()
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.infra.db.postgresql.task_alerts.TaskAlertRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.task_alerts.TaskAlertService", lambda *_args, **_kwargs: SimpleNamespace(push=push))
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=AsyncMock()))
|
||||
def event(rule, category=None):
|
||||
return {"event_id": str(uuid4()), "rule_id": rule, "change": "raised", "occurred_at": START.isoformat(),
|
||||
"detail": "检测信号", "category": category, "task_id": None}
|
||||
pressure, water, quality = event("p"), event("water"), event("quality-P1", "data_quality")
|
||||
document = {"replay_id": str(uuid4()), "monitoring_config": {"decision_policy": "weighted"},
|
||||
"config": {"rules": [{"rule_id": "p", "kind": "relative_change"}, {"rule_id": "water", "kind": "bounds"}]},
|
||||
"rule_states": {item["rule_id"]: {"event": deepcopy(item)} for item in (pressure, water, quality)}}
|
||||
|
||||
async def run():
|
||||
await ScadaReplayService._create_investigations(service, document, [pressure, water, quality], {"sequence": 1}, create_tasks=False)
|
||||
assert push.await_count == 1 # Measurement signals share diagnosis; telemetry quality remains independent.
|
||||
assert document["rule_states"]["p"]["event"]["notification_suppressed"]
|
||||
assert document["rule_states"]["water"]["event"]["notification_suppressed"]
|
||||
resolved = {**document["rule_states"]["p"]["event"], "change": "resolved"}
|
||||
await ScadaReplayService._create_investigations(service, document, [resolved], {"sequence": 2}, create_tasks=False)
|
||||
assert push.await_count == 1
|
||||
resolved.pop("notification_suppressed") # An old alert raised before this change still resolves.
|
||||
await ScadaReplayService._create_investigations(service, document, [resolved], {"sequence": 2}, create_tasks=False)
|
||||
assert push.await_count == 2
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_failed_incident_publication_rolls_back_state_and_retries_same_task_request(monkeypatch):
|
||||
repo, service, _, _, create = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
|
||||
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
|
||||
return {**incident_monitoring(0), "period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
|
||||
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
|
||||
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
|
||||
service._publish_monitoring_alert.side_effect = [RuntimeError("alert unavailable"), None]
|
||||
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
with pytest.raises(RuntimeError, match="alert unavailable"):
|
||||
await service.process_pending_step(first.replay_id)
|
||||
assert repo.document["status"] == "failed"
|
||||
assert "monitoring_incident" not in repo.document
|
||||
assert repo.document["monitoring_count"] == 0
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry"))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.last_monitoring.incident.action == "raised"
|
||||
assert result.monitoring_count == 1
|
||||
first_command, retried_command = [call.args[0] for call in create_task.await_args_list]
|
||||
assert first_command == retried_command
|
||||
assert result.last_monitoring.task_id == create_task.return_value.task_id
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("indicator", ["pressure", "outlet_flow", "burst_detection", "quality", "partial_data"])
|
||||
def test_any_single_measurement_anomaly_enters_diagnosis_below_burst_threshold(monkeypatch, indicator):
|
||||
_, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
|
||||
monitoring = incident_monitoring(0, status="attention")
|
||||
for key in ("pressure", "outlet_flow", "burst_detection"):
|
||||
monitoring[key]["hit"] = key == indicator
|
||||
monitoring["weighted"] = {"score": 0.1, "threshold": 0.8,
|
||||
"weights": {"burst_detection": 0.5, "pressure": 0.4, "outlet_flow": 0.1}}
|
||||
if indicator == "quality":
|
||||
monitoring["status"] = "insufficient_data"
|
||||
monitoring["anomaly_signals"] = [{"event_id": str(uuid4()), "rule_id": "quality_bounds",
|
||||
"device_ids": ["Q1"], "detail": "水质测点超出已配置上限"}]
|
||||
monitoring["burst_detection"] = {"status": "not_run", "hit": False}
|
||||
if indicator == "partial_data":
|
||||
monitoring["status"] = "insufficient_data"
|
||||
monitoring["pressure"]["hit"] = True
|
||||
document = {"replay_id": str(uuid4())}
|
||||
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 1}))
|
||||
command = create_task.await_args.args[0]
|
||||
assert command.workflow == "scada_anomaly"
|
||||
assert command.source.event_kind == "scada_anomaly"
|
||||
assert "任一指标异常即启动调查" in command.summary
|
||||
assert "爆管嫌疑门槛 0.80" in command.summary
|
||||
assert monitoring["task_id"] == str(create_task.return_value.task_id)
|
||||
assert monitoring["incident"]["action"] == "raised"
|
||||
|
||||
|
||||
def test_flow_only_incident_does_not_recover_while_absolute_flow_remains_high():
|
||||
from app.services.scada_monitoring_incidents import update_incident
|
||||
document = {"replay_id": str(uuid4())}
|
||||
first = incident_monitoring(0, status="attention", value=100.)
|
||||
first["outlet_flow"] = {"status": "ready", "hit": True, "devices": [
|
||||
{"device_id": "F1", "hit": True, "baseline": 100., "threshold_percent": 10., "value": 150.}
|
||||
]}
|
||||
incident, _ = update_incident(document, first, 1)
|
||||
incident["task_id"] = str(uuid4())
|
||||
for step in (1, 2):
|
||||
normal = incident_monitoring(step, status="normal", value=100.)
|
||||
normal["outlet_flow"] = {"status": "ready", "hit": False, "devices": [
|
||||
{"device_id": "F1", "value": 150., "observed_at": normal["period_end"]}
|
||||
]}
|
||||
_, action = update_incident(document, normal, step + 1)
|
||||
assert action == "ongoing"
|
||||
for step in (3, 4):
|
||||
normal = incident_monitoring(step, status="normal", value=100.)
|
||||
normal["outlet_flow"] = {"status": "ready", "hit": False, "devices": [
|
||||
{"device_id": "F1", "value": 100., "observed_at": normal["period_end"]}
|
||||
]}
|
||||
_, action = update_incident(document, normal, step + 1)
|
||||
assert action == "resolved"
|
||||
|
||||
|
||||
def test_transient_measurement_anomaly_still_enters_diagnosis_at_period_end(monkeypatch):
|
||||
repo, service, query, _, create = setup(monkeypatch)
|
||||
readings = [(START, 20.), (START + timedelta(minutes=5), 5.), (START + timedelta(minutes=10), 5.)]
|
||||
async def raw_page(_conn, _devices, _start, _end, through, _limit, after=None, **_):
|
||||
return [{"time": at, "device_id": "P1", "monitored_value": value, "source_hash": str(index)}
|
||||
for index, (at, value) in enumerate(readings) if at <= through and (after is None or at > after[0])][:1]
|
||||
query.side_effect = raw_page
|
||||
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
|
||||
return {**incident_monitoring(0, status="normal", value=5.),
|
||||
"period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
|
||||
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
|
||||
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
|
||||
async def run():
|
||||
initial = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
result = await service.process_pending_step(initial.replay_id)
|
||||
assert result.active_events == []
|
||||
assert result.last_monitoring.status == "attention"
|
||||
assert result.last_monitoring.anomaly_signals[0]["rule_id"] == "p"
|
||||
assert result.last_monitoring.task_id is not None
|
||||
assert result.last_monitoring.incident.action == "raised"
|
||||
assert "step_anomaly_signals" not in repo.document
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action,status", [("step", "paused"), ("retry", "failed")])
|
||||
def test_warmed_weighted_replay_with_old_calibration_requires_rebuild(monkeypatch, action, status):
|
||||
repo, service, _, _, create = setup(monkeypatch)
|
||||
async def run():
|
||||
initial = await service.create(create, monitoring_interval_seconds=900,
|
||||
monitoring_config=ScadaMonitoringSettings())
|
||||
repo.document.update(warmed=True, status=status,
|
||||
monitoring_calibration={"version": "scada-monitor-calibration@1"})
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action=action))
|
||||
assert error.value.status_code == 409
|
||||
assert "新建" in error.value.detail
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_old_active_percentage_event_does_not_enter_new_period_diagnosis(monkeypatch):
|
||||
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
|
||||
repo, service, _, _, _ = setup(monkeypatch)
|
||||
config = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
|
||||
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "relative_change",
|
||||
"direction": "both", "threshold": 30, "unit": "m"}])
|
||||
monkeypatch.setattr(service, "_create_monitoring_task", AsyncMock())
|
||||
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
|
||||
return {**incident_monitoring(0, status="normal", value=20.),
|
||||
"period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
|
||||
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
|
||||
monkeypatch.setattr("app.services.scada_replay.evaluate_monitoring_period", evaluate)
|
||||
async def run():
|
||||
initial = await service.create(config, monitoring_interval_seconds=900,
|
||||
monitoring_config=ScadaMonitoringSettings())
|
||||
old = {"event_id": str(uuid4()), "rule_id": "p", "device_ids": ["P1"], "status": "active",
|
||||
"occurred_at": START.isoformat(), "last_observed_at": START.isoformat(),
|
||||
"detail": "旧30%事件", "task_id": None}
|
||||
repo.document.update(warmed=True, rule_states={"p": {"event": old, "hits": 1, "recoveries": 0}},
|
||||
monitoring_calibration={"version": CALIBRATION_VERSION, "diagnostic_policy": DIAGNOSTIC_POLICY,
|
||||
"knowledge_cutoff": START.isoformat(), "diagnostic": {"P1": {
|
||||
"phase_origin": START.isoformat(), "interval_seconds": 900, "lower": [10.], "upper": [30.]}}})
|
||||
await service.command(initial.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"))
|
||||
result = await service.process_pending_step(initial.replay_id)
|
||||
assert result.last_monitoring.status == "normal"
|
||||
assert result.last_monitoring.anomaly_signals == []
|
||||
assert repo.document["rule_states"]["p"]["event"] == old
|
||||
asyncio.run(run())
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Real spawned workers share a listener while established streams drain."""
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import uvicorn
|
||||
|
||||
from scripts.server_handoff import HandoffSupervisor
|
||||
|
||||
|
||||
def wait_for(predicate, *, timeout=5.):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(0.02)
|
||||
raise AssertionError("test-owned worker did not reach the expected state")
|
||||
|
||||
|
||||
def next_event(lines, *, after_ns=0):
|
||||
for line in lines:
|
||||
if line.startswith("data: "):
|
||||
event = json.loads(line[len("data: "):])
|
||||
if event.get("time_ns", 0) >= after_ns:
|
||||
return event
|
||||
raise AssertionError("SSE ended before its fixture release marker")
|
||||
|
||||
|
||||
def owned_workers(supervisor):
|
||||
return [worker for worker in [supervisor.current, *supervisor.retiring] if worker is not None]
|
||||
|
||||
|
||||
def close_owned_supervisor(supervisor, directory, original_children):
|
||||
"""Release every fixture stream before waiting; bound cleanup even on failure."""
|
||||
(directory / "release-all").touch()
|
||||
(directory / "allow-startup").touch()
|
||||
workers = owned_workers(supervisor)
|
||||
errors = []
|
||||
|
||||
def close():
|
||||
try:
|
||||
supervisor.close()
|
||||
except BaseException as error:
|
||||
errors.append(error)
|
||||
|
||||
closing = Thread(target=close, daemon=True)
|
||||
closing.start()
|
||||
closing.join(timeout=8)
|
||||
leaked = [process for process in multiprocessing.active_children()
|
||||
if process.pid not in original_children and process.is_alive()]
|
||||
if closing.is_alive() or leaked:
|
||||
# This is emergency teardown of processes created by this test, never
|
||||
# production reload policy or an operation on the developer's service.
|
||||
for process in leaked:
|
||||
process.terminate()
|
||||
for process in leaked:
|
||||
process.join(timeout=3)
|
||||
closing.join(timeout=3)
|
||||
pytest.fail("handoff cleanup left a test-owned worker running")
|
||||
for worker in workers:
|
||||
worker.process.join(timeout=1)
|
||||
assert not worker.process.is_alive()
|
||||
if errors:
|
||||
raise errors[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def handoff(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "fixtures"))
|
||||
monkeypatch.setenv("SCADA_HANDOFF_FIXTURE_DIR", str(tmp_path))
|
||||
(tmp_path / "version").write_text("v1")
|
||||
(tmp_path / "startup-mode").write_text("ready")
|
||||
(tmp_path / "readiness").write_text("true")
|
||||
original_children = {process.pid for process in multiprocessing.active_children()}
|
||||
|
||||
@contextmanager
|
||||
def running(*, startup_timeout=5., readiness_check=None):
|
||||
config = uvicorn.Config(
|
||||
"server_handoff_app:app", host="127.0.0.1", port=0,
|
||||
lifespan="on", log_level="error", access_log=False,
|
||||
timeout_graceful_shutdown=None,
|
||||
)
|
||||
supervisor = HandoffSupervisor(config, startup_timeout=startup_timeout, poll_interval=.02,
|
||||
readiness_check=readiness_check)
|
||||
try:
|
||||
yield supervisor, tmp_path
|
||||
finally:
|
||||
close_owned_supervisor(supervisor, tmp_path, original_children)
|
||||
|
||||
return running
|
||||
|
||||
|
||||
def base_url(supervisor):
|
||||
return f"http://127.0.0.1:{supervisor.socket.getsockname()[1]}"
|
||||
|
||||
|
||||
def health(supervisor, *, timeout=3.):
|
||||
# A fresh TCP connection makes each assertion exercise current acceptance.
|
||||
with httpx.Client(timeout=timeout, trust_env=False) as client:
|
||||
response = client.get(base_url(supervisor) + "/health")
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def test_first_worker_is_spawned_and_ready_before_start_returns(handoff):
|
||||
with handoff() as (supervisor, _):
|
||||
assert supervisor.start() is True
|
||||
worker = supervisor.current
|
||||
assert worker.process._start_method == "spawn"
|
||||
assert worker.process.is_alive()
|
||||
assert health(supervisor) == {"pid": worker.process.pid, "version": "v1"}
|
||||
assert supervisor.retiring == []
|
||||
|
||||
|
||||
def test_reload_keeps_old_sse_until_release_and_new_requests_use_new_worker(handoff):
|
||||
with handoff() as (supervisor, directory):
|
||||
assert supervisor.start() is True
|
||||
old = supervisor.current
|
||||
with httpx.Client(timeout=3., trust_env=False) as client:
|
||||
with client.stream("GET", base_url(supervisor) + "/events") as stream:
|
||||
stream.raise_for_status()
|
||||
lines = stream.iter_lines()
|
||||
assert next_event(lines)["pid"] == old.process.pid
|
||||
(directory / "version").write_text("v2")
|
||||
(directory / "startup-mode").write_text("delay")
|
||||
stop_probe = Event()
|
||||
samples, errors = [], []
|
||||
|
||||
def probe_health():
|
||||
while not stop_probe.is_set():
|
||||
try:
|
||||
samples.append(health(supervisor, timeout=.3))
|
||||
except Exception as error:
|
||||
errors.append(repr(error))
|
||||
stop_probe.wait(.01)
|
||||
|
||||
probing = Thread(target=probe_health, daemon=True)
|
||||
probing.start()
|
||||
try:
|
||||
wait_for(lambda: samples)
|
||||
assert supervisor.reload() is True
|
||||
wait_for(lambda: any(sample["version"] == "v2" for sample in samples))
|
||||
finally:
|
||||
stop_probe.set()
|
||||
probing.join(timeout=2)
|
||||
assert not probing.is_alive()
|
||||
assert errors == [], f"HTTP availability gap during reload: {errors}"
|
||||
assert {sample["version"] for sample in samples} == {"v1", "v2"}
|
||||
|
||||
new = supervisor.current
|
||||
assert new.process.pid != old.process.pid
|
||||
assert old in supervisor.retiring
|
||||
assert old.process.is_alive()
|
||||
for _ in range(3):
|
||||
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
|
||||
after_handoff = time.time_ns()
|
||||
event = next_event(lines, after_ns=after_handoff)
|
||||
assert (event["pid"], event["version"]) == (old.process.pid, "v1")
|
||||
assert not (directory / f"stopped-{old.process.pid}").exists()
|
||||
|
||||
(directory / f"release-{old.process.pid}").touch()
|
||||
assert any(line == "event: complete" for line in lines)
|
||||
old.process.join(timeout=5)
|
||||
assert old.process.exitcode == 0
|
||||
assert (directory / f"stopped-{old.process.pid}").read_text() == "v1"
|
||||
supervisor.reap()
|
||||
assert old not in supervisor.retiring
|
||||
assert health(supervisor) == {"pid": new.process.pid, "version": "v2"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["fail", "wait"])
|
||||
def test_candidate_startup_failure_keeps_previous_worker_serving(handoff, mode):
|
||||
with handoff() as (supervisor, directory):
|
||||
assert supervisor.start() is True
|
||||
old = supervisor.current
|
||||
(directory / "version").write_text("broken-v2")
|
||||
(directory / "startup-mode").write_text(mode)
|
||||
if mode == "wait":
|
||||
supervisor.startup_timeout = .75
|
||||
assert supervisor.reload() is False
|
||||
assert supervisor.current is old
|
||||
assert old.process.is_alive()
|
||||
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
||||
assert supervisor.retiring == []
|
||||
assert len(list(directory.glob("started-*"))) == 2
|
||||
|
||||
|
||||
def test_candidate_process_start_error_keeps_previous_worker_healthy(handoff, monkeypatch):
|
||||
def fail_start():
|
||||
raise OSError("test process spawn rejected")
|
||||
|
||||
with handoff() as (supervisor, _):
|
||||
assert supervisor.start() is True
|
||||
old = supervisor.current
|
||||
candidate = SimpleNamespace(pid=None, start=fail_start)
|
||||
monkeypatch.setattr("scripts.server_handoff.get_subprocess", lambda **_: candidate)
|
||||
assert supervisor.reload() is False
|
||||
assert supervisor.current is old
|
||||
assert supervisor.retiring == []
|
||||
assert old.process.is_alive()
|
||||
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
||||
|
||||
|
||||
def test_readiness_check_rejects_unusable_candidate_before_handoff(handoff):
|
||||
with handoff(readiness_check="server_handoff_app:readiness_check") as (supervisor, directory):
|
||||
assert supervisor.start() is True
|
||||
old = supervisor.current
|
||||
assert (directory / f"checked-{old.process.pid}").read_text() == "true"
|
||||
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
||||
(directory / "version").write_text("not-ready-v2")
|
||||
(directory / "readiness").write_text("false")
|
||||
stop_probe = Event()
|
||||
samples, errors = [], []
|
||||
|
||||
def probe_health():
|
||||
while not stop_probe.is_set():
|
||||
try:
|
||||
samples.append(health(supervisor, timeout=.3))
|
||||
except Exception as error:
|
||||
errors.append(repr(error))
|
||||
stop_probe.wait(.01)
|
||||
|
||||
probing = Thread(target=probe_health, daemon=True)
|
||||
probing.start()
|
||||
try:
|
||||
wait_for(lambda: samples)
|
||||
assert supervisor.reload() is False
|
||||
finally:
|
||||
stop_probe.set()
|
||||
probing.join(timeout=2)
|
||||
assert not probing.is_alive()
|
||||
assert errors == []
|
||||
assert all(sample == {"pid": old.process.pid, "version": "v1"} for sample in samples)
|
||||
rejected = [path for path in directory.glob("checked-*") if path.name != f"checked-{old.process.pid}"]
|
||||
assert len(rejected) == 1
|
||||
assert rejected[0].read_text() == "false"
|
||||
assert supervisor.current is old
|
||||
assert old.process.is_alive()
|
||||
assert supervisor.retiring == []
|
||||
assert health(supervisor) == {"pid": old.process.pid, "version": "v1"}
|
||||
|
||||
(directory / "version").write_text("ready-v3")
|
||||
(directory / "readiness").write_text("true")
|
||||
assert supervisor.reload() is True
|
||||
new = supervisor.current
|
||||
assert new.process.pid != old.process.pid
|
||||
assert (directory / f"checked-{new.process.pid}").read_text() == "true"
|
||||
assert health(supervisor) == {"pid": new.process.pid, "version": "ready-v3"}
|
||||
|
||||
|
||||
def test_close_waits_for_existing_stream_to_finish_without_forcing_worker(handoff):
|
||||
with handoff() as (supervisor, directory):
|
||||
assert supervisor.start() is True
|
||||
worker = supervisor.current
|
||||
with httpx.Client(timeout=3., trust_env=False) as client:
|
||||
with client.stream("GET", base_url(supervisor) + "/events") as stream:
|
||||
lines = stream.iter_lines()
|
||||
assert next_event(lines)["pid"] == worker.process.pid
|
||||
errors = []
|
||||
|
||||
def close():
|
||||
try:
|
||||
supervisor.close()
|
||||
except BaseException as error:
|
||||
errors.append(error)
|
||||
|
||||
closing = Thread(target=close, daemon=True)
|
||||
closing.start()
|
||||
try:
|
||||
after_close = time.time_ns()
|
||||
assert next_event(lines, after_ns=after_close)["pid"] == worker.process.pid
|
||||
assert closing.is_alive()
|
||||
assert worker.process.is_alive()
|
||||
(directory / f"release-{worker.process.pid}").touch()
|
||||
assert any(line == "event: complete" for line in lines)
|
||||
finally:
|
||||
(directory / "release-all").touch()
|
||||
closing.join(timeout=5)
|
||||
assert not closing.is_alive()
|
||||
assert errors == []
|
||||
assert worker.process.exitcode == 0
|
||||
assert (directory / f"stopped-{worker.process.pid}").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("active_alive,edited,expected_reloads", [(False, False, 1), (True, True, 0)])
|
||||
def test_retiring_limit_coalesces_edits_but_never_blocks_crash_recovery(
|
||||
monkeypatch, active_alive, edited, expected_reloads,
|
||||
):
|
||||
class OneWatchIteration:
|
||||
calls = 0
|
||||
|
||||
def wait(self, _timeout):
|
||||
self.calls += 1
|
||||
return self.calls > 1
|
||||
|
||||
def set(self):
|
||||
self.calls = 2
|
||||
|
||||
supervisor = HandoffSupervisor.__new__(HandoffSupervisor)
|
||||
supervisor.current = SimpleNamespace(process=SimpleNamespace(is_alive=lambda: active_alive))
|
||||
supervisor.retiring = [object(), object()]
|
||||
supervisor.should_exit = OneWatchIteration()
|
||||
supervisor.poll_interval = .001
|
||||
supervisor._files = {}
|
||||
calls = []
|
||||
monkeypatch.setattr(supervisor, "start", lambda: True)
|
||||
monkeypatch.setattr(supervisor, "reap", lambda: None)
|
||||
monkeypatch.setattr(supervisor, "_snapshot", lambda: {"edited.py": (1, 1)} if edited else {})
|
||||
monkeypatch.setattr(supervisor, "reload", lambda: calls.append("reload"))
|
||||
monkeypatch.setattr(supervisor, "close", lambda: calls.append("close"))
|
||||
supervisor.run()
|
||||
assert calls.count("reload") == expected_reloads
|
||||
assert calls[-1] == "close"
|
||||
@@ -281,6 +281,30 @@ def test_temporary_project_database_cleans_up_after_failure(monkeypatch) -> None
|
||||
]
|
||||
|
||||
|
||||
def test_temporary_project_database_copies_snapshot_without_template(monkeypatch) -> None:
|
||||
calls: list[tuple[str, ...]] = []
|
||||
|
||||
@contextmanager
|
||||
def temporary_template(project, purpose):
|
||||
calls.append(("schema", project, purpose))
|
||||
yield "isolated_run"
|
||||
|
||||
monkeypatch.setattr(projects, "have_project", lambda name: False)
|
||||
monkeypatch.setattr(projects, "temporary_template_database", temporary_template)
|
||||
monkeypatch.setattr(
|
||||
"app.native.wndb.core.model_replace.copy_network_tables",
|
||||
lambda source, target: calls.append(("model", source, target)),
|
||||
)
|
||||
|
||||
with projects.temporary_project_database("project_a", "burst") as name:
|
||||
assert name == "isolated_run"
|
||||
|
||||
assert calls == [
|
||||
("schema", "project_a", "burst"),
|
||||
("model", "project_a", "isolated_run"),
|
||||
]
|
||||
|
||||
|
||||
def test_temporary_template_database_does_not_clone_a_project(monkeypatch) -> None:
|
||||
calls: list[tuple[str, ...]] = []
|
||||
monkeypatch.setattr(
|
||||
|
||||
Reference in New Issue
Block a user