OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scada): add calibrated monitoring and incident recovery

Browse Source
This commit is contained in:
jiang
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_mcp_query.py
+203
View File
@@ -1,5 +1,6 @@
import asyncio
import base64
import json
import time
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
@@ -72,6 +73,57 @@ def test_calculation_limit_is_advertised_and_enforced_before_execution(runtime):
assert calls == []
def test_case3_location_mock_skips_queue_and_preserves_evidence_and_receipt(runtime, monkeypatch):
from app.mcp import calculation_jobs
app, client, _, _ = runtime
original_scope = app.scope_factory
@asynccontextmanager
async def demo_scope(identity, project):
async with original_scope(identity, project) as scope:
scope.project.project_code = "tjwater2_v2"
yield scope
app.scope_factory = demo_scope
monkeypatch.setattr(application, "authorize_query", AsyncMock())
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
monkeypatch.setattr(application, "verify_task", AsyncMock())
submit = AsyncMock(side_effect=AssertionError("demo entered calculation queue"))
monkeypatch.setattr(calculation_jobs, "submit_calculation", submit)
app.executor = AsyncMock(return_value=QueryResult(
source="run_burst_location", source_ref="demo", queried_at=datetime.now(timezone.utc),
semantics="analysis", origin="synthetic", data={"result": {"located_pipe": "515870",
"top_candidates": [{"pipe_id": "515870", "similarity": 0.9708, "similarity_percent": 97.08}]}},
))
saved = []
class TaskService:
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
async def record_data_snapshot(self, task_id, snapshot):
saved.append(snapshot)
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
value = binding(task=True)
query = {"name": "run_burst_location", "arguments": {
"context_handle": value.context_handle,
"scada_burst_start": "2026-07-16T07:45:00+08:00",
"scada_burst_end": "2026-07-16T08:00:00+08:00",
}}
first = request(client, "tools/call", query).json()["result"]["structuredContent"]
second = request(client, "tools/call", query).json()["result"]["structuredContent"]
assert first == second
assert first["origin"] == "synthetic"
assert first["data"]["result"]["top_candidates"][0]["similarity_percent"] == 97.08
assert "mock" not in first["data"]["result"]
assert first["evidence_id"] == saved[0].source_ref
assert len(saved) == app.executor.await_count == 1
assert app.executor.await_args.args[3].burst_leakage is None
submit.assert_not_awaited()
def test_queue_submission_errors_preserve_mcp_error_mapping(runtime, monkeypatch):
from app.mcp import calculation_jobs
_, client, _, _ = runtime
@@ -321,6 +373,45 @@ def test_replay_binding_denies_future_and_allows_device_metadata(monkeypatch):
asyncio.run(query_service.authorize_query(SimpleNamespace(project=object()), bound, TOOLS["list_scada_devices"], SimpleNamespace()))
def test_model_response_displays_utc8_without_changing_query_instants_or_source(runtime):
app, client, _, _ = runtime
at = datetime(2026, 12, 31, 16, tzinfo=timezone.utc)
original = QueryResult(source="query_scada_readings", source_ref="saved:utc", queried_at=at,
semantics="observed", data=[{"device_id": "P1", "time": "2026-12-31T16:00:00.123456789Z", "value": 20}],
window={"start": at - timedelta(minutes=15), "end": at})
snapshot = original.model_dump(mode="json")
app.executor = AsyncMock(return_value=original)
result = request(client, "tools/call", {"name": "query_scada_readings", "arguments": {
"context_handle": binding().context_handle, "device_ids": ["P1"],
"start": "2026-12-31T15:45:00Z", "end": "2026-12-31T16:00:00Z",
}}).json()["result"]
output = result["structuredContent"]
assert output["data"][0]["time"] == "2027-01-01T00:00:00.123456789+08:00"
assert output["display_time_zone"] == "Asia/Shanghai"
assert output["window"]["end"] == "2027-01-01T00:00:00+08:00"
assert json.loads(result["content"][0]["text"]) == output
assert app.executor.await_args.args[3].end == at
assert original.model_dump(mode="json") == snapshot
@pytest.mark.parametrize("tool_name", ["get_element_properties", "list_elements", "query_nearby_elements",
"get_network_options", "get_node_connections", "get_element_gis_properties",
"query_network_extent", "list_regions", "get_region", "get_region_nodes",
"analyze_valve_isolation", "generate_node_coverage"])
def test_replay_allows_asset_and_topology_analysis_without_live_telemetry(monkeypatch, tool_name):
bound = binding(task=True)
bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00Z", "batch_sequence": 1}
permissions = PERMISSIONS | {"simulation.run"}
bound.permission_cap = sorted(permissions)
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: permissions)
scope = SimpleNamespace(project=SimpleNamespace(project_role="member"))
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
bound.permission_cap = []
with pytest.raises(HTTPException) as caught:
asyncio.run(query_service.authorize_query(scope, bound, TOOLS[tool_name], SimpleNamespace()))
assert caught.value.status_code == 403
@pytest.mark.parametrize("seconds", [30, 120, 240])
def test_all_tools_advertise_the_effective_host_request_budget(runtime, seconds):
app, client, _, _ = runtime
@@ -600,6 +691,89 @@ def test_large_node_coverage_retains_publishable_hint_and_saves_complete_geometr
assert saved[0].data["data"]["result"]["map_payload"] == geometry
@pytest.mark.parametrize("task", [False, True])
@pytest.mark.parametrize("wide_ids", [False, True])
def test_scada_catalog_keeps_all_devices_or_full_evidence_within_budget(runtime, monkeypatch, task, wide_ids):
app, client, _, _ = runtime
value = binding(task=task)
saved = []
monkeypatch.setattr(contexts, "verify_task", AsyncMock())
monkeypatch.setattr(application, "verify_task", AsyncMock())
class TaskService:
repo = SimpleNamespace(lock=lambda key: asyncio.sleep(0))
async def record_data_snapshot(self, task_id, snapshot):
saved.append(snapshot)
return SimpleNamespace(evidence_id=snapshot.source_ref), snapshot
monkeypatch.setattr(application, "task_service", lambda scope: TaskService())
devices = [{
"device_id": ("d" * 150 if wide_ids else "") + str(100000 + index),
"device_type": "pressure", "measurement_unit": "m",
"node_id": str(index), "link_id": None, "longitude": 121.5, "latitude": 31.0,
"transmission_mode": "non_realtime", "transmission_frequency": "0:15:00",
"reliability": 1, "api_query_id": str(index), "x": 1000, "y": -20000,
} for index in range(100)]
app.executor = AsyncMock(return_value=QueryResult(
source="list_scada_devices", source_ref="mcp:catalog", queried_at=datetime.now(timezone.utc),
semantics="asset", data=devices,
page={"limit": 100, "offset": 0, "total": None, "has_more": True},
))
response = request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
"context_handle": value.context_handle,
}}).json()["result"]
assert app.executor.await_args.args[3].limit == 100
if wide_ids and not task:
assert response["isError"] is True
assert response["structuredContent"]["error"]["code"] == "RESULT_TOO_LARGE"
assert not saved
return
assert not response.get("isError"), response
output = response["structuredContent"]
if wide_ids:
assert output["data"]["resource_uri"].endswith(output["evidence_id"])
else:
assert [row["device_id"] for row in output["data"]] == [row["device_id"] for row in devices]
assert all(row["device_type"] == "pressure" and row["measurement_unit"] == "m" for row in output["data"])
assert "get_scada_device" in " ".join(output["data_gaps"])
assert output["page"] == {"limit": 100, "offset": 0, "total": None, "has_more": True}
assert output["truncated"] is True
assert len(json.dumps(output, ensure_ascii=False).encode()) <= 12000
if task:
assert saved[0].data["data"] == devices
assert output["evidence_id"] == saved[0].source_ref
assert not saved[0].truncated
else:
assert not saved and output["evidence_id"] is None
def test_scada_catalog_default_100_paginates_without_changing_other_queries(runtime, monkeypatch):
app, client, _, _ = runtime
value = binding()
devices = [{"device_id": str(index), "device_type": "pressure", "measurement_unit": "m"}
for index in range(101)]
async def read_devices(connection, *, device_id, limit, offset):
return devices[offset:offset + limit]
reader = AsyncMock(side_effect=read_devices)
monkeypatch.setattr(query_service.reads, "scada_devices", reader)
app.executor = query_service.execute_query
schema = next(tool["inputSchema"] for tool in request(client).json()["result"]["tools"]
if tool["name"] == "list_scada_devices")
assert schema["properties"]["limit"]["default"] == 100
assert schema["properties"]["limit"]["maximum"] == 500
pages = [request(client, "tools/call", {"name": "list_scada_devices", "arguments": {
"context_handle": value.context_handle, "offset": offset,
}}).json()["result"]["structuredContent"] for offset in [0, 100]]
assert [row for page in pages for row in page["data"]] == devices
assert pages[0]["page"]["has_more"] is True and pages[1]["page"]["has_more"] is False
assert [call.kwargs["limit"] for call in reader.await_args_list] == [101, 101]
assert [call.kwargs["offset"] for call in reader.await_args_list] == [0, 100]
assert TOOLS["list_elements"].model(context_handle=value.context_handle, element_type="pipe").limit == 20
def test_business_skill_is_delivered_over_mcp_without_recording_event_evidence(runtime, monkeypatch):
app, client, _, _ = runtime
value = binding(task=True)
@@ -653,6 +827,35 @@ def test_old_task_run_and_reader_parent_are_rejected(monkeypatch):
asyncio.run(contexts.verify_task(scope, value))
@pytest.mark.parametrize("purpose", ["task_execution", "task_collaboration"])
@pytest.mark.parametrize("replay", [False, True])
def test_work_order_task_excludes_location_but_scada_replay_keeps_it(monkeypatch, purpose, replay):
monkeypatch.setattr(contexts, "permissions_for_context", lambda _: PERMISSIONS)
value = binding(task=True)
value.purpose = purpose
replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-07-16T00:00:00Z", "batch_sequence": 3} if replay else None
class Tasks:
async def get_task(self, task_id):
return {"task_id": task_id, "workflow": "burst_report", "source": {"replay_context": replay_context}}
async def get_active_run(self, task_id):
return {"run_id": str(value.run_id), "execution_request_id": str(uuid4()),
"status": "running", "agent_session_id": value.native_session_id,
"lease_expires_at": datetime.now(timezone.utc) + timedelta(minutes=1)}
monkeypatch.setattr(contexts, "task_repository", lambda _: Tasks())
scope = SimpleNamespace(project=object(), connection=Connection())
asyncio.run(contexts.verify_task(scope, value))
assert ("run_burst_location" in value.tool_cap) is replay
assert "analyze_valve_isolation" in value.tool_cap
assert "analyze_valve_closure" in value.tool_cap
if not replay:
with pytest.raises(HTTPException) as caught:
asyncio.run(query_service.authorize_query(scope, value, TOOLS["run_burst_location"], SimpleNamespace()))
assert caught.value.status_code == 403
def test_jwt_signature_issuer_audience_lifetime_and_key_cache():
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
pem = key.private_bytes(