feat(scada): add project monitoring replay

2026-11-04 14:27:21 +08:00
parent f25423dec4
commit 26fa5d1909
53 changed files with 5153 additions and 20 deletions
@@ -0,0 +1,242 @@
+import asyncio
+from contextlib import asynccontextmanager
+from copy import deepcopy
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+from uuid import uuid4
+
+import pytest
+from fastapi import HTTPException
+
+from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
+from app.infra.db.postgresql.scada import ScadaInfoRepository
+from app.infra.db.timescaledb.repositories.scada import ScadaRepository
+from app.services.scada_replay import ScadaReplayService
+
+START = datetime(2026, 10, 8, tzinfo=timezone.utc)
+
+
+class Connection:
+    @asynccontextmanager
+    async def transaction(self):
+        yield
+
+
+class Repository:
+    def __init__(self):
+        self.project_id = str(uuid4())
+        self.connection = Connection()
+        self.document = None
+        self.batches = {}
+
+    async def available(self): return True
+    @asynccontextmanager
+    async def control(self): yield
+    @asynccontextmanager
+    async def processing(self): yield True
+    async def by_request(self, key):
+        return deepcopy(self.document) if self.document and self.document["config"]["request_id"] == str(key) else None
+    async def active_other(self, key): return False
+    async def insert(self, document): self.document = deepcopy(document)
+    async def save(self, document): self.document = deepcopy(document)
+    async def get(self, _): return deepcopy(self.document)
+    async def current(self): return deepcopy(self.document)
+    async def pending_batch(self, _):
+        return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None)
+    async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch)
+    async def monitoring_outputs(self, *_args): return []
+
+
+def setup(monkeypatch):
+    repo = Repository()
+    context = SimpleNamespace(
+        project_id=repo.project_id,
+        user_id=uuid4(),
+    )
+    service = ScadaReplayService(repo, context, Connection(), clock=lambda: START + timedelta(seconds=2))
+    monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
+        "device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00"
+    }]))
+    async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_):
+        return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}]
+                if after is None else [])
+    query = AsyncMock(side_effect=raw_page)
+    cleaning = AsyncMock()
+    monkeypatch.setattr(ScadaRepository, "get_raw_page", query)
+    monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning)
+    monkeypatch.setattr(service, "_create_investigations", AsyncMock())
+    create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
+        device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}])
+    return repo, service, query, cleaning, create
+
+
+def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        restored = await service.create(create, monitoring_interval_seconds=300)
+        assert restored.replay_id == first.replay_id
+        assert restored.monitoring_interval_seconds == 900
+        command = ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")
+        played = await service.command(first.replay_id, command)
+        assert played.status == "paused" and played.step_pending and played.revision == 2
+        assert (await service.command(first.replay_id, command)).revision == 2
+        with pytest.raises(HTTPException) as conflict:
+            await service.command(first.replay_id, command.model_copy(update={"request_id": uuid4()}))
+        assert conflict.value.status_code == 409
+    asyncio.run(run())
+    query.assert_not_awaited(); cleaning.assert_not_awaited()
+
+
+def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        repo.document["warmed"] = True
+        await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
+        result = await service.process_pending_step(first.replay_id)
+        assert result.processed_count == 1
+        assert result.anomaly_count == 1
+        assert repo.batches[1]["stage"] == "detected"
+    asyncio.run(run())
+    assert query.await_count == 2
+    assert cleaning.await_count == 2
+    assert cleaning.await_args_list[0].args[1][0]["cleaned_value"] == 20
+
+
+def test_detection_failure_resumes_cleaned_batch_without_duplicate_reads_or_writes(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    publish = AsyncMock(side_effect=[RuntimeError("Agent storage unavailable"), None, None])
+    monkeypatch.setattr(service, "_create_investigations", publish)
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        repo.document["warmed"] = True
+        await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
+        with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
+        assert repo.document["status"] == "failed"
+        assert repo.batches[1]["stage"] == "cleaned"
+        assert repo.document["processed_count"] == 0
+        await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry"))
+        result = await service.process_pending_step(first.replay_id)
+        assert result.processed_count == 1 and result.anomaly_count == 1
+    asyncio.run(run())
+    assert query.await_count == 2
+    assert cleaning.await_count == 2
+
+
+def test_cleaning_failure_never_runs_detection_and_preserves_prepared_result(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    cleaning.side_effect = RuntimeError("Timescale unavailable")
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        repo.document["warmed"] = True
+        await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
+        with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
+        assert repo.batches[1]["stage"] == "clean_prepared"
+        assert repo.document["processed_count"] == 0
+    asyncio.run(run())
+    service._create_investigations.assert_not_awaited()
+
+
+def test_creation_requires_explicit_project_monitoring_interval(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+
+    async def run():
+        with pytest.raises(HTTPException) as missing:
+            await service.create(create, monitoring_interval_seconds=None)
+        assert missing.value.status_code == 422
+        assert repo.document is None
+
+    asyncio.run(run())
+    query.assert_not_awaited()
+    cleaning.assert_not_awaited()
+
+
+def test_step_processes_every_observation_within_one_monitoring_interval(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    rows = [{"time": START + timedelta(minutes=minute), "device_id": "P1",
+             "monitored_value": 20., "source_hash": f"h{minute}"} for minute in (0, 5, 10, 15, 20)]
+
+    async def raw_page(_connection, _devices, _start, _end, through, _limit, after=None, **_):
+        return next(([row] for row in rows if row["time"] <= through
+                     and (after is None or (row["time"], row["device_id"]) > after)), [])
+
+    query.side_effect = raw_page
+
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        repo.document["warmed"] = True
+        stepping = await service.command(first.replay_id, ScadaReplayCommand(
+            request_id=uuid4(), expected_revision=1, action="step"
+        ))
+        assert stepping.status == "paused" and stepping.step_pending
+        with pytest.raises(HTTPException):
+            await service.command(first.replay_id, ScadaReplayCommand(
+                request_id=uuid4(), expected_revision=2, action="step"
+            ))
+        result = await service.process_pending_step(first.replay_id)
+        assert result.processed_count == 4
+        assert result.virtual_now == START + timedelta(minutes=15)
+        assert result.status == "paused"
+        assert not result.step_pending
+        assert result.processed_until == START + timedelta(minutes=15)
+        assert result.monitoring_count == 1
+        assert result.last_monitoring.status == "insufficient_data"
+        assert "step_target" not in repo.document
+        assert all(call.args[4] == START + timedelta(minutes=15) for call in query.await_args_list)
+
+    asyncio.run(run())
+    assert cleaning.await_count == 4
+
+
+def test_partial_final_step_checks_empty_interval_and_completes_without_fabricated_rows(monkeypatch):
+    repo, service, query, cleaning, create = setup(monkeypatch)
+    create = create.model_copy(update={"end_time": START + timedelta(minutes=22)})
+    query.side_effect = None
+    query.return_value = []
+
+    async def run():
+        first = await service.create(create, monitoring_interval_seconds=900)
+        repo.document.update(warmed=True, virtual_now=(START + timedelta(minutes=15)).isoformat())
+        await service.command(first.replay_id, ScadaReplayCommand(
+            request_id=uuid4(), expected_revision=1, action="step"
+        ))
+        result = await service.process_pending_step(first.replay_id)
+        assert result.virtual_now == create.end_time
+        assert result.processed_until == create.end_time
+        assert result.status == "completed" and not result.step_pending
+        assert result.processed_count == 0
+
+    asyncio.run(run())
+    assert query.await_args.args[4] == create.end_time
+
+
+def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch):
+    repo, service, *_ = setup(monkeypatch)
+    create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
+    monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
+    monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
+    monkeypatch.setattr(
+        "app.services.operational_tasks.OperationalTaskService",
+        lambda *_: SimpleNamespace(create=create_task),
+    )
+    monitoring = {
+        "status": "suspected_burst",
+        "period_end": (START + timedelta(minutes=15)).isoformat(),
+        "knowledge_cutoff": (START + timedelta(minutes=15)).isoformat(),
+        "rule_version": "burst-monitor-v1",
+        "pressure": {"hit_device_ids": ["P1", "P2"]},
+        "outlet_flow": {"hit_device_ids": ["F1"]},
+        "burst_detection": {"target_score": -0.05},
+        "task_id": None,
+    }
+    document = {"replay_id": str(uuid4())}
+
+    asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8}))
+
+    command = create_task.await_args.args[0]
+    assert command.workflow == "burst_report"
+    assert command.source.replay_context.batch_sequence == 8
+    assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
+    assert monitoring["task_id"] == str(create_task.return_value.task_id)