243 lines
11 KiB
Python
243 lines
11 KiB
Python
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
from copy import deepcopy
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
|
|
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
|
from app.services.scada_replay import ScadaReplayService
|
|
|
|
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
|
|
|
|
|
class Connection:
|
|
@asynccontextmanager
|
|
async def transaction(self):
|
|
yield
|
|
|
|
|
|
class Repository:
|
|
def __init__(self):
|
|
self.project_id = str(uuid4())
|
|
self.connection = Connection()
|
|
self.document = None
|
|
self.batches = {}
|
|
|
|
async def available(self): return True
|
|
@asynccontextmanager
|
|
async def control(self): yield
|
|
@asynccontextmanager
|
|
async def processing(self): yield True
|
|
async def by_request(self, key):
|
|
return deepcopy(self.document) if self.document and self.document["config"]["request_id"] == str(key) else None
|
|
async def active_other(self, key): return False
|
|
async def insert(self, document): self.document = deepcopy(document)
|
|
async def save(self, document): self.document = deepcopy(document)
|
|
async def get(self, _): return deepcopy(self.document)
|
|
async def current(self): return deepcopy(self.document)
|
|
async def pending_batch(self, _):
|
|
return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None)
|
|
async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch)
|
|
async def monitoring_outputs(self, *_args): return []
|
|
|
|
|
|
def setup(monkeypatch):
|
|
repo = Repository()
|
|
context = SimpleNamespace(
|
|
project_id=repo.project_id,
|
|
user_id=uuid4(),
|
|
)
|
|
service = ScadaReplayService(repo, context, Connection(), clock=lambda: START + timedelta(seconds=2))
|
|
monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
|
|
"device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00"
|
|
}]))
|
|
async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_):
|
|
return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}]
|
|
if after is None else [])
|
|
query = AsyncMock(side_effect=raw_page)
|
|
cleaning = AsyncMock()
|
|
monkeypatch.setattr(ScadaRepository, "get_raw_page", query)
|
|
monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning)
|
|
monkeypatch.setattr(service, "_create_investigations", AsyncMock())
|
|
create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
|
|
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}])
|
|
return repo, service, query, cleaning, create
|
|
|
|
|
|
def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
restored = await service.create(create, monitoring_interval_seconds=300)
|
|
assert restored.replay_id == first.replay_id
|
|
assert restored.monitoring_interval_seconds == 900
|
|
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")
|
|
played = await service.command(first.replay_id, command)
|
|
assert played.status == "paused" and played.step_pending and played.revision == 2
|
|
assert (await service.command(first.replay_id, command)).revision == 2
|
|
with pytest.raises(HTTPException) as conflict:
|
|
await service.command(first.replay_id, command.model_copy(update={"request_id": uuid4()}))
|
|
assert conflict.value.status_code == 409
|
|
asyncio.run(run())
|
|
query.assert_not_awaited(); cleaning.assert_not_awaited()
|
|
|
|
|
|
def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document["warmed"] = True
|
|
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
|
result = await service.process_pending_step(first.replay_id)
|
|
assert result.processed_count == 1
|
|
assert result.anomaly_count == 1
|
|
assert repo.batches[1]["stage"] == "detected"
|
|
asyncio.run(run())
|
|
assert query.await_count == 2
|
|
assert cleaning.await_count == 2
|
|
assert cleaning.await_args_list[0].args[1][0]["cleaned_value"] == 20
|
|
|
|
|
|
def test_detection_failure_resumes_cleaned_batch_without_duplicate_reads_or_writes(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
publish = AsyncMock(side_effect=[RuntimeError("Agent storage unavailable"), None, None])
|
|
monkeypatch.setattr(service, "_create_investigations", publish)
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document["warmed"] = True
|
|
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
|
with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
|
|
assert repo.document["status"] == "failed"
|
|
assert repo.batches[1]["stage"] == "cleaned"
|
|
assert repo.document["processed_count"] == 0
|
|
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry"))
|
|
result = await service.process_pending_step(first.replay_id)
|
|
assert result.processed_count == 1 and result.anomaly_count == 1
|
|
asyncio.run(run())
|
|
assert query.await_count == 2
|
|
assert cleaning.await_count == 2
|
|
|
|
|
|
def test_cleaning_failure_never_runs_detection_and_preserves_prepared_result(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
cleaning.side_effect = RuntimeError("Timescale unavailable")
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document["warmed"] = True
|
|
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
|
with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
|
|
assert repo.batches[1]["stage"] == "clean_prepared"
|
|
assert repo.document["processed_count"] == 0
|
|
asyncio.run(run())
|
|
service._create_investigations.assert_not_awaited()
|
|
|
|
|
|
def test_creation_requires_explicit_project_monitoring_interval(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
|
|
async def run():
|
|
with pytest.raises(HTTPException) as missing:
|
|
await service.create(create, monitoring_interval_seconds=None)
|
|
assert missing.value.status_code == 422
|
|
assert repo.document is None
|
|
|
|
asyncio.run(run())
|
|
query.assert_not_awaited()
|
|
cleaning.assert_not_awaited()
|
|
|
|
|
|
def test_step_processes_every_observation_within_one_monitoring_interval(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
rows = [{"time": START + timedelta(minutes=minute), "device_id": "P1",
|
|
"monitored_value": 20., "source_hash": f"h{minute}"} for minute in (0, 5, 10, 15, 20)]
|
|
|
|
async def raw_page(_connection, _devices, _start, _end, through, _limit, after=None, **_):
|
|
return next(([row] for row in rows if row["time"] <= through
|
|
and (after is None or (row["time"], row["device_id"]) > after)), [])
|
|
|
|
query.side_effect = raw_page
|
|
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document["warmed"] = True
|
|
stepping = await service.command(first.replay_id, ScadaReplayCommand(
|
|
request_id=uuid4(), expected_revision=1, action="step"
|
|
))
|
|
assert stepping.status == "paused" and stepping.step_pending
|
|
with pytest.raises(HTTPException):
|
|
await service.command(first.replay_id, ScadaReplayCommand(
|
|
request_id=uuid4(), expected_revision=2, action="step"
|
|
))
|
|
result = await service.process_pending_step(first.replay_id)
|
|
assert result.processed_count == 4
|
|
assert result.virtual_now == START + timedelta(minutes=15)
|
|
assert result.status == "paused"
|
|
assert not result.step_pending
|
|
assert result.processed_until == START + timedelta(minutes=15)
|
|
assert result.monitoring_count == 1
|
|
assert result.last_monitoring.status == "insufficient_data"
|
|
assert "step_target" not in repo.document
|
|
assert all(call.args[4] == START + timedelta(minutes=15) for call in query.await_args_list)
|
|
|
|
asyncio.run(run())
|
|
assert cleaning.await_count == 4
|
|
|
|
|
|
def test_partial_final_step_checks_empty_interval_and_completes_without_fabricated_rows(monkeypatch):
|
|
repo, service, query, cleaning, create = setup(monkeypatch)
|
|
create = create.model_copy(update={"end_time": START + timedelta(minutes=22)})
|
|
query.side_effect = None
|
|
query.return_value = []
|
|
|
|
async def run():
|
|
first = await service.create(create, monitoring_interval_seconds=900)
|
|
repo.document.update(warmed=True, virtual_now=(START + timedelta(minutes=15)).isoformat())
|
|
await service.command(first.replay_id, ScadaReplayCommand(
|
|
request_id=uuid4(), expected_revision=1, action="step"
|
|
))
|
|
result = await service.process_pending_step(first.replay_id)
|
|
assert result.virtual_now == create.end_time
|
|
assert result.processed_until == create.end_time
|
|
assert result.status == "completed" and not result.step_pending
|
|
assert result.processed_count == 0
|
|
|
|
asyncio.run(run())
|
|
assert query.await_args.args[4] == create.end_time
|
|
|
|
|
|
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch):
|
|
repo, service, *_ = setup(monkeypatch)
|
|
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
|
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
|
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
|
monkeypatch.setattr(
|
|
"app.services.operational_tasks.OperationalTaskService",
|
|
lambda *_: SimpleNamespace(create=create_task),
|
|
)
|
|
monitoring = {
|
|
"status": "suspected_burst",
|
|
"period_end": (START + timedelta(minutes=15)).isoformat(),
|
|
"knowledge_cutoff": (START + timedelta(minutes=15)).isoformat(),
|
|
"rule_version": "burst-monitor-v1",
|
|
"pressure": {"hit_device_ids": ["P1", "P2"]},
|
|
"outlet_flow": {"hit_device_ids": ["F1"]},
|
|
"burst_detection": {"target_score": -0.05},
|
|
"task_id": None,
|
|
}
|
|
document = {"replay_id": str(uuid4())}
|
|
|
|
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8}))
|
|
|
|
command = create_task.await_args.args[0]
|
|
assert command.workflow == "burst_report"
|
|
assert command.source.replay_context.batch_sequence == 8
|
|
assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
|
|
assert monitoring["task_id"] == str(create_task.return_value.task_id)
|