refactor(db)!: clean up business SQL access
- make realtime replacement and analysis result writes transactional\n- consolidate SCADA repositories and remove process-global project state\n- validate SCADA batches and use indexed GIS-backed business queries\n\nBREAKING CHANGE: remove the public analysis result writer and the pipeline-health network_name query parameter.
This commit is contained in:
@@ -7,6 +7,18 @@ from uuid import uuid4
|
||||
import pytest
|
||||
|
||||
|
||||
def _empty_scada_mappings(simulation):
|
||||
return simulation.ScadaElementMappings(
|
||||
reservoirs={},
|
||||
tanks={},
|
||||
fixed_pumps={},
|
||||
variable_pumps={},
|
||||
pressure={},
|
||||
demand={},
|
||||
quality={},
|
||||
)
|
||||
|
||||
|
||||
def test_run_simulation_exposes_explicit_valve_control():
|
||||
from app.services import simulation
|
||||
|
||||
@@ -174,6 +186,7 @@ def test_extended_simulation_stores_results_by_run_id(monkeypatch):
|
||||
modify_total_duration=900,
|
||||
scheme_type="burst_analysis",
|
||||
scheme_name="case",
|
||||
scada_mappings=_empty_scada_mappings(simulation),
|
||||
)
|
||||
|
||||
args, kwargs = storage_calls[0]
|
||||
@@ -250,6 +263,7 @@ def test_extended_simulation_marks_run_failed_when_result_storage_fails(monkeypa
|
||||
modify_total_duration=900,
|
||||
scheme_type="burst_analysis",
|
||||
scheme_name="case",
|
||||
scada_mappings=_empty_scada_mappings(simulation),
|
||||
)
|
||||
|
||||
assert [call[1]["status"] for call in lifecycle_calls] == ["failed"]
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import asyncio
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
|
||||
from app.infra.db.postgresql import scada
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
|
||||
|
||||
@@ -63,3 +67,24 @@ def test_get_scadas_normalizes_id_and_type():
|
||||
assert "node_id" in conn.cursor_instance.query
|
||||
assert "link_id" in conn.cursor_instance.query
|
||||
assert "FROM gis.scada_devices" in conn.cursor_instance.query
|
||||
|
||||
|
||||
def test_realtime_element_mappings_are_project_local_and_immutable(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
scada,
|
||||
"read_all",
|
||||
lambda *_args: [
|
||||
{
|
||||
"device_type": " PRESSURE ",
|
||||
"element_id": " J1 ",
|
||||
"api_query_id": " sensor-1 ",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
mappings = scada.load_realtime_element_mappings("project-a")
|
||||
|
||||
assert mappings.pressure == {"J1": "sensor-1"}
|
||||
assert isinstance(mappings.pressure, MappingProxyType)
|
||||
with pytest.raises(TypeError):
|
||||
mappings.pressure["J2"] = "sensor-2"
|
||||
|
||||
@@ -94,13 +94,13 @@ def test_realtime_node_and_link_replacement_share_outer_transaction(monkeypatch)
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
RealtimeRepository,
|
||||
"insert_nodes_batch_sync",
|
||||
lambda _conn, _data: calls.append("nodes"),
|
||||
"_copy_nodes_sync",
|
||||
lambda _cur, _data, _time: calls.append("nodes"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
RealtimeRepository,
|
||||
"insert_links_batch_sync",
|
||||
lambda _conn, _data: calls.append("links"),
|
||||
"_copy_links_sync",
|
||||
lambda _cur, _data, _time: calls.append("links"),
|
||||
)
|
||||
|
||||
RealtimeRepository.store_realtime_simulation_result_sync(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
@@ -10,6 +11,12 @@ from app.api.v1.endpoints.timeseries import composite as composite_endpoint
|
||||
from app.infra.db.timescaledb import composite_queries
|
||||
|
||||
|
||||
class _FakeTimescaleConnection:
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
yield
|
||||
|
||||
|
||||
def test_clean_scada_uses_current_project_metadata(monkeypatch):
|
||||
"""Fengyang data must not be classified with the global tjwater metadata."""
|
||||
|
||||
@@ -51,7 +58,7 @@ def test_clean_scada_uses_current_project_metadata(monkeypatch):
|
||||
|
||||
result = asyncio.run(
|
||||
composite_queries.CompositeQueries.clean_scada_data(
|
||||
object(),
|
||||
_FakeTimescaleConnection(),
|
||||
object(),
|
||||
["fengyang-pressure-1"],
|
||||
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||
@@ -79,8 +86,8 @@ def test_clean_scada_rejects_devices_missing_from_project_metadata(monkeypatch):
|
||||
with pytest.raises(ValueError, match="缺少元数据"):
|
||||
asyncio.run(
|
||||
composite_queries.CompositeQueries.clean_scada_data(
|
||||
object(),
|
||||
object(),
|
||||
_FakeTimescaleConnection(),
|
||||
_FakeTimescaleConnection(),
|
||||
["fengyang-pressure-1"],
|
||||
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||
datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||
@@ -126,7 +133,7 @@ def test_clean_scada_rejects_zero_database_updates(monkeypatch):
|
||||
with pytest.raises(ValueError, match="未产生任何数据库更新"):
|
||||
asyncio.run(
|
||||
composite_queries.CompositeQueries.clean_scada_data(
|
||||
object(),
|
||||
_FakeTimescaleConnection(),
|
||||
object(),
|
||||
["fengyang-pressure-1"],
|
||||
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||
@@ -170,7 +177,7 @@ def test_clean_scada_propagates_write_failures(monkeypatch):
|
||||
with pytest.raises(RuntimeError, match="database write failed"):
|
||||
asyncio.run(
|
||||
composite_queries.CompositeQueries.clean_scada_data(
|
||||
object(),
|
||||
_FakeTimescaleConnection(),
|
||||
object(),
|
||||
["fengyang-pressure-1"],
|
||||
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||
|
||||
@@ -31,7 +31,7 @@ def test_repeated_analysis_names_create_distinct_run_ids(monkeypatch) -> None:
|
||||
assert first != second
|
||||
assert cursor.execute.call_count == 2
|
||||
assert all(
|
||||
"insert into analysis.runs" in call.args[0]
|
||||
"insert into analysis.runs" in call.args[0].lower()
|
||||
for call in cursor.execute.call_args_list
|
||||
)
|
||||
|
||||
@@ -57,6 +57,43 @@ def test_update_analysis_run_targets_execution_id(monkeypatch) -> None:
|
||||
)
|
||||
|
||||
statement, params = cursor.execute.call_args.args
|
||||
assert "where run_id = %s" in statement
|
||||
assert "where run_id = %s" in statement.lower()
|
||||
assert params[-1] == run_id
|
||||
assert params[1] == "completed"
|
||||
|
||||
|
||||
def test_run_and_business_result_share_one_transaction(monkeypatch) -> None:
|
||||
connection = object()
|
||||
context = MagicMock()
|
||||
context.__enter__.return_value = connection
|
||||
monkeypatch.setattr(
|
||||
scheme_management,
|
||||
"project_transaction",
|
||||
lambda _name: context,
|
||||
)
|
||||
created_ids = []
|
||||
result_ids = []
|
||||
monkeypatch.setattr(
|
||||
scheme_management.AnalysisRepository,
|
||||
"create_run_sync",
|
||||
lambda conn, **kwargs: created_ids.append((conn, kwargs["run_id"])),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
scheme_management.AnalysisRepository,
|
||||
"insert_result_sync",
|
||||
lambda conn, run_id, **_kwargs: result_ids.append((conn, run_id)),
|
||||
)
|
||||
|
||||
run_id = scheme_management.store_analysis_run_with_result(
|
||||
name="tjwater_v2",
|
||||
scheme_name="leak-run",
|
||||
scheme_type="dma_leak_identification",
|
||||
username="alice",
|
||||
scheme_start_time="2026-08-24T00:00:00Z",
|
||||
scheme_detail={},
|
||||
result_type="leakage_identification",
|
||||
result_payload={"rows": []},
|
||||
)
|
||||
|
||||
assert created_ids == [(connection, run_id)]
|
||||
assert result_ids == [(connection, run_id)]
|
||||
|
||||
@@ -97,6 +97,31 @@ def test_update_validates_nodes_before_write(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def test_list_sensor_placements_batches_node_lookup(monkeypatch):
|
||||
first = _run()
|
||||
second = {**_run(), "run_id": uuid4(), "sensor_locations": ["J2", "J3"]}
|
||||
lookup_calls = []
|
||||
monkeypatch.setattr(
|
||||
sensor_placement.sensor_placement_repository,
|
||||
"get_all_sensor_placements",
|
||||
lambda _network: [first, second],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sensor_placement.sensor_placement_repository,
|
||||
"get_sensor_placement_nodes",
|
||||
lambda _network, node_ids: lookup_calls.append(node_ids)
|
||||
or [
|
||||
{**_point(), "node_id": node_id}
|
||||
for node_id in node_ids
|
||||
],
|
||||
)
|
||||
|
||||
runs = sensor_placement.list_sensor_placement_runs("tjwater_v2")
|
||||
|
||||
assert lookup_calls == [["J1", "J2", "J3"]]
|
||||
assert [len(run["sensor_points"]) for run in runs] == [2, 2]
|
||||
|
||||
|
||||
def test_sensor_nodes_query_uses_new_network_and_gis_schemas(monkeypatch):
|
||||
cursor = _mock_project_cursor(monkeypatch)
|
||||
cursor.fetchall.return_value = []
|
||||
@@ -108,6 +133,9 @@ def test_sensor_nodes_query_uses_new_network_and_gis_schemas(monkeypatch):
|
||||
assert "network.links" in query
|
||||
assert "gis.node_geometries" in query
|
||||
assert "ST_Transform(g.geom, 3857)" in query
|
||||
assert "l.start_node_id = ANY(%s)" in query
|
||||
assert "l.end_node_id = ANY(%s)" in query
|
||||
assert "CROSS JOIN LATERAL" not in query
|
||||
assert cursor.execute.call_args.args[1] == (["J1"], ["J1"], ["J1"])
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from app.infra.db.postgresql import scada_assets
|
||||
from app.infra.db.postgresql import scada
|
||||
from app.native.wndb.core.database import ChangeSet, sql_literal
|
||||
from app.native.wndb.gis import coordinates
|
||||
from app.native.wndb.model import controls, junctions, patterns
|
||||
@@ -48,9 +48,9 @@ def test_get_all_scada_info_reads_materialized_view(monkeypatch) -> None:
|
||||
statements.append(statement)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(scada_assets, "read_all", fake_read_all)
|
||||
monkeypatch.setattr(scada, "read_all", fake_read_all)
|
||||
|
||||
assert scada_assets.get_all_scada_info("project_a") == []
|
||||
assert scada.get_all_scada_info("project_a") == []
|
||||
assert "FROM gis.scada_devices" in statements[0]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user