refactor(db)!: clean up business SQL access

- make realtime replacement and analysis result writes transactional\n- consolidate SCADA repositories and remove process-global project state\n- validate SCADA batches and use indexed GIS-backed business queries\n\nBREAKING CHANGE: remove the public analysis result writer and the pipeline-health network_name query parameter.
This commit is contained in:
2026-08-28 11:37:36 +08:00
parent b74799a39d
commit 9b095c7439
34 changed files with 859 additions and 921 deletions
+14
View File
@@ -7,6 +7,18 @@ from uuid import uuid4
import pytest
def _empty_scada_mappings(simulation):
return simulation.ScadaElementMappings(
reservoirs={},
tanks={},
fixed_pumps={},
variable_pumps={},
pressure={},
demand={},
quality={},
)
def test_run_simulation_exposes_explicit_valve_control():
from app.services import simulation
@@ -174,6 +186,7 @@ def test_extended_simulation_stores_results_by_run_id(monkeypatch):
modify_total_duration=900,
scheme_type="burst_analysis",
scheme_name="case",
scada_mappings=_empty_scada_mappings(simulation),
)
args, kwargs = storage_calls[0]
@@ -250,6 +263,7 @@ def test_extended_simulation_marks_run_failed_when_result_storage_fails(monkeypa
modify_total_duration=900,
scheme_type="burst_analysis",
scheme_name="case",
scada_mappings=_empty_scada_mappings(simulation),
)
assert [call[1]["status"] for call in lifecycle_calls] == ["failed"]
@@ -1,5 +1,9 @@
import asyncio
from types import MappingProxyType
import pytest
from app.infra.db.postgresql import scada
from app.infra.db.postgresql.scada import ScadaInfoRepository
@@ -63,3 +67,24 @@ def test_get_scadas_normalizes_id_and_type():
assert "node_id" in conn.cursor_instance.query
assert "link_id" in conn.cursor_instance.query
assert "FROM gis.scada_devices" in conn.cursor_instance.query
def test_realtime_element_mappings_are_project_local_and_immutable(monkeypatch):
monkeypatch.setattr(
scada,
"read_all",
lambda *_args: [
{
"device_type": " PRESSURE ",
"element_id": " J1 ",
"api_query_id": " sensor-1 ",
}
],
)
mappings = scada.load_realtime_element_mappings("project-a")
assert mappings.pressure == {"J1": "sensor-1"}
assert isinstance(mappings.pressure, MappingProxyType)
with pytest.raises(TypeError):
mappings.pressure["J2"] = "sensor-2"
+4 -4
View File
@@ -94,13 +94,13 @@ def test_realtime_node_and_link_replacement_share_outer_transaction(monkeypatch)
calls: list[str] = []
monkeypatch.setattr(
RealtimeRepository,
"insert_nodes_batch_sync",
lambda _conn, _data: calls.append("nodes"),
"_copy_nodes_sync",
lambda _cur, _data, _time: calls.append("nodes"),
)
monkeypatch.setattr(
RealtimeRepository,
"insert_links_batch_sync",
lambda _conn, _data: calls.append("links"),
"_copy_links_sync",
lambda _cur, _data, _time: calls.append("links"),
)
RealtimeRepository.store_realtime_simulation_result_sync(
+12 -5
View File
@@ -1,4 +1,5 @@
import asyncio
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from unittest.mock import AsyncMock
@@ -10,6 +11,12 @@ from app.api.v1.endpoints.timeseries import composite as composite_endpoint
from app.infra.db.timescaledb import composite_queries
class _FakeTimescaleConnection:
@asynccontextmanager
async def transaction(self):
yield
def test_clean_scada_uses_current_project_metadata(monkeypatch):
"""Fengyang data must not be classified with the global tjwater metadata."""
@@ -51,7 +58,7 @@ def test_clean_scada_uses_current_project_metadata(monkeypatch):
result = asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
_FakeTimescaleConnection(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
@@ -79,8 +86,8 @@ def test_clean_scada_rejects_devices_missing_from_project_metadata(monkeypatch):
with pytest.raises(ValueError, match="缺少元数据"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
object(),
_FakeTimescaleConnection(),
_FakeTimescaleConnection(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
@@ -126,7 +133,7 @@ def test_clean_scada_rejects_zero_database_updates(monkeypatch):
with pytest.raises(ValueError, match="未产生任何数据库更新"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
_FakeTimescaleConnection(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
@@ -170,7 +177,7 @@ def test_clean_scada_propagates_write_failures(monkeypatch):
with pytest.raises(RuntimeError, match="database write failed"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
_FakeTimescaleConnection(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
+39 -2
View File
@@ -31,7 +31,7 @@ def test_repeated_analysis_names_create_distinct_run_ids(monkeypatch) -> None:
assert first != second
assert cursor.execute.call_count == 2
assert all(
"insert into analysis.runs" in call.args[0]
"insert into analysis.runs" in call.args[0].lower()
for call in cursor.execute.call_args_list
)
@@ -57,6 +57,43 @@ def test_update_analysis_run_targets_execution_id(monkeypatch) -> None:
)
statement, params = cursor.execute.call_args.args
assert "where run_id = %s" in statement
assert "where run_id = %s" in statement.lower()
assert params[-1] == run_id
assert params[1] == "completed"
def test_run_and_business_result_share_one_transaction(monkeypatch) -> None:
connection = object()
context = MagicMock()
context.__enter__.return_value = connection
monkeypatch.setattr(
scheme_management,
"project_transaction",
lambda _name: context,
)
created_ids = []
result_ids = []
monkeypatch.setattr(
scheme_management.AnalysisRepository,
"create_run_sync",
lambda conn, **kwargs: created_ids.append((conn, kwargs["run_id"])),
)
monkeypatch.setattr(
scheme_management.AnalysisRepository,
"insert_result_sync",
lambda conn, run_id, **_kwargs: result_ids.append((conn, run_id)),
)
run_id = scheme_management.store_analysis_run_with_result(
name="tjwater_v2",
scheme_name="leak-run",
scheme_type="dma_leak_identification",
username="alice",
scheme_start_time="2026-08-24T00:00:00Z",
scheme_detail={},
result_type="leakage_identification",
result_payload={"rows": []},
)
assert created_ids == [(connection, run_id)]
assert result_ids == [(connection, run_id)]
@@ -97,6 +97,31 @@ def test_update_validates_nodes_before_write(monkeypatch):
)
def test_list_sensor_placements_batches_node_lookup(monkeypatch):
first = _run()
second = {**_run(), "run_id": uuid4(), "sensor_locations": ["J2", "J3"]}
lookup_calls = []
monkeypatch.setattr(
sensor_placement.sensor_placement_repository,
"get_all_sensor_placements",
lambda _network: [first, second],
)
monkeypatch.setattr(
sensor_placement.sensor_placement_repository,
"get_sensor_placement_nodes",
lambda _network, node_ids: lookup_calls.append(node_ids)
or [
{**_point(), "node_id": node_id}
for node_id in node_ids
],
)
runs = sensor_placement.list_sensor_placement_runs("tjwater_v2")
assert lookup_calls == [["J1", "J2", "J3"]]
assert [len(run["sensor_points"]) for run in runs] == [2, 2]
def test_sensor_nodes_query_uses_new_network_and_gis_schemas(monkeypatch):
cursor = _mock_project_cursor(monkeypatch)
cursor.fetchall.return_value = []
@@ -108,6 +133,9 @@ def test_sensor_nodes_query_uses_new_network_and_gis_schemas(monkeypatch):
assert "network.links" in query
assert "gis.node_geometries" in query
assert "ST_Transform(g.geom, 3857)" in query
assert "l.start_node_id = ANY(%s)" in query
assert "l.end_node_id = ANY(%s)" in query
assert "CROSS JOIN LATERAL" not in query
assert cursor.execute.call_args.args[1] == (["J1"], ["J1"], ["J1"])
+3 -3
View File
@@ -1,7 +1,7 @@
import ast
from pathlib import Path
from app.infra.db.postgresql import scada_assets
from app.infra.db.postgresql import scada
from app.native.wndb.core.database import ChangeSet, sql_literal
from app.native.wndb.gis import coordinates
from app.native.wndb.model import controls, junctions, patterns
@@ -48,9 +48,9 @@ def test_get_all_scada_info_reads_materialized_view(monkeypatch) -> None:
statements.append(statement)
return []
monkeypatch.setattr(scada_assets, "read_all", fake_read_all)
monkeypatch.setattr(scada, "read_all", fake_read_all)
assert scada_assets.get_all_scada_info("project_a") == []
assert scada.get_all_scada_info("project_a") == []
assert "FROM gis.scada_devices" in statements[0]