OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scada): use project-scoped metadata

Browse Source
This commit is contained in:
jiang
2026-07-17 16:28:40 +08:00
parent a204980944
commit b4ecfbb87a
11 changed files with 694 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_burst_location_service.py
+23 -7
View File
@@ -12,12 +12,19 @@ def _load_burst_location_module():
Path(__file__).resolve().parents[2] / "app" / "services" / "burst_location.py"
)
missing = object()
previous_modules = {}
def install_module(name: str, module: types.ModuleType) -> None:
previous_modules.setdefault(name, sys.modules.get(name, missing))
sys.modules[name] = module
def ensure_package(name: str) -> types.ModuleType:
module = sys.modules.get(name)
if module is None:
module = types.ModuleType(name)
module.__path__ = []
sys.modules[name] = module
install_module(name, module)
return module
for package_name in [
@@ -46,11 +53,11 @@ def _load_burst_location_module():
)
)
time_api_module.utc_now = lambda: datetime.now(timezone.utc)
sys.modules["app.services.time_api"] = time_api_module
install_module("app.services.time_api", time_api_module)
algorithms_module = types.ModuleType("app.algorithms.burst_location")
algorithms_module.run_burst_location = lambda **kwargs: {}
sys.modules["app.algorithms.burst_location"] = algorithms_module
install_module("app.algorithms.burst_location", algorithms_module)
internal_queries_module = types.ModuleType(
"app.infra.db.timescaledb.internal_queries"
@@ -70,7 +77,9 @@ def _load_burst_location_module():
return {}
internal_queries_module.InternalQueries = DummyInternalQueries
sys.modules["app.infra.db.timescaledb.internal_queries"] = internal_queries_module
install_module(
"app.infra.db.timescaledb.internal_queries", internal_queries_module
)
scheme_management_module = types.ModuleType("app.services.scheme_management")
scheme_management_module.query_burst_location_scheme_detail = lambda *args, **kwargs: {}
@@ -78,18 +87,25 @@ def _load_burst_location_module():
scheme_management_module.query_scheme_list = lambda *args, **kwargs: []
scheme_management_module.scheme_name_exists = lambda *args, **kwargs: False
scheme_management_module.store_scheme_info = lambda *args, **kwargs: None
sys.modules["app.services.scheme_management"] = scheme_management_module
install_module("app.services.scheme_management", scheme_management_module)
tjnetwork_module = types.ModuleType("app.services.tjnetwork")
tjnetwork_module.dump_inp = lambda *args, **kwargs: None
tjnetwork_module.get_all_scada_info = lambda *args, **kwargs: []
sys.modules["app.services.tjnetwork"] = tjnetwork_module
install_module("app.services.tjnetwork", tjnetwork_module)
module_name = "tests_burst_location_under_test"
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module)
try:
spec.loader.exec_module(module)
finally:
for name, previous in reversed(previous_modules.items()):
if previous is missing:
sys.modules.pop(name, None)
else:
sys.modules[name] = previous
return module
tests/unit/test_postgres_scada_repository.py
+62
View File
@@ -0,0 +1,62 @@
import asyncio
from app.infra.db.postgresql.scada import ScadaInfoRepository
class _FakeCursor:
def __init__(self):
self.query = None
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
async def execute(self, query):
self.query = query
async def fetchall(self):
return [
{
"id": " 25470001 ",
"type": " PRESSURE ",
"associated_element_id": " J1 ",
"api_query_id": "query-1",
"transmission_mode": "realtime",
"transmission_frequency": None,
"reliability": "0.95",
"x_coor": "117.1",
"y_coor": "32.9",
}
]
class _FakeConnection:
def __init__(self):
self.cursor_instance = _FakeCursor()
def cursor(self):
return self.cursor_instance
def test_get_scadas_normalizes_id_and_type():
conn = _FakeConnection()
result = asyncio.run(ScadaInfoRepository.get_scadas(conn))
assert result == [
{
"id": "25470001",
"type": "pressure",
"associated_element_id": "J1",
"api_query_id": "query-1",
"transmission_mode": "realtime",
"transmission_frequency": None,
"reliability": 0.95,
"x": 117.1,
"y": 32.9,
}
]
assert "associated_element_id" in conn.cursor_instance.query
assert "FROM public.scada_info" in conn.cursor_instance.query
tests/unit/test_pressure_cleaning.py
+35 -44
View File
@@ -1,47 +1,25 @@
import importlib.util
import sys
import types
from pathlib import Path
import numpy as np
import pandas as pd
import pytest
from app.algorithms.cleaning import pressure as pressure_cleaning
def _load_pressure_cleaning_module():
project_root = Path(__file__).resolve().parents[2]
utils_path = project_root / "app" / "algorithms" / "_utils.py"
pressure_path = project_root / "app" / "algorithms" / "cleaning" / "pressure.py"
app_module = sys.modules.setdefault("app", types.ModuleType("app"))
algorithms_module = sys.modules.setdefault(
"app.algorithms",
types.ModuleType("app.algorithms"),
)
setattr(app_module, "algorithms", algorithms_module)
utils_spec = importlib.util.spec_from_file_location("app.algorithms._utils", utils_path)
assert utils_spec and utils_spec.loader
utils_module = importlib.util.module_from_spec(utils_spec)
sys.modules["app.algorithms._utils"] = utils_module
utils_spec.loader.exec_module(utils_module)
pressure_spec = importlib.util.spec_from_file_location(
"tests_pressure_under_test",
pressure_path,
)
assert pressure_spec and pressure_spec.loader
pressure_module = importlib.util.module_from_spec(pressure_spec)
pressure_spec.loader.exec_module(pressure_module)
return pressure_module
DATA_DIR = Path(__file__).resolve().parents[3] / "data"
RAW_DATA_PATH = DATA_DIR / "node_simulation.csv"
NOISY_DATA_PATH = DATA_DIR / "node_simulation_noisy.csv"
REQUIRES_PRESSURE_SAMPLES = pytest.mark.skipif(
not RAW_DATA_PATH.exists() or not NOISY_DATA_PATH.exists(),
reason="pressure cleaning sample CSV files are not available",
)
@REQUIRES_PRESSURE_SAMPLES
def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
module = _load_pressure_cleaning_module()
repo_root = Path(__file__).resolve().parents[3]
raw_df = pd.read_csv(repo_root / "data" / "node_simulation.csv")
noisy_df = pd.read_csv(repo_root / "data" / "node_simulation_noisy.csv")
cleaned_df = module.clean_pressure_data_df_km(noisy_df)
raw_df = pd.read_csv(RAW_DATA_PATH)
noisy_df = pd.read_csv(NOISY_DATA_PATH)
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(noisy_df)
for df in (raw_df, noisy_df, cleaned_df):
df["time"] = pd.to_datetime(df["time"])
@@ -50,7 +28,12 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
assert set(cleaned_df.columns) == {"time", "id", "pressure"}
assert cleaned_df["pressure"].isna().sum() == 0
noisy_joined = raw_df.merge(noisy_df, on=["time", "id"], how="inner", suffixes=("_raw", "_noisy"))
noisy_joined = raw_df.merge(
noisy_df,
on=["time", "id"],
how="inner",
suffixes=("_raw", "_noisy"),
)
cleaned_joined = raw_df.merge(
cleaned_df,
on=["time", "id"],
@@ -59,10 +42,20 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
)
noisy_rmse = float(
np.sqrt(np.mean((noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]) ** 2))
np.sqrt(
np.mean(
(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"])
** 2
)
)
)
cleaned_rmse = float(
np.sqrt(np.mean((cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"]) ** 2))
np.sqrt(
np.mean(
(cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"])
** 2
)
)
)
noisy_mae = float(
np.mean(np.abs(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]))
@@ -88,11 +81,9 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
assert abs(spike_row - 28.018701553344727) < 2.0
@REQUIRES_PRESSURE_SAMPLES
def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_strings():
module = _load_pressure_cleaning_module()
repo_root = Path(__file__).resolve().parents[3]
noisy_df = pd.read_csv(repo_root / "data" / "node_simulation_noisy.csv")
noisy_df = pd.read_csv(NOISY_DATA_PATH)
single_sensor = (
noisy_df[noisy_df["id"] == 170490][["time", "pressure"]]
.rename(columns={"pressure": "170490"})
@@ -102,7 +93,7 @@ def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_str
pd.to_datetime(single_sensor["time"], utc=True).dt.strftime("%Y-%m-%dT%H:%M:%SZ")
)
cleaned_df = module.clean_pressure_data_df_km(single_sensor)
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(single_sensor)
assert len(cleaned_df) == 192
assert cleaned_df["170490"].isna().sum() == 0
tests/unit/test_project_scada_metadata.py
+121
View File
@@ -0,0 +1,121 @@
import asyncio
from datetime import datetime, timezone
from unittest.mock import AsyncMock
from app.api.v1.endpoints import project_data
from app.infra.db.timescaledb import composite_queries
PROJECT_SCADA = {
"id": "fengyang-pressure-1",
"type": "pressure",
"associated_element_id": "J1",
"api_query_id": "query-1",
"transmission_mode": "realtime",
"transmission_frequency": None,
"reliability": 1.0,
"x": 117.1,
"y": 32.9,
}
START_TIME = datetime(2026, 6, 1, tzinfo=timezone.utc)
END_TIME = datetime(2026, 6, 2, tzinfo=timezone.utc)
def _patch_project_scadas(monkeypatch):
monkeypatch.setattr(
composite_queries.ScadaInfoRepository,
"get_scadas",
AsyncMock(return_value=[PROJECT_SCADA.copy()]),
)
def test_realtime_scada_simulation_uses_current_project_metadata(monkeypatch):
_patch_project_scadas(monkeypatch)
query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}])
monkeypatch.setattr(
composite_queries.RealtimeRepository,
"get_node_field_by_time_range",
query_mock,
)
result = asyncio.run(
composite_queries.CompositeQueries.get_scada_associated_realtime_simulation_data(
object(),
object(),
[PROJECT_SCADA["id"]],
START_TIME,
END_TIME,
)
)
assert result[PROJECT_SCADA["id"]][0]["scada_id"] == PROJECT_SCADA["id"]
assert query_mock.await_count == 1
assert query_mock.await_args.args[1:] == (
START_TIME,
END_TIME,
"J1",
"pressure",
)
def test_scheme_scada_simulation_uses_current_project_metadata(monkeypatch):
_patch_project_scadas(monkeypatch)
query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}])
monkeypatch.setattr(
composite_queries.SchemeRepository,
"get_node_field_by_scheme_and_time_range",
query_mock,
)
result = asyncio.run(
composite_queries.CompositeQueries.get_scada_associated_scheme_simulation_data(
object(),
object(),
[PROJECT_SCADA["id"]],
START_TIME,
END_TIME,
"baseline",
"scheme-1",
)
)
assert result[PROJECT_SCADA["id"]][0]["scada_id"] == PROJECT_SCADA["id"]
assert query_mock.await_args.args[5:] == ("J1", "pressure")
def test_element_scada_query_uses_current_project_metadata(monkeypatch):
_patch_project_scadas(monkeypatch)
query_mock = AsyncMock(
return_value={
PROJECT_SCADA["id"]: [{"time": START_TIME, "value": 26.5}]
}
)
monkeypatch.setattr(
composite_queries.ScadaRepository,
"get_scada_field_by_id_time_range",
query_mock,
)
result = asyncio.run(
composite_queries.CompositeQueries.get_element_associated_scada_data(
object(),
object(),
"J1",
START_TIME,
END_TIME,
)
)
assert result == {"J1": [{"time": START_TIME, "value": 26.5}]}
def test_scada_info_endpoint_uses_current_project_connection(monkeypatch):
monkeypatch.setattr(
project_data.ScadaInfoRepository,
"get_scadas",
AsyncMock(return_value=[PROJECT_SCADA.copy()]),
)
result = asyncio.run(project_data.get_scada_info_with_connection(object()))
assert result == {"success": True, "data": [PROJECT_SCADA], "count": 1}
tests/unit/test_realtime_repository.py
+1 -44
View File
@@ -1,48 +1,7 @@
import asyncio
from datetime import datetime, timezone
import importlib.util
from pathlib import Path
import sys
from types import ModuleType
def _load_time_api_module():
module_path = (
Path(__file__).resolve().parents[2] / "app" / "services" / "time_api.py"
)
spec = importlib.util.spec_from_file_location("tests_time_api_under_test", module_path)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module)
return module
def _load_realtime_repository():
time_api_module = _load_time_api_module()
app_module = ModuleType("app")
services_module = ModuleType("app.services")
services_module.time_api = time_api_module
app_module.services = services_module
sys.modules["app"] = app_module
sys.modules["app.services"] = services_module
sys.modules["app.services.time_api"] = time_api_module
module_path = (
Path(__file__).resolve().parents[2]
/ "app"
/ "infra"
/ "db"
/ "timescaledb"
/ "repositories"
/ "realtime.py"
)
spec = importlib.util.spec_from_file_location(
"tests_realtime_repo_under_test", module_path
)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module)
return module.RealtimeRepository
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
class _FakeCursor:
@@ -71,7 +30,6 @@ class _FakeConnection:
def test_get_links_by_time_range_normalizes_inputs_to_utc():
RealtimeRepository = _load_realtime_repository()
conn = _FakeConnection()
asyncio.run(
@@ -91,7 +49,6 @@ def test_get_links_by_time_range_normalizes_inputs_to_utc():
def test_get_nodes_by_time_range_normalizes_inputs_to_utc():
RealtimeRepository = _load_realtime_repository()
conn = _FakeConnection()
asyncio.run(
tests/unit/test_scada_cleaning.py
+200
View File
@@ -0,0 +1,200 @@
import asyncio
from datetime import datetime, timezone
from unittest.mock import AsyncMock
import pandas as pd
import pytest
from fastapi import HTTPException
from app.api.v1.endpoints.timeseries import composite as composite_endpoint
from app.infra.db.timescaledb import composite_queries
def test_clean_scada_uses_current_project_metadata(monkeypatch):
"""Fengyang data must not be classified with the global tjwater metadata."""
monkeypatch.setattr(
composite_queries.ScadaInfoRepository,
"get_scadas",
AsyncMock(
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
),
)
monkeypatch.setattr(
composite_queries.ScadaRepository,
"get_scada_field_by_id_time_range",
AsyncMock(
return_value={
"fengyang-pressure-1": [
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
]
}
),
)
update_mock = AsyncMock()
monkeypatch.setattr(
composite_queries.ScadaRepository,
"update_scada_field",
update_mock,
)
monkeypatch.setattr(
composite_queries,
"clean_pressure_data_df_km",
lambda frame: pd.DataFrame(
{
"time": frame["time"],
"fengyang-pressure-1": frame["fengyang-pressure-1"],
}
),
)
result = asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
)
)
assert result == "success"
update_mock.assert_awaited_once()
def test_clean_scada_rejects_devices_missing_from_project_metadata(monkeypatch):
monkeypatch.setattr(
composite_queries.ScadaInfoRepository,
"get_scadas",
AsyncMock(return_value=[{"id": "other-device", "type": "pressure"}]),
)
query_mock = AsyncMock()
monkeypatch.setattr(
composite_queries.ScadaRepository,
"get_scada_field_by_id_time_range",
query_mock,
)
with pytest.raises(ValueError, match="缺少元数据"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
)
)
query_mock.assert_not_awaited()
def test_clean_scada_rejects_zero_database_updates(monkeypatch):
monkeypatch.setattr(
composite_queries.ScadaInfoRepository,
"get_scadas",
AsyncMock(
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
),
)
monkeypatch.setattr(
composite_queries.ScadaRepository,
"get_scada_field_by_id_time_range",
AsyncMock(
return_value={
"fengyang-pressure-1": [
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
]
}
),
)
update_mock = AsyncMock()
monkeypatch.setattr(
composite_queries.ScadaRepository,
"update_scada_field",
update_mock,
)
monkeypatch.setattr(
composite_queries,
"clean_pressure_data_df_km",
lambda _frame: pd.DataFrame(
{"time": [], "fengyang-pressure-1": []}
),
)
with pytest.raises(ValueError, match="未产生任何数据库更新"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
)
)
update_mock.assert_not_awaited()
def test_clean_scada_propagates_write_failures(monkeypatch):
monkeypatch.setattr(
composite_queries.ScadaInfoRepository,
"get_scadas",
AsyncMock(
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
),
)
monkeypatch.setattr(
composite_queries.ScadaRepository,
"get_scada_field_by_id_time_range",
AsyncMock(
return_value={
"fengyang-pressure-1": [
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
]
}
),
)
monkeypatch.setattr(
composite_queries.ScadaRepository,
"update_scada_field",
AsyncMock(side_effect=RuntimeError("database write failed")),
)
monkeypatch.setattr(
composite_queries,
"clean_pressure_data_df_km",
lambda frame: frame,
)
with pytest.raises(RuntimeError, match="database write failed"):
asyncio.run(
composite_queries.CompositeQueries.clean_scada_data(
object(),
object(),
["fengyang-pressure-1"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
)
)
def test_clean_scada_endpoint_returns_http_400_for_validation_error(monkeypatch):
monkeypatch.setattr(
composite_endpoint.CompositeQueries,
"clean_scada_data",
AsyncMock(side_effect=ValueError("当前项目没有可清洗的 SCADA 设备")),
)
with pytest.raises(HTTPException) as exc_info:
asyncio.run(
composite_endpoint.clean_scada_data(
device_ids="all",
start_time=datetime(2026, 6, 1, tzinfo=timezone.utc),
end_time=datetime(2026, 6, 2, tzinfo=timezone.utc),
timescale_conn=object(),
postgres_conn=object(),
)
)
assert exc_info.value.status_code == 400
assert exc_info.value.detail == "当前项目没有可清洗的 SCADA 设备"