fix(scada): use project-scoped metadata
This commit is contained in:
+44
-35
@@ -1,36 +1,45 @@
|
|||||||
from app.algorithms.cleaning import flow_data_clean, pressure_data_clean
|
"""Algorithm package with side-effect-free, lazy compatibility exports."""
|
||||||
from app.algorithms.sensor import (
|
|
||||||
pressure_sensor_placement_sensitivity,
|
|
||||||
pressure_sensor_placement_kmeans,
|
|
||||||
)
|
|
||||||
from app.algorithms.isolation.valve import valve_isolation_analysis
|
|
||||||
from app.algorithms.leakage import LeakageIdentifier
|
|
||||||
from app.algorithms.health import PipelineHealthAnalyzer
|
|
||||||
from app.algorithms.burst_location import run_burst_location
|
|
||||||
from app.algorithms.simulation.scenarios import (
|
|
||||||
convert_to_local_unit,
|
|
||||||
burst_analysis,
|
|
||||||
valve_close_analysis,
|
|
||||||
flushing_analysis,
|
|
||||||
contaminant_simulation,
|
|
||||||
age_analysis,
|
|
||||||
pressure_regulation,
|
|
||||||
)
|
|
||||||
|
|
||||||
__all__ = [
|
from importlib import import_module
|
||||||
"flow_data_clean",
|
from typing import Any
|
||||||
"pressure_data_clean",
|
|
||||||
"pressure_sensor_placement_sensitivity",
|
|
||||||
"pressure_sensor_placement_kmeans",
|
_EXPORT_MODULES = {
|
||||||
"convert_to_local_unit",
|
"flow_data_clean": "app.algorithms.cleaning",
|
||||||
"burst_analysis",
|
"pressure_data_clean": "app.algorithms.cleaning",
|
||||||
"valve_close_analysis",
|
"pressure_sensor_placement_sensitivity": "app.algorithms.sensor",
|
||||||
"flushing_analysis",
|
"pressure_sensor_placement_kmeans": "app.algorithms.sensor",
|
||||||
"contaminant_simulation",
|
"valve_isolation_analysis": "app.algorithms.isolation.valve",
|
||||||
"age_analysis",
|
"LeakageIdentifier": "app.algorithms.leakage",
|
||||||
"pressure_regulation",
|
"PipelineHealthAnalyzer": "app.algorithms.health",
|
||||||
"valve_isolation_analysis",
|
"run_burst_location": "app.algorithms.burst_location",
|
||||||
"LeakageIdentifier",
|
**{
|
||||||
"PipelineHealthAnalyzer",
|
name: "app.algorithms.simulation.scenarios"
|
||||||
"run_burst_location",
|
for name in (
|
||||||
]
|
"convert_to_local_unit",
|
||||||
|
"burst_analysis",
|
||||||
|
"valve_close_analysis",
|
||||||
|
"flushing_analysis",
|
||||||
|
"contaminant_simulation",
|
||||||
|
"age_analysis",
|
||||||
|
"pressure_regulation",
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
__all__ = list(_EXPORT_MODULES)
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> Any:
|
||||||
|
try:
|
||||||
|
module_name = _EXPORT_MODULES[name]
|
||||||
|
except KeyError as exc:
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc
|
||||||
|
|
||||||
|
value = getattr(import_module(module_name), name)
|
||||||
|
globals()[name] = value
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def __dir__() -> list[str]:
|
||||||
|
return sorted({*globals(), *__all__})
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
from fastapi import APIRouter, Depends, HTTPException, Path, Query
|
from fastapi import APIRouter, Depends, HTTPException, Path, Query
|
||||||
from psycopg import AsyncConnection
|
from psycopg import AsyncConnection
|
||||||
|
|
||||||
import app.native.wndb as wndb
|
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||||
from app.infra.db.postgresql.scheme import SchemeRepository
|
from app.infra.db.postgresql.scheme import SchemeRepository
|
||||||
from app.auth.project_dependencies import get_project_pg_connection
|
from app.auth.project_dependencies import get_project_pg_connection
|
||||||
from app.services import project_info
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@@ -26,9 +25,7 @@ async def get_scada_info_with_connection(
|
|||||||
返回项目中所有的SCADA设备信息
|
返回项目中所有的SCADA设备信息
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
_ = conn
|
scada_data = await ScadaInfoRepository.get_scadas(conn)
|
||||||
network_name = project_info.name
|
|
||||||
scada_data = wndb.get_all_scada_info(network_name) if network_name else []
|
|
||||||
return {"success": True, "data": scada_data, "count": len(scada_data)}
|
return {"success": True, "data": scada_data, "count": len(scada_data)}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from psycopg import AsyncConnection
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: Any) -> str | None:
|
||||||
|
return str(value).strip() if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_float(value: Any) -> float | None:
|
||||||
|
return float(value) if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
class ScadaInfoRepository:
|
||||||
|
"""Read SCADA metadata from the current project's business database."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def get_scadas(conn: AsyncConnection) -> list[dict[str, Any]]:
|
||||||
|
async with conn.cursor() as cur:
|
||||||
|
await cur.execute(
|
||||||
|
"""
|
||||||
|
SELECT id,
|
||||||
|
type,
|
||||||
|
associated_element_id,
|
||||||
|
api_query_id,
|
||||||
|
transmission_mode,
|
||||||
|
transmission_frequency,
|
||||||
|
reliability,
|
||||||
|
x_coor,
|
||||||
|
y_coor
|
||||||
|
FROM public.scada_info
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
records = await cur.fetchall()
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": str(record["id"]).strip(),
|
||||||
|
"type": str(record["type"]).strip().lower(),
|
||||||
|
"associated_element_id": _optional_text(
|
||||||
|
record["associated_element_id"]
|
||||||
|
),
|
||||||
|
"api_query_id": record["api_query_id"],
|
||||||
|
"transmission_mode": record["transmission_mode"],
|
||||||
|
"transmission_frequency": record["transmission_frequency"],
|
||||||
|
"reliability": _optional_float(record["reliability"]),
|
||||||
|
"x": _optional_float(record["x_coor"]),
|
||||||
|
"y": _optional_float(record["y_coor"]),
|
||||||
|
}
|
||||||
|
for record in records
|
||||||
|
]
|
||||||
@@ -1,18 +1,19 @@
|
|||||||
import time
|
import time
|
||||||
from typing import List, Optional, Any, Dict, Tuple
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from psycopg import AsyncConnection
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
import pandas as pd
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
from psycopg import AsyncConnection
|
||||||
|
|
||||||
|
import app.native.wndb as wndb
|
||||||
from app.algorithms.cleaning.flow import clean_flow_data_df_kf
|
from app.algorithms.cleaning.flow import clean_flow_data_df_kf
|
||||||
from app.algorithms.cleaning.pressure import clean_pressure_data_df_km
|
from app.algorithms.cleaning.pressure import clean_pressure_data_df_km
|
||||||
from app.algorithms.health.analyzer import PipelineHealthAnalyzer
|
from app.algorithms.health.analyzer import PipelineHealthAnalyzer
|
||||||
import app.native.wndb as wndb
|
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||||
|
|
||||||
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
||||||
from app.infra.db.timescaledb.repositories.scheme import SchemeRepository
|
from app.infra.db.timescaledb.repositories.scheme import SchemeRepository
|
||||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||||
from app.services import project_info
|
|
||||||
|
|
||||||
|
|
||||||
class CompositeQueries:
|
class CompositeQueries:
|
||||||
@@ -20,6 +21,13 @@ class CompositeQueries:
|
|||||||
复合查询类,提供跨表查询功能
|
复合查询类,提供跨表查询功能
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _get_project_scada_index(
|
||||||
|
postgres_conn: AsyncConnection,
|
||||||
|
) -> Dict[str, Dict[str, Any]]:
|
||||||
|
scadas = await ScadaInfoRepository.get_scadas(postgres_conn)
|
||||||
|
return {scada["id"]: scada for scada in scadas}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def get_scada_associated_realtime_simulation_data(
|
async def get_scada_associated_realtime_simulation_data(
|
||||||
timescale_conn: AsyncConnection,
|
timescale_conn: AsyncConnection,
|
||||||
@@ -48,31 +56,22 @@ class CompositeQueries:
|
|||||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||||
"""
|
"""
|
||||||
result = {}
|
result = {}
|
||||||
# 1. 查询所有 SCADA 信息
|
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||||||
network_name = project_info.name
|
|
||||||
scada_infos = wndb.get_all_scada_info(network_name) if network_name else []
|
|
||||||
|
|
||||||
for device_id in device_ids:
|
for device_id in device_ids:
|
||||||
# 2. 根据 device_id 找到对应的 SCADA 信息
|
target_scada = scada_by_id.get(device_id)
|
||||||
target_scada = None
|
|
||||||
for scada in scada_infos:
|
|
||||||
if scada["id"] == device_id:
|
|
||||||
target_scada = scada
|
|
||||||
break
|
|
||||||
|
|
||||||
if not target_scada:
|
if not target_scada:
|
||||||
raise ValueError(f"SCADA device {device_id} not found")
|
raise ValueError(f"SCADA device {device_id} not found")
|
||||||
|
|
||||||
# 3. 根据 type 和 associated_element_id 查询对应的模拟数据
|
|
||||||
element_id = target_scada["associated_element_id"]
|
element_id = target_scada["associated_element_id"]
|
||||||
scada_type = target_scada["type"]
|
scada_type = target_scada["type"]
|
||||||
|
|
||||||
if scada_type.lower() == "pipe_flow":
|
if scada_type == "pipe_flow":
|
||||||
# 查询 link 模拟数据
|
# 查询 link 模拟数据
|
||||||
res = await RealtimeRepository.get_link_field_by_time_range(
|
res = await RealtimeRepository.get_link_field_by_time_range(
|
||||||
timescale_conn, start_time, end_time, element_id, "flow"
|
timescale_conn, start_time, end_time, element_id, "flow"
|
||||||
)
|
)
|
||||||
elif scada_type.lower() == "pressure":
|
elif scada_type == "pressure":
|
||||||
# 查询 node 模拟数据
|
# 查询 node 模拟数据
|
||||||
res = await RealtimeRepository.get_node_field_by_time_range(
|
res = await RealtimeRepository.get_node_field_by_time_range(
|
||||||
timescale_conn, start_time, end_time, element_id, "pressure"
|
timescale_conn, start_time, end_time, element_id, "pressure"
|
||||||
@@ -115,26 +114,17 @@ class CompositeQueries:
|
|||||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||||
"""
|
"""
|
||||||
result = {}
|
result = {}
|
||||||
# 1. 查询所有 SCADA 信息
|
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||||||
network_name = project_info.name
|
|
||||||
scada_infos = wndb.get_all_scada_info(network_name) if network_name else []
|
|
||||||
|
|
||||||
for device_id in device_ids:
|
for device_id in device_ids:
|
||||||
# 2. 根据 device_id 找到对应的 SCADA 信息
|
target_scada = scada_by_id.get(device_id)
|
||||||
target_scada = None
|
|
||||||
for scada in scada_infos:
|
|
||||||
if scada["id"] == device_id:
|
|
||||||
target_scada = scada
|
|
||||||
break
|
|
||||||
|
|
||||||
if not target_scada:
|
if not target_scada:
|
||||||
raise ValueError(f"SCADA device {device_id} not found")
|
raise ValueError(f"SCADA device {device_id} not found")
|
||||||
|
|
||||||
# 3. 根据 type 和 associated_element_id 查询对应的模拟数据
|
|
||||||
element_id = target_scada["associated_element_id"]
|
element_id = target_scada["associated_element_id"]
|
||||||
scada_type = target_scada["type"]
|
scada_type = target_scada["type"]
|
||||||
|
|
||||||
if scada_type.lower() == "pipe_flow":
|
if scada_type == "pipe_flow":
|
||||||
# 查询 link 模拟数据
|
# 查询 link 模拟数据
|
||||||
res = await SchemeRepository.get_link_field_by_scheme_and_time_range(
|
res = await SchemeRepository.get_link_field_by_scheme_and_time_range(
|
||||||
timescale_conn,
|
timescale_conn,
|
||||||
@@ -145,7 +135,7 @@ class CompositeQueries:
|
|||||||
element_id,
|
element_id,
|
||||||
"flow",
|
"flow",
|
||||||
)
|
)
|
||||||
elif scada_type.lower() == "pressure":
|
elif scada_type == "pressure":
|
||||||
# 查询 node 模拟数据
|
# 查询 node 模拟数据
|
||||||
res = await SchemeRepository.get_node_field_by_scheme_and_time_range(
|
res = await SchemeRepository.get_node_field_by_scheme_and_time_range(
|
||||||
timescale_conn,
|
timescale_conn,
|
||||||
@@ -167,19 +157,19 @@ class CompositeQueries:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
async def get_realtime_simulation_data(
|
async def get_realtime_simulation_data(
|
||||||
timescale_conn: AsyncConnection,
|
timescale_conn: AsyncConnection,
|
||||||
featureInfos: List[Tuple[str, str]],
|
feature_infos: List[Tuple[str, str]],
|
||||||
start_time: datetime,
|
start_time: datetime,
|
||||||
end_time: datetime,
|
end_time: datetime,
|
||||||
) -> Dict[str, List[Dict[str, Any]]]:
|
) -> Dict[str, List[Dict[str, Any]]]:
|
||||||
"""
|
"""
|
||||||
获取 link/node 模拟值
|
获取 link/node 模拟值
|
||||||
|
|
||||||
根据传入的 featureInfos,找到关联的 link/node,
|
根据传入的 feature_infos,找到关联的 link/node,
|
||||||
并根据对应的 type,查询对应的模拟数据
|
并根据对应的 type,查询对应的模拟数据
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
timescale_conn: TimescaleDB 异步连接
|
timescale_conn: TimescaleDB 异步连接
|
||||||
featureInfos: 传入的 feature 信息列表,包含 (element_id, type)
|
feature_infos: 传入的 feature 信息列表,包含 (element_id, type)
|
||||||
start_time: 开始时间
|
start_time: 开始时间
|
||||||
end_time: 结束时间
|
end_time: 结束时间
|
||||||
|
|
||||||
@@ -190,20 +180,20 @@ class CompositeQueries:
|
|||||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||||
"""
|
"""
|
||||||
result = {}
|
result = {}
|
||||||
for feature_id, type in featureInfos:
|
for feature_id, feature_type in feature_infos:
|
||||||
|
|
||||||
if type.lower() == "pipe":
|
if feature_type.lower() == "pipe":
|
||||||
# 查询 link 模拟数据
|
# 查询 link 模拟数据
|
||||||
res = await RealtimeRepository.get_link_field_by_time_range(
|
res = await RealtimeRepository.get_link_field_by_time_range(
|
||||||
timescale_conn, start_time, end_time, feature_id, "flow"
|
timescale_conn, start_time, end_time, feature_id, "flow"
|
||||||
)
|
)
|
||||||
elif type.lower() == "junction":
|
elif feature_type.lower() == "junction":
|
||||||
# 查询 node 模拟数据
|
# 查询 node 模拟数据
|
||||||
res = await RealtimeRepository.get_node_field_by_time_range(
|
res = await RealtimeRepository.get_node_field_by_time_range(
|
||||||
timescale_conn, start_time, end_time, feature_id, "pressure"
|
timescale_conn, start_time, end_time, feature_id, "pressure"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown type: {type}")
|
raise ValueError(f"Unknown type: {feature_type}")
|
||||||
# 添加 scada_id 到每个数据项
|
# 添加 scada_id 到每个数据项
|
||||||
for item in res:
|
for item in res:
|
||||||
item["feature_id"] = feature_id
|
item["feature_id"] = feature_id
|
||||||
@@ -213,7 +203,7 @@ class CompositeQueries:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
async def get_scheme_simulation_data(
|
async def get_scheme_simulation_data(
|
||||||
timescale_conn: AsyncConnection,
|
timescale_conn: AsyncConnection,
|
||||||
featureInfos: List[Tuple[str, str]],
|
feature_infos: List[Tuple[str, str]],
|
||||||
start_time: datetime,
|
start_time: datetime,
|
||||||
end_time: datetime,
|
end_time: datetime,
|
||||||
scheme_type: str,
|
scheme_type: str,
|
||||||
@@ -222,12 +212,12 @@ class CompositeQueries:
|
|||||||
"""
|
"""
|
||||||
获取 link/node scheme 模拟值
|
获取 link/node scheme 模拟值
|
||||||
|
|
||||||
根据传入的 featureInfos,找到关联的 link/node,
|
根据传入的 feature_infos,找到关联的 link/node,
|
||||||
并根据对应的 type,查询对应的模拟数据
|
并根据对应的 type,查询对应的模拟数据
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
timescale_conn: TimescaleDB 异步连接
|
timescale_conn: TimescaleDB 异步连接
|
||||||
featureInfos: 传入的 feature 信息列表,包含 (element_id, type)
|
feature_infos: 传入的 feature 信息列表,包含 (element_id, type)
|
||||||
start_time: 开始时间
|
start_time: 开始时间
|
||||||
end_time: 结束时间
|
end_time: 结束时间
|
||||||
scheme_type: 工况类型
|
scheme_type: 工况类型
|
||||||
@@ -240,8 +230,8 @@ class CompositeQueries:
|
|||||||
ValueError: 当类型无效时
|
ValueError: 当类型无效时
|
||||||
"""
|
"""
|
||||||
result = {}
|
result = {}
|
||||||
for feature_id, type in featureInfos:
|
for feature_id, feature_type in feature_infos:
|
||||||
if type.lower() == "pipe":
|
if feature_type.lower() == "pipe":
|
||||||
# 查询 link 模拟数据
|
# 查询 link 模拟数据
|
||||||
res = await SchemeRepository.get_link_field_by_scheme_and_time_range(
|
res = await SchemeRepository.get_link_field_by_scheme_and_time_range(
|
||||||
timescale_conn,
|
timescale_conn,
|
||||||
@@ -252,7 +242,7 @@ class CompositeQueries:
|
|||||||
feature_id,
|
feature_id,
|
||||||
"flow",
|
"flow",
|
||||||
)
|
)
|
||||||
elif type.lower() == "junction":
|
elif feature_type.lower() == "junction":
|
||||||
# 查询 node 模拟数据
|
# 查询 node 模拟数据
|
||||||
res = await SchemeRepository.get_node_field_by_scheme_and_time_range(
|
res = await SchemeRepository.get_node_field_by_scheme_and_time_range(
|
||||||
timescale_conn,
|
timescale_conn,
|
||||||
@@ -264,7 +254,7 @@ class CompositeQueries:
|
|||||||
"pressure",
|
"pressure",
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown type: {type}")
|
raise ValueError(f"Unknown type: {feature_type}")
|
||||||
# 添加 feature_id 到每个数据项
|
# 添加 feature_id 到每个数据项
|
||||||
for item in res:
|
for item in res:
|
||||||
item["feature_id"] = feature_id
|
item["feature_id"] = feature_id
|
||||||
@@ -301,33 +291,27 @@ class CompositeQueries:
|
|||||||
ValueError: 当元素类型无效时
|
ValueError: 当元素类型无效时
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# 1. 查询所有 SCADA 信息
|
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||||||
network_name = project_info.name
|
associated_scada = next(
|
||||||
scada_infos = wndb.get_all_scada_info(network_name) if network_name else []
|
(
|
||||||
|
scada
|
||||||
# 2. 根据 element_type 和 element_id 找到关联的 SCADA 设备
|
for scada in scada_by_id.values()
|
||||||
associated_scada = None
|
if scada["associated_element_id"] == element_id
|
||||||
for scada in scada_infos:
|
),
|
||||||
if scada["associated_element_id"] == element_id:
|
None,
|
||||||
associated_scada = scada
|
)
|
||||||
break
|
|
||||||
|
|
||||||
if not associated_scada:
|
if not associated_scada:
|
||||||
# 没有找到关联的 SCADA 设备
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 3. 通过 SCADA device_id 获取监测数据
|
|
||||||
device_id = associated_scada["id"]
|
device_id = associated_scada["id"]
|
||||||
|
|
||||||
# 根据 use_cleaned 参数选择字段
|
|
||||||
data_field = "cleaned_value" if use_cleaned else "monitored_value"
|
data_field = "cleaned_value" if use_cleaned else "monitored_value"
|
||||||
|
|
||||||
# 保证 device_id 以列表形式传递
|
|
||||||
res = await ScadaRepository.get_scada_field_by_id_time_range(
|
res = await ScadaRepository.get_scada_field_by_id_time_range(
|
||||||
timescale_conn, [device_id], start_time, end_time, data_field
|
timescale_conn, [device_id], start_time, end_time, data_field
|
||||||
)
|
)
|
||||||
|
|
||||||
# 将 device_id 替换为 element_id 返回
|
|
||||||
return {element_id: res.get(device_id, [])}
|
return {element_id: res.get(device_id, [])}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -351,108 +335,124 @@ class CompositeQueries:
|
|||||||
end_time: 结束时间
|
end_time: 结束时间
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
"success" 或错误信息
|
"success"
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: 当前项目没有可清洗设备或指定时间范围内没有监测数据
|
||||||
"""
|
"""
|
||||||
try:
|
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||||||
# 获取所有 SCADA 信息
|
supported_types = {"pressure", "pipe_flow", "flow"}
|
||||||
network_name = project_info.name
|
|
||||||
scada_infos = wndb.get_all_scada_info(network_name) if network_name else []
|
|
||||||
# 将列表转换为字典,以 device_id 为键
|
|
||||||
scada_device_info_dict = {info["id"]: info for info in scada_infos}
|
|
||||||
|
|
||||||
# 如果 device_ids 为空,则处理所有 SCADA 设备
|
if device_ids:
|
||||||
if not device_ids:
|
device_ids = [str(device_id).strip() for device_id in device_ids]
|
||||||
device_ids = list(scada_device_info_dict.keys())
|
missing_metadata_ids = [
|
||||||
|
device_id
|
||||||
|
for device_id in device_ids
|
||||||
|
if device_id not in scada_by_id
|
||||||
|
]
|
||||||
|
if missing_metadata_ids:
|
||||||
|
raise ValueError(
|
||||||
|
f"当前项目中有 {len(missing_metadata_ids)} 个 SCADA 设备缺少元数据"
|
||||||
|
)
|
||||||
|
|
||||||
# 批量查询所有设备的数据
|
unsupported_ids = [
|
||||||
data = await ScadaRepository.get_scada_field_by_id_time_range(
|
device_id
|
||||||
timescale_conn, device_ids, start_time, end_time, "monitored_value"
|
for device_id in device_ids
|
||||||
|
if scada_by_id[device_id]["type"] not in supported_types
|
||||||
|
]
|
||||||
|
if unsupported_ids:
|
||||||
|
raise ValueError(
|
||||||
|
f"当前项目中有 {len(unsupported_ids)} 个 SCADA 设备类型不支持清洗"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
device_ids = [
|
||||||
|
device_id
|
||||||
|
for device_id, info in scada_by_id.items()
|
||||||
|
if info["type"] in supported_types
|
||||||
|
]
|
||||||
|
|
||||||
|
if not device_ids:
|
||||||
|
raise ValueError("当前项目没有可清洗的 SCADA 设备")
|
||||||
|
|
||||||
|
data = await ScadaRepository.get_scada_field_by_id_time_range(
|
||||||
|
timescale_conn, device_ids, start_time, end_time, "monitored_value"
|
||||||
|
)
|
||||||
|
if not data:
|
||||||
|
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
||||||
|
|
||||||
|
normalized_data = {
|
||||||
|
str(device_id): records for device_id, records in data.items()
|
||||||
|
}
|
||||||
|
missing_data_ids = [
|
||||||
|
device_id for device_id in device_ids if not normalized_data.get(device_id)
|
||||||
|
]
|
||||||
|
if missing_data_ids:
|
||||||
|
raise ValueError(
|
||||||
|
f"指定时间范围内有 {len(missing_data_ids)} 个 SCADA 设备没有监测数据"
|
||||||
)
|
)
|
||||||
|
|
||||||
if not data:
|
all_records = [
|
||||||
return "error: fetch none scada data" # 没有数据,直接返回
|
{
|
||||||
|
"time": record["time"],
|
||||||
|
"device_id": device_id,
|
||||||
|
"value": record["value"],
|
||||||
|
}
|
||||||
|
for device_id, records in normalized_data.items()
|
||||||
|
for record in records
|
||||||
|
]
|
||||||
|
if not all_records:
|
||||||
|
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
||||||
|
|
||||||
# 将嵌套字典转换为 DataFrame,使用 time 作为索引
|
df_long = pd.DataFrame(all_records)
|
||||||
# data 格式: {device_id: [{"time": "...", "value": ...}, ...]}
|
df = df_long.pivot(index="time", columns="device_id", values="value")
|
||||||
all_records = []
|
|
||||||
for device_id, records in data.items():
|
pressure_ids = [
|
||||||
for record in records:
|
device_id
|
||||||
all_records.append(
|
for device_id in df.columns
|
||||||
{
|
if scada_by_id[device_id]["type"] == "pressure"
|
||||||
"time": record["time"],
|
]
|
||||||
"device_id": device_id,
|
flow_ids = [
|
||||||
"value": record["value"],
|
device_id
|
||||||
}
|
for device_id in df.columns
|
||||||
|
if scada_by_id[device_id]["type"] in {"pipe_flow", "flow"}
|
||||||
|
]
|
||||||
|
|
||||||
|
updated_rows = 0
|
||||||
|
for grouped_ids, cleaning_function in (
|
||||||
|
(pressure_ids, clean_pressure_data_df_km),
|
||||||
|
(flow_ids, clean_flow_data_df_kf),
|
||||||
|
):
|
||||||
|
if not grouped_ids:
|
||||||
|
continue
|
||||||
|
|
||||||
|
source_df = df[grouped_ids].reset_index()
|
||||||
|
cleaned_df = cleaning_function(source_df)
|
||||||
|
time_values = cleaned_df["time"].tolist()
|
||||||
|
|
||||||
|
for device_id in grouped_ids:
|
||||||
|
if device_id not in cleaned_df.columns:
|
||||||
|
raise ValueError(f"设备 {device_id} 的清洗结果缺少数据列")
|
||||||
|
|
||||||
|
cleaned_values = cleaned_df[device_id].tolist()
|
||||||
|
for time_value, value in zip(time_values, cleaned_values):
|
||||||
|
time_dt = (
|
||||||
|
time_value
|
||||||
|
if isinstance(time_value, datetime)
|
||||||
|
else datetime.fromisoformat(str(time_value))
|
||||||
)
|
)
|
||||||
|
await ScadaRepository.update_scada_field(
|
||||||
|
timescale_conn,
|
||||||
|
time_dt,
|
||||||
|
device_id,
|
||||||
|
"cleaned_value",
|
||||||
|
value,
|
||||||
|
)
|
||||||
|
updated_rows += 1
|
||||||
|
|
||||||
if not all_records:
|
if updated_rows == 0:
|
||||||
return "error: fetch none scada data" # 没有数据,直接返回
|
raise ValueError("SCADA 数据清洗未产生任何数据库更新")
|
||||||
|
|
||||||
# 创建 DataFrame 并透视,使 device_id 成为列
|
return "success"
|
||||||
df_long = pd.DataFrame(all_records)
|
|
||||||
df = df_long.pivot(index="time", columns="device_id", values="value")
|
|
||||||
|
|
||||||
# 根据type分类设备
|
|
||||||
pressure_ids = [
|
|
||||||
id
|
|
||||||
for id in df.columns
|
|
||||||
if scada_device_info_dict.get(id, {}).get("type") == "pressure"
|
|
||||||
]
|
|
||||||
flow_ids = [
|
|
||||||
id
|
|
||||||
for id in df.columns
|
|
||||||
if scada_device_info_dict.get(id, {}).get("type") == "pipe_flow"
|
|
||||||
]
|
|
||||||
|
|
||||||
# 处理pressure数据
|
|
||||||
if pressure_ids:
|
|
||||||
pressure_df = df[pressure_ids]
|
|
||||||
# 重置索引,将 time 变为普通列
|
|
||||||
pressure_df = pressure_df.reset_index()
|
|
||||||
# 调用清洗方法
|
|
||||||
cleaned_df = clean_pressure_data_df_km(pressure_df)
|
|
||||||
# 将清洗后的数据写回数据库
|
|
||||||
for device_id in pressure_ids:
|
|
||||||
if device_id in cleaned_df.columns:
|
|
||||||
cleaned_values = cleaned_df[device_id].tolist()
|
|
||||||
time_values = cleaned_df["time"].tolist()
|
|
||||||
for i, time_str in enumerate(time_values):
|
|
||||||
time_dt = datetime.fromisoformat(time_str)
|
|
||||||
value = cleaned_values[i]
|
|
||||||
await ScadaRepository.update_scada_field(
|
|
||||||
timescale_conn,
|
|
||||||
time_dt,
|
|
||||||
device_id,
|
|
||||||
"cleaned_value",
|
|
||||||
value,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 处理flow数据
|
|
||||||
if flow_ids:
|
|
||||||
flow_df = df[flow_ids]
|
|
||||||
# 重置索引,将 time 变为普通列
|
|
||||||
flow_df = flow_df.reset_index()
|
|
||||||
# 调用清洗方法
|
|
||||||
cleaned_df = clean_flow_data_df_kf(flow_df)
|
|
||||||
# 将清洗后的数据写回数据库
|
|
||||||
for device_id in flow_ids:
|
|
||||||
if device_id in cleaned_df.columns:
|
|
||||||
cleaned_values = cleaned_df[device_id].tolist()
|
|
||||||
time_values = cleaned_df["time"].tolist()
|
|
||||||
for i, time_str in enumerate(time_values):
|
|
||||||
time_dt = datetime.fromisoformat(time_str)
|
|
||||||
value = cleaned_values[i]
|
|
||||||
await ScadaRepository.update_scada_field(
|
|
||||||
timescale_conn,
|
|
||||||
time_dt,
|
|
||||||
device_id,
|
|
||||||
"cleaned_value",
|
|
||||||
value,
|
|
||||||
)
|
|
||||||
|
|
||||||
return "success"
|
|
||||||
except Exception as e:
|
|
||||||
return f"error: {str(e)}"
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def predict_pipeline_health(
|
async def predict_pipeline_health(
|
||||||
|
|||||||
+1
-1
@@ -56,7 +56,7 @@ def install_stub(monkeypatch, name: str, attrs: dict | None = None, package: boo
|
|||||||
parent = types.ModuleType(parent_name)
|
parent = types.ModuleType(parent_name)
|
||||||
parent.__path__ = []
|
parent.__path__ = []
|
||||||
monkeypatch.setitem(sys.modules, parent_name, parent)
|
monkeypatch.setitem(sys.modules, parent_name, parent)
|
||||||
setattr(parent, child_name, module)
|
monkeypatch.setattr(parent, child_name, module, raising=False)
|
||||||
|
|
||||||
return module
|
return module
|
||||||
|
|
||||||
|
|||||||
@@ -12,12 +12,19 @@ def _load_burst_location_module():
|
|||||||
Path(__file__).resolve().parents[2] / "app" / "services" / "burst_location.py"
|
Path(__file__).resolve().parents[2] / "app" / "services" / "burst_location.py"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
missing = object()
|
||||||
|
previous_modules = {}
|
||||||
|
|
||||||
|
def install_module(name: str, module: types.ModuleType) -> None:
|
||||||
|
previous_modules.setdefault(name, sys.modules.get(name, missing))
|
||||||
|
sys.modules[name] = module
|
||||||
|
|
||||||
def ensure_package(name: str) -> types.ModuleType:
|
def ensure_package(name: str) -> types.ModuleType:
|
||||||
module = sys.modules.get(name)
|
module = sys.modules.get(name)
|
||||||
if module is None:
|
if module is None:
|
||||||
module = types.ModuleType(name)
|
module = types.ModuleType(name)
|
||||||
module.__path__ = []
|
module.__path__ = []
|
||||||
sys.modules[name] = module
|
install_module(name, module)
|
||||||
return module
|
return module
|
||||||
|
|
||||||
for package_name in [
|
for package_name in [
|
||||||
@@ -46,11 +53,11 @@ def _load_burst_location_module():
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
time_api_module.utc_now = lambda: datetime.now(timezone.utc)
|
time_api_module.utc_now = lambda: datetime.now(timezone.utc)
|
||||||
sys.modules["app.services.time_api"] = time_api_module
|
install_module("app.services.time_api", time_api_module)
|
||||||
|
|
||||||
algorithms_module = types.ModuleType("app.algorithms.burst_location")
|
algorithms_module = types.ModuleType("app.algorithms.burst_location")
|
||||||
algorithms_module.run_burst_location = lambda **kwargs: {}
|
algorithms_module.run_burst_location = lambda **kwargs: {}
|
||||||
sys.modules["app.algorithms.burst_location"] = algorithms_module
|
install_module("app.algorithms.burst_location", algorithms_module)
|
||||||
|
|
||||||
internal_queries_module = types.ModuleType(
|
internal_queries_module = types.ModuleType(
|
||||||
"app.infra.db.timescaledb.internal_queries"
|
"app.infra.db.timescaledb.internal_queries"
|
||||||
@@ -70,7 +77,9 @@ def _load_burst_location_module():
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
internal_queries_module.InternalQueries = DummyInternalQueries
|
internal_queries_module.InternalQueries = DummyInternalQueries
|
||||||
sys.modules["app.infra.db.timescaledb.internal_queries"] = internal_queries_module
|
install_module(
|
||||||
|
"app.infra.db.timescaledb.internal_queries", internal_queries_module
|
||||||
|
)
|
||||||
|
|
||||||
scheme_management_module = types.ModuleType("app.services.scheme_management")
|
scheme_management_module = types.ModuleType("app.services.scheme_management")
|
||||||
scheme_management_module.query_burst_location_scheme_detail = lambda *args, **kwargs: {}
|
scheme_management_module.query_burst_location_scheme_detail = lambda *args, **kwargs: {}
|
||||||
@@ -78,18 +87,25 @@ def _load_burst_location_module():
|
|||||||
scheme_management_module.query_scheme_list = lambda *args, **kwargs: []
|
scheme_management_module.query_scheme_list = lambda *args, **kwargs: []
|
||||||
scheme_management_module.scheme_name_exists = lambda *args, **kwargs: False
|
scheme_management_module.scheme_name_exists = lambda *args, **kwargs: False
|
||||||
scheme_management_module.store_scheme_info = lambda *args, **kwargs: None
|
scheme_management_module.store_scheme_info = lambda *args, **kwargs: None
|
||||||
sys.modules["app.services.scheme_management"] = scheme_management_module
|
install_module("app.services.scheme_management", scheme_management_module)
|
||||||
|
|
||||||
tjnetwork_module = types.ModuleType("app.services.tjnetwork")
|
tjnetwork_module = types.ModuleType("app.services.tjnetwork")
|
||||||
tjnetwork_module.dump_inp = lambda *args, **kwargs: None
|
tjnetwork_module.dump_inp = lambda *args, **kwargs: None
|
||||||
tjnetwork_module.get_all_scada_info = lambda *args, **kwargs: []
|
tjnetwork_module.get_all_scada_info = lambda *args, **kwargs: []
|
||||||
sys.modules["app.services.tjnetwork"] = tjnetwork_module
|
install_module("app.services.tjnetwork", tjnetwork_module)
|
||||||
|
|
||||||
module_name = "tests_burst_location_under_test"
|
module_name = "tests_burst_location_under_test"
|
||||||
spec = importlib.util.spec_from_file_location(module_name, module_path)
|
spec = importlib.util.spec_from_file_location(module_name, module_path)
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
assert spec and spec.loader
|
assert spec and spec.loader
|
||||||
spec.loader.exec_module(module)
|
try:
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
finally:
|
||||||
|
for name, previous in reversed(previous_modules.items()):
|
||||||
|
if previous is missing:
|
||||||
|
sys.modules.pop(name, None)
|
||||||
|
else:
|
||||||
|
sys.modules[name] = previous
|
||||||
return module
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import asyncio
|
||||||
|
|
||||||
|
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeCursor:
|
||||||
|
def __init__(self):
|
||||||
|
self.query = None
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def execute(self, query):
|
||||||
|
self.query = query
|
||||||
|
|
||||||
|
async def fetchall(self):
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": " 25470001 ",
|
||||||
|
"type": " PRESSURE ",
|
||||||
|
"associated_element_id": " J1 ",
|
||||||
|
"api_query_id": "query-1",
|
||||||
|
"transmission_mode": "realtime",
|
||||||
|
"transmission_frequency": None,
|
||||||
|
"reliability": "0.95",
|
||||||
|
"x_coor": "117.1",
|
||||||
|
"y_coor": "32.9",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeConnection:
|
||||||
|
def __init__(self):
|
||||||
|
self.cursor_instance = _FakeCursor()
|
||||||
|
|
||||||
|
def cursor(self):
|
||||||
|
return self.cursor_instance
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_scadas_normalizes_id_and_type():
|
||||||
|
conn = _FakeConnection()
|
||||||
|
|
||||||
|
result = asyncio.run(ScadaInfoRepository.get_scadas(conn))
|
||||||
|
|
||||||
|
assert result == [
|
||||||
|
{
|
||||||
|
"id": "25470001",
|
||||||
|
"type": "pressure",
|
||||||
|
"associated_element_id": "J1",
|
||||||
|
"api_query_id": "query-1",
|
||||||
|
"transmission_mode": "realtime",
|
||||||
|
"transmission_frequency": None,
|
||||||
|
"reliability": 0.95,
|
||||||
|
"x": 117.1,
|
||||||
|
"y": 32.9,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
assert "associated_element_id" in conn.cursor_instance.query
|
||||||
|
assert "FROM public.scada_info" in conn.cursor_instance.query
|
||||||
@@ -1,47 +1,25 @@
|
|||||||
import importlib.util
|
|
||||||
import sys
|
|
||||||
import types
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.algorithms.cleaning import pressure as pressure_cleaning
|
||||||
|
|
||||||
|
|
||||||
def _load_pressure_cleaning_module():
|
DATA_DIR = Path(__file__).resolve().parents[3] / "data"
|
||||||
project_root = Path(__file__).resolve().parents[2]
|
RAW_DATA_PATH = DATA_DIR / "node_simulation.csv"
|
||||||
utils_path = project_root / "app" / "algorithms" / "_utils.py"
|
NOISY_DATA_PATH = DATA_DIR / "node_simulation_noisy.csv"
|
||||||
pressure_path = project_root / "app" / "algorithms" / "cleaning" / "pressure.py"
|
REQUIRES_PRESSURE_SAMPLES = pytest.mark.skipif(
|
||||||
|
not RAW_DATA_PATH.exists() or not NOISY_DATA_PATH.exists(),
|
||||||
app_module = sys.modules.setdefault("app", types.ModuleType("app"))
|
reason="pressure cleaning sample CSV files are not available",
|
||||||
algorithms_module = sys.modules.setdefault(
|
)
|
||||||
"app.algorithms",
|
|
||||||
types.ModuleType("app.algorithms"),
|
|
||||||
)
|
|
||||||
setattr(app_module, "algorithms", algorithms_module)
|
|
||||||
|
|
||||||
utils_spec = importlib.util.spec_from_file_location("app.algorithms._utils", utils_path)
|
|
||||||
assert utils_spec and utils_spec.loader
|
|
||||||
utils_module = importlib.util.module_from_spec(utils_spec)
|
|
||||||
sys.modules["app.algorithms._utils"] = utils_module
|
|
||||||
utils_spec.loader.exec_module(utils_module)
|
|
||||||
|
|
||||||
pressure_spec = importlib.util.spec_from_file_location(
|
|
||||||
"tests_pressure_under_test",
|
|
||||||
pressure_path,
|
|
||||||
)
|
|
||||||
assert pressure_spec and pressure_spec.loader
|
|
||||||
pressure_module = importlib.util.module_from_spec(pressure_spec)
|
|
||||||
pressure_spec.loader.exec_module(pressure_module)
|
|
||||||
return pressure_module
|
|
||||||
|
|
||||||
|
|
||||||
|
@REQUIRES_PRESSURE_SAMPLES
|
||||||
def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
||||||
module = _load_pressure_cleaning_module()
|
raw_df = pd.read_csv(RAW_DATA_PATH)
|
||||||
repo_root = Path(__file__).resolve().parents[3]
|
noisy_df = pd.read_csv(NOISY_DATA_PATH)
|
||||||
|
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(noisy_df)
|
||||||
raw_df = pd.read_csv(repo_root / "data" / "node_simulation.csv")
|
|
||||||
noisy_df = pd.read_csv(repo_root / "data" / "node_simulation_noisy.csv")
|
|
||||||
cleaned_df = module.clean_pressure_data_df_km(noisy_df)
|
|
||||||
|
|
||||||
for df in (raw_df, noisy_df, cleaned_df):
|
for df in (raw_df, noisy_df, cleaned_df):
|
||||||
df["time"] = pd.to_datetime(df["time"])
|
df["time"] = pd.to_datetime(df["time"])
|
||||||
@@ -50,7 +28,12 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
|||||||
assert set(cleaned_df.columns) == {"time", "id", "pressure"}
|
assert set(cleaned_df.columns) == {"time", "id", "pressure"}
|
||||||
assert cleaned_df["pressure"].isna().sum() == 0
|
assert cleaned_df["pressure"].isna().sum() == 0
|
||||||
|
|
||||||
noisy_joined = raw_df.merge(noisy_df, on=["time", "id"], how="inner", suffixes=("_raw", "_noisy"))
|
noisy_joined = raw_df.merge(
|
||||||
|
noisy_df,
|
||||||
|
on=["time", "id"],
|
||||||
|
how="inner",
|
||||||
|
suffixes=("_raw", "_noisy"),
|
||||||
|
)
|
||||||
cleaned_joined = raw_df.merge(
|
cleaned_joined = raw_df.merge(
|
||||||
cleaned_df,
|
cleaned_df,
|
||||||
on=["time", "id"],
|
on=["time", "id"],
|
||||||
@@ -59,10 +42,20 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
|||||||
)
|
)
|
||||||
|
|
||||||
noisy_rmse = float(
|
noisy_rmse = float(
|
||||||
np.sqrt(np.mean((noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]) ** 2))
|
np.sqrt(
|
||||||
|
np.mean(
|
||||||
|
(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"])
|
||||||
|
** 2
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
cleaned_rmse = float(
|
cleaned_rmse = float(
|
||||||
np.sqrt(np.mean((cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"]) ** 2))
|
np.sqrt(
|
||||||
|
np.mean(
|
||||||
|
(cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"])
|
||||||
|
** 2
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
noisy_mae = float(
|
noisy_mae = float(
|
||||||
np.mean(np.abs(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]))
|
np.mean(np.abs(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]))
|
||||||
@@ -88,11 +81,9 @@ def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
|||||||
assert abs(spike_row - 28.018701553344727) < 2.0
|
assert abs(spike_row - 28.018701553344727) < 2.0
|
||||||
|
|
||||||
|
|
||||||
|
@REQUIRES_PRESSURE_SAMPLES
|
||||||
def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_strings():
|
def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_strings():
|
||||||
module = _load_pressure_cleaning_module()
|
noisy_df = pd.read_csv(NOISY_DATA_PATH)
|
||||||
repo_root = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
noisy_df = pd.read_csv(repo_root / "data" / "node_simulation_noisy.csv")
|
|
||||||
single_sensor = (
|
single_sensor = (
|
||||||
noisy_df[noisy_df["id"] == 170490][["time", "pressure"]]
|
noisy_df[noisy_df["id"] == 170490][["time", "pressure"]]
|
||||||
.rename(columns={"pressure": "170490"})
|
.rename(columns={"pressure": "170490"})
|
||||||
@@ -102,7 +93,7 @@ def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_str
|
|||||||
pd.to_datetime(single_sensor["time"], utc=True).dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
pd.to_datetime(single_sensor["time"], utc=True).dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
)
|
)
|
||||||
|
|
||||||
cleaned_df = module.clean_pressure_data_df_km(single_sensor)
|
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(single_sensor)
|
||||||
|
|
||||||
assert len(cleaned_df) == 192
|
assert len(cleaned_df) == 192
|
||||||
assert cleaned_df["170490"].isna().sum() == 0
|
assert cleaned_df["170490"].isna().sum() == 0
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
from app.api.v1.endpoints import project_data
|
||||||
|
from app.infra.db.timescaledb import composite_queries
|
||||||
|
|
||||||
|
|
||||||
|
PROJECT_SCADA = {
|
||||||
|
"id": "fengyang-pressure-1",
|
||||||
|
"type": "pressure",
|
||||||
|
"associated_element_id": "J1",
|
||||||
|
"api_query_id": "query-1",
|
||||||
|
"transmission_mode": "realtime",
|
||||||
|
"transmission_frequency": None,
|
||||||
|
"reliability": 1.0,
|
||||||
|
"x": 117.1,
|
||||||
|
"y": 32.9,
|
||||||
|
}
|
||||||
|
START_TIME = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
END_TIME = datetime(2026, 6, 2, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_project_scadas(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(return_value=[PROJECT_SCADA.copy()]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_scada_simulation_uses_current_project_metadata(monkeypatch):
|
||||||
|
_patch_project_scadas(monkeypatch)
|
||||||
|
query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.RealtimeRepository,
|
||||||
|
"get_node_field_by_time_range",
|
||||||
|
query_mock,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.get_scada_associated_realtime_simulation_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
[PROJECT_SCADA["id"]],
|
||||||
|
START_TIME,
|
||||||
|
END_TIME,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result[PROJECT_SCADA["id"]][0]["scada_id"] == PROJECT_SCADA["id"]
|
||||||
|
assert query_mock.await_count == 1
|
||||||
|
assert query_mock.await_args.args[1:] == (
|
||||||
|
START_TIME,
|
||||||
|
END_TIME,
|
||||||
|
"J1",
|
||||||
|
"pressure",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scheme_scada_simulation_uses_current_project_metadata(monkeypatch):
|
||||||
|
_patch_project_scadas(monkeypatch)
|
||||||
|
query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.SchemeRepository,
|
||||||
|
"get_node_field_by_scheme_and_time_range",
|
||||||
|
query_mock,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.get_scada_associated_scheme_simulation_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
[PROJECT_SCADA["id"]],
|
||||||
|
START_TIME,
|
||||||
|
END_TIME,
|
||||||
|
"baseline",
|
||||||
|
"scheme-1",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result[PROJECT_SCADA["id"]][0]["scada_id"] == PROJECT_SCADA["id"]
|
||||||
|
assert query_mock.await_args.args[5:] == ("J1", "pressure")
|
||||||
|
|
||||||
|
|
||||||
|
def test_element_scada_query_uses_current_project_metadata(monkeypatch):
|
||||||
|
_patch_project_scadas(monkeypatch)
|
||||||
|
query_mock = AsyncMock(
|
||||||
|
return_value={
|
||||||
|
PROJECT_SCADA["id"]: [{"time": START_TIME, "value": 26.5}]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"get_scada_field_by_id_time_range",
|
||||||
|
query_mock,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.get_element_associated_scada_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
"J1",
|
||||||
|
START_TIME,
|
||||||
|
END_TIME,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {"J1": [{"time": START_TIME, "value": 26.5}]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_scada_info_endpoint_uses_current_project_connection(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
project_data.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(return_value=[PROJECT_SCADA.copy()]),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(project_data.get_scada_info_with_connection(object()))
|
||||||
|
|
||||||
|
assert result == {"success": True, "data": [PROJECT_SCADA], "count": 1}
|
||||||
@@ -1,48 +1,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
import importlib.util
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
from types import ModuleType
|
|
||||||
|
|
||||||
|
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
||||||
def _load_time_api_module():
|
|
||||||
module_path = (
|
|
||||||
Path(__file__).resolve().parents[2] / "app" / "services" / "time_api.py"
|
|
||||||
)
|
|
||||||
spec = importlib.util.spec_from_file_location("tests_time_api_under_test", module_path)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
assert spec and spec.loader
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def _load_realtime_repository():
|
|
||||||
time_api_module = _load_time_api_module()
|
|
||||||
app_module = ModuleType("app")
|
|
||||||
services_module = ModuleType("app.services")
|
|
||||||
services_module.time_api = time_api_module
|
|
||||||
app_module.services = services_module
|
|
||||||
sys.modules["app"] = app_module
|
|
||||||
sys.modules["app.services"] = services_module
|
|
||||||
sys.modules["app.services.time_api"] = time_api_module
|
|
||||||
|
|
||||||
module_path = (
|
|
||||||
Path(__file__).resolve().parents[2]
|
|
||||||
/ "app"
|
|
||||||
/ "infra"
|
|
||||||
/ "db"
|
|
||||||
/ "timescaledb"
|
|
||||||
/ "repositories"
|
|
||||||
/ "realtime.py"
|
|
||||||
)
|
|
||||||
spec = importlib.util.spec_from_file_location(
|
|
||||||
"tests_realtime_repo_under_test", module_path
|
|
||||||
)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
assert spec and spec.loader
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return module.RealtimeRepository
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeCursor:
|
class _FakeCursor:
|
||||||
@@ -71,7 +30,6 @@ class _FakeConnection:
|
|||||||
|
|
||||||
|
|
||||||
def test_get_links_by_time_range_normalizes_inputs_to_utc():
|
def test_get_links_by_time_range_normalizes_inputs_to_utc():
|
||||||
RealtimeRepository = _load_realtime_repository()
|
|
||||||
conn = _FakeConnection()
|
conn = _FakeConnection()
|
||||||
|
|
||||||
asyncio.run(
|
asyncio.run(
|
||||||
@@ -91,7 +49,6 @@ def test_get_links_by_time_range_normalizes_inputs_to_utc():
|
|||||||
|
|
||||||
|
|
||||||
def test_get_nodes_by_time_range_normalizes_inputs_to_utc():
|
def test_get_nodes_by_time_range_normalizes_inputs_to_utc():
|
||||||
RealtimeRepository = _load_realtime_repository()
|
|
||||||
conn = _FakeConnection()
|
conn = _FakeConnection()
|
||||||
|
|
||||||
asyncio.run(
|
asyncio.run(
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import pytest
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from app.api.v1.endpoints.timeseries import composite as composite_endpoint
|
||||||
|
from app.infra.db.timescaledb import composite_queries
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_scada_uses_current_project_metadata(monkeypatch):
|
||||||
|
"""Fengyang data must not be classified with the global tjwater metadata."""
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(
|
||||||
|
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"get_scada_field_by_id_time_range",
|
||||||
|
AsyncMock(
|
||||||
|
return_value={
|
||||||
|
"fengyang-pressure-1": [
|
||||||
|
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
update_mock = AsyncMock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"update_scada_field",
|
||||||
|
update_mock,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries,
|
||||||
|
"clean_pressure_data_df_km",
|
||||||
|
lambda frame: pd.DataFrame(
|
||||||
|
{
|
||||||
|
"time": frame["time"],
|
||||||
|
"fengyang-pressure-1": frame["fengyang-pressure-1"],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.clean_scada_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
["fengyang-pressure-1"],
|
||||||
|
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "success"
|
||||||
|
update_mock.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_scada_rejects_devices_missing_from_project_metadata(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(return_value=[{"id": "other-device", "type": "pressure"}]),
|
||||||
|
)
|
||||||
|
query_mock = AsyncMock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"get_scada_field_by_id_time_range",
|
||||||
|
query_mock,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="缺少元数据"):
|
||||||
|
asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.clean_scada_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
["fengyang-pressure-1"],
|
||||||
|
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
query_mock.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_scada_rejects_zero_database_updates(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(
|
||||||
|
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"get_scada_field_by_id_time_range",
|
||||||
|
AsyncMock(
|
||||||
|
return_value={
|
||||||
|
"fengyang-pressure-1": [
|
||||||
|
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
update_mock = AsyncMock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"update_scada_field",
|
||||||
|
update_mock,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries,
|
||||||
|
"clean_pressure_data_df_km",
|
||||||
|
lambda _frame: pd.DataFrame(
|
||||||
|
{"time": [], "fengyang-pressure-1": []}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="未产生任何数据库更新"):
|
||||||
|
asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.clean_scada_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
["fengyang-pressure-1"],
|
||||||
|
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
update_mock.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_scada_propagates_write_failures(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaInfoRepository,
|
||||||
|
"get_scadas",
|
||||||
|
AsyncMock(
|
||||||
|
return_value=[{"id": "fengyang-pressure-1", "type": "pressure"}]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"get_scada_field_by_id_time_range",
|
||||||
|
AsyncMock(
|
||||||
|
return_value={
|
||||||
|
"fengyang-pressure-1": [
|
||||||
|
{"time": "2026-06-01T00:00:00+08:00", "value": 26.5}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries.ScadaRepository,
|
||||||
|
"update_scada_field",
|
||||||
|
AsyncMock(side_effect=RuntimeError("database write failed")),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_queries,
|
||||||
|
"clean_pressure_data_df_km",
|
||||||
|
lambda frame: frame,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="database write failed"):
|
||||||
|
asyncio.run(
|
||||||
|
composite_queries.CompositeQueries.clean_scada_data(
|
||||||
|
object(),
|
||||||
|
object(),
|
||||||
|
["fengyang-pressure-1"],
|
||||||
|
datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_scada_endpoint_returns_http_400_for_validation_error(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
composite_endpoint.CompositeQueries,
|
||||||
|
"clean_scada_data",
|
||||||
|
AsyncMock(side_effect=ValueError("当前项目没有可清洗的 SCADA 设备")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
asyncio.run(
|
||||||
|
composite_endpoint.clean_scada_data(
|
||||||
|
device_ids="all",
|
||||||
|
start_time=datetime(2026, 6, 1, tzinfo=timezone.utc),
|
||||||
|
end_time=datetime(2026, 6, 2, tzinfo=timezone.utc),
|
||||||
|
timescale_conn=object(),
|
||||||
|
postgres_conn=object(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.status_code == 400
|
||||||
|
assert exc_info.value.detail == "当前项目没有可清洗的 SCADA 设备"
|
||||||
Reference in New Issue
Block a user