jiang
9b095c7439
- make realtime replacement and analysis result writes transactional\n- consolidate SCADA repositories and remove process-global project state\n- validate SCADA batches and use indexed GIS-backed business queries\n\nBREAKING CHANGE: remove the public analysis result writer and the pipeline-health network_name query parameter.
620 lines
23 KiB
Python
620 lines
23 KiB
Python
import time
|
||
from datetime import datetime, timedelta
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
from psycopg import AsyncConnection
|
||
from uuid import UUID
|
||
|
||
from app.algorithms.cleaning.flow import clean_flow_data_df_kf
|
||
from app.algorithms.cleaning.pressure import clean_pressure_data_df_km
|
||
from app.algorithms.health.analyzer import PipelineHealthAnalyzer
|
||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
||
from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository
|
||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||
|
||
|
||
class CompositeQueries:
|
||
"""
|
||
复合查询类，提供跨表查询功能
|
||
"""
|
||
|
||
@staticmethod
|
||
async def _get_project_scada_index(
|
||
postgres_conn: AsyncConnection,
|
||
) -> Dict[str, Dict[str, Any]]:
|
||
scadas = await ScadaInfoRepository.get_scadas(postgres_conn)
|
||
return {scada["device_id"]: scada for scada in scadas}
|
||
|
||
@staticmethod
|
||
async def get_scada_associated_realtime_simulation_data(
|
||
timescale_conn: AsyncConnection,
|
||
postgres_conn: AsyncConnection,
|
||
device_ids: List[str],
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
) -> Dict[str, List[Dict[str, Any]]]:
|
||
"""
|
||
获取 SCADA 关联的 link/node 模拟值
|
||
|
||
根据传入的 SCADA device_ids，找到关联的 link/node，
|
||
并根据对应的 type，查询对应的模拟数据
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
postgres_conn: PostgreSQL 异步连接
|
||
device_ids: SCADA 设备ID列表
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
|
||
Returns:
|
||
模拟数据字典，以 device_id 为键，值为数据列表，每个数据包含 time, value 和 scada_id
|
||
|
||
Raises:
|
||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||
"""
|
||
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||
link_devices: dict[str, str] = {}
|
||
node_devices: dict[str, str] = {}
|
||
for device_id in device_ids:
|
||
target_scada = scada_by_id.get(device_id)
|
||
if not target_scada:
|
||
raise ValueError(f"SCADA device {device_id} not found")
|
||
scada_type = target_scada["device_type"]
|
||
if scada_type in {"pipe_flow", "flow"}:
|
||
link_devices[device_id] = target_scada["link_id"]
|
||
elif scada_type == "pressure":
|
||
node_devices[device_id] = target_scada["node_id"]
|
||
else:
|
||
raise ValueError(f"Unknown SCADA type: {scada_type}")
|
||
|
||
link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
|
||
timescale_conn, start_time, end_time,
|
||
list(dict.fromkeys(link_devices.values())), "flow",
|
||
)
|
||
node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
|
||
timescale_conn, start_time, end_time,
|
||
list(dict.fromkeys(node_devices.values())), "pressure",
|
||
)
|
||
return {
|
||
device_id: [
|
||
{**item, "scada_id": device_id}
|
||
for item in (
|
||
link_series.get(element_id, [])
|
||
if device_id in link_devices
|
||
else node_series.get(element_id, [])
|
||
)
|
||
]
|
||
for device_id, element_id in (link_devices | node_devices).items()
|
||
}
|
||
|
||
@staticmethod
|
||
async def get_scada_associated_analysis_simulation_data(
|
||
timescale_conn: AsyncConnection,
|
||
postgres_conn: AsyncConnection,
|
||
device_ids: List[str],
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
run_id: UUID,
|
||
) -> Dict[str, List[Dict[str, Any]]]:
|
||
"""
|
||
获取 SCADA 关联的 link/node 分析模拟值
|
||
|
||
根据传入的 SCADA device_ids，找到关联的 link/node，
|
||
并根据对应的 type，查询对应的模拟数据
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
postgres_conn: PostgreSQL 异步连接
|
||
device_ids: SCADA 设备ID列表
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
|
||
Returns:
|
||
模拟数据字典，以 device_id 为键，值为数据列表，每个数据包含 time, value 和 scada_id
|
||
|
||
Raises:
|
||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||
"""
|
||
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||
link_devices: dict[str, str] = {}
|
||
node_devices: dict[str, str] = {}
|
||
for device_id in device_ids:
|
||
target_scada = scada_by_id.get(device_id)
|
||
if not target_scada:
|
||
raise ValueError(f"SCADA device {device_id} not found")
|
||
|
||
scada_type = target_scada["device_type"]
|
||
if scada_type in {"pipe_flow", "flow"}:
|
||
link_devices[device_id] = target_scada["link_id"]
|
||
elif scada_type == "pressure":
|
||
node_devices[device_id] = target_scada["node_id"]
|
||
else:
|
||
raise ValueError(f"Unknown SCADA type: {scada_type}")
|
||
|
||
link_series = await AnalysisResultsRepository.get_series_by_ids(
|
||
timescale_conn, run_id, "link",
|
||
list(dict.fromkeys(link_devices.values())), start_time, end_time, "flow",
|
||
)
|
||
node_series = await AnalysisResultsRepository.get_series_by_ids(
|
||
timescale_conn, run_id, "node",
|
||
list(dict.fromkeys(node_devices.values())), start_time, end_time, "pressure",
|
||
)
|
||
return {
|
||
device_id: [
|
||
{**item, "scada_id": device_id}
|
||
for item in (
|
||
link_series.get(element_id, [])
|
||
if device_id in link_devices
|
||
else node_series.get(element_id, [])
|
||
)
|
||
]
|
||
for device_id, element_id in (link_devices | node_devices).items()
|
||
}
|
||
|
||
@staticmethod
|
||
async def get_realtime_simulation_data(
|
||
timescale_conn: AsyncConnection,
|
||
feature_infos: List[Tuple[str, str]],
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
) -> Dict[str, List[Dict[str, Any]]]:
|
||
"""
|
||
获取 link/node 模拟值
|
||
|
||
根据传入的 feature_infos，找到关联的 link/node，
|
||
并根据对应的 type，查询对应的模拟数据
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
feature_infos: 传入的 feature 信息列表，包含 (element_id, type)
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
|
||
Returns:
|
||
模拟数据字典，以 feature_id 为键，值为数据列表，每个数据包含 time, value 和 feature_id
|
||
|
||
Raises:
|
||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||
"""
|
||
pipe_ids: list[str] = []
|
||
junction_ids: list[str] = []
|
||
for feature_id, feature_type in feature_infos:
|
||
if feature_type.lower() == "pipe":
|
||
pipe_ids.append(feature_id)
|
||
elif feature_type.lower() == "junction":
|
||
junction_ids.append(feature_id)
|
||
else:
|
||
raise ValueError(f"Unknown type: {feature_type}")
|
||
link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
|
||
timescale_conn, start_time, end_time, list(dict.fromkeys(pipe_ids)), "flow"
|
||
)
|
||
node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
|
||
timescale_conn, start_time, end_time,
|
||
list(dict.fromkeys(junction_ids)), "pressure",
|
||
)
|
||
return {
|
||
feature_id: [
|
||
{**item, "feature_id": feature_id}
|
||
for item in (
|
||
link_series.get(feature_id, [])
|
||
if feature_type.lower() == "pipe"
|
||
else node_series.get(feature_id, [])
|
||
)
|
||
]
|
||
for feature_id, feature_type in feature_infos
|
||
}
|
||
|
||
@staticmethod
|
||
async def get_analysis_simulation_data(
|
||
timescale_conn: AsyncConnection,
|
||
feature_infos: List[Tuple[str, str]],
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
run_id: UUID,
|
||
) -> Dict[str, List[Dict[str, Any]]]:
|
||
"""
|
||
获取 link/node 分析模拟值
|
||
|
||
根据传入的 feature_infos，找到关联的 link/node，
|
||
并根据对应的 type，查询对应的模拟数据
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
feature_infos: 传入的 feature 信息列表，包含 (element_id, type)
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
run_id: 分析运行 ID
|
||
|
||
Returns:
|
||
模拟数据字典，以 feature_id 为键，值为数据列表，每个数据包含 time, value 和 feature_id
|
||
|
||
Raises:
|
||
ValueError: 当类型无效时
|
||
"""
|
||
pipe_ids: list[str] = []
|
||
junction_ids: list[str] = []
|
||
for feature_id, feature_type in feature_infos:
|
||
if feature_type.lower() == "pipe":
|
||
pipe_ids.append(feature_id)
|
||
elif feature_type.lower() == "junction":
|
||
junction_ids.append(feature_id)
|
||
else:
|
||
raise ValueError(f"Unknown type: {feature_type}")
|
||
link_series = await AnalysisResultsRepository.get_series_by_ids(
|
||
timescale_conn, run_id, "link", list(dict.fromkeys(pipe_ids)),
|
||
start_time, end_time, "flow",
|
||
)
|
||
node_series = await AnalysisResultsRepository.get_series_by_ids(
|
||
timescale_conn, run_id, "node", list(dict.fromkeys(junction_ids)),
|
||
start_time, end_time, "pressure",
|
||
)
|
||
return {
|
||
feature_id: [
|
||
{**item, "feature_id": feature_id}
|
||
for item in (
|
||
link_series.get(feature_id, [])
|
||
if feature_type.lower() == "pipe"
|
||
else node_series.get(feature_id, [])
|
||
)
|
||
]
|
||
for feature_id, feature_type in feature_infos
|
||
}
|
||
|
||
@staticmethod
|
||
async def get_element_associated_scada_data(
|
||
timescale_conn: AsyncConnection,
|
||
postgres_conn: AsyncConnection,
|
||
element_id: str,
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
use_cleaned: bool = False,
|
||
) -> Optional[Any]:
|
||
"""
|
||
获取 link/node 关联的 SCADA 监测值
|
||
|
||
根据传入的 link/node id，匹配 SCADA 信息，
|
||
如果存在关联的 SCADA device_id，获取实际的监测数据
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
postgres_conn: PostgreSQL 异步连接
|
||
element_id: link 或 node 的 ID
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
use_cleaned: 是否使用清洗后的数据 (True: "cleaned_value", False: "monitored_value")
|
||
|
||
Returns:
|
||
SCADA 监测数据值，如果没有找到则返回 None
|
||
|
||
Raises:
|
||
ValueError: 当元素类型无效时
|
||
"""
|
||
|
||
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||
associated_scada = next(
|
||
(
|
||
scada
|
||
for scada in scada_by_id.values()
|
||
if (scada.get("node_id") or scada.get("link_id")) == element_id
|
||
),
|
||
None,
|
||
)
|
||
|
||
if not associated_scada:
|
||
return None
|
||
|
||
device_id = associated_scada["device_id"]
|
||
|
||
data_field = "cleaned_value" if use_cleaned else "monitored_value"
|
||
|
||
res = await ScadaRepository.get_scada_field_by_id_time_range(
|
||
timescale_conn, [device_id], start_time, end_time, data_field
|
||
)
|
||
|
||
return {element_id: res.get(device_id, [])}
|
||
|
||
@staticmethod
|
||
async def clean_scada_data(
|
||
timescale_conn: AsyncConnection,
|
||
postgres_conn: AsyncConnection,
|
||
device_ids: List[str],
|
||
start_time: datetime,
|
||
end_time: datetime,
|
||
) -> str:
|
||
"""
|
||
清洗 SCADA 数据
|
||
|
||
根据 device_ids 查询 monitored_value，清洗后更新 cleaned_value
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 连接
|
||
postgres_conn: PostgreSQL 连接
|
||
device_ids: 设备 ID 列表
|
||
start_time: 开始时间
|
||
end_time: 结束时间
|
||
|
||
Returns:
|
||
"success"
|
||
|
||
Raises:
|
||
ValueError: 当前项目没有可清洗设备或指定时间范围内没有监测数据
|
||
"""
|
||
scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
|
||
supported_types = {"pressure", "pipe_flow", "flow"}
|
||
|
||
if device_ids:
|
||
device_ids = [str(device_id).strip() for device_id in device_ids]
|
||
missing_metadata_ids = [
|
||
device_id
|
||
for device_id in device_ids
|
||
if device_id not in scada_by_id
|
||
]
|
||
if missing_metadata_ids:
|
||
raise ValueError(
|
||
f"当前项目中有 {len(missing_metadata_ids)} 个 SCADA 设备缺少元数据"
|
||
)
|
||
|
||
unsupported_ids = [
|
||
device_id
|
||
for device_id in device_ids
|
||
if scada_by_id[device_id]["device_type"] not in supported_types
|
||
]
|
||
if unsupported_ids:
|
||
raise ValueError(
|
||
f"当前项目中有 {len(unsupported_ids)} 个 SCADA 设备类型不支持清洗"
|
||
)
|
||
else:
|
||
device_ids = [
|
||
device_id
|
||
for device_id, info in scada_by_id.items()
|
||
if info["device_type"] in supported_types
|
||
]
|
||
|
||
if not device_ids:
|
||
raise ValueError("当前项目没有可清洗的 SCADA 设备")
|
||
|
||
data = await ScadaRepository.get_scada_field_by_id_time_range(
|
||
timescale_conn, device_ids, start_time, end_time, "monitored_value"
|
||
)
|
||
if not data:
|
||
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
||
|
||
normalized_data = {
|
||
str(device_id): records for device_id, records in data.items()
|
||
}
|
||
missing_data_ids = [
|
||
device_id for device_id in device_ids if not normalized_data.get(device_id)
|
||
]
|
||
if missing_data_ids:
|
||
raise ValueError(
|
||
f"指定时间范围内有 {len(missing_data_ids)} 个 SCADA 设备没有监测数据"
|
||
)
|
||
|
||
all_records = [
|
||
{
|
||
"time": record["time"],
|
||
"device_id": device_id,
|
||
"value": record["value"],
|
||
}
|
||
for device_id, records in normalized_data.items()
|
||
for record in records
|
||
]
|
||
if not all_records:
|
||
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
||
|
||
df_long = pd.DataFrame(all_records)
|
||
df = df_long.pivot(index="time", columns="device_id", values="value")
|
||
|
||
pressure_ids = [
|
||
device_id
|
||
for device_id in df.columns
|
||
if scada_by_id[device_id]["device_type"] == "pressure"
|
||
]
|
||
flow_ids = [
|
||
device_id
|
||
for device_id in df.columns
|
||
if scada_by_id[device_id]["device_type"] in {"pipe_flow", "flow"}
|
||
]
|
||
|
||
cleaned_rows: list[tuple[datetime, str, float | None]] = []
|
||
for grouped_ids, cleaning_function in (
|
||
(pressure_ids, clean_pressure_data_df_km),
|
||
(flow_ids, clean_flow_data_df_kf),
|
||
):
|
||
if not grouped_ids:
|
||
continue
|
||
|
||
source_df = df[grouped_ids].reset_index()
|
||
cleaned_df = cleaning_function(source_df)
|
||
time_values = cleaned_df["time"].tolist()
|
||
|
||
for device_id in grouped_ids:
|
||
if device_id not in cleaned_df.columns:
|
||
raise ValueError(f"设备 {device_id} 的清洗结果缺少数据列")
|
||
|
||
cleaned_values = cleaned_df[device_id].tolist()
|
||
for time_value, value in zip(time_values, cleaned_values):
|
||
time_dt = (
|
||
time_value
|
||
if isinstance(time_value, datetime)
|
||
else datetime.fromisoformat(str(time_value))
|
||
)
|
||
cleaned_rows.append(
|
||
(
|
||
time_dt,
|
||
device_id,
|
||
None if pd.isna(value) else float(value),
|
||
)
|
||
)
|
||
|
||
if not cleaned_rows:
|
||
raise ValueError("SCADA 数据清洗未产生任何数据库更新")
|
||
|
||
expected_rows = len({(row[0], row[1]) for row in cleaned_rows})
|
||
async with timescale_conn.transaction():
|
||
updated_rows = await ScadaRepository.update_scada_field_batch(
|
||
timescale_conn,
|
||
cleaned_rows,
|
||
"cleaned_value",
|
||
)
|
||
if updated_rows != expected_rows:
|
||
raise ValueError(
|
||
"SCADA 清洗目标在写入期间发生变化，"
|
||
f"预期更新 {expected_rows} 行，实际更新 {updated_rows} 行"
|
||
)
|
||
|
||
return "success"
|
||
|
||
@staticmethod
|
||
async def predict_pipeline_health(
|
||
timescale_conn: AsyncConnection,
|
||
postgres_conn: AsyncConnection,
|
||
query_time: datetime,
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
预测管道健康状况
|
||
|
||
根据管网名称和当前时间，查询管道信息和实时数据，
|
||
使用随机生存森林模型预测管道的生存概率
|
||
|
||
Args:
|
||
timescale_conn: TimescaleDB 异步连接
|
||
query_time: 查询时间
|
||
property_conditions: 可选的管道筛选条件，如 {"diameter": 300}
|
||
|
||
Returns:
|
||
预测结果列表，每个元素包含 link_id 和对应的生存函数
|
||
|
||
Raises:
|
||
ValueError: 当参数无效或数据不足时
|
||
FileNotFoundError: 当模型文件未找到时
|
||
"""
|
||
try:
|
||
# 1. 准备时间范围（查询时间前后1秒）
|
||
start_time = query_time - timedelta(seconds=1)
|
||
end_time = query_time + timedelta(seconds=1)
|
||
|
||
# 2. 先查询流速数据（velocity），获取有数据的管道ID列表
|
||
velocity_data = await RealtimeRepository.get_links_field_by_time_range(
|
||
timescale_conn, start_time, end_time, "velocity"
|
||
)
|
||
|
||
if not velocity_data:
|
||
raise ValueError("未找到流速数据")
|
||
|
||
# 3. 只查询有流速数据的管道的基本信息
|
||
valid_link_ids = list(velocity_data.keys())
|
||
|
||
# GIS 物化视图是低频更新管网的查询面；只读取本次有结果的管道。
|
||
async with postgres_conn.cursor() as cur:
|
||
await cur.execute(
|
||
"""
|
||
SELECT id, diameter, start_node_id AS node1,
|
||
end_node_id AS node2
|
||
FROM gis.pipes
|
||
WHERE id = ANY(%s)
|
||
""",
|
||
(valid_link_ids,),
|
||
)
|
||
all_links = await cur.fetchall()
|
||
|
||
# 转换为字典以快速查找
|
||
links_dict = {str(link["id"]): link for link in all_links}
|
||
|
||
# 获取所有需要查询的节点ID
|
||
node_ids = set()
|
||
for link_id in valid_link_ids:
|
||
if link_id in links_dict:
|
||
link = links_dict[link_id]
|
||
node_ids.add(link["node1"])
|
||
node_ids.add(link["node2"])
|
||
|
||
# 4. 批量查询压力数据（pressure）
|
||
pressure_data = await RealtimeRepository.get_nodes_field_by_time_range(
|
||
timescale_conn, start_time, end_time, "pressure"
|
||
)
|
||
|
||
# 5. 组合数据结构
|
||
materials = []
|
||
diameters = []
|
||
velocities = []
|
||
pressures = []
|
||
link_ids = []
|
||
|
||
for link_id in valid_link_ids:
|
||
# 跳过不在管道字典中的ID（如泵等其他元素）
|
||
if link_id not in links_dict:
|
||
continue
|
||
|
||
link = links_dict[link_id]
|
||
diameter = link["diameter"]
|
||
node1 = link["node1"]
|
||
node2 = link["node2"]
|
||
|
||
# 获取流速数据
|
||
velocity_values = velocity_data[link_id]
|
||
velocity = velocity_values[-1]["value"] if velocity_values else 0
|
||
|
||
# 获取node1和node2的压力数据，计算平均值
|
||
node1_pressure = 0
|
||
node2_pressure = 0
|
||
|
||
if node1 in pressure_data and pressure_data[node1]:
|
||
pressure_values = pressure_data[node1]
|
||
node1_pressure = (
|
||
pressure_values[-1]["value"] if pressure_values else 0
|
||
)
|
||
|
||
if node2 in pressure_data and pressure_data[node2]:
|
||
pressure_values = pressure_data[node2]
|
||
node2_pressure = (
|
||
pressure_values[-1]["value"] if pressure_values else 0
|
||
)
|
||
|
||
# 计算平均压力
|
||
avg_pressure = (node1_pressure + node2_pressure) / 2
|
||
|
||
# 添加到列表
|
||
link_ids.append(link_id)
|
||
materials.append(7) # 默认材料类型为7，可根据实际情况调整
|
||
diameters.append(diameter)
|
||
velocities.append(velocity)
|
||
pressures.append(avg_pressure)
|
||
|
||
if not link_ids:
|
||
raise ValueError("没有找到有效的管道数据用于预测")
|
||
|
||
# 6. 创建DataFrame
|
||
data = pd.DataFrame(
|
||
{
|
||
"Material": materials,
|
||
"Diameter": diameters,
|
||
"Flow Velocity": velocities,
|
||
"Pressure": pressures,
|
||
}
|
||
)
|
||
|
||
# 7. 使用PipelineHealthAnalyzer进行预测
|
||
analyzer = PipelineHealthAnalyzer()
|
||
survival_functions = analyzer.predict_survival(data)
|
||
# 8. 组合结果
|
||
results = []
|
||
for i, link_id in enumerate(link_ids):
|
||
sf = survival_functions[i]
|
||
results.append(
|
||
{
|
||
"link_id": link_id,
|
||
"survival_function": {
|
||
"x": sf.x.tolist(), # 时间点（年）
|
||
"y": sf.y.tolist(), # 生存概率
|
||
},
|
||
}
|
||
)
|
||
return results
|
||
|
||
except Exception as e:
|
||
raise ValueError(f"管道健康预测失败: {str(e)}")
|