jiang
6a4478bfe4
Remove retired query paths and repository compatibility branches, and share task test fixtures. Cancel pending questions and resolve their notifications when recovering expired runs.
323 lines
12 KiB
Python
323 lines
12 KiB
Python
import time
|
|
from datetime import datetime, timedelta
|
|
from typing import Any, Dict, List
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from psycopg import AsyncConnection
|
|
|
|
from app.algorithms.pipe_health_prediction.survival_predictor import (
|
|
PipeHealthSurvivalPredictor,
|
|
)
|
|
from app.algorithms.scada_cleaning.flow_series import clean_flow_data_df_kf
|
|
from app.algorithms.scada_cleaning.pressure_series import clean_pressure_data_df_km
|
|
from app.infra.db.postgresql.network_assets import NetworkAssetRepository
|
|
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
|
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
|
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
|
|
|
|
|
class TimeseriesAnalysisService:
|
|
"""SCADA 清洗与管道健康预测；历史查询由 TimeseriesHistoryService 负责。"""
|
|
|
|
@staticmethod
|
|
async def _get_project_scada_index(
|
|
postgres_conn: AsyncConnection,
|
|
) -> Dict[str, Dict[str, Any]]:
|
|
scadas = await ScadaInfoRepository.get_scadas(postgres_conn)
|
|
return {scada["device_id"]: scada for scada in scadas}
|
|
|
|
@staticmethod
|
|
async def clean_scada_data(
|
|
timescale_conn: AsyncConnection,
|
|
postgres_conn: AsyncConnection,
|
|
device_ids: List[str],
|
|
start_time: datetime,
|
|
end_time: datetime,
|
|
) -> str:
|
|
"""
|
|
清洗 SCADA 数据
|
|
|
|
根据 device_ids 查询 monitored_value，清洗后更新 cleaned_value
|
|
|
|
Args:
|
|
timescale_conn: TimescaleDB 连接
|
|
postgres_conn: PostgreSQL 连接
|
|
device_ids: 设备 ID 列表
|
|
start_time: 开始时间
|
|
end_time: 结束时间
|
|
|
|
Returns:
|
|
"success"
|
|
|
|
Raises:
|
|
ValueError: 当前项目没有可清洗设备或指定时间范围内没有监测数据
|
|
"""
|
|
scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn)
|
|
supported_types = {"pressure", "pipe_flow", "flow"}
|
|
|
|
if device_ids:
|
|
device_ids = [str(device_id).strip() for device_id in device_ids]
|
|
missing_metadata_ids = [
|
|
device_id
|
|
for device_id in device_ids
|
|
if device_id not in scada_by_id
|
|
]
|
|
if missing_metadata_ids:
|
|
raise ValueError(
|
|
f"当前项目中有 {len(missing_metadata_ids)} 个 SCADA 设备缺少元数据"
|
|
)
|
|
|
|
unsupported_ids = [
|
|
device_id
|
|
for device_id in device_ids
|
|
if scada_by_id[device_id]["device_type"] not in supported_types
|
|
]
|
|
if unsupported_ids:
|
|
raise ValueError(
|
|
f"当前项目中有 {len(unsupported_ids)} 个 SCADA 设备类型不支持清洗"
|
|
)
|
|
else:
|
|
device_ids = [
|
|
device_id
|
|
for device_id, info in scada_by_id.items()
|
|
if info["device_type"] in supported_types
|
|
]
|
|
|
|
if not device_ids:
|
|
raise ValueError("当前项目没有可清洗的 SCADA 设备")
|
|
|
|
data = await ScadaRepository.get_scada_field_by_id_time_range(
|
|
timescale_conn, device_ids, start_time, end_time, "monitored_value"
|
|
)
|
|
if not data:
|
|
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
|
|
|
normalized_data = {
|
|
str(device_id): records for device_id, records in data.items()
|
|
}
|
|
missing_data_ids = [
|
|
device_id for device_id in device_ids if not normalized_data.get(device_id)
|
|
]
|
|
if missing_data_ids:
|
|
raise ValueError(
|
|
f"指定时间范围内有 {len(missing_data_ids)} 个 SCADA 设备没有监测数据"
|
|
)
|
|
|
|
all_records = [
|
|
{
|
|
"time": record["time"],
|
|
"device_id": device_id,
|
|
"value": record["value"],
|
|
}
|
|
for device_id, records in normalized_data.items()
|
|
for record in records
|
|
]
|
|
if not all_records:
|
|
raise ValueError("指定时间范围内没有 SCADA 监测数据")
|
|
|
|
df_long = pd.DataFrame(all_records)
|
|
df = df_long.pivot(index="time", columns="device_id", values="value")
|
|
|
|
pressure_ids = [
|
|
device_id
|
|
for device_id in df.columns
|
|
if scada_by_id[device_id]["device_type"] == "pressure"
|
|
]
|
|
flow_ids = [
|
|
device_id
|
|
for device_id in df.columns
|
|
if scada_by_id[device_id]["device_type"] in {"pipe_flow", "flow"}
|
|
]
|
|
|
|
cleaned_rows: list[tuple[datetime, str, float | None]] = []
|
|
for grouped_ids, cleaning_function in (
|
|
(pressure_ids, clean_pressure_data_df_km),
|
|
(flow_ids, clean_flow_data_df_kf),
|
|
):
|
|
if not grouped_ids:
|
|
continue
|
|
|
|
source_df = df[grouped_ids].reset_index()
|
|
cleaned_df = cleaning_function(source_df)
|
|
time_values = cleaned_df["time"].tolist()
|
|
|
|
for device_id in grouped_ids:
|
|
if device_id not in cleaned_df.columns:
|
|
raise ValueError(f"设备 {device_id} 的清洗结果缺少数据列")
|
|
|
|
cleaned_values = cleaned_df[device_id].tolist()
|
|
for time_value, value in zip(time_values, cleaned_values):
|
|
time_dt = (
|
|
time_value
|
|
if isinstance(time_value, datetime)
|
|
else datetime.fromisoformat(str(time_value))
|
|
)
|
|
cleaned_rows.append(
|
|
(
|
|
time_dt,
|
|
device_id,
|
|
None if pd.isna(value) else float(value),
|
|
)
|
|
)
|
|
|
|
if not cleaned_rows:
|
|
raise ValueError("SCADA 数据清洗未产生任何数据库更新")
|
|
|
|
expected_rows = len({(row[0], row[1]) for row in cleaned_rows})
|
|
async with timescale_conn.transaction():
|
|
updated_rows = await ScadaRepository.update_scada_field_batch(
|
|
timescale_conn,
|
|
cleaned_rows,
|
|
"cleaned_value",
|
|
)
|
|
if updated_rows != expected_rows:
|
|
raise ValueError(
|
|
"SCADA 清洗目标在写入期间发生变化，"
|
|
f"预期更新 {expected_rows} 行，实际更新 {updated_rows} 行"
|
|
)
|
|
|
|
return "success"
|
|
|
|
@staticmethod
|
|
async def predict_pipeline_health(
|
|
timescale_conn: AsyncConnection,
|
|
postgres_conn: AsyncConnection,
|
|
query_time: datetime,
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
预测管道健康状况
|
|
|
|
根据管网名称和当前时间，查询管道信息和实时数据，
|
|
使用随机生存森林模型预测管道的生存概率
|
|
|
|
Args:
|
|
timescale_conn: TimescaleDB 异步连接
|
|
query_time: 查询时间
|
|
property_conditions: 可选的管道筛选条件，如 {"diameter": 300}
|
|
|
|
Returns:
|
|
预测结果列表，每个元素包含 link_id 和对应的生存函数
|
|
|
|
Raises:
|
|
ValueError: 当参数无效或数据不足时
|
|
FileNotFoundError: 当模型文件未找到时
|
|
"""
|
|
try:
|
|
# 1. 准备时间范围（查询时间前后1秒）
|
|
start_time = query_time - timedelta(seconds=1)
|
|
end_time = query_time + timedelta(seconds=1)
|
|
|
|
# 2. 先查询流速数据（velocity），获取有数据的管道ID列表
|
|
velocity_data = await RealtimeRepository.get_links_field_by_time_range(
|
|
timescale_conn, start_time, end_time, "velocity"
|
|
)
|
|
|
|
if not velocity_data:
|
|
raise ValueError("未找到流速数据")
|
|
|
|
# 3. 只查询有流速数据的管道的基本信息
|
|
valid_link_ids = list(velocity_data.keys())
|
|
|
|
# GIS 物化视图是低频更新管网的查询面；只读取本次有结果的管道。
|
|
all_links = await NetworkAssetRepository.get_pipes_by_ids(
|
|
postgres_conn, valid_link_ids
|
|
)
|
|
|
|
# 转换为字典以快速查找
|
|
links_dict = {str(link["id"]): link for link in all_links}
|
|
|
|
# 获取所有需要查询的节点ID
|
|
node_ids = set()
|
|
for link_id in valid_link_ids:
|
|
if link_id in links_dict:
|
|
link = links_dict[link_id]
|
|
node_ids.add(link["node1"])
|
|
node_ids.add(link["node2"])
|
|
|
|
# 4. 批量查询压力数据（pressure）
|
|
pressure_data = await RealtimeRepository.get_nodes_field_by_time_range(
|
|
timescale_conn, start_time, end_time, "pressure"
|
|
)
|
|
|
|
# 5. 组合数据结构
|
|
materials = []
|
|
diameters = []
|
|
velocities = []
|
|
pressures = []
|
|
link_ids = []
|
|
|
|
for link_id in valid_link_ids:
|
|
# 跳过不在管道字典中的ID（如泵等其他元素）
|
|
if link_id not in links_dict:
|
|
continue
|
|
|
|
link = links_dict[link_id]
|
|
diameter = link["diameter"]
|
|
node1 = link["node1"]
|
|
node2 = link["node2"]
|
|
|
|
# 获取流速数据
|
|
velocity_values = velocity_data[link_id]
|
|
velocity = velocity_values[-1]["value"] if velocity_values else 0
|
|
|
|
# 获取node1和node2的压力数据，计算平均值
|
|
node1_pressure = 0
|
|
node2_pressure = 0
|
|
|
|
if node1 in pressure_data and pressure_data[node1]:
|
|
pressure_values = pressure_data[node1]
|
|
node1_pressure = (
|
|
pressure_values[-1]["value"] if pressure_values else 0
|
|
)
|
|
|
|
if node2 in pressure_data and pressure_data[node2]:
|
|
pressure_values = pressure_data[node2]
|
|
node2_pressure = (
|
|
pressure_values[-1]["value"] if pressure_values else 0
|
|
)
|
|
|
|
# 计算平均压力
|
|
avg_pressure = (node1_pressure + node2_pressure) / 2
|
|
|
|
# 添加到列表
|
|
link_ids.append(link_id)
|
|
materials.append(7) # 默认材料类型为7，可根据实际情况调整
|
|
diameters.append(diameter)
|
|
velocities.append(velocity)
|
|
pressures.append(avg_pressure)
|
|
|
|
if not link_ids:
|
|
raise ValueError("没有找到有效的管道数据用于预测")
|
|
|
|
# 6. 创建DataFrame
|
|
data = pd.DataFrame(
|
|
{
|
|
"Material": materials,
|
|
"Diameter": diameters,
|
|
"Flow Velocity": velocities,
|
|
"Pressure": pressures,
|
|
}
|
|
)
|
|
|
|
# 7. 使用生存模型进行预测
|
|
analyzer = PipeHealthSurvivalPredictor()
|
|
survival_functions = analyzer.predict_survival(data)
|
|
# 8. 组合结果
|
|
results = []
|
|
for i, link_id in enumerate(link_ids):
|
|
sf = survival_functions[i]
|
|
results.append(
|
|
{
|
|
"link_id": link_id,
|
|
"survival_function": {
|
|
"x": sf.x.tolist(), # 时间点（年）
|
|
"y": sf.y.tolist(), # 生存概率
|
|
},
|
|
}
|
|
)
|
|
return results
|
|
|
|
except Exception as e:
|
|
raise ValueError(f"管道健康预测失败: {str(e)}")
|