import time from datetime import datetime, timedelta from typing import Any, Dict, List import numpy as np import pandas as pd from psycopg import AsyncConnection from app.algorithms.pipe_health_prediction.survival_predictor import ( PipeHealthSurvivalPredictor, ) from app.algorithms.scada_cleaning.flow_series import clean_flow_data_df_kf from app.algorithms.scada_cleaning.pressure_series import clean_pressure_data_df_km from app.infra.db.postgresql.network_assets import NetworkAssetRepository from app.infra.db.postgresql.scada import ScadaInfoRepository from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository class TimeseriesAnalysisService: """SCADA 清洗与管道健康预测；历史查询由 TimeseriesHistoryService 负责。""" @staticmethod async def _get_project_scada_index( postgres_conn: AsyncConnection, ) -> Dict[str, Dict[str, Any]]: scadas = await ScadaInfoRepository.get_scadas(postgres_conn) return {scada["device_id"]: scada for scada in scadas} @staticmethod async def clean_scada_data( timescale_conn: AsyncConnection, postgres_conn: AsyncConnection, device_ids: List[str], start_time: datetime, end_time: datetime, ) -> str: """ 清洗 SCADA 数据 根据 device_ids 查询 monitored_value，清洗后更新 cleaned_value Args: timescale_conn: TimescaleDB 连接 postgres_conn: PostgreSQL 连接 device_ids: 设备 ID 列表 start_time: 开始时间 end_time: 结束时间 Returns: "success" Raises: ValueError: 当前项目没有可清洗设备或指定时间范围内没有监测数据 """ scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn) supported_types = {"pressure", "pipe_flow", "flow"} if device_ids: device_ids = [str(device_id).strip() for device_id in device_ids] missing_metadata_ids = [ device_id for device_id in device_ids if device_id not in scada_by_id ] if missing_metadata_ids: raise ValueError( f"当前项目中有 {len(missing_metadata_ids)} 个 SCADA 设备缺少元数据" ) unsupported_ids = [ device_id for device_id in device_ids if scada_by_id[device_id]["device_type"] not in supported_types ] if unsupported_ids: raise ValueError( f"当前项目中有 {len(unsupported_ids)} 个 SCADA 设备类型不支持清洗" ) else: device_ids = [ device_id for device_id, info in scada_by_id.items() if info["device_type"] in supported_types ] if not device_ids: raise ValueError("当前项目没有可清洗的 SCADA 设备") data = await ScadaRepository.get_scada_field_by_id_time_range( timescale_conn, device_ids, start_time, end_time, "monitored_value" ) if not data: raise ValueError("指定时间范围内没有 SCADA 监测数据") normalized_data = { str(device_id): records for device_id, records in data.items() } missing_data_ids = [ device_id for device_id in device_ids if not normalized_data.get(device_id) ] if missing_data_ids: raise ValueError( f"指定时间范围内有 {len(missing_data_ids)} 个 SCADA 设备没有监测数据" ) all_records = [ { "time": record["time"], "device_id": device_id, "value": record["value"], } for device_id, records in normalized_data.items() for record in records ] if not all_records: raise ValueError("指定时间范围内没有 SCADA 监测数据") df_long = pd.DataFrame(all_records) df = df_long.pivot(index="time", columns="device_id", values="value") pressure_ids = [ device_id for device_id in df.columns if scada_by_id[device_id]["device_type"] == "pressure" ] flow_ids = [ device_id for device_id in df.columns if scada_by_id[device_id]["device_type"] in {"pipe_flow", "flow"} ] cleaned_rows: list[tuple[datetime, str, float | None]] = [] for grouped_ids, cleaning_function in ( (pressure_ids, clean_pressure_data_df_km), (flow_ids, clean_flow_data_df_kf), ): if not grouped_ids: continue source_df = df[grouped_ids].reset_index() cleaned_df = cleaning_function(source_df) time_values = cleaned_df["time"].tolist() for device_id in grouped_ids: if device_id not in cleaned_df.columns: raise ValueError(f"设备 {device_id} 的清洗结果缺少数据列") cleaned_values = cleaned_df[device_id].tolist() for time_value, value in zip(time_values, cleaned_values): time_dt = ( time_value if isinstance(time_value, datetime) else datetime.fromisoformat(str(time_value)) ) cleaned_rows.append( ( time_dt, device_id, None if pd.isna(value) else float(value), ) ) if not cleaned_rows: raise ValueError("SCADA 数据清洗未产生任何数据库更新") expected_rows = len({(row[0], row[1]) for row in cleaned_rows}) async with timescale_conn.transaction(): updated_rows = await ScadaRepository.update_scada_field_batch( timescale_conn, cleaned_rows, "cleaned_value", ) if updated_rows != expected_rows: raise ValueError( "SCADA 清洗目标在写入期间发生变化，" f"预期更新 {expected_rows} 行，实际更新 {updated_rows} 行" ) return "success" @staticmethod async def predict_pipeline_health( timescale_conn: AsyncConnection, postgres_conn: AsyncConnection, query_time: datetime, ) -> List[Dict[str, Any]]: """ 预测管道健康状况 根据管网名称和当前时间，查询管道信息和实时数据， 使用随机生存森林模型预测管道的生存概率 Args: timescale_conn: TimescaleDB 异步连接 query_time: 查询时间 property_conditions: 可选的管道筛选条件，如 {"diameter": 300} Returns: 预测结果列表，每个元素包含 link_id 和对应的生存函数 Raises: ValueError: 当参数无效或数据不足时 FileNotFoundError: 当模型文件未找到时 """ try: # 1. 准备时间范围（查询时间前后1秒） start_time = query_time - timedelta(seconds=1) end_time = query_time + timedelta(seconds=1) # 2. 先查询流速数据（velocity），获取有数据的管道ID列表 velocity_data = await RealtimeRepository.get_links_field_by_time_range( timescale_conn, start_time, end_time, "velocity" ) if not velocity_data: raise ValueError("未找到流速数据") # 3. 只查询有流速数据的管道的基本信息 valid_link_ids = list(velocity_data.keys()) # GIS 物化视图是低频更新管网的查询面；只读取本次有结果的管道。 all_links = await NetworkAssetRepository.get_pipes_by_ids( postgres_conn, valid_link_ids ) # 转换为字典以快速查找 links_dict = {str(link["id"]): link for link in all_links} # 获取所有需要查询的节点ID node_ids = set() for link_id in valid_link_ids: if link_id in links_dict: link = links_dict[link_id] node_ids.add(link["node1"]) node_ids.add(link["node2"]) # 4. 批量查询压力数据（pressure） pressure_data = await RealtimeRepository.get_nodes_field_by_time_range( timescale_conn, start_time, end_time, "pressure" ) # 5. 组合数据结构 materials = [] diameters = [] velocities = [] pressures = [] link_ids = [] for link_id in valid_link_ids: # 跳过不在管道字典中的ID（如泵等其他元素） if link_id not in links_dict: continue link = links_dict[link_id] diameter = link["diameter"] node1 = link["node1"] node2 = link["node2"] # 获取流速数据 velocity_values = velocity_data[link_id] velocity = velocity_values[-1]["value"] if velocity_values else 0 # 获取node1和node2的压力数据，计算平均值 node1_pressure = 0 node2_pressure = 0 if node1 in pressure_data and pressure_data[node1]: pressure_values = pressure_data[node1] node1_pressure = ( pressure_values[-1]["value"] if pressure_values else 0 ) if node2 in pressure_data and pressure_data[node2]: pressure_values = pressure_data[node2] node2_pressure = ( pressure_values[-1]["value"] if pressure_values else 0 ) # 计算平均压力 avg_pressure = (node1_pressure + node2_pressure) / 2 # 添加到列表 link_ids.append(link_id) materials.append(7) # 默认材料类型为7，可根据实际情况调整 diameters.append(diameter) velocities.append(velocity) pressures.append(avg_pressure) if not link_ids: raise ValueError("没有找到有效的管道数据用于预测") # 6. 创建DataFrame data = pd.DataFrame( { "Material": materials, "Diameter": diameters, "Flow Velocity": velocities, "Pressure": pressures, } ) # 7. 使用生存模型进行预测 analyzer = PipeHealthSurvivalPredictor() survival_functions = analyzer.predict_survival(data) # 8. 组合结果 results = [] for i, link_id in enumerate(link_ids): sf = survival_functions[i] results.append( { "link_id": link_id, "survival_function": { "x": sf.x.tolist(), # 时间点（年） "y": sf.y.tolist(), # 生存概率 }, } ) return results except Exception as e: raise ValueError(f"管道健康预测失败: {str(e)}")