import os from datetime import datetime from typing import Any import pandas as pd from app.algorithms.dma_leakage_estimation.genetic_optimizer import DmaLeakageOptimizer from app.algorithms.dma_leakage_estimation.topology_partitioning import ( build_dma_partitions, ) from app.infra.db.timescaledb.internal_queries import InternalQueries from app.infra.db.postgresql.scada import get_all_scada_info from app.native.wndb.gis.network_views import ( get_network_link_nodes, get_network_node_coords, ) from app.native.wndb.inp.exporter import dump_inp from app.services.scheme_management import store_analysis_run_with_result from app.domain.time import parse_utc_time, utc_now DEFAULT_N_WORKERS = max(1, min((os.cpu_count() or 1) - 1, 4)) def run_leakage_identification( network: str, username: str, observed_pressure_data: ( pd.DataFrame | dict[str, list[Any]] | list[dict[str, Any]] | None ) = None, start_time: float = 0, duration: float = 24, timestep: float = 5, q_sum: float = 0.2, q_sum_unit: str = "m3/s", pop_size: int = 50, max_gen: int = 100, n_workers: int = DEFAULT_N_WORKERS, output_flow_unit: str = "m3/s", dma_count: int | None = None, scada_start: datetime | str | None = None, scada_end: datetime | str | None = None, sensor_nodes: list[str] | None = None, scheme_name: str | None = None, ) -> dict[str, Any]: inp_path = _prepare_leakage_inp(network) selected_sensor_nodes = ( list(dict.fromkeys([node for node in (sensor_nodes or []) if node])) if sensor_nodes else _get_pressure_sensor_nodes(network) ) if not selected_sensor_nodes: raise ValueError("未提供有效传感器节点,且系统未识别到可用压力传感器。") area_map, areas, node_coords = _build_area_map_by_topology( network, selected_sensor_nodes, dma_count ) observed_source = "request_payload" if scada_start is not None or scada_end is not None: observed_df = _build_observed_pressure_from_scada( network=network, sensor_nodes=selected_sensor_nodes, scada_start=scada_start, scada_end=scada_end, ) observed_source = "backend_timerange" else: if observed_pressure_data is None: raise ValueError( "未提供 observed_pressure_data,且未提供 scada_start/scada_end。" ) observed_df = observed_pressure_data q_sum_m3s = DmaLeakageOptimizer._flow_to_m3s(q_sum, q_sum_unit) identifier = DmaLeakageOptimizer( inp_path=inp_path, sensor_nodes=selected_sensor_nodes, area_map=area_map, start_time=start_time, duration=duration, timestep=timestep, q_sum=q_sum_m3s, ) result_df = identifier.run_identification( observed_pressure_data=observed_df, pop_size=pop_size, max_gen=max_gen, n_workers=n_workers, output_flow_unit=output_flow_unit, save_result=False, ) rows = result_df.to_dict(orient="records") # node_visual_payload = _build_node_visual_payload(area_map, node_coords, rows) # drawing_payload = _build_drawing_payload(node_visual_payload) payload = { "result_path": result_df.attrs.get("result_path"), "sensor_nodes": selected_sensor_nodes, "observed_source": observed_source, "area_count": len(set(area_map.values())), "node_area_map": area_map, "areas": areas, # "node_visual_payload": node_visual_payload, # "drawing_payload": drawing_payload, "rows": rows, } if scheme_name: scheme_start_time = ( _to_datetime(scada_start).isoformat() if scada_start is not None else utc_now().isoformat() ) scheme_detail = { "network": network, "dma_count": dma_count, "sensor_nodes": selected_sensor_nodes, "scada_start": ( _to_datetime(scada_start).isoformat() if scada_start is not None else None ), "scada_end": ( _to_datetime(scada_end).isoformat() if scada_end is not None else None ), "algorithm_params": { "start_time": start_time, "duration": duration, "timestep": timestep, "q_sum": q_sum, "q_sum_unit": q_sum_unit, "output_flow_unit": output_flow_unit, "pop_size": pop_size, "max_gen": max_gen, "n_workers": n_workers, }, "result_summary": { "area_count": len(set(area_map.values())), "max_leakage": max( (float(row.get("LeakageFlow_m3_per_s", 0.0)) for row in rows), default=0.0, ), }, } store_analysis_run_with_result( name=network, scheme_name=scheme_name, scheme_type="dma_leak_identification", username=username, scheme_start_time=scheme_start_time, scheme_detail=scheme_detail, result_type="leakage_identification", result_payload={ "network": network, "run_status": "completed", "error_message": None, "sensor_nodes": selected_sensor_nodes, "rows": rows, "node_area_map": area_map, "areas": areas, "drawing_payload": {}, }, ) payload["scheme_name"] = scheme_name return payload def _get_pressure_sensor_nodes(network: str) -> list[str]: scada_devices = get_all_scada_info(network) sensor_nodes: list[str] = [] for item in scada_devices: scada_type = str(item.get("device_type", "")).lower() if scada_type != "pressure": continue node_id = item.get("node_id") if isinstance(node_id, str) and node_id: sensor_nodes.append(node_id) sensor_nodes = list(dict.fromkeys(sensor_nodes)) if not sensor_nodes: raise ValueError("未找到关联节点的压力 SCADA 设备。") return sensor_nodes def _build_area_map_by_topology( network: str, sensor_nodes: list[str], dma_count: int | None ) -> tuple[dict[str, str], list[dict[str, Any]], dict[str, dict[str, float]]]: node_coords = get_network_node_coords(network) area_map, areas = build_dma_partitions( sensor_nodes, node_coords, get_network_link_nodes(network), dma_count, ) return area_map, areas, node_coords def _build_area_node_map(area_map: dict[str, str]) -> dict[str, list[str]]: area_node_map: dict[str, list[str]] = {} for node_id, area_id in area_map.items(): area_node_map.setdefault(area_id, []).append(node_id) for area_id in list(area_node_map.keys()): area_node_map[area_id] = sorted(area_node_map[area_id]) return area_node_map def _build_node_visual_payload( area_map: dict[str, str], node_coords: dict[str, dict[str, float]], rows: list[dict[str, Any]], ) -> dict[str, Any]: area_leakage_map = _build_area_leakage_map(rows) max_leakage = max(area_leakage_map.values(), default=0.0) features: list[dict[str, Any]] = [] for node_id, area_id in area_map.items(): coord = node_coords.get(node_id) if not coord: continue leakage_flow = float(area_leakage_map.get(area_id, 0.0)) leakage_level = _classify_leakage_level(leakage_flow, max_leakage) features.append( { "type": "Feature", "properties": { "node_id": node_id, "area_id": area_id, "leakage_flow_m3_per_s": leakage_flow, "leakage_level": leakage_level, }, "geometry": { "type": "Point", "coordinates": [float(coord["x"]), float(coord["y"])], }, } ) return {"type": "FeatureCollection", "features": features} def _build_area_leakage_map(rows: list[dict[str, Any]]) -> dict[str, float]: area_leakage_map: dict[str, float] = {} for row in rows: area_id = str(row.get("Area", "")).strip() if not area_id: continue area_leakage_map[area_id] = float(row.get("LeakageFlow_m3_per_s", 0.0)) return area_leakage_map def _classify_leakage_level(leakage_flow: float, max_leakage: float) -> str: if max_leakage <= 0: return "normal" ratio = leakage_flow / max_leakage if ratio >= 0.75: return "high" if ratio >= 0.4: return "medium" if ratio > 0: return "low" return "normal" def _build_drawing_payload(node_visual_payload: dict[str, Any]) -> dict[str, Any]: return node_visual_payload def _build_observed_pressure_from_scada( network: str, sensor_nodes: list[str], scada_start: datetime | str | None, scada_end: datetime | str | None, ) -> pd.DataFrame: if scada_start is None or scada_end is None: raise ValueError("使用后端 SCADA 查询时必须同时提供 scada_start 与 scada_end。") start_dt = _to_datetime(scada_start) end_dt = _to_datetime(scada_end) if start_dt >= end_dt: raise ValueError("SCADA 时间窗非法:scada_start 必须早于 scada_end。") node_query_id: dict[str, str] = {} for item in get_all_scada_info(network): if str(item.get("device_type", "")).lower() != "pressure": continue node_id = item.get("node_id") query_id = item.get("api_query_id") if ( isinstance(node_id, str) and node_id and isinstance(query_id, str) and query_id ): node_query_id[node_id] = query_id query_ids = [node_query_id[node] for node in sensor_nodes if node in node_query_id] if not query_ids: raise ValueError("未找到可用于压力观测的 SCADA api_query_id。") scada_data = InternalQueries.query_scada_by_ids_timerange( db_name=network, device_ids=query_ids, start_time=start_dt.isoformat(), end_time=end_dt.isoformat(), ) available_lengths = [ len(scada_data.get(query_id, [])) for query_id in query_ids if len(scada_data.get(query_id, [])) > 0 ] if not available_lengths: raise ValueError("指定时间窗内未查询到压力 SCADA 数据。") min_len = min(available_lengths) obs_df = pd.DataFrame() for node_id in sensor_nodes: query_id = node_query_id.get(node_id) if not query_id: continue records = scada_data.get(query_id, [])[:min_len] if len(records) < min_len: continue obs_df[node_id] = [float(item["value"]) for item in records] if obs_df.empty: raise ValueError("SCADA 压力数据无法构建观测矩阵。") return obs_df def _to_datetime(value: datetime | str) -> datetime: return parse_utc_time(value) def _prepare_leakage_inp(network: str) -> str: project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) db_inp_dir = os.path.join(project_root, "db_inp") os.makedirs(db_inp_dir, exist_ok=True) inp_path = os.path.join(db_inp_dir, f"{network}.leakage.inp") if os.path.isfile(inp_path) and os.path.getsize(inp_path) > 0: return inp_path dump_inp(network, inp_path, "2") if not os.path.isfile(inp_path) or os.path.getsize(inp_path) <= 0: raise ValueError(f"漏损识别 INP 文件无效: {inp_path}") return inp_path