jiang
5966d039de
Reorganize algorithm packages by business responsibility, move orchestration into services, and keep database access behind pooled repositories. Harden analysis API validation, remove unsafe legacy simulation endpoints, and add regression and architecture boundary coverage. BREAKING CHANGE: legacy algorithm module paths and obsolete simulation endpoints are removed.
342 lines
12 KiB
Python
342 lines
12 KiB
Python
import os
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import pandas as pd
|
|
|
|
from app.algorithms.dma_leakage_estimation.genetic_optimizer import DmaLeakageOptimizer
|
|
from app.algorithms.dma_leakage_estimation.topology_partitioning import (
|
|
build_dma_partitions,
|
|
)
|
|
from app.infra.db.timescaledb.internal_queries import InternalQueries
|
|
from app.infra.db.postgresql.scada import get_all_scada_info
|
|
from app.native.wndb.gis.network_views import (
|
|
get_network_link_nodes,
|
|
get_network_node_coords,
|
|
)
|
|
from app.native.wndb.inp.exporter import dump_inp
|
|
from app.services.scheme_management import store_analysis_run_with_result
|
|
from app.domain.time import parse_utc_time, utc_now
|
|
|
|
DEFAULT_N_WORKERS = max(1, min((os.cpu_count() or 1) - 1, 4))
|
|
|
|
|
|
def run_leakage_identification(
|
|
network: str,
|
|
username: str,
|
|
observed_pressure_data: (
|
|
pd.DataFrame | dict[str, list[Any]] | list[dict[str, Any]] | None
|
|
) = None,
|
|
start_time: float = 0,
|
|
duration: float = 24,
|
|
timestep: float = 5,
|
|
q_sum: float = 0.2,
|
|
q_sum_unit: str = "m3/s",
|
|
pop_size: int = 50,
|
|
max_gen: int = 100,
|
|
n_workers: int = DEFAULT_N_WORKERS,
|
|
output_flow_unit: str = "m3/s",
|
|
dma_count: int | None = None,
|
|
scada_start: datetime | str | None = None,
|
|
scada_end: datetime | str | None = None,
|
|
sensor_nodes: list[str] | None = None,
|
|
scheme_name: str | None = None,
|
|
) -> dict[str, Any]:
|
|
inp_path = _prepare_leakage_inp(network)
|
|
|
|
selected_sensor_nodes = (
|
|
list(dict.fromkeys([node for node in (sensor_nodes or []) if node]))
|
|
if sensor_nodes
|
|
else _get_pressure_sensor_nodes(network)
|
|
)
|
|
if not selected_sensor_nodes:
|
|
raise ValueError("未提供有效传感器节点，且系统未识别到可用压力传感器。")
|
|
|
|
area_map, areas, node_coords = _build_area_map_by_topology(
|
|
network, selected_sensor_nodes, dma_count
|
|
)
|
|
|
|
observed_source = "request_payload"
|
|
if scada_start is not None or scada_end is not None:
|
|
observed_df = _build_observed_pressure_from_scada(
|
|
network=network,
|
|
sensor_nodes=selected_sensor_nodes,
|
|
scada_start=scada_start,
|
|
scada_end=scada_end,
|
|
)
|
|
observed_source = "backend_timerange"
|
|
else:
|
|
if observed_pressure_data is None:
|
|
raise ValueError(
|
|
"未提供 observed_pressure_data，且未提供 scada_start/scada_end。"
|
|
)
|
|
observed_df = observed_pressure_data
|
|
|
|
q_sum_m3s = DmaLeakageOptimizer._flow_to_m3s(q_sum, q_sum_unit)
|
|
identifier = DmaLeakageOptimizer(
|
|
inp_path=inp_path,
|
|
sensor_nodes=selected_sensor_nodes,
|
|
area_map=area_map,
|
|
start_time=start_time,
|
|
duration=duration,
|
|
timestep=timestep,
|
|
q_sum=q_sum_m3s,
|
|
)
|
|
result_df = identifier.run_identification(
|
|
observed_pressure_data=observed_df,
|
|
pop_size=pop_size,
|
|
max_gen=max_gen,
|
|
n_workers=n_workers,
|
|
output_flow_unit=output_flow_unit,
|
|
save_result=False,
|
|
)
|
|
rows = result_df.to_dict(orient="records")
|
|
# node_visual_payload = _build_node_visual_payload(area_map, node_coords, rows)
|
|
# drawing_payload = _build_drawing_payload(node_visual_payload)
|
|
payload = {
|
|
"result_path": result_df.attrs.get("result_path"),
|
|
"sensor_nodes": selected_sensor_nodes,
|
|
"observed_source": observed_source,
|
|
"area_count": len(set(area_map.values())),
|
|
"node_area_map": area_map,
|
|
"areas": areas,
|
|
# "node_visual_payload": node_visual_payload,
|
|
# "drawing_payload": drawing_payload,
|
|
"rows": rows,
|
|
}
|
|
if scheme_name:
|
|
scheme_start_time = (
|
|
_to_datetime(scada_start).isoformat()
|
|
if scada_start is not None
|
|
else utc_now().isoformat()
|
|
)
|
|
scheme_detail = {
|
|
"network": network,
|
|
"dma_count": dma_count,
|
|
"sensor_nodes": selected_sensor_nodes,
|
|
"scada_start": (
|
|
_to_datetime(scada_start).isoformat()
|
|
if scada_start is not None
|
|
else None
|
|
),
|
|
"scada_end": (
|
|
_to_datetime(scada_end).isoformat() if scada_end is not None else None
|
|
),
|
|
"algorithm_params": {
|
|
"start_time": start_time,
|
|
"duration": duration,
|
|
"timestep": timestep,
|
|
"q_sum": q_sum,
|
|
"q_sum_unit": q_sum_unit,
|
|
"output_flow_unit": output_flow_unit,
|
|
"pop_size": pop_size,
|
|
"max_gen": max_gen,
|
|
"n_workers": n_workers,
|
|
},
|
|
"result_summary": {
|
|
"area_count": len(set(area_map.values())),
|
|
"max_leakage": max(
|
|
(float(row.get("LeakageFlow_m3_per_s", 0.0)) for row in rows),
|
|
default=0.0,
|
|
),
|
|
},
|
|
}
|
|
store_analysis_run_with_result(
|
|
name=network,
|
|
scheme_name=scheme_name,
|
|
scheme_type="dma_leak_identification",
|
|
username=username,
|
|
scheme_start_time=scheme_start_time,
|
|
scheme_detail=scheme_detail,
|
|
result_type="leakage_identification",
|
|
result_payload={
|
|
"network": network,
|
|
"run_status": "completed",
|
|
"error_message": None,
|
|
"sensor_nodes": selected_sensor_nodes,
|
|
"rows": rows,
|
|
"node_area_map": area_map,
|
|
"areas": areas,
|
|
"drawing_payload": {},
|
|
},
|
|
)
|
|
payload["scheme_name"] = scheme_name
|
|
return payload
|
|
|
|
|
|
def _get_pressure_sensor_nodes(network: str) -> list[str]:
|
|
scada_devices = get_all_scada_info(network)
|
|
sensor_nodes: list[str] = []
|
|
for item in scada_devices:
|
|
scada_type = str(item.get("device_type", "")).lower()
|
|
if scada_type != "pressure":
|
|
continue
|
|
node_id = item.get("node_id")
|
|
if isinstance(node_id, str) and node_id:
|
|
sensor_nodes.append(node_id)
|
|
sensor_nodes = list(dict.fromkeys(sensor_nodes))
|
|
if not sensor_nodes:
|
|
raise ValueError("未找到关联节点的压力 SCADA 设备。")
|
|
return sensor_nodes
|
|
|
|
|
|
def _build_area_map_by_topology(
|
|
network: str, sensor_nodes: list[str], dma_count: int | None
|
|
) -> tuple[dict[str, str], list[dict[str, Any]], dict[str, dict[str, float]]]:
|
|
node_coords = get_network_node_coords(network)
|
|
area_map, areas = build_dma_partitions(
|
|
sensor_nodes,
|
|
node_coords,
|
|
get_network_link_nodes(network),
|
|
dma_count,
|
|
)
|
|
return area_map, areas, node_coords
|
|
|
|
|
|
def _build_area_node_map(area_map: dict[str, str]) -> dict[str, list[str]]:
|
|
area_node_map: dict[str, list[str]] = {}
|
|
for node_id, area_id in area_map.items():
|
|
area_node_map.setdefault(area_id, []).append(node_id)
|
|
for area_id in list(area_node_map.keys()):
|
|
area_node_map[area_id] = sorted(area_node_map[area_id])
|
|
return area_node_map
|
|
|
|
|
|
def _build_node_visual_payload(
|
|
area_map: dict[str, str],
|
|
node_coords: dict[str, dict[str, float]],
|
|
rows: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
area_leakage_map = _build_area_leakage_map(rows)
|
|
max_leakage = max(area_leakage_map.values(), default=0.0)
|
|
features: list[dict[str, Any]] = []
|
|
for node_id, area_id in area_map.items():
|
|
coord = node_coords.get(node_id)
|
|
if not coord:
|
|
continue
|
|
leakage_flow = float(area_leakage_map.get(area_id, 0.0))
|
|
leakage_level = _classify_leakage_level(leakage_flow, max_leakage)
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"properties": {
|
|
"node_id": node_id,
|
|
"area_id": area_id,
|
|
"leakage_flow_m3_per_s": leakage_flow,
|
|
"leakage_level": leakage_level,
|
|
},
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [float(coord["x"]), float(coord["y"])],
|
|
},
|
|
}
|
|
)
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def _build_area_leakage_map(rows: list[dict[str, Any]]) -> dict[str, float]:
|
|
area_leakage_map: dict[str, float] = {}
|
|
for row in rows:
|
|
area_id = str(row.get("Area", "")).strip()
|
|
if not area_id:
|
|
continue
|
|
area_leakage_map[area_id] = float(row.get("LeakageFlow_m3_per_s", 0.0))
|
|
return area_leakage_map
|
|
|
|
|
|
def _classify_leakage_level(leakage_flow: float, max_leakage: float) -> str:
|
|
if max_leakage <= 0:
|
|
return "normal"
|
|
ratio = leakage_flow / max_leakage
|
|
if ratio >= 0.75:
|
|
return "high"
|
|
if ratio >= 0.4:
|
|
return "medium"
|
|
if ratio > 0:
|
|
return "low"
|
|
return "normal"
|
|
|
|
|
|
def _build_drawing_payload(node_visual_payload: dict[str, Any]) -> dict[str, Any]:
|
|
return node_visual_payload
|
|
|
|
|
|
def _build_observed_pressure_from_scada(
|
|
network: str,
|
|
sensor_nodes: list[str],
|
|
scada_start: datetime | str | None,
|
|
scada_end: datetime | str | None,
|
|
) -> pd.DataFrame:
|
|
if scada_start is None or scada_end is None:
|
|
raise ValueError("使用后端 SCADA 查询时必须同时提供 scada_start 与 scada_end。")
|
|
|
|
start_dt = _to_datetime(scada_start)
|
|
end_dt = _to_datetime(scada_end)
|
|
if start_dt >= end_dt:
|
|
raise ValueError("SCADA 时间窗非法：scada_start 必须早于 scada_end。")
|
|
|
|
node_query_id: dict[str, str] = {}
|
|
for item in get_all_scada_info(network):
|
|
if str(item.get("device_type", "")).lower() != "pressure":
|
|
continue
|
|
node_id = item.get("node_id")
|
|
query_id = item.get("api_query_id")
|
|
if (
|
|
isinstance(node_id, str)
|
|
and node_id
|
|
and isinstance(query_id, str)
|
|
and query_id
|
|
):
|
|
node_query_id[node_id] = query_id
|
|
|
|
query_ids = [node_query_id[node] for node in sensor_nodes if node in node_query_id]
|
|
if not query_ids:
|
|
raise ValueError("未找到可用于压力观测的 SCADA api_query_id。")
|
|
|
|
scada_data = InternalQueries.query_scada_by_ids_timerange(
|
|
db_name=network,
|
|
device_ids=query_ids,
|
|
start_time=start_dt.isoformat(),
|
|
end_time=end_dt.isoformat(),
|
|
)
|
|
|
|
available_lengths = [
|
|
len(scada_data.get(query_id, []))
|
|
for query_id in query_ids
|
|
if len(scada_data.get(query_id, [])) > 0
|
|
]
|
|
if not available_lengths:
|
|
raise ValueError("指定时间窗内未查询到压力 SCADA 数据。")
|
|
min_len = min(available_lengths)
|
|
|
|
obs_df = pd.DataFrame()
|
|
for node_id in sensor_nodes:
|
|
query_id = node_query_id.get(node_id)
|
|
if not query_id:
|
|
continue
|
|
records = scada_data.get(query_id, [])[:min_len]
|
|
if len(records) < min_len:
|
|
continue
|
|
obs_df[node_id] = [float(item["value"]) for item in records]
|
|
|
|
if obs_df.empty:
|
|
raise ValueError("SCADA 压力数据无法构建观测矩阵。")
|
|
return obs_df
|
|
|
|
|
|
def _to_datetime(value: datetime | str) -> datetime:
|
|
return parse_utc_time(value)
|
|
|
|
|
|
def _prepare_leakage_inp(network: str) -> str:
|
|
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
db_inp_dir = os.path.join(project_root, "db_inp")
|
|
os.makedirs(db_inp_dir, exist_ok=True)
|
|
inp_path = os.path.join(db_inp_dir, f"{network}.leakage.inp")
|
|
if os.path.isfile(inp_path) and os.path.getsize(inp_path) > 0:
|
|
return inp_path
|
|
dump_inp(network, inp_path, "2")
|
|
if not os.path.isfile(inp_path) or os.path.getsize(inp_path) <= 0:
|
|
raise ValueError(f"漏损识别 INP 文件无效: {inp_path}")
|
|
return inp_path
|