diff --git a/.dockerignore b/.dockerignore index af63b08..961ae6e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,4 +19,4 @@ logs/ coverage/ *.pyc *.dump -app/algorithms/health/model/my_survival_forest_model_quxi.joblib +app/algorithms/pipe_health_prediction/model/my_survival_forest_model_quxi.joblib diff --git a/.gitignore b/.gitignore index cafd240..ea985b6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ build/ .env *.dump .vscode/ -app/algorithms/health/model/my_survival_forest_model_quxi.joblib +app/algorithms/pipe_health_prediction/model/my_survival_forest_model_quxi.joblib /inp/ diff --git a/Dockerfile b/Dockerfile index cfdb907..7993cda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ COPY infra ./infra COPY resources ./resources COPY scripts/check_openapi.py ./scripts/check_openapi.py COPY tests ./tests -RUN python -c "from pathlib import Path; from zipfile import ZipFile; model_dir = Path('app/algorithms/health/model'); zip_path = model_dir / 'my_survival_forest_model_quxi.zip'; joblib_name = 'my_survival_forest_model_quxi.joblib'; joblib_path = model_dir / joblib_name; assert zip_path.exists(), f'Model archive not found: {zip_path}'; archive = ZipFile(zip_path); archive.extract(joblib_name, model_dir); archive.close(); assert joblib_path.exists(), f'Model file not extracted: {joblib_path}'" && \ - rm -f app/algorithms/health/model/my_survival_forest_model_quxi.zip +RUN python -c "from pathlib import Path; from zipfile import ZipFile; model_dir = Path('app/algorithms/pipe_health_prediction/model'); zip_path = model_dir / 'my_survival_forest_model_quxi.zip'; joblib_name = 'my_survival_forest_model_quxi.joblib'; joblib_path = model_dir / joblib_name; assert zip_path.exists(), f'Model archive not found: {zip_path}'; archive = ZipFile(zip_path); archive.extract(joblib_name, model_dir); archive.close(); assert joblib_path.exists(), f'Model file not extracted: {joblib_path}'" && \ + rm -f app/algorithms/pipe_health_prediction/model/my_survival_forest_model_quxi.zip RUN mkdir -p db_inp temp data inp # 设置 PYTHONPATH 以便 uvicorn 找到 app 模块 diff --git a/app/algorithms/__init__.py b/app/algorithms/__init__.py index a31f4b3..4e590ad 100644 --- a/app/algorithms/__init__.py +++ b/app/algorithms/__init__.py @@ -1,45 +1,5 @@ -"""Algorithm package with side-effect-free, lazy compatibility exports.""" +"""Pure water-network calculation packages. -from importlib import import_module -from typing import Any - - -_EXPORT_MODULES = { - "flow_data_clean": "app.algorithms.cleaning", - "pressure_data_clean": "app.algorithms.cleaning", - "pressure_sensor_placement_sensitivity": "app.algorithms.sensor", - "pressure_sensor_placement_kmeans": "app.algorithms.sensor", - "valve_isolation_analysis": "app.algorithms.isolation.valve", - "LeakageIdentifier": "app.algorithms.leakage", - "PipelineHealthAnalyzer": "app.algorithms.health", - "run_burst_location": "app.algorithms.burst_location", - **{ - name: "app.algorithms.simulation.scenarios" - for name in ( - "convert_to_local_unit", - "burst_analysis", - "valve_close_analysis", - "flushing_analysis", - "contaminant_simulation", - "age_analysis", - "pressure_regulation", - ) - }, -} - -__all__ = list(_EXPORT_MODULES) - - -def __getattr__(name: str) -> Any: - try: - module_name = _EXPORT_MODULES[name] - except KeyError as exc: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc - - value = getattr(import_module(module_name), name) - globals()[name] = value - return value - - -def __dir__() -> list[str]: - return sorted({*globals(), *__all__}) +Application workflows belong in :mod:`app.services`; database and external +system access belongs in :mod:`app.infra` or :mod:`app.native`. +""" diff --git a/app/algorithms/burst_detection/__init__.py b/app/algorithms/burst_detection/__init__.py index 226dc73..b11c844 100644 --- a/app/algorithms/burst_detection/__init__.py +++ b/app/algorithms/burst_detection/__init__.py @@ -1,3 +1,3 @@ -from app.algorithms.burst_detection.burst_detector import BurstDetector +from app.algorithms.burst_detection.pressure_anomaly import PressureAnomalyDetector -__all__ = ["BurstDetector"] +__all__ = ["PressureAnomalyDetector"] diff --git a/app/algorithms/burst_detection/burst_detector.py b/app/algorithms/burst_detection/pressure_anomaly.py similarity index 99% rename from app/algorithms/burst_detection/burst_detector.py rename to app/algorithms/burst_detection/pressure_anomaly.py index 530ae6b..2193f44 100644 --- a/app/algorithms/burst_detection/burst_detector.py +++ b/app/algorithms/burst_detection/pressure_anomaly.py @@ -17,7 +17,7 @@ PressureDataInput = ( IGNORED_OBSERVATION_COLUMNS = {"time", "timestamp", "datetime", "date"} -class BurstDetector: +class PressureAnomalyDetector: """FFT + IsolationForest based burst detection for daily aligned pressure data.""" def __init__( diff --git a/app/algorithms/burst_localization/__init__.py b/app/algorithms/burst_localization/__init__.py new file mode 100644 index 0000000..575cd8a --- /dev/null +++ b/app/algorithms/burst_localization/__init__.py @@ -0,0 +1,3 @@ +from .pipeline import run_burst_location + +__all__ = ["run_burst_location"] diff --git a/app/algorithms/burst_location/burst_locator.py b/app/algorithms/burst_localization/candidate_ranking.py similarity index 99% rename from app/algorithms/burst_location/burst_locator.py rename to app/algorithms/burst_localization/candidate_ranking.py index dd5906c..dda98cd 100644 --- a/app/algorithms/burst_location/burst_locator.py +++ b/app/algorithms/burst_localization/candidate_ranking.py @@ -11,13 +11,13 @@ import networkx as nx import numpy as np import pandas as pd -from .leak_simulator import cal_signature_pipe_multi_pf -from .network_partitioner import ( +from .leak_signature import cal_signature_pipe_multi_pf +from .topology_partitioning import ( cal_group_num, metis_grouping_pipe_weight, visualize_metis_partition, ) -from .similarity_calculator import ( +from .similarity_metrics import ( adjust_ratio, cal_similarity_all_multi_new_sq_improve_double_lzr, decode_mode, @@ -769,4 +769,3 @@ def DN_search_multi_simple_add_flow_count_new( final_candidates_csv, ) - diff --git a/app/algorithms/burst_location/leak_simulator.py b/app/algorithms/burst_localization/leak_signature.py similarity index 100% rename from app/algorithms/burst_location/leak_simulator.py rename to app/algorithms/burst_localization/leak_signature.py diff --git a/app/algorithms/burst_location/burst_location.py b/app/algorithms/burst_localization/pipeline.py similarity index 67% rename from app/algorithms/burst_location/burst_location.py rename to app/algorithms/burst_localization/pipeline.py index 54a0130..d70035a 100644 --- a/app/algorithms/burst_location/burst_location.py +++ b/app/algorithms/burst_localization/pipeline.py @@ -1,5 +1,3 @@ -import argparse -import json import logging from multiprocessing import cpu_count from pathlib import Path @@ -7,12 +5,12 @@ from typing import Any, Iterable import pandas as pd -from app.algorithms.burst_location import leak_simulator +from app.algorithms.burst_localization import leak_signature -from .burst_locator import ( +from .candidate_ranking import ( DN_search_multi_simple_add_flow_count_new, ) -from .network_model import ( +from .topology_model import ( _build_node_pipe_maps, cal_node_coordinate, construct_graph, @@ -26,35 +24,6 @@ DEFAULT_N_WORKERS = max(1, min(cpu_count() - 1, 4)) logger = logging.getLogger(__name__) -def _read_id_list_json(path): - if path is None: - return None - data = json.loads(Path(path).read_text(encoding="utf-8")) - if isinstance(data, list): - return [str(item) for item in data] - if isinstance(data, dict): - if "ids" in data and isinstance(data["ids"], list): - return [str(item) for item in data["ids"]] - raise ValueError(f"ID JSON must be list or dict with key 'ids': {path}") - raise ValueError(f"Unsupported ID JSON format: {path}") - - -def _read_series_csv(path): - if path is None: - return None - df = pd.read_csv(path) - if df.shape[1] < 2: - raise ValueError(f"CSV must contain at least two columns (id,value): {path}") - if {"id", "value"}.issubset(df.columns): - id_col, value_col = "id", "value" - else: - id_col, value_col = df.columns[0], df.columns[1] - series = pd.Series( - df[value_col].values, index=df[id_col].astype(str).values, dtype=float - ) - return series - - def _align_scada_series( series: pd.Series, ids: Iterable[str], series_name: str ) -> pd.Series: @@ -159,7 +128,7 @@ def run_burst_location( pipe_diameter, ) = read_inf_inp(wn) - candidate_pipe, _ = leak_simulator.cal_possible_pipe( + candidate_pipe, _ = leak_signature.cal_possible_pipe( burst_leakage, all_pipe, pipe_diameter ) @@ -267,76 +236,3 @@ def run_burst_location( "final_candidates_csv": final_candidates_csv, "stage_timing_seconds": stage_timing, } - - -def _parse_args(): - parser = argparse.ArgumentParser(description="爆管定位主函数入口") - parser.add_argument("--wn-inp", required=True, help="EPANET inp 文件路径") - parser.add_argument( - "--pressure-ids-json", required=True, help="压力SCADA ID列表 JSON 文件" - ) - parser.add_argument( - "--flow-ids-json", default=None, help="（可选）流量SCADA ID列表 JSON 文件" - ) - parser.add_argument( - "--burst-pressure-csv", required=True, help="爆管时压力 CSV（id,value）" - ) - parser.add_argument( - "--normal-pressure-csv", required=True, help="正常时压力 CSV（id,value）" - ) - parser.add_argument( - "--burst-flow-csv", default=None, help="（可选）爆管时流量 CSV（id,value）" - ) - parser.add_argument( - "--normal-flow-csv", default=None, help="（可选）正常时流量 CSV（id,value）" - ) - parser.add_argument( - "--burst-leakage", type=float, required=True, help="爆管漏损流量" - ) - parser.add_argument( - "--min-dpressure", - type=float, - default=2.0, - help="（可选）最小压降阈值，默认 2.0", - ) - parser.add_argument( - "--basic-pressure", - type=float, - default=10.0, - help="（可选）基础服务压力，默认 10.0", - ) - parser.add_argument( - "--n-workers", - type=int, - default=DEFAULT_N_WORKERS, - help="（可选）特征中心模拟进程数，默认 max(1, min(cpu_count()-1, 4))", - ) - parser.add_argument( - "--final-candidates-csv-path", - default="temp/burst_location/final_round_candidates.csv", - help="（可选）最后一轮候选管道明细 CSV 输出路径", - ) - return parser.parse_args() - - -def main(): - args = _parse_args() - result = run_burst_location( - wn_inp_path=args.wn_inp, - pressure_scada_ids=_read_id_list_json(args.pressure_ids_json), - burst_pressure=_read_series_csv(args.burst_pressure_csv), - normal_pressure=_read_series_csv(args.normal_pressure_csv), - burst_leakage=args.burst_leakage, - flow_scada_ids=_read_id_list_json(args.flow_ids_json), - burst_flow=_read_series_csv(args.burst_flow_csv), - normal_flow=_read_series_csv(args.normal_flow_csv), - min_dpressure=args.min_dpressure, - basic_pressure=args.basic_pressure, - n_workers=args.n_workers, - final_candidates_csv_path=args.final_candidates_csv_path, - ) - print(json.dumps(result, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/app/algorithms/burst_location/similarity_calculator.py b/app/algorithms/burst_localization/similarity_metrics.py similarity index 100% rename from app/algorithms/burst_location/similarity_calculator.py rename to app/algorithms/burst_localization/similarity_metrics.py diff --git a/app/algorithms/burst_location/noise_generator.py b/app/algorithms/burst_localization/synthetic_observations.py similarity index 99% rename from app/algorithms/burst_location/noise_generator.py rename to app/algorithms/burst_localization/synthetic_observations.py index cef6d27..af249b9 100644 --- a/app/algorithms/burst_location/noise_generator.py +++ b/app/algorithms/burst_localization/synthetic_observations.py @@ -6,7 +6,7 @@ import random import numpy as np import pandas as pd -from .leak_simulator import simple_add_leak, simple_recover_wn, simple_simulation_pf +from .leak_signature import simple_add_leak, simple_recover_wn, simple_simulation_pf def add_noise_pd(data, noise_type, noise_para): @@ -195,4 +195,3 @@ def change_para_of_wn(wn, pipe_roughness_change): pipe.roughness = pipe_roughness_change[pipe_name] return wn - diff --git a/app/algorithms/burst_location/network_model.py b/app/algorithms/burst_localization/topology_model.py similarity index 100% rename from app/algorithms/burst_location/network_model.py rename to app/algorithms/burst_localization/topology_model.py diff --git a/app/algorithms/burst_location/network_partitioner.py b/app/algorithms/burst_localization/topology_partitioning.py similarity index 100% rename from app/algorithms/burst_location/network_partitioner.py rename to app/algorithms/burst_localization/topology_partitioning.py diff --git a/app/algorithms/burst_location/__init__.py b/app/algorithms/burst_location/__init__.py deleted file mode 100644 index c7ec7a1..0000000 --- a/app/algorithms/burst_location/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .burst_location import run_burst_location - -__all__ = ["run_burst_location"] diff --git a/app/algorithms/cleaning/__init__.py b/app/algorithms/cleaning/__init__.py deleted file mode 100644 index c6f9a28..0000000 --- a/app/algorithms/cleaning/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -import os - -from app.algorithms.cleaning import flow as _flow_module -from app.algorithms.cleaning import pressure as _pressure_module - - -############################################################ -# 流量监测数据清洗 ***卡尔曼滤波法*** -############################################################ -# 2025/08/21 hxyan - - -def flow_data_clean(input_csv_file: str) -> str: - """ - 读取 input_csv_path 中的每列时间序列，使用一维 Kalman 滤波平滑并用预测值替换基于 3σ 检测出的异常点。 - 保存输出为：_cleaned.xlsx（与输入同目录），并返回输出文件的绝对路径。如有同名文件存在，则覆盖。 - :param: input_csv_file: 输入的 CSV 文件明或路径 - :return: 输出文件的绝对路径 - """ - - # 提供的 input_csv_path 绝对路径，以下为 默认脚本目录下同名 CSV 文件，构建绝对路径，可根据情况修改 - # 使用 algorithms 根目录保持与原 data_cleaning.py 一致的行为 - script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - input_csv_path = os.path.join(script_dir, input_csv_file) - - # 检查文件是否存在 - if not os.path.exists(input_csv_path): - raise FileNotFoundError(f"指定的文件不存在: {input_csv_path}") - # 调用 clean_flow_data_kf 函数进行数据清洗 - out_xlsx_path = _flow_module.clean_flow_data_kf(input_csv_path) - print("清洗后的数据已保存到:", out_xlsx_path) - - -############################################################ -# 压力监测数据清洗 ***kmean++法*** -############################################################ -# 2025/08/21 hxyan - - -def pressure_data_clean(input_csv_file: str) -> str: - """ - 读取 input_csv_path 中的每列时间序列，使用Kmean++清洗数据。 - 保存输出为：_cleaned.xlsx（与输入同目录），并返回输出文件的绝对路径。如有同名文件存在，则覆盖。 - 原始数据在 sheet 'raw_pressure_data'，处理后数据在 sheet 'cleaned_pressusre_data'。 - :param input_csv_path: 输入的 CSV 文件路径 - :return: 输出文件的绝对路径 - """ - - # 提供的 input_csv_path 绝对路径，以下为 默认脚本目录下同名 CSV 文件，构建绝对路径，可根据情况修改 - # 使用 algorithms 根目录保持与原 data_cleaning.py 一致的行为 - script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - input_csv_path = os.path.join(script_dir, input_csv_file) - - # 检查文件是否存在 - if not os.path.exists(input_csv_path): - raise FileNotFoundError(f"指定的文件不存在: {input_csv_path}") - # 调用 clean_pressure_data_km 函数进行数据清洗 - out_xlsx_path = _pressure_module.clean_pressure_data_km(input_csv_path) - print("清洗后的数据已保存到:", out_xlsx_path) diff --git a/app/algorithms/demand_allocation/__init__.py b/app/algorithms/demand_allocation/__init__.py new file mode 100644 index 0000000..d9ced52 --- /dev/null +++ b/app/algorithms/demand_allocation/__init__.py @@ -0,0 +1,5 @@ +"""Demand allocation calculations.""" + +from .pipe_length_weighted import allocate_demand_by_pipe_length + +__all__ = ["allocate_demand_by_pipe_length"] diff --git a/app/algorithms/demand_allocation/pipe_length_weighted.py b/app/algorithms/demand_allocation/pipe_length_weighted.py new file mode 100644 index 0000000..e7db22b --- /dev/null +++ b/app/algorithms/demand_allocation/pipe_length_weighted.py @@ -0,0 +1,36 @@ +"""Pipe-length-weighted demand allocation. + +This module deliberately accepts plain topology data and performs no database +or file access. Application services are responsible for loading topology. +""" + +from typing import Any, Mapping + + +def allocate_demand_by_pipe_length( + demand: float, + topology_nodes: Mapping[str, Mapping[str, Any]], + topology_links: Mapping[str, Mapping[str, Any]], +) -> dict[str, float]: + """Allocate total demand to junctions by half of each incident link length.""" + + if not topology_nodes or not topology_links or demand == 0.0: + return {} + + total_link_length = sum( + abs(float(link["length"])) for link in topology_links.values() + ) + if total_link_length <= 0.0: + return {} + + demand_per_length = demand / total_link_length + result: dict[str, float] = {} + for node_id, node in topology_nodes.items(): + if node["type"] != "junction": + continue + incident_length = sum( + abs(float(topology_links[link_id]["length"])) + for link_id in node["links"] + ) + result[node_id] = incident_length * demand_per_length * 0.5 + return result diff --git a/app/algorithms/dma_leakage_estimation/__init__.py b/app/algorithms/dma_leakage_estimation/__init__.py new file mode 100644 index 0000000..e0a063e --- /dev/null +++ b/app/algorithms/dma_leakage_estimation/__init__.py @@ -0,0 +1,3 @@ +from app.algorithms.dma_leakage_estimation.genetic_optimizer import DmaLeakageOptimizer + +__all__ = ["DmaLeakageOptimizer"] diff --git a/app/algorithms/leakage/identifier.py b/app/algorithms/dma_leakage_estimation/genetic_optimizer.py similarity index 89% rename from app/algorithms/leakage/identifier.py rename to app/algorithms/dma_leakage_estimation/genetic_optimizer.py index c02a705..fa20a58 100644 --- a/app/algorithms/leakage/identifier.py +++ b/app/algorithms/dma_leakage_estimation/genetic_optimizer.py @@ -3,7 +3,6 @@ import numpy as np import pandas as pd import os import time -import argparse from multiprocessing import Pool, cpu_count from typing import Any, List, Dict, Union @@ -70,7 +69,7 @@ def _worker_init( def _worker_evaluate(raw_ratios: np.ndarray) -> float: d = _worker_data - effective_ratio_map = LeakageIdentifier._effective_area_ratios( + effective_ratio_map = DmaLeakageOptimizer._effective_area_ratios( raw_ratios, d["area_ids"], d["nodes_by_area"], @@ -121,7 +120,7 @@ def _worker_evaluate(raw_ratios: np.ndarray) -> float: _cleanup_temp_files(prefix) -class LeakageIdentifier: +class DmaLeakageOptimizer: FLOW_UNIT_TO_M3S = { "m3/s": 1.0, "m³/s": 1.0, @@ -543,7 +542,7 @@ class LeakageProblem(Problem): leak_ratios = x # 将漏损分布归一化 - effective_ratio_map = LeakageIdentifier._effective_area_ratios( + effective_ratio_map = DmaLeakageOptimizer._effective_area_ratios( leak_ratios, self.area_ids, self.nodes_by_area, @@ -605,51 +604,6 @@ class LeakageProblem(Problem): def close(self) -> None: if self._pool is not None: - self._pool.close() - self._pool.join() - self._pool = None - - -def main() -> int: - parser = argparse.ArgumentParser(description="漏损区域识别") - parser.add_argument("--inp", required=True, help=".inp 文件路径") - parser.add_argument("--map", help="节点-区域映射 CSV 路径") - parser.add_argument("--scada", help="SCADA 压力 CSV 路径 (观测数据)") - parser.add_argument("--sensors", help="传感器节点 ID 列表 (逗号分隔)") - parser.add_argument("--output", default="Results", help="输出目录") - - parser.add_argument("--pop_size", type=int, default=50, help="种群大小") - parser.add_argument("--max_gen", type=int, default=100, help="最大代数") - parser.add_argument("--duration", type=float, default=24, help="模拟时长(小时)") - parser.add_argument("--q_sum", type=float, default=0.241, help="总漏损流量") - parser.add_argument( - "--q_sum_unit", - default="m3/s", - choices=list(LeakageIdentifier.FLOW_UNIT_TO_M3S.keys()), - help="q_sum 输入单位（建议与现场习惯一致，内部统一换算为 m3/s）", - ) - - args = parser.parse_args() - - if not args.map or not args.scada or not args.sensors: - parser.error("--map、--scada、--sensors 为必填") - - q_sum_m3s = LeakageIdentifier._flow_to_m3s(args.q_sum, args.q_sum_unit) - - sensors = [sensor.strip() for sensor in args.sensors.split(",") if sensor.strip()] - identifier = LeakageIdentifier( - args.inp, sensors, args.map, duration=args.duration, q_sum=q_sum_m3s - ) - - identifier.run_identification( - args.scada, - args.output, - pop_size=args.pop_size, - max_gen=args.max_gen, - output_flow_unit=args.q_sum_unit, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) + self._pool.close() + self._pool.join() + self._pool = None diff --git a/app/algorithms/dma_leakage_estimation/topology_partitioning.py b/app/algorithms/dma_leakage_estimation/topology_partitioning.py new file mode 100644 index 0000000..419f939 --- /dev/null +++ b/app/algorithms/dma_leakage_estimation/topology_partitioning.py @@ -0,0 +1,206 @@ +"""Pure topology partitioning used by DMA leakage estimation.""" + +import math +from collections import deque +from typing import Any, Iterable, Mapping + +import numpy as np + + +def build_dma_partitions( + sensor_nodes: list[str], + node_coords: Mapping[str, Mapping[str, Any]], + link_entries: Iterable[str], + dma_count: int | None, +) -> tuple[dict[str, str], list[dict[str, Any]]]: + """Assign every topology node to a sensor-seeded virtual DMA.""" + + all_nodes = list(node_coords) + if not all_nodes: + raise ValueError("管网中未获取到可分区节点。") + available_sensors = [node for node in sensor_nodes if node in node_coords] + if not available_sensors: + raise ValueError("无可用压力传感器，无法生成虚拟分区。") + + area_count = _resolve_dma_count(dma_count, available_sensors, all_nodes) + sensor_area_map = _cluster_sensors_to_areas( + available_sensors, node_coords, area_count + ) + adjacency = _build_adjacency(link_entries, all_nodes) + distance_by_sensor = { + sensor: _bfs_distances(adjacency, sensor) for sensor in available_sensors + } + + assignment_count = {sensor: 0 for sensor in available_sensors} + area_map: dict[str, str] = {} + for node_id in sorted(all_nodes): + sensor = _choose_sensor_for_node( + node_id, + available_sensors, + node_coords, + distance_by_sensor, + assignment_count, + ) + assignment_count[sensor] += 1 + area_map[node_id] = sensor_area_map[sensor] + + return area_map, _build_area_meta(area_map, sensor_area_map) + + +def _resolve_dma_count( + dma_count: int | None, sensor_nodes: list[str], all_nodes: list[str] +) -> int: + if dma_count is None: + return min(len(sensor_nodes), len(all_nodes)) + if dma_count <= 0: + raise ValueError("dma_count 必须大于 0。") + if dma_count > len(all_nodes): + raise ValueError("dma_count 不能大于可分区节点数量。") + if dma_count > len(sensor_nodes): + raise ValueError("dma_count 不能大于可用传感器数量。") + return dma_count + + +def _cluster_sensors_to_areas( + sensor_nodes: list[str], + node_coords: Mapping[str, Mapping[str, Any]], + area_count: int, +) -> dict[str, str]: + if area_count >= len(sensor_nodes): + return {sensor: str(index + 1) for index, sensor in enumerate(sensor_nodes)} + + points = np.array( + [ + [float(node_coords[sensor]["x"]), float(node_coords[sensor]["y"])] + for sensor in sensor_nodes + ], + dtype=float, + ) + centers = points[:area_count].copy() + labels = np.full(points.shape[0], -1, dtype=int) + for _ in range(20): + distances_squared = ( + (points[:, None, :] - centers[None, :, :]) ** 2 + ).sum(axis=2) + next_labels = distances_squared.argmin(axis=1) + if np.array_equal(labels, next_labels): + break + labels = next_labels + for index in range(area_count): + cluster_points = points[labels == index] + if cluster_points.size > 0: + centers[index] = cluster_points.mean(axis=0) + labels = _restore_empty_area_labels(labels, points, centers, area_count) + return { + sensor: str(int(labels[index]) + 1) + for index, sensor in enumerate(sensor_nodes) + } + + +def _restore_empty_area_labels( + labels: np.ndarray, + points: np.ndarray, + centers: np.ndarray, + area_count: int, +) -> np.ndarray: + """Keep every requested area represented when coordinates are degenerate.""" + + labels = labels.copy() + for missing_area in sorted(set(range(area_count)) - set(labels.tolist())): + area_sizes = { + area: int(np.count_nonzero(labels == area)) for area in range(area_count) + } + donor_area = max( + (area for area, size in area_sizes.items() if size > 1), + key=lambda area: (area_sizes[area], -area), + ) + donor_indices = np.flatnonzero(labels == donor_area) + replacement_index = max( + (int(index) for index in donor_indices), + key=lambda index: ( + float(np.sum((points[index] - centers[donor_area]) ** 2)), + index, + ), + ) + labels[replacement_index] = missing_area + centers[missing_area] = points[replacement_index] + return labels + + +def _build_adjacency( + link_entries: Iterable[str], all_nodes: list[str] +) -> dict[str, set[str]]: + adjacency: dict[str, set[str]] = {node: set() for node in all_nodes} + for link in link_entries: + parts = str(link).split(":") + if len(parts) < 4: + continue + node1, node2 = parts[-2], parts[-1] + if node1 in adjacency and node2 in adjacency: + adjacency[node1].add(node2) + adjacency[node2].add(node1) + return adjacency + + +def _bfs_distances(adjacency: Mapping[str, set[str]], start: str) -> dict[str, int]: + distances = {start: 0} + queue: deque[str] = deque([start]) + while queue: + node = queue.popleft() + for neighbor in adjacency.get(node, set()): + if neighbor in distances: + continue + distances[neighbor] = distances[node] + 1 + queue.append(neighbor) + return distances + + +def _choose_sensor_for_node( + node_id: str, + sensors: list[str], + node_coords: Mapping[str, Mapping[str, Any]], + distance_by_sensor: Mapping[str, Mapping[str, int]], + assignment_count: Mapping[str, int], +) -> str: + min_distance: int | None = None + candidates: list[str] = [] + for sensor in sensors: + distance = distance_by_sensor.get(sensor, {}).get(node_id) + if distance is None: + continue + if min_distance is None or distance < min_distance: + min_distance = distance + candidates = [sensor] + elif distance == min_distance: + candidates.append(sensor) + if not candidates: + node_coord = node_coords[node_id] + return min( + sensors, + key=lambda sensor: math.hypot( + float(node_coord["x"]) - float(node_coords[sensor]["x"]), + float(node_coord["y"]) - float(node_coords[sensor]["y"]), + ), + ) + return min(candidates, key=lambda sensor: (assignment_count[sensor], sensor)) + + +def _build_area_meta( + area_map: Mapping[str, str], sensor_area_map: Mapping[str, str] +) -> list[dict[str, Any]]: + nodes_by_area: dict[str, list[str]] = {} + for node_id, area_id in area_map.items(): + nodes_by_area.setdefault(area_id, []).append(node_id) + sensors_by_area: dict[str, list[str]] = {} + for sensor, area_id in sensor_area_map.items(): + sensors_by_area.setdefault(area_id, []).append(sensor) + + return [ + { + "area_id": area_id, + "sensor_nodes": sorted(sensors_by_area.get(area_id, [])), + "node_ids": sorted(nodes_by_area[area_id]), + "node_count": len(nodes_by_area[area_id]), + } + for area_id in sorted(nodes_by_area, key=int) + ] diff --git a/app/algorithms/health/__init__.py b/app/algorithms/health/__init__.py deleted file mode 100644 index d0e216f..0000000 --- a/app/algorithms/health/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.algorithms.health.analyzer import PipelineHealthAnalyzer - -__all__ = ["PipelineHealthAnalyzer"] diff --git a/app/algorithms/isolation/__init__.py b/app/algorithms/isolation/__init__.py deleted file mode 100644 index 12d5a35..0000000 --- a/app/algorithms/isolation/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.algorithms.isolation.valve import valve_isolation_analysis - -__all__ = ["valve_isolation_analysis"] diff --git a/app/algorithms/isolation/valve.py b/app/algorithms/isolation/valve.py deleted file mode 100644 index c53c0f4..0000000 --- a/app/algorithms/isolation/valve.py +++ /dev/null @@ -1,167 +0,0 @@ -from collections import defaultdict, deque -from functools import lru_cache -from typing import Any - -from app.services.tjnetwork import ( - get_network_link_nodes, - is_node, - get_link_properties, -) - - -VALVE_LINK_TYPE = "valve" - - -def _parse_link_entry(link_entry: str) -> tuple[str, str, str, str]: - parts = link_entry.split(":", 3) - if len(parts) != 4: - raise ValueError(f"Invalid link entry format: {link_entry}") - return parts[0], parts[1], parts[2], parts[3] - - -@lru_cache(maxsize=16) -def _get_network_topology(network: str): - """ - 解析并缓存网络拓扑，大幅减少重复的 API 调用和字符串解析开销。 - 返回: - - pipe_adj: 永久连通的管道/泵邻接表 (dict[str, set]) - - all_valves: 所有阀门字典 {id: (n1, n2)} - - link_lookup: 链路快速查表 {id: (n1, n2, type)} 用于快速定位事故点 - - node_set: 所有已知节点集合 - """ - pipe_adj = defaultdict(set) - all_valves = {} - link_lookup = {} - node_set = set() - - # 此处假设 get_network_link_nodes 获取全网数据 - for link_entry in get_network_link_nodes(network): - link_id, link_type, node1, node2 = _parse_link_entry(link_entry) - link_type_name = str(link_type).lower() - - link_lookup[link_id] = (node1, node2, link_type_name) - node_set.add(node1) - node_set.add(node2) - - if link_type_name == VALVE_LINK_TYPE: - all_valves[link_id] = (node1, node2) - else: - # 只有非阀门（管道/泵）才进入永久连通图 - pipe_adj[node1].add(node2) - pipe_adj[node2].add(node1) - - return pipe_adj, all_valves, link_lookup, node_set - - -def valve_isolation_analysis( - network: str, accident_elements: str | list[str], disabled_valves: list[str] = None -) -> dict[str, Any]: - """ - 关阀搜索/分析：基于拓扑结构确定事故隔离所需关阀。 - :param network: 模型名称 - :param accident_elements: 事故点（节点或管道/泵/阀门ID），可以是单个ID字符串或ID列表 - :param disabled_valves: 故障/无法关闭的阀门ID列表 - :return: dict，包含受影响节点、必须关闭阀门、可选阀门等信息 - """ - if disabled_valves is None: - disabled_valves_set = set() - else: - disabled_valves_set = set(disabled_valves) - - if isinstance(accident_elements, str): - target_elements = [accident_elements] - else: - target_elements = accident_elements - - # 1. 获取缓存拓扑 (极快，无 IO) - pipe_adj, all_valves, link_lookup, node_set = _get_network_topology(network) - - # 2. 确定起点，优先查表避免 API 调用 - start_nodes = set() - for element in target_elements: - if element in node_set: - start_nodes.add(element) - elif element in link_lookup: - n1, n2, _ = link_lookup[element] - start_nodes.add(n1) - start_nodes.add(n2) - else: - # 仅当缓存中没找到时（极少见），才回退到慢速 API - if is_node(network, element): - start_nodes.add(element) - else: - props = get_link_properties(network, element) - n1, n2 = props.get("node1"), props.get("node2") - if n1 and n2: - start_nodes.add(n1) - start_nodes.add(n2) - else: - raise ValueError( - f"Accident element {element} invalid or missing endpoints" - ) - - # 3. 处理故障阀门 (构建临时增量图) - # 我们不修改 cached pipe_adj，而是建立一个 extra_adj - extra_adj = defaultdict(list) - boundary_valves = {} # 当前有效的边界阀门 - - for vid, (n1, n2) in all_valves.items(): - if vid in disabled_valves_set: - # 故障阀门：视为连通管道 - extra_adj[n1].append(n2) - extra_adj[n2].append(n1) - else: - # 正常阀门：视为潜在边界 - boundary_valves[vid] = (n1, n2) - - # 4. BFS 搜索 (叠加 pipe_adj 和 extra_adj) - affected_nodes: set[str] = set() - queue = deque(start_nodes) - while queue: - node = queue.popleft() - if node in affected_nodes: - continue - affected_nodes.add(node) - - # 遍历永久管道邻居 - if node in pipe_adj: - for neighbor in pipe_adj[node]: - if neighbor not in affected_nodes: - queue.append(neighbor) - - # 遍历故障阀门带来的额外邻居 - if node in extra_adj: - for neighbor in extra_adj[node]: - if neighbor not in affected_nodes: - queue.append(neighbor) - - # 5. 结果聚合 - must_close_valves: list[str] = [] - optional_valves: list[str] = [] - - for valve_id, (n1, n2) in boundary_valves.items(): - in_n1 = n1 in affected_nodes - in_n2 = n2 in affected_nodes - if in_n1 and in_n2: - optional_valves.append(valve_id) - elif in_n1 or in_n2: - must_close_valves.append(valve_id) - - must_close_valves.sort() - optional_valves.sort() - isolatable = bool(must_close_valves) - - result = { - "accident_elements": target_elements, - "disabled_valves": disabled_valves, - "affected_nodes": sorted(affected_nodes) if isolatable else [], - "affected_node_count": len(affected_nodes), - "must_close_valves": must_close_valves, - "optional_valves": optional_valves, - "isolatable": isolatable, - } - - if len(target_elements) == 1: - result["accident_element"] = target_elements[0] - - return result diff --git a/app/algorithms/leakage/__init__.py b/app/algorithms/leakage/__init__.py deleted file mode 100644 index 9ec8b91..0000000 --- a/app/algorithms/leakage/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.algorithms.leakage.identifier import LeakageIdentifier - -__all__ = ["LeakageIdentifier"] diff --git a/app/algorithms/pipe_health_prediction/__init__.py b/app/algorithms/pipe_health_prediction/__init__.py new file mode 100644 index 0000000..5621545 --- /dev/null +++ b/app/algorithms/pipe_health_prediction/__init__.py @@ -0,0 +1,5 @@ +from app.algorithms.pipe_health_prediction.survival_predictor import ( + PipeHealthSurvivalPredictor, +) + +__all__ = ["PipeHealthSurvivalPredictor"] diff --git a/app/algorithms/health/model/my_survival_forest_model_quxi.zip b/app/algorithms/pipe_health_prediction/model/my_survival_forest_model_quxi.zip similarity index 100% rename from app/algorithms/health/model/my_survival_forest_model_quxi.zip rename to app/algorithms/pipe_health_prediction/model/my_survival_forest_model_quxi.zip diff --git a/app/algorithms/health/analyzer.py b/app/algorithms/pipe_health_prediction/survival_predictor.py similarity index 84% rename from app/algorithms/health/analyzer.py rename to app/algorithms/pipe_health_prediction/survival_predictor.py index ed1caca..432504e 100644 --- a/app/algorithms/health/analyzer.py +++ b/app/algorithms/pipe_health_prediction/survival_predictor.py @@ -4,7 +4,7 @@ import pandas as pd import matplotlib.pyplot as plt -class PipelineHealthAnalyzer: +class PipeHealthSurvivalPredictor: """ 管道健康分析器类，使用随机生存森林模型预测管道的生存概率。 @@ -28,11 +28,6 @@ class PipelineHealthAnalyzer: "model", "my_survival_forest_model_quxi.joblib", ) - # 确保 model 目录存在 - model_dir = os.path.dirname(model_path) - if model_dir and not os.path.exists(model_dir): - os.makedirs(model_dir, exist_ok=True) - if not os.path.exists(model_path): raise FileNotFoundError(f"模型文件未找到: {model_path}") @@ -102,7 +97,7 @@ class PipelineHealthAnalyzer: # 调用说明示例 """ -在其他项目中使用PipelineHealthAnalyzer类的步骤： +在其他项目中使用 PipeHealthSurvivalPredictor 类的步骤： 1. 安装依赖（在requirements.txt中添加）： joblib==1.5.0 @@ -112,34 +107,29 @@ class PipelineHealthAnalyzer: matplotlib==3.9.4 2. 导入类： - from pipeline_health_analyzer import PipelineHealthAnalyzer + from survival_predictor import PipeHealthSurvivalPredictor 3. 初始化分析器（替换为实际模型路径）： - analyzer = PipelineHealthAnalyzer(model_path='path/to/my_survival_forest_model3-10.joblib') + predictor = PipeHealthSurvivalPredictor(model_path='path/to/model.joblib') -4. 准备数据（pandas DataFrame，包含9个特征列）： +4. 准备数据（pandas DataFrame，包含4个特征列）： import pandas as pd data = pd.DataFrame({ 'Material': [1, 2], # 示例数据 'Diameter': [100, 150], 'Flow Velocity': [1.5, 2.0], - 'Pressure': [50, 60], - 'Temperature': [20, 25], - 'Precipitation': [0.1, 0.2], - 'Location': [1, 2], - 'Structural Defects': [0, 1], - 'Functional Defects': [0, 0] + 'Pressure': [50, 60] }) 5. 进行预测： - survival_funcs = analyzer.predict_survival(data) + survival_funcs = predictor.predict_survival(data) 6. 查看结果（每个样本的生存概率随时间变化）： for i, sf in enumerate(survival_funcs): print(f"样本 {i+1}: 时间点: {sf.x[:5]}..., 生存概率: {sf.y[:5]}...") 7. 可视化（可选）： - analyzer.plot_survival(survival_funcs, save_path='survival_plot.png') + predictor.plot_survival(survival_funcs, save_path='survival_plot.png') 注意： - 数据格式必须匹配特征列表，特征值为数值型。 diff --git a/app/algorithms/pressure_sensor_placement/__init__.py b/app/algorithms/pressure_sensor_placement/__init__.py new file mode 100644 index 0000000..9750126 --- /dev/null +++ b/app/algorithms/pressure_sensor_placement/__init__.py @@ -0,0 +1 @@ +"""Pressure sensor placement calculation implementations.""" diff --git a/app/algorithms/pressure_sensor_placement/kmeans_placement.py b/app/algorithms/pressure_sensor_placement/kmeans_placement.py new file mode 100644 index 0000000..c089894 --- /dev/null +++ b/app/algorithms/pressure_sensor_placement/kmeans_placement.py @@ -0,0 +1,96 @@ +import matplotlib.pyplot as plt +import numpy as np +import sklearn.cluster +import wntr + + +class KMeansPlacement: + def __init__(self, wn, num_monitors: int, min_diameter_mm: float): + self.cluster_num = num_monitors + self.wn = wn + self.monitor_nodes: list[str] = [] + self.coords: list[tuple[float, float]] = [] + self.candidate_nodes: list[str] = [] + self.min_diameter_mm = min_diameter_mm + + def get_junctions_coordinates(self) -> None: + eligible_nodes: set[str] = set() + junction_names = set(self.wn.junction_name_list) + for pipe_name in self.wn.pipe_name_list: + pipe = self.wn.get_link(pipe_name) + if float(pipe.diameter) * 1000 < self.min_diameter_mm: + continue + eligible_nodes.update( + node_id + for node_id in (pipe.start_node_name, pipe.end_node_name) + if node_id in junction_names + ) + + for junction_name in self.wn.junction_name_list: + if junction_name not in eligible_nodes: + continue + junction = self.wn.get_node(junction_name) + self.candidate_nodes.append(junction_name) + self.coords.append(junction.coordinates) + + def select_monitoring_points(self) -> list[str]: + if not self.coords: + self.get_junctions_coordinates() + if self.cluster_num <= 0: + raise ValueError("sensor_count must be greater than zero") + if self.cluster_num > len(self.candidate_nodes): + raise ValueError("符合最小管径条件的候选节点数量少于请求的监测点数量") + coords = np.array(self.coords) + coordinate_span = coords.max(axis=0) - coords.min(axis=0) + coordinate_span[coordinate_span == 0] = 1.0 + coords_normalized = (coords - coords.min(axis=0)) / coordinate_span + kmeans = sklearn.cluster.KMeans(n_clusters=self.cluster_num, random_state=42) + kmeans.fit(coords_normalized) + + selected_indices: set[int] = set() + for cluster_index, center in enumerate(kmeans.cluster_centers_): + cluster_indices = np.flatnonzero(kmeans.labels_ == cluster_index) + available_indices = [ + int(index) + for index in cluster_indices + if int(index) not in selected_indices + ] + if not available_indices: + available_indices = [ + index + for index in range(len(self.candidate_nodes)) + if index not in selected_indices + ] + nearest_index = min( + available_indices, + key=lambda index: ( + float(np.sum((coords_normalized[index] - center) ** 2)), + index, + ), + ) + selected_indices.add(nearest_index) + nearest_node = self.candidate_nodes[nearest_index] + self.monitor_nodes.append(nearest_node) + + return self.monitor_nodes + + def visualize_network(self) -> None: + """Visualize network with monitoring points.""" + wntr.graphics.plot_network( + self.wn, + node_attribute=self.monitor_nodes, + node_size=30, + title="Optimal sensor", + ) + plt.show() + + +def optimize_sensor_placement( + network_model: wntr.network.WaterNetworkModel, + sensor_count: int, + min_diameter_mm: float, +) -> list[str]: + """Select sensor nodes from an already loaded network model.""" + + placement = KMeansPlacement(network_model, sensor_count, min_diameter_mm) + return placement.select_monitoring_points() diff --git a/app/algorithms/sensor/sensitivity.py b/app/algorithms/pressure_sensor_placement/sensitivity_placement.py similarity index 98% rename from app/algorithms/sensor/sensitivity.py rename to app/algorithms/pressure_sensor_placement/sensitivity_placement.py index 618c08a..7ad2a94 100644 --- a/app/algorithms/sensor/sensitivity.py +++ b/app/algorithms/pressure_sensor_placement/sensitivity_placement.py @@ -903,14 +903,3 @@ def optimize_sensor_placement_from_inp( sensor_num=sensor_num, min_diameter=min_diameter, ) - - -def get_ID(name: str, sensor_num: int, min_diameter: int) -> list[str]: - """Compatibility entry point used by the sensor placement service.""" - - inp_path = Path("db_inp") / f"{name}.db.inp" - return optimize_sensor_placement_from_inp( - inp_path, - sensor_num=sensor_num, - min_diameter=min_diameter, - ) diff --git a/app/algorithms/scada_cleaning/__init__.py b/app/algorithms/scada_cleaning/__init__.py new file mode 100644 index 0000000..d019df7 --- /dev/null +++ b/app/algorithms/scada_cleaning/__init__.py @@ -0,0 +1,6 @@ +"""SCADA time-series cleaning algorithms.""" + +from .flow_series import clean_flow_data_df_kf +from .pressure_series import clean_pressure_data_df_km + +__all__ = ["clean_flow_data_df_kf", "clean_pressure_data_df_km"] diff --git a/app/algorithms/cleaning/flow.py b/app/algorithms/scada_cleaning/flow_series.py similarity index 86% rename from app/algorithms/cleaning/flow.py rename to app/algorithms/scada_cleaning/flow_series.py index ee361b9..9c9c777 100644 --- a/app/algorithms/cleaning/flow.py +++ b/app/algorithms/scada_cleaning/flow_series.py @@ -142,11 +142,13 @@ def clean_flow_data_kf( return os.path.abspath(output_path) -def clean_flow_data_df_kf(data: pd.DataFrame, show_plot: bool = False) -> dict: +def clean_flow_data_df_kf( + data: pd.DataFrame, show_plot: bool = False +) -> pd.DataFrame: """ 接收一个 DataFrame 数据结构，使用一维 Kalman 滤波平滑并用预测值替换基于 IQR 检测出的异常点。 区分合理的0值（流量转换）和异常的0值（连续多个0或孤立0）。 - 返回完整的清洗后的字典数据结构。 + 返回完整的清洗后 DataFrame。 Args: data: 输入 DataFrame（可包含 time 列） @@ -305,42 +307,3 @@ def clean_flow_data_df_kf(data: pd.DataFrame, show_plot: bool = False) -> dict: # 返回完整的修复后字典 return cleaned_data - - -# # 测试 -# if __name__ == "__main__": -# # 默认：脚本目录下同名 CSV 文件 -# script_dir = os.path.dirname(os.path.abspath(__file__)) -# default_csv = os.path.join(script_dir, "pipe_flow_data_to_clean2.0.csv") -# out = clean_flow_data_kf(default_csv) -# print("清洗后的数据已保存到:", out) - -# 测试 clean_flow_data_dict 函数 -if __name__ == "__main__": - import random - - # 读取 szh_flow_scada.csv 文件 - script_dir = os.path.dirname(os.path.abspath(__file__)) - csv_path = os.path.join(script_dir, "szh_flow_scada.csv") - data = pd.read_csv(csv_path, header=0, index_col=None, encoding="utf-8") - - # 排除 Time 列，随机选择 5 列 - columns_to_exclude = ["Time"] - available_columns = [col for col in data.columns if col not in columns_to_exclude] - selected_columns = random.sample(available_columns, 1) - - # 将选中的列转换为字典 - data_dict = {col: data[col].tolist() for col in selected_columns} - - print("选中的列:", selected_columns) - print("原始数据长度:", len(data_dict[selected_columns[0]])) - - # 调用函数进行清洗 - cleaned_dict = clean_flow_data_df_kf(data_dict, show_plot=True) - # 将清洗后的字典写回 CSV - out_csv = os.path.join(script_dir, f"{selected_columns[0]}_clean.csv") - pd.DataFrame(cleaned_dict).to_csv(out_csv, index=False, encoding="utf-8-sig") - print("已保存清洗结果到:", out_csv) - print("清洗后的字典键:", list(cleaned_dict.keys())) - print("清洗后的数据长度:", len(cleaned_dict[selected_columns[0]])) - print("测试完成：函数运行正常") diff --git a/app/algorithms/cleaning/pressure.py b/app/algorithms/scada_cleaning/pressure_series.py similarity index 93% rename from app/algorithms/cleaning/pressure.py rename to app/algorithms/scada_cleaning/pressure_series.py index 2287ba3..4ba722e 100644 --- a/app/algorithms/cleaning/pressure.py +++ b/app/algorithms/scada_cleaning/pressure_series.py @@ -541,39 +541,3 @@ def clean_pressure_data_df_km(data: pd.DataFrame, show_plot: bool = False) -> pd plt.show() return data_repaired - - -# 测试 -# if __name__ == "__main__": -# # 默认使用脚本目录下的 pressure_raw_data.csv -# script_dir = os.path.dirname(os.path.abspath(__file__)) -# default_csv = os.path.join(script_dir, "pressure_raw_data.csv") -# out_path = clean_pressure_data_km(default_csv, show_plot=False) -# print("保存路径：", out_path) - -# 测试 clean_pressure_data_dict_km 函数 -if __name__ == "__main__": - import random - - # 读取 szh_pressure_scada.csv 文件 - script_dir = os.path.dirname(os.path.abspath(__file__)) - csv_path = os.path.join(script_dir, "szh_pressure_scada.csv") - data = pd.read_csv(csv_path, header=0, index_col=None, encoding="utf-8") - - # 排除 Time 列，随机选择 5 列 - columns_to_exclude = ["Time"] - available_columns = [col for col in data.columns if col not in columns_to_exclude] - selected_columns = random.sample(available_columns, 5) - - # 将选中的列转换为字典 - data_dict = {col: data[col].tolist() for col in selected_columns} - - print("选中的列:", selected_columns) - print("原始数据长度:", len(data_dict[selected_columns[0]])) - - # 调用函数进行清洗 - cleaned_dict = clean_pressure_data_df_km(data_dict, show_plot=True) - - print("清洗后的字典键:", list(cleaned_dict.keys())) - print("清洗后的数据长度:", len(cleaned_dict[selected_columns[0]])) - print("测试完成：函数运行正常") diff --git a/app/algorithms/sensor/__init__.py b/app/algorithms/sensor/__init__.py deleted file mode 100644 index b5737cd..0000000 --- a/app/algorithms/sensor/__init__.py +++ /dev/null @@ -1,131 +0,0 @@ -from contextlib import contextmanager -import fcntl -from pathlib import Path -from typing import Any - -from app.algorithms.sensor import kmeans as kmeans_sensor -from app.algorithms.sensor import sensitivity -from app.infra.db.postgresql.sensor_placement import create_sensor_placement -from app.services.sensor_placement import ( - SensorPlacementConflictError, - SensorPlacementValidationError, - validate_sensor_placement_nodes, -) -from app.services.tjnetwork import dump_inp - - -def _sensor_inp_path(name: str) -> Path: - if ( - not name - or name in {".", ".."} - or "/" in name - or "\\" in name - or "\x00" in name - ): - raise SensorPlacementValidationError("管网名称不是有效的项目标识") - return Path("db_inp") / f"{name}.db.inp" - - -@contextmanager -def _sensor_inp_lock(name: str): - inp_path = _sensor_inp_path(name) - inp_path.parent.mkdir(parents=True, exist_ok=True) - lock_path = inp_path.with_suffix(".sensor.lock") - with lock_path.open("w", encoding="utf-8") as lock_file: - try: - fcntl.flock( - lock_file.fileno(), - fcntl.LOCK_EX | fcntl.LOCK_NB, - ) - except BlockingIOError as exc: - raise SensorPlacementConflictError( - "当前项目已有监测点优化任务正在运行，请稍后重试" - ) from exc - try: - yield inp_path - finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) - - -def _create_validated_placement( - name: str, - *, - run_name: str, - min_diameter: int, - created_by: str, - sensor_locations: list[str], -) -> dict[str, Any]: - validate_sensor_placement_nodes(name, sensor_locations) - return create_sensor_placement( - name, - run_name=run_name, - min_diameter=min_diameter, - created_by=created_by, - sensor_locations=sensor_locations, - ) - - -def pressure_sensor_placement_sensitivity( - name: str, - scheme_name: str, - sensor_number: int, - min_diameter: int, - username: str, -) -> dict[str, Any]: - """ - 基于改进灵敏度法进行压力监测点优化布置 - :param name: 数据库名称 - :param scheme_name: 监测优化布置方案名称 - :param sensor_number: 传感器数目 - :param min_diameter: 最小管径 - :param username: 用户名 - :return: 新建的监测点方案 - """ - with _sensor_inp_lock(name): - sensor_location = sensitivity.get_ID( - name=name, - sensor_num=sensor_number, - min_diameter=min_diameter, - ) - return _create_validated_placement( - name, - run_name=scheme_name, - min_diameter=min_diameter, - created_by=username, - sensor_locations=sensor_location, - ) - - -# 2025/08/21 -# 基于kmeans聚类法进行压力监测点优化布置 -def pressure_sensor_placement_kmeans( - name: str, - scheme_name: str, - sensor_number: int, - min_diameter: int, - username: str, -) -> dict[str, Any]: - """ - 基于聚类法进行压力监测点优化布置 - :param name: 数据库名称（注意，此处数据库名称也是inp文件名称，inp文件与pg库名要一样） - :param scheme_name: 监测优化布置方案名称 - :param sensor_number: 传感器数目 - :param min_diameter: 最小管径 - :param username: 用户名 - :return: 新建的监测点方案 - """ - # dump_inp - with _sensor_inp_lock(name) as inp_path: - dump_inp(name, str(inp_path), "2") - sensor_location = kmeans_sensor.kmeans_sensor_placement( - name=name, - sensor_num=sensor_number, - min_diameter=min_diameter, - ) - return _create_validated_placement( - name, - run_name=scheme_name, - min_diameter=min_diameter, - created_by=username, - sensor_locations=sensor_location, - ) diff --git a/app/algorithms/sensor/kmeans.py b/app/algorithms/sensor/kmeans.py deleted file mode 100644 index 84c37d3..0000000 --- a/app/algorithms/sensor/kmeans.py +++ /dev/null @@ -1,71 +0,0 @@ -import wntr -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt -import sklearn.cluster -import os - - -class QD_KMeans(object): - def __init__(self, wn, num_monitors): - # self.inp = inp - self.cluster_num = num_monitors # 聚类中心个数，也即测压点个数 - self.wn = wn - self.monitor_nodes = [] - self.coords = [] - self.junction_nodes = {} # Added missing initialization - - def get_junctions_coordinates(self): - - for junction_name in self.wn.junction_name_list: - junction = self.wn.get_node(junction_name) - self.junction_nodes[junction_name] = junction.coordinates - self.coords.append(junction.coordinates) - - # print(f"Total junctions: {self.junction_coordinates}") - - def select_monitoring_points(self): - if not self.coords: # Add check if coordinates are collected - self.get_junctions_coordinates() - coords = np.array(self.coords) - coords_normalized = (coords - coords.min(axis=0)) / ( - coords.max(axis=0) - coords.min(axis=0) - ) - kmeans = sklearn.cluster.KMeans(n_clusters=self.cluster_num, random_state=42) - kmeans.fit(coords_normalized) - - for center in kmeans.cluster_centers_: - distances = np.sum((coords_normalized - center) ** 2, axis=1) - nearest_node = self.wn.junction_name_list[np.argmin(distances)] - self.monitor_nodes.append(nearest_node) - - return self.monitor_nodes - - def visualize_network(self): - """Visualize network with monitoring points""" - ax = wntr.graphics.plot_network( - self.wn, - node_attribute=self.monitor_nodes, - node_size=30, - title="Optimal sensor", - ) - plt.show() - - -def kmeans_sensor_placement(name: str, sensor_num: int, min_diameter: int) -> list: - inp_name = f"./db_inp/{name}.db.inp" - wn = wntr.network.WaterNetworkModel(inp_name) - wn_cluster = QD_KMeans(wn, sensor_num) - - # Select monitoring pointse - sensor_ids = wn_cluster.select_monitoring_points() - - # wn_cluster.visualize_network() - - return sensor_ids - - -if __name__ == "__main__": - # sensorindex = get_ID(name='suzhouhe_2024_cloud_0817', sensor_num=30, min_diameter=500) - sensorindex = kmeans_sensor_placement(name="szh", sensor_num=50, min_diameter=300) - print(sensorindex) diff --git a/app/algorithms/simulation/__init__.py b/app/algorithms/simulation/__init__.py deleted file mode 100644 index b8e0fa6..0000000 --- a/app/algorithms/simulation/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from app.algorithms.simulation.scenarios import ( - convert_to_local_unit, - burst_analysis, - valve_close_analysis, - flushing_analysis, - contaminant_simulation, - age_analysis, - pressure_regulation, -) - -__all__ = [ - "convert_to_local_unit", - "burst_analysis", - "valve_close_analysis", - "flushing_analysis", - "contaminant_simulation", - "age_analysis", - "pressure_regulation", -] diff --git a/app/algorithms/simulation/runner.py b/app/algorithms/simulation/runner.py deleted file mode 100644 index d2fe1a9..0000000 --- a/app/algorithms/simulation/runner.py +++ /dev/null @@ -1,867 +0,0 @@ -import numpy as np -from functools import wraps -from app.services.tjnetwork import ( - ChangeSet, - get_pattern, - get_patterns, - get_pump, - get_reservoir, - get_status, - get_tank, - get_time, - read_all, - run_project, - set_pattern, - set_status, - set_tank, - set_time, -) -# from get_real_status import * -from datetime import datetime,timedelta -from math import modf -import json -import pytz -import requests -import time -import app.services.project_info as project_info -from app.native.wndb.core.projects import temporary_project_database -from app.services.time_api import parse_clock_duration_seconds - -url_path = 'http://10.101.15.16:9000/loong' # 内网 -# url_path = 'http://183.64.62.100:9057/loong' # 外网 -url_real = url_path + '/api/mpoints/realValue' -url_hist = url_path + '/api/curves/data' - -PATTERN_TIME_STEP=15.0 -DN_900_ID='2498' -DN_500_ID='3854' -DN_1000_ID='3853' -H_RESSURE='2510' -L_PRESURE='2514' -H_TANK='4780' -L_TANK='4854' - -H_REGION_1='SA_ZBBDJSCP000002' -H_REGION_2='' #to do -L_REGION_1='SA_ZBBDTJSC000001' -L_REGION_2='SA_R00003' - -# reservoir basic height -RESERVOIR_BASIC_HEIGHT = float(250.35) - -# regions -regions = ['hp', 'lp'] -regions_demand_patterns = {'hp': ['DN900', 'DN500'], 'lp': ['DN1000']} # 出厂水量近似表示用水量 -regions_patterns = {'hp': ['ChuanYiJiXiao', 'BeiQuanHuaYuan', 'ZhuangYuanFuDi', 'JingNingJiaYuan', - '308', 'JiaYinYuan', 'XinChengGuoJi', 'YiJingBeiChen', 'ZhongYangXinDu', - 'XinHaiJiaYuan', 'DongFengJie', 'DingYaXinYu', 'ZiYunTai', 'XieMaGuangChang', - 'YongJinFu', 'BianDianZhan', 'BeiNanDaDao', 'TianShengLiJie', 'XueYuanXiaoQu', - 'YunHuaLu', 'GaoJiaQiao', 'LuZuoFuLuXiaDuan', 'TianRunCheng', 'CaoJiaBa', - 'PuLingChang', 'QiLongXiaoQu', 'TuanXiao', - 'TuanShanBaoZhongShiHua', 'XieMa', 'BeiWenQuanJiuHaoErQi', 'LaiYinHuSiQi', - 'DN500', 'DN900'], - 'lp': ['PanXiMingDu', 'WanKeJinYuHuaFuGaoCeng', 'KeJiXiao', - 'LuGouQiao', 'LongJiangHuaYuan', 'LaoQiZhongDui', 'ShiYanCun', 'TianQiDaSha', - 'TianShengPaiChuSuo', 'TianShengShangPin', 'JiaoTang', 'RenMinHuaYuan', - 'TaiJiBinJiangYiQi', 'TianQiHuaYuan', 'TaiJiBinJiangErQi', '122Zhong', - 'WanKeJinYuHuaFuYangFang', 'ChengBeiCaiShiKou', 'WenXingShe', 'YueLiangTianBBGJCZ', - 'YueLiangTian', 'YueLiangTian200', 'ChengTaoChang', 'HuoCheZhan', 'LiangKu', 'QunXingLu', - 'JiuYuanErTongYiYuan', 'TangDouHua', 'TaiJiBinJiangErQi(SanJi)', - 'ZhangDouHua', 'JinYunXiaoQuDN400', - 'DN1000']} - -# nodes -monitor_single_patterns = ['ChuanYiJiXiao', 'BeiQuanHuaYuan', 'ZhuangYuanFuDi', 'JingNingJiaYuan', - '308', 'JiaYinYuan', 'XinChengGuoJi', 'YiJingBeiChen', 'ZhongYangXinDu', - 'XinHaiJiaYuan', 'DongFengJie', 'DingYaXinYu', 'ZiYunTai', 'XieMaGuangChang', - 'YongJinFu', 'PanXiMingDu', 'WanKeJinYuHuaFuGaoCeng', 'KeJiXiao', - 'LuGouQiao', 'LongJiangHuaYuan', 'LaoQiZhongDui', 'ShiYanCun', 'TianQiDaSha', - 'TianShengPaiChuSuo', 'TianShengShangPin', 'JiaoTang', 'RenMinHuaYuan', - 'TaiJiBinJiangYiQi', 'TianQiHuaYuan', 'TaiJiBinJiangErQi', '122Zhong', - 'WanKeJinYuHuaFuYangFang'] -monitor_single_patterns_id = {'ChuanYiJiXiao': '7338', 'BeiQuanHuaYuan': '7315', 'ZhuangYuanFuDi': '7316', - 'JingNingJiaYuan': '7528', '308': '8272', 'JiaYinYuan': '7304', - 'XinChengGuoJi': '7325', 'YiJingBeiChen': '7328', 'ZhongYangXinDu': '7329', - 'XinHaiJiaYuan': '9138', 'DongFengJie': '7302', 'DingYaXinYu': '7331', - 'ZiYunTai': '7420,9059', 'XieMaGuangChang': '7326', 'YongJinFu': '9059', - 'PanXiMingDu': '7320', 'WanKeJinYuHuaFuGaoCeng': '7419', - 'KeJiXiao': '7305', 'LuGouQiao': '7306', 'LongJiangHuaYuan': '7318', - 'LaoQiZhongDui': '9075', 'ShiYanCun': '7309', 'TianQiDaSha': '7323', - 'TianShengPaiChuSuo': '7335', 'TianShengShangPin': '7324', 'JiaoTang': '7332', - 'RenMinHuaYuan': '7322', 'TaiJiBinJiangYiQi': '7333', 'TianQiHuaYuan': '8235', - 'TaiJiBinJiangErQi': '7334', '122Zhong': '7314', 'WanKeJinYuHuaFuYangFang': '7418'} - -monitor_unity_patterns = ['BianDianZhan', 'BeiNanDaDao', 'TianShengLiJie', 'XueYuanXiaoQu', - 'YunHuaLu', 'GaoJiaQiao', 'LuZuoFuLuXiaDuan', 'TianRunCheng', - 'CaoJiaBa', 'PuLingChang', 'QiLongXiaoQu', 'TuanXiao', - 'ChengBeiCaiShiKou', 'WenXingShe', 'YueLiangTianBBGJCZ', - 'YueLiangTian', 'YueLiangTian200', - 'ChengTaoChang', 'HuoCheZhan', 'LiangKu', 'QunXingLu', - 'TuanShanBaoZhongShiHua', 'XieMa', 'BeiWenQuanJiuHaoErQi', 'LaiYinHuSiQi', - 'JiuYuanErTongYiYuan', 'TangDouHua', 'TaiJiBinJiangErQi(SanJi)', - 'ZhangDouHua', 'JinYunXiaoQuDN400', - 'DN500', 'DN900', 'DN1000'] -monitor_unity_patterns_id = {'BianDianZhan': '7339', 'BeiNanDaDao': '7319', 'TianShengLiJie': '8242', - 'XueYuanXiaoQu': '7327', 'YunHuaLu': '7312', 'GaoJiaQiao': '7340', - 'LuZuoFuLuXiaDuan': '7343', 'TianRunCheng': '7310', 'CaoJiaBa': '7300', - 'PuLingChang': '7307', 'QiLongXiaoQu': '7321', 'TuanXiao': '8963', - 'ChengBeiCaiShiKou': '7330', 'WenXingShe': '7311', - 'YueLiangTianBBGJCZ': '7313', 'YueLiangTian': '7313', 'YueLiangTian200': '7313', - 'ChengTaoChang': '7301', 'HuoCheZhan': '7303', - 'LiangKu': '7296', 'QunXingLu': '7308', - 'DN500': '3854', 'DN900': '2498', 'DN1000': '3853'} -monitor_patterns = monitor_single_patterns + monitor_unity_patterns -monitor_patterns_id = {**monitor_single_patterns_id, **monitor_unity_patterns_id} -# pumps -pumps_name = ['1#', '2#', '3#', '4#', '5#', '6#', '7#'] -pumps = ['PU00000', 'PU00001', 'PU00002', 'PU00003', 'PU00004', 'PU00005', 'PU00006'] -variable_frequency_pumps = ['PU00004', 'PU00005', 'PU00006'] -pumps_id = {'PU00000': '2747', 'PU00001': '2776', 'PU00002': '2730', 'PU00003': '2787', - 'PU00004': '2500', 'PU00005': '2502', 'PU00006': '2504'} -# reservoirs -reservoirs = ['ZBBDJSCP000002', 'R00003'] -reservoirs_id = {'ZBBDJSCP000002': '2497', 'R00003': '2571'} -# tanks -tanks = ['ZBBDTJSC000002', 'ZBBDTJSC000001'] -tanks_id = {'ZBBDTJSC000002': '4780', 'ZBBDTJSC000001': '9774'} - - -class DataLoader: - """数据加载器""" - def __init__(self, project_name, start_time: datetime, end_time: datetime, - pumps_control: dict = None, tank_initial_level_control: dict = None, - region_demand_control: dict = None, downloading_prohibition: bool = False): - self.project_name = project_name # 数据库名 - self.current_time = self.round_time(datetime.now(pytz.timezone('Asia/Shanghai')), 1) # 圆整至整分钟 - self.current_round_time = self.round_time(self.current_time, int(PATTERN_TIME_STEP)) - self.updating_data_flag = True \ - if self.current_round_time == self.round_time(start_time, int(PATTERN_TIME_STEP)) \ - else False # 判断是否从当前时刻开始模拟(是否更新最新监测数据) - self.downloading_prohibition = downloading_prohibition # 是否禁止下载数据(默认False: 允许下载) - self.updating_data_flag = False if self.downloading_prohibition else self.updating_data_flag - self.pattern_start_index = get_pattern_index( - self.round_time(start_time, int(PATTERN_TIME_STEP)).strftime("%Y-%m-%d %H:%M:%S")) # pattern起始索引 - self.pattern_end_index = get_pattern_index( - self.round_time(end_time, int(PATTERN_TIME_STEP)).strftime("%Y-%m-%d %H:%M:%S")) # pattern结束索引 - self.pattern_index_list = list(range(self.pattern_start_index, self.pattern_end_index + 1)) # pattern索引列表 - self.download_id = self.get_download_id() # 数据下载接口id '7338,7315,7316,...' - self.current_time_download_data = dict( - zip(self.download_id.split(','), - [np.nan]*len(list(self.download_id.split(',')))) - ) # {id(str): value(float)} - self.current_time_download_data_flag = dict( - zip(self.download_id.split(','), - [False]*len(list(self.download_id.split(',')))) - ) # 下载数据是否具备实时性, {id(str): flag(bool)} - self.old_flow_data = self.init_dict_of_list(dict( - zip(monitor_patterns, - [[np.nan]] * (len(monitor_patterns))) - )) # {pattern_name(str): flow(float)} - self.old_pattern_factor = self.init_dict_of_list(dict( - zip(monitor_patterns, - [[np.nan]] * (len(monitor_patterns))) - )) # {pattern_name(str): [pattern_factor(float)]} - self.new_flow_data = self.init_dict_of_list(dict( - zip(monitor_patterns, - [[np.nan]] * (len(monitor_patterns))) - )) # {pattern_name(str): flow(float)} - self.new_pattern_factor = self.init_dict_of_list(dict( - zip(monitor_patterns, - [[np.nan]] * (len(monitor_patterns))) - )) # {pattern_name(str): [pattern_factor(float)]} - self.reservoir_data = dict(zip(reservoirs, [np.nan]*len(reservoirs))) # {reservoir_name(str): level(float)} - self.tank_data = dict(zip(tanks, [np.nan] * len(tanks))) # {tank_name(str): level(float)} - self.pump_data = self.init_dict_of_list( - dict(zip(pumps, [[np.nan]]*len(pumps)))) # {pump_name(str): [frequency(float)]} - self.pump_control = pumps_control # {pump_name(str): [frequency(float)]} - self.tank_initial_level_control = tank_initial_level_control # {tank_name(str): level(float)} - self.region_demand_current = dict(zip(regions, [0]*len(regions))) # {region_name(str): total_demand(float)} - self.region_demand_control = region_demand_control # {region_name(str): total_demand(float)} - self.region_demand_control_factor = dict( - zip(regions, [1]*len(regions))) # 区域流量控制系数(用于调整用水量), {region_name(str): factor(float)} - - def load_data(self): - """生成数据集""" - self.download_data() # 下载实时数据 - self.get_old_pattern_and_flow() # 读取历史记录pattern信息 - self.cal_demand_convert_factor() # 计算用水量转换系数(设定用水量时) - self.set_new_flow() # 设置'更新'流量 - self.set_new_pattern_factor() # 设置'更新'pattern factors - self.set_reservoirs() # 设置清水池 - self.set_tanks() # 设置调节池 - self.set_pumps() # 设置水泵 - return self.pattern_start_index - - def download_data(self): - """下载数据""" - if self.updating_data_flag is True: - print('{} -- Start downloading data.'.format( - datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) - data_wait_flag = True - while data_wait_flag: - try: - newest_data_time = self.download_real_data(self.download_id) # 获取实时数据 - except Exception as e: - print('{}\nWaiting for real data.'.format(e)) - time.sleep(1) - else: - print('{} -- Downloading data ok. Newest timestamp: {}.'.format( - datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - newest_data_time.strftime('%Y-%m-%d %H:%M:%S'))) - data_wait_flag = False - - def cal_current_region_demand(self): - """计算区域当前用水量""" - if self.updating_data_flag is True: - for region in self.region_demand_current.keys(): - total_demand = 0 - for pipe in regions_demand_patterns[region]: - total_demand += self.current_time_download_data[monitor_patterns_id[pipe]] # 出厂流量 - self.region_demand_current[region] = total_demand - - def cal_history_region_demand(self, pattern_index_list): - """计算区域历史用水量(对应记录的pattern)""" - old_demand = {} - for region in regions: - total_demand = 0 - for pipe_pattern_name in regions_demand_patterns[region]: - old_flows, old_patterns = self.get_history_pattern_info(self.project_name, pipe_pattern_name) - for idx in pattern_index_list: - total_demand += old_flows[idx] / 4 # 15分钟水量 - old_demand[region] = total_demand - return old_demand - - def cal_demand_convert_factor(self): - """计算用水量转换系数(设定用水量时)""" - self.cal_current_region_demand() # 计算区域当前时刻用水量 - old_demand_moment = self.cal_history_region_demand([self.pattern_start_index]) # 计算区域目标时刻总用水量 - old_demand_period = self.cal_history_region_demand(self.pattern_index_list) # 计算区域目标时段总用水量 - for region in regions: - self.region_demand_control_factor[region] \ - = (self.region_demand_current[region] / 4) / old_demand_moment[region] \ - if self.updating_data_flag is True else 1 - self.region_demand_control_factor[region] = self.region_demand_control[region] / old_demand_period[region] \ - if (self.region_demand_control is not None) and (region in self.region_demand_control.keys()) \ - else self.region_demand_control_factor[region] - - def get_old_pattern_and_flow(self): - """获取所有pattern的选定时段的历史记录的pattern和flow""" - for idx in monitor_patterns: # 遍历patterns - old_flows, old_patterns = self.get_history_pattern_info(self.project_name, idx) - for pattern_idx in self.pattern_index_list: - old_flow_data = old_flows[pattern_idx] - old_pattern_factor = old_patterns[pattern_idx] - if pattern_idx == self.pattern_start_index: # 起始时刻 - self.old_flow_data[idx][0] = old_flow_data - self.old_pattern_factor[idx][0] = old_pattern_factor - else: - self.old_flow_data[idx].append(old_flow_data) - self.old_pattern_factor[idx].append(old_pattern_factor) - - def set_new_flow(self): - """计算模拟时段新流量(相较于历史记录)""" - for idx in self.new_flow_data.keys(): # 遍历patterns - region_name = None - for region in regions_patterns.keys(): - if idx in regions_patterns[region]: - region_name = region # pattern所属分区 - break - # 实时流量 - if self.updating_data_flag is True: - if idx in monitor_unity_patterns[-3:]: # 出水管流量 - self.new_flow_data[idx][0] = self.current_time_download_data[monitor_patterns_id[idx]] - else: # 其余流量 - self.new_flow_data[idx][0] \ - = self.region_demand_control_factor[region_name] * self.old_flow_data[idx][0] - # if idx == 'ZiYunTai': - # idx_a, idx_b = monitor_patterns_id[idx].split(',') - # self.new_flow_data[idx][0] \ - # = self.current_time_download_data[idx_a] - self.current_time_download_data[idx_b] - # else: - # self.new_flow_data[idx][0] = self.current_time_download_data[monitor_patterns_id[idx]] - # for data_id in monitor_patterns_id[idx].split(','): - # if (self.current_time_download_data_flag[data_id] is False) \ - # and (idx not in [pipe for pipe_list in regions_demand_patterns.values() - # for pipe in pipe_list]): # 无法获取实时数据 - # self.new_flow_data[idx][0] \ - # = self.region_demand_control_factor[region_name] * self.old_flow_data[idx][0] - # break - # 根据设定用水量修改新流量 - if (self.region_demand_control is not None) \ - and (region_name in self.region_demand_control.keys()): - for pattern_idx in self.pattern_index_list: - if pattern_idx == self.pattern_start_index: # 起始时刻 - self.new_flow_data[idx][0] \ - = self.region_demand_control_factor[region_name] * self.old_flow_data[idx][0] - else: - self.new_flow_data[idx].append( - self.region_demand_control_factor[region_name] - * self.old_flow_data[idx][self.pattern_index_list.index(pattern_idx)] - ) - - def set_new_pattern_factor(self): - """更新计算选定时段(设定用水量)/时刻的pattern factor""" - pattern_index_list = self.pattern_index_list \ - if self.region_demand_control is not None \ - else [self.pattern_start_index] - for idx in monitor_patterns: # 遍历patterns - for pattern_idx in pattern_index_list: # 遍历需要修改的pattern(index) - pattern_idx_cls = pattern_index_list.index(pattern_idx) # 转换index(类表存储结构) - old_flow_data = self.old_flow_data[idx][pattern_idx_cls] - old_pattern_factor = self.old_pattern_factor[idx][pattern_idx_cls] - if pattern_idx_cls == 0: # 起始时刻 - if idx in monitor_single_patterns: - if not np.isnan(self.new_flow_data[idx][0]): - self.new_pattern_factor[idx][0] = (self.new_flow_data[idx][0] * 1000 / 3600) # m3/h to L/s - if idx in monitor_unity_patterns: - if not np.isnan(self.new_flow_data[idx][0]): - self.new_pattern_factor[idx][0] \ - = old_pattern_factor * self.new_flow_data[idx][0] / old_flow_data - else: - if idx in monitor_single_patterns: - if len(self.new_flow_data[idx]) > pattern_idx_cls: - self.new_pattern_factor[idx].append( - (self.new_flow_data[idx][pattern_idx_cls] * 1000 / 3600)) # m3/h to L/s - if idx in monitor_unity_patterns: - if len(self.new_flow_data[idx]) > pattern_idx_cls: - self.new_pattern_factor[idx].append( - old_pattern_factor - * self.new_flow_data[idx][pattern_idx_cls] - / old_flow_data) - - def set_reservoirs(self): - """设置清水池""" - if self.updating_data_flag is True: - for idx in self.reservoir_data.keys(): - if self.current_time_download_data_flag[reservoirs_id[idx]] is False: # 无法获取实时数据 - print('There is no current data of reservoir: {}.'.format(idx)) - else: - self.reservoir_data[idx] \ - = self.current_time_download_data[reservoirs_id[idx]] + RESERVOIR_BASIC_HEIGHT - - def set_tanks(self): - """设置调节池""" - for idx in self.tank_data.keys(): - if self.updating_data_flag is True: - if self.current_time_download_data_flag[tanks_id[idx]] is False: # 无法获取实时数据 - print('There is no current data of tank: {}.'.format(idx)) - else: - self.tank_data[idx] = self.current_time_download_data[tanks_id[idx]] - self.tank_data[idx] = self.tank_initial_level_control[idx] \ - if (self.tank_initial_level_control is not None) and (idx in self.tank_initial_level_control) \ - else self.tank_data[idx] - - def set_pumps(self): - """设置水泵""" - for idx in self.pump_data.keys(): - if self.updating_data_flag is True: - if self.current_time_download_data_flag[pumps_id[idx]] is False: # 无法获取实时数据 - print('There is no current data of pump: {}.'.format(idx)) - if (self.pump_control is not None) and (idx in self.pump_control.keys()): - self.pump_data[idx] = self.pump_control[idx] - else: - self.pump_data[idx] = [self.current_time_download_data[pumps_id[idx]]] - if (self.pump_control is not None) and (idx in self.pump_control.keys()): - self.pump_data[idx] = self.pump_data[idx] + self.pump_control[idx] \ - if len(self.pump_control[idx]) < len(self.pattern_index_list) \ - else self.pump_control[idx] # 水泵设定 - else: - if (self.pump_control is not None) and (idx in self.pump_control.keys()): - self.pump_data[idx] = self.pump_control[idx] - self.pump_data[idx] \ - = list(np.array(self.pump_data[idx]) / 50) \ - if idx in variable_frequency_pumps else self.pump_data[idx] - - def set_valves(self): - """设置阀门""" - pass - - def download_real_data(self, ids: str): - """加载实时数据""" - # 数据接口的地址 - global url_real - # 设置GET请求的参数 - params = {'ids': ids} - # 发送GET请求获取数据 - response = requests.get(url_real, params=params) - # 检查响应状态码，200表示请求成功 - if response.status_code == 200: - newest_data_time = None # 下载记录数据的最新时间 - # 解析响应的JSON数据 - data = response.json() - for realValue in data: # 取出逐个id的数据 - data_time = convert_utc_to_bj(realValue['datadt']) # datetime - self.current_time_download_data[str(realValue['id'])] \ - = float(realValue['realValue']) # {id(str): value(float)} - if data_time > self.current_round_time.replace(tzinfo=None) - timedelta(minutes=5): # 下载数据为实时数据 - self.current_time_download_data_flag[str(realValue['id'])] = True - if newest_data_time is None: - newest_data_time = data_time - else: - newest_data_time = data_time if data_time > newest_data_time else newest_data_time # 更新最新时间 - if newest_data_time <= self.current_round_time.replace(tzinfo=None) - timedelta(minutes=5): # 最新记录时间早于当前时间 - warning_text = 'There is no current data with newest timestamp: {}.'.format( - newest_data_time.strftime('%Y-%m-%d %H:%M:%S')) - delta_time = self.current_round_time.replace(tzinfo=None) - newest_data_time - if delta_time < timedelta(minutes=PATTERN_TIME_STEP): # 时间接近(可等待再次下载) - raise Exception(warning_text) - else: - print(warning_text) - self.updating_data_flag = False - else: - for idx in monitor_unity_patterns[-3:]: # 出水管流量 - if self.current_time_download_data_flag[monitor_patterns_id[idx]] is False: # 无法获取出水管流量的实时数据 - print('There is no current data of outflow: {}.'.format(idx)) - self.updating_data_flag = False - if self.updating_data_flag is False: - print('Abandon updating data with downloaded data.') - return newest_data_time - else: - # 如果请求不成功，打印错误信息 - print("请求失败，状态码:", response.status_code) - raise ConnectionError('Cannot download data.') - - @ staticmethod - def init_dict_of_list(dict_of_list): - """初始化值为列表的字典(重新生成列表地址, 防止指向同一列表)""" - for idx in dict_of_list.keys(): - dict_of_list[idx] = dict_of_list[idx].copy() - return dict_of_list - - @ staticmethod - def get_download_id(): - """生成下载数据项的id""" - # id_list = (list(monitor_single_patterns_id.values()) - # + list(monitor_unity_patterns_id.values()) - # + list(tanks_id.values()) - # + list(reservoirs_id.values()) - # + list(pumps_id.values())) - id_list = (list(monitor_unity_patterns_id.values())[-3:] - + list(tanks_id.values()) - + list(reservoirs_id.values()) - + list(pumps_id.values())) - id_list = sorted(set(id_list), key=id_list.index) - if None in id_list: - id_list.remove(None) - return ','.join(id_list) - - @ staticmethod - def get_history_pattern_info(project_name, pattern_name): - """读取选定pattern的保存的历史pattern信息(flow, factor)""" - factors_list = [] - flow_list = [] - patterns_info = read_all( - project_name, - "select flow, factor from network.pattern_flow_samples " - "where pattern_id = %s order by sequence_no", - (pattern_name,), - ) - for item in patterns_info: - flow_list.append(float(item['flow'])) - factors_list.append(float(item['factor'])) - return flow_list, factors_list - - @ staticmethod - def judge_time(current_time, time_index_list): - """时间判断""" - current_index \ - = time_index_list.index(current_time) if (current_time in time_index_list) else None - return current_index - - @staticmethod - def get_time_index_list(start_time: datetime, end_time: datetime, step: int): - """生成时间索引""" - time_index_list = [] # 时间索引[str] - time_index = start_time - while time_index <= end_time: - time_index_list.append(time_index) - time_index += timedelta(minutes=step) - return time_index_list - - @ staticmethod - def round_time(time_: datetime, interval=5): - """时间向下取整到整n分钟(北京时间): 四舍六入五留双/向下取整""" - # return datetime.fromtimestamp(round(time_.timestamp() / (60 * interval)) * (60 * interval)) - return datetime.fromtimestamp(int((time_.timestamp()) // (60 * interval)) * (60 * interval)) - - -def convert_utc_to_bj(utc_time_str): - """将utc时间(str)转换成北京时间(datetime)""" - # 解析UTC时间字符串为datetime对象 - utc_time = datetime.strptime(utc_time_str, '%Y-%m-%dT%H:%M:%SZ') - # 设定UTC时区 - utc_timezone = pytz.timezone('UTC') - # 转换为北京时间 - beijing_timezone = pytz.timezone('Asia/Shanghai') - beijing_time = utc_time.replace(tzinfo=utc_timezone).astimezone(beijing_timezone).replace(tzinfo=None) - return beijing_time - - -def get_datetime(cur_datetime:str): - str_format = "%Y-%m-%d %H:%M:%S" - return datetime.strptime(cur_datetime, str_format) - - -def get_strftime(cur_datetime: datetime): - str_format = "%Y-%m-%d %H:%M:%S" - return cur_datetime.strftime(str_format) - - -def step_time(cur_datetime:str, step=5): - str_format="%Y-%m-%d %H:%M:%S" - dt=datetime.strptime(cur_datetime,str_format) - dt=dt+timedelta(minutes=step) - return datetime.strftime(dt,str_format) - - -def get_pattern_index(cur_datetime:str)->int: - str_format="%Y-%m-%d %H:%M:%S" - dt=datetime.strptime(cur_datetime,str_format) - hr=dt.hour - mnt=dt.minute - i=int((hr*60+mnt)/PATTERN_TIME_STEP) - return i - -def get_pattern_index_str(cur_datetime:str)->str: - i=get_pattern_index(cur_datetime) - [minN,hrN]=modf(i*PATTERN_TIME_STEP/60) - minN_str=str(int(minN*60)) - minN_str=minN_str.zfill(2) - hrN_str=str(int(hrN)) - hrN_str=hrN_str.zfill(2) - str_i='{}:{}:00'.format(hrN_str,minN_str) - return str_i - - -def from_seconds_to_clock (secs: int)->str: - hrs=int(secs/3600) - minutes=int((secs-hrs*3600)/60) - seconds=(secs-hrs*3600-minutes*60) - hrs_str=str(hrs).zfill(2) - minutes_str=str(minutes).zfill(2) - seconds_str=str(seconds).zfill(2) - str_clock='{}:{}:{}'.format(hrs_str,minutes_str,seconds_str) - return str_clock - -def from_clock_to_seconds (clock: str)->int: - str_format="%Y-%m-%d %H:%M:%S" - dt=datetime.strptime(clock,str_format) - hr=dt.hour - mnt=dt.minute - seconds=dt.second - return hr*3600+mnt*60+seconds - -def from_clock_to_seconds_2 (clock: str)->int: - return parse_clock_duration_seconds(clock) - - -def from_clock_to_seconds_3 (clock: str)->int: - return parse_clock_duration_seconds(clock) - - -###convert datetimestring -##"XXXX-XX-XXT00:00:00Z" ->"XXXX-XX-XX 00:00:00" -def trim_time_flag(url_date_time:str)->str: - str_datetime=str.replace(url_date_time,'T',' ') - str_datetime=str.replace(str_datetime,'Z','') - return str_datetime - -# 单时间步长模拟 -def run_simulation(name:str,start_datetime:str,end_datetime:str=None, duration:int=900)->str: - #get_current_data(cur_datetime) - #extract the patternindex from datetime - #e.g. 0: the first time step for 00:00-00:14; 1: the second step for 00:15-00:30 - start_datetime=trim_time_flag(start_datetime) - if(end_datetime!=None): - end_datetime=trim_time_flag(end_datetime) - - - - ## redistribute the basedemand according to the currentTotalQ and the base_totalQ - # step 1. get_real _data - if end_datetime==None or start_datetime==end_datetime: - end_datetime=step_time(start_datetime) - - # # ids=['2498','3854','3853','2510','2514','4780','4854'] - # # real_data=get_real_data(ids,start_datetime,end_datetime) - # print(datetime.now(pytz.timezone('Asia/Shanghai')).strftime("%Y-%m-%d %H:%M:%S")+"--获取实时数据完毕\n") - # #step 2. re-distribute the real q to base demand of the node region_sa by region_sa - # regions=get_all_service_area_ids(name) - # total_demands={} - # for region in regions: - # total_demands[region]=get_total_base_demand(name,region) - # region_demand_factor={} - # #Region_ID:SA_ZBBDJSCP000002 高区;SA_R00003+SA_ZBBDTJSC000001 低区 - # H_region_real_demands=real_data[DN_900_ID][start_datetime]+real_data[DN_500_ID][start_datetime] - # L_region_real_demands=real_data[DN_1000_ID][start_datetime] - # factor_H_zone=H_region_real_demands/total_demands[H_REGION_1]/3.6 #3.6: m3/h->L/s - # factor_L_zone=L_region_real_demands/(total_demands[L_REGION_1]+total_demands[L_REGION_2])/3.6 - # print(datetime.now(pytz.timezone('Asia/Shanghai')).strftime("%Y-%m-%d %H:%M:%S")+"--流量因子计算完毕完毕\n") - # for region in regions: - # region_nodes=get_nodes_in_region(name,region) - # factor=1 - # if region==H_REGION_1 or H_REGION_2: - # factor=factor_H_zone - # else: - # factor=factor_L_zone - # - # for node in region_nodes: - # d=get_demand(name,node) - # for r in d['demands']: - # r['demand']=factor*r['demand'] - # cs=ChangeSet() - # cs.append(d) - # set_demand(name,cs) - # - # # - # # - # print(datetime.now(pytz.timezone('Asia/Shanghai')).strftime("%Y-%m-%d %H:%M:%S")+"--节点流量重分配完毕\n") - - #step 3. set pattern index to the current time,and set duration to 300 secs - # - str_pattern_start=get_pattern_index_str(start_datetime) - dic_time=get_time(name) - dic_time['PATTERN START']=str_pattern_start - - if duration !=None: - dic_time['DURATION']=from_seconds_to_clock(duration) - else: - dic_time['DURATION']=dic_time['HYDRAULIC TIMESTEP'] - cs=ChangeSet() - cs.operations.append(dic_time) - set_time(name,cs) - - # step4. run simulation and save the result to name-time.out for download - #inp_file = 'inp\\'+name+'.inp' - #db_name=name - #dump_inp(db_name,inp_file,'2') - # result=run_inp(db_name) - result=run_project(name) - #json string format - # simulation_result, output, report - result_data=json.loads(result) - #print(result_data['simulation_result']) - print(datetime.now(pytz.timezone('Asia/Shanghai')).strftime("%Y-%m-%d %H:%M:%S")+'run finished successfully\n') - #print(result_data['report']) - return result - - -# 在线模拟 -def _clean_extended_simulation(func): - @wraps(func) - def wrapper(name: str, simulation_type: str, *args, **kwargs): - if simulation_type.upper() != "EXTENDED": - return func(name, simulation_type, *args, **kwargs) - with temporary_project_database(name, "extended_simulation") as temporary: - kwargs["_temporary_project"] = temporary - return func(name, simulation_type, *args, **kwargs) - - return wrapper - - -@_clean_extended_simulation -def run_simulation_ex(name: str, simulation_type: str, start_datetime: str, - end_datetime: str = None, duration: int = 0, - pump_control: dict[str, list] = None, tank_initial_level_control: dict[str, float] = None, - region_demand_control: dict[str, float] = None, valve_control: dict[str, dict] = None, - downloading_prohibition: bool = False, - _temporary_project: str | None = None) -> str: - time_cost_start = time.perf_counter() - print('{} -- Hydraulic simulation started.'.format( - datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S'))) - - if simulation_type.upper() == 'REALTIME': # 实时模拟(修改原数据库) - name_c = name - elif simulation_type.upper() == 'EXTENDED': # 扩展模拟(复制数据库) - if _temporary_project is None: - raise RuntimeError("Extended simulation isolation was not prepared") - name_c = _temporary_project - else: - raise Exception('Incorrect simulation type, choose in (realtime, extended)') - - # 时间处理 - # extract the pattern index from datetime - # e.g. 0: the first time step for 00:00-00:14; 1: the second step for 00:15-00:30 - # start_datetime = get_strftime(convert_utc_to_bj(start_datetime)) - start_datetime = trim_time_flag(start_datetime) - if end_datetime is not None: - # end_datetime = get_strftime(convert_utc_to_bj(end_datetime)) - end_datetime = trim_time_flag(end_datetime) - - # pump name转化/输入值规范化 - if pump_control is not None: - for key in list(pump_control.keys()): - pump_control[key] = [pump_control[key]] if type(pump_control[key]) is not list else pump_control[key] - pump_control[pumps[pumps_name.index(key)]] = pump_control.pop(key) - - # 重新分配节点(nodes)水量 - # 1) (single)base_demand_new=1, pattern_new=real_data - # 2) (unity)base_demand_new=base_demand_old, pattern_new=factor*pattern_old(factor=flow_new/flow_old) - # 获取需水量数据 - # a) 历史pattern对应水量(读取保存数据库) - # b) 实时水量(数据接口下载) - - # 修改node demand = 1, pattern factor *= demand(monitor single patterns对应node) - # nodes = get_nodes(name_c) # nodes - # for node_name in nodes: # 遍历nodes - # demands_dict = get_demand(name_c, node_name) # {'demands':[{'demand':, 'pattern':}]} - # for demands in demands_dict['demands']: - # if (demands['pattern'] in monitor_single_patterns) and (demands['demand'] != 1): # 1) - # pattern = get_pattern(name_c, demands['pattern']) - # pattern['factors'] = list(demands['demand'] * np.array(pattern['factors'])) # 修改pattern - # cs = ChangeSet() - # cs.append(pattern) - # set_pattern(name_c, cs) - # demands_dict['demands'][ - # demands_dict['demands'].index(demands) - # ]['demand'] = 1 # 修改demand - # cs = ChangeSet() - # cs.append(demands_dict) - # set_demand(name_c, cs) - - start_time = get_datetime(start_datetime) # datetime - end_time = get_datetime(end_datetime) \ - if end_datetime is not None \ - else get_datetime(start_datetime) + timedelta(seconds=duration) # datetime - # modify_pattern_start_index = get_pattern_index(start_datetime) # 待修改pattern的起始索引(int) - - dataset_loader = DataLoader(project_name=name_c, - start_time=start_time, end_time=end_time, - pumps_control=pump_control, tank_initial_level_control=tank_initial_level_control, - region_demand_control=region_demand_control, - downloading_prohibition=downloading_prohibition) # 实例化数据加载器 - modify_index \ - = dataset_loader.load_data() # 加载数据(index: 需要修改pattern的factor index, None: 无需修改除水泵和调节池外pattern) - new_patterns \ - = dataset_loader.new_pattern_factor # {name: float,} pattern factor(实时: 更新, 其他: 保持/更新(设定用水量时)) - tank_init_level = dataset_loader.tank_data # {name: float,} 调节池初始液位(实时: 更新, 其他: 保持/更新(设定液位时)) - reservoir_level = dataset_loader.reservoir_data # {name: float,} 水库液位(实时: 更新, 其他: 保持) - pump_freq = dataset_loader.pump_data # {name: [float,]} 水泵频率(实时: 更新, 其他: 保持/更新(设定状态时)) - print(datetime.now(pytz.timezone('Asia/Shanghai')).strftime("%Y-%m-%d %H:%M:%S") + " -- Loading data ok.\n") - - pattern_name_list = get_patterns(name_c) # 所有pattern - # 修改node pattern/demand - # nodes = get_nodes(name_c) # nodes - # for node_name in nodes: # 遍历nodes - # demands_dict = get_demand(name_c, node_name) # {'demands':[{'demand':, 'pattern':}]} - # for demands in demands_dict['demands']: - # if demands['pattern'] in monitor_single_patterns: # 1) - # demands_dict['demands'][ - # demands_dict['demands'].index(demands) - # ]['demand'] = 1 # 修改demand - # pattern = get_pattern(name_c, demands['pattern']) - # pattern['factors'][modify_index] = flow_new[demands['pattern']] # 修改pattern - # cs = ChangeSet() - # cs.append(pattern) - # set_pattern(name_c, cs) - # if demands['pattern'] in pattern_name_list: - # pattern_name_list.remove(demands['pattern']) # 移出待修改pattern列表 - # else: # 2) - # continue - # cs = ChangeSet() - # cs.append(demands_dict) - # set_demand(name_c, cs) - for pattern_name in monitor_patterns: # 遍历patterns - if not np.isnan(new_patterns[pattern_name][0]): - pattern = get_pattern(name_c, pattern_name) - pattern['factors'][modify_index: - modify_index + len(new_patterns[pattern_name])] \ - = new_patterns[pattern_name] - cs = ChangeSet() - cs.append(pattern) - set_pattern(name_c, cs) - if pattern_name in pattern_name_list: - pattern_name_list.remove(pattern_name) # 移出待修改pattern列表 - - # 修改清水池(reservoir)液位pattern - for reservoir_name in reservoirs: # 遍历reservoirs - if (not np.isnan(reservoir_level[reservoir_name])) and (reservoir_level[reservoir_name] != 0): - reservoir_pattern = get_pattern(name_c, get_reservoir(name_c, reservoir_name)['pattern']) - reservoir_pattern['factors'][modify_index] = reservoir_level[reservoir_name] - cs = ChangeSet() - cs.append(reservoir_pattern) - set_pattern(name_c, cs) - if reservoir_pattern['id'] in pattern_name_list: - pattern_name_list.remove(reservoir_pattern['id']) # 移出待修改pattern列表 - - # 修改调节池(tank)初始液位 - for tank_name in tanks: # 遍历tanks - if (not np.isnan(tank_init_level[tank_name])) and (tank_init_level[tank_name] != 0): - tank = get_tank(name_c, tank_name) - tank['init_level'] = tank_init_level[tank_name] - cs = ChangeSet() - cs.append(tank) - set_tank(name_c, cs) - - # 修改水泵(pump)pattern - for pump_name in pumps: # 遍历pumps - if not np.isnan(pump_freq[pump_name][0]): - pump_pattern = get_pattern(name_c, get_pump(name_c, pump_name)['pattern']) - pump_pattern['factors'][modify_index - :modify_index + len(pump_freq[pump_name])] \ - = pump_freq[pump_name] - cs = ChangeSet() - cs.append(pump_pattern) - set_pattern(name_c, cs) - if pump_pattern['id'] in pattern_name_list: - pattern_name_list.remove(pump_pattern['id']) # 移出待修改pattern列表 - # 修改阀门(valve)status和setting - if valve_control is not None: - for valve in valve_control.keys(): - status = get_status(name_c, valve) - if 'status' in valve_control[valve].keys(): - status['status'] = valve_control[valve]['status'] - if 'setting' in valve_control[valve].keys(): - status['setting'] = valve_control[valve]['setting'] - if 'k' in valve_control[valve].keys(): - valve_k = valve_control[valve]['k'] - if valve_k == 0: - status['status'] = 'CLOSED' - else: - status['setting'] = 0.1036 * pow(valve_k, -3.105) - cs = ChangeSet() - cs.append(status) - set_status(name_c, cs) - - print('Finish demands amending, unmodified patterns: {}.'.format(pattern_name_list)) - - # 修改时间信息 - str_pattern_start = get_pattern_index_str( - DataLoader.round_time(start_time, int(PATTERN_TIME_STEP)).strftime("%Y-%m-%d %H:%M:%S")) - dic_time = get_time(name_c) - dic_time['PATTERN START'] = str_pattern_start - - if duration is not None: - dic_time['DURATION'] = from_seconds_to_clock(duration) - else: - dic_time['DURATION'] = dic_time['HYDRAULIC TIMESTEP'] - cs = ChangeSet() - cs.operations.append(dic_time) - set_time(name_c, cs) - - # 运行并返回结果 - result = run_project(name_c) - - time_cost_end = time.perf_counter() - print('{} -- Hydraulic simulation finished, cost time: {:.2f} s.'.format( - datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S'), - time_cost_end - time_cost_start)) - - return result - - -if __name__ == '__main__': - # if get_current_data()==True: - # tQ=get_current_total_Q() - # print(f"the current tQ is {tQ}\n") - # data=get_hist_data(ids,conver_beingtime_to_ucttime('2024-04-10 15:05:00'),conver_beingtime_to_ucttime('2024-04-10 15:10:00')) - # read_inp("beibeizone","beibeizone-export_nochinese.inp") - # run_simulation("beibeizone","2024-04-01T08:00:00Z") - # read_inp('bb_server', 'model20_en.inp') - run_simulation_ex( - name=project_info.name, simulation_type='extended', start_datetime='2024-11-09T02:30:00Z', - # end_datetime='2024-05-30T16:00:00Z', - # duration=0, - # pump_control={'PU00006': [45, 40]} - # region_demand_control={'hp': 6000, 'lp': 2000} - ) diff --git a/app/algorithms/valve_isolation/__init__.py b/app/algorithms/valve_isolation/__init__.py new file mode 100644 index 0000000..0323bc3 --- /dev/null +++ b/app/algorithms/valve_isolation/__init__.py @@ -0,0 +1,3 @@ +from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis + +__all__ = ["valve_isolation_analysis"] diff --git a/app/algorithms/valve_isolation/topology_search.py b/app/algorithms/valve_isolation/topology_search.py new file mode 100644 index 0000000..9baee6f --- /dev/null +++ b/app/algorithms/valve_isolation/topology_search.py @@ -0,0 +1,103 @@ +"""Topology-only valve isolation search.""" + +from collections import defaultdict, deque +from typing import Any, Iterable + + +VALVE_LINK_TYPE = "valve" + + +def _parse_link_entry(link_entry: str) -> tuple[str, str, str, str]: + parts = link_entry.split(":", 3) + if len(parts) != 4: + raise ValueError(f"Invalid link entry format: {link_entry}") + return parts[0], parts[1], parts[2], parts[3] + + +def valve_isolation_analysis( + link_entries: Iterable[str], + accident_elements: str | list[str], + disabled_valves: list[str] | None = None, +) -> dict[str, Any]: + """Determine boundary valves and affected nodes from a topology snapshot.""" + + disabled_valves_set = set(disabled_valves or []) + target_elements = ( + [accident_elements] + if isinstance(accident_elements, str) + else accident_elements + ) + + pipe_adj: dict[str, set[str]] = defaultdict(set) + all_valves: dict[str, tuple[str, str]] = {} + link_lookup: dict[str, tuple[str, str, str]] = {} + node_set: set[str] = set() + for link_entry in link_entries: + link_id, link_type, node1, node2 = _parse_link_entry(link_entry) + link_type_name = str(link_type).lower() + link_lookup[link_id] = (node1, node2, link_type_name) + node_set.update((node1, node2)) + if link_type_name == VALVE_LINK_TYPE: + all_valves[link_id] = (node1, node2) + else: + pipe_adj[node1].add(node2) + pipe_adj[node2].add(node1) + + start_nodes: set[str] = set() + for element in target_elements: + if element in node_set: + start_nodes.add(element) + elif element in link_lookup: + node1, node2, _ = link_lookup[element] + start_nodes.update((node1, node2)) + else: + raise ValueError(f"Accident element {element} was not found in topology") + + extra_adj: dict[str, list[str]] = defaultdict(list) + boundary_valves: dict[str, tuple[str, str]] = {} + for valve_id, (node1, node2) in all_valves.items(): + if valve_id in disabled_valves_set: + extra_adj[node1].append(node2) + extra_adj[node2].append(node1) + else: + boundary_valves[valve_id] = (node1, node2) + + affected_nodes: set[str] = set() + queue = deque(start_nodes) + while queue: + node = queue.popleft() + if node in affected_nodes: + continue + affected_nodes.add(node) + queue.extend(pipe_adj.get(node, set()) - affected_nodes) + queue.extend( + neighbor + for neighbor in extra_adj.get(node, ()) + if neighbor not in affected_nodes + ) + + must_close_valves: list[str] = [] + optional_valves: list[str] = [] + for valve_id, (node1, node2) in boundary_valves.items(): + node1_affected = node1 in affected_nodes + node2_affected = node2 in affected_nodes + if node1_affected and node2_affected: + optional_valves.append(valve_id) + elif node1_affected or node2_affected: + must_close_valves.append(valve_id) + + must_close_valves.sort() + optional_valves.sort() + isolatable = bool(must_close_valves) + result: dict[str, Any] = { + "accident_elements": target_elements, + "disabled_valves": disabled_valves, + "affected_nodes": sorted(affected_nodes) if isolatable else [], + "affected_node_count": len(affected_nodes), + "must_close_valves": must_close_valves, + "optional_valves": optional_valves, + "isolatable": isolatable, + } + if len(target_elements) == 1: + result["accident_element"] = target_elements[0] + return result diff --git a/app/algorithms/water_demand/__init__.py b/app/algorithms/water_demand/__init__.py deleted file mode 100644 index 0d7bf25..0000000 --- a/app/algorithms/water_demand/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Water-demand distribution algorithms.""" - -from .service import ( - calculate_demand_to_network, - calculate_demand_to_nodes, - calculate_demand_to_region, - distribute_demand_to_nodes, - distribute_demand_to_region, - get_total_base_demand, -) - -__all__ = [ - "calculate_demand_to_network", - "calculate_demand_to_nodes", - "calculate_demand_to_region", - "distribute_demand_to_nodes", - "distribute_demand_to_region", - "get_total_base_demand", -] diff --git a/app/algorithms/water_demand/service.py b/app/algorithms/water_demand/service.py deleted file mode 100644 index 3a1a8af..0000000 --- a/app/algorithms/water_demand/service.py +++ /dev/null @@ -1,104 +0,0 @@ -from app.native.wndb.commands.executor import execute_batch_command -from app.native.wndb.core.database import ChangeSet -from app.native.wndb.gis.region_geometry import Topology, get_nodes_in_region -from app.native.wndb.gis.network_views import ( - get_junction_demands, - sum_junction_base_demand, -) -from app.native.wndb.model.elements import get_nodes - - -DISTRIBUTION_TYPE_ADD = 'ADD' -DISTRIBUTION_TYPE_OVERRIDE = 'OVERRIDE' - - -def calculate_demand_to_nodes(name: str, demand: float, nodes: list[str]) -> dict[str, float]: - if len(nodes) == 0 or demand == 0.0: - return {} - - topology = Topology(name, nodes) - t_nodes = topology.nodes() - t_links = topology.links() - - length_sum = 0.0 - for value in t_links.values(): - length_sum += abs(value['length']) - - if length_sum <= 0.0: - return {} - - demand_per_length = demand / length_sum - - result: dict[str, float] = {} - for node, value in t_nodes.items(): - if value["type"] != "junction": - continue - demand_per_node = 0.0 - for link in value['links']: - demand_per_node += abs(t_links[link]['length']) * demand_per_length * 0.5 - result[node] = demand_per_node - - return result - - -def calculate_demand_to_region(name: str, demand: float, region: str) -> dict[str, float]: - nodes = get_nodes_in_region(name, region) - return calculate_demand_to_nodes(name, demand, nodes) - - -def calculate_demand_to_network(name: str, demand: float) -> dict[str, float]: - nodes = get_nodes(name) - return calculate_demand_to_nodes(name, demand, nodes) - - -def distribute_demand_to_nodes(name: str, demand: float, nodes: list[str], type: str = DISTRIBUTION_TYPE_ADD) -> ChangeSet: - if len(nodes) == 0 or demand == 0.0: - return ChangeSet() - if type != DISTRIBUTION_TYPE_ADD and type != DISTRIBUTION_TYPE_OVERRIDE: - return ChangeSet() - - topology = Topology(name, nodes) - t_nodes = topology.nodes() - t_links = topology.links() - - length_sum = 0.0 - for value in t_links.values(): - length_sum += abs(value['length']) - - if length_sum <= 0.0: - return ChangeSet() - - demand_per_length = demand / length_sum - - cs = ChangeSet() - demands_by_junction = get_junction_demands( - name, - [node for node, value in t_nodes.items() if value["type"] == "junction"], - ) - - for node, value in t_nodes.items(): - if value["type"] != "junction": - continue - demand_per_node = 0.0 - for link in value['links']: - demand_per_node += abs(t_links[link]['length']) * demand_per_length * 0.5 - - ds = demands_by_junction.get(node, []) - if len(ds) == 0: - ds = [{'demand': demand_per_node, 'pattern': None, 'category': None}] - elif type == DISTRIBUTION_TYPE_ADD: - ds[0]['demand'] += demand_per_node - else: - ds[0]['demand'] = demand_per_node - cs.update({'type': 'demand', 'junction': node, 'demands': ds}) - - return execute_batch_command(name, cs) - - -def distribute_demand_to_region(name: str, demand: float, region: str, type: str = DISTRIBUTION_TYPE_ADD) -> ChangeSet: - nodes = get_nodes_in_region(name, region) - return distribute_demand_to_nodes(name, demand, nodes, type) - -def get_total_base_demand(name: str, region: str) -> float: - nodes = get_nodes_in_region(name, region) - return sum_junction_base_demand(name, nodes) diff --git a/app/api/v1/endpoints/burst_detection.py b/app/api/v1/endpoints/burst_detection.py index 98163de..b2097c4 100644 --- a/app/api/v1/endpoints/burst_detection.py +++ b/app/api/v1/endpoints/burst_detection.py @@ -4,6 +4,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Body from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool from app.auth.keycloak_dependencies import get_current_keycloak_username from app.services.burst_detection import ( @@ -73,6 +74,10 @@ async def detect_burst( HTTPException: 当处理过程中发生错误时 """ try: - return run_burst_detection(**data.model_dump(), username=username) + return await run_in_threadpool( + run_burst_detection, + **data.model_dump(), + username=username, + ) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) diff --git a/app/api/v1/endpoints/burst_location.py b/app/api/v1/endpoints/burst_location.py index 669acbb..a4e0154 100644 --- a/app/api/v1/endpoints/burst_location.py +++ b/app/api/v1/endpoints/burst_location.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Body from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool from app.auth.keycloak_dependencies import get_current_keycloak_username from app.services.burst_location import ( @@ -63,6 +64,10 @@ async def locate_burst( HTTPException: 当数据类型或值不正确时 """ try: - return run_burst_location_by_network(**data.model_dump(), username=username) + return await run_in_threadpool( + run_burst_location_by_network, + **data.model_dump(), + username=username, + ) except (TypeError, ValueError) as exc: raise HTTPException(status_code=400, detail=str(exc)) diff --git a/app/api/v1/endpoints/leakage.py b/app/api/v1/endpoints/leakage.py index 1af055e..3a630a1 100644 --- a/app/api/v1/endpoints/leakage.py +++ b/app/api/v1/endpoints/leakage.py @@ -3,34 +3,49 @@ from typing import Any from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, Body -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field +from starlette.concurrency import run_in_threadpool from app.auth.keycloak_dependencies import get_current_keycloak_username -from app.services.leakage_identifier import ( +from app.services.dma_leakage_estimation import ( run_leakage_identification, ) router = APIRouter() DEFAULT_N_WORKERS = max(1, min((os.cpu_count() or 1) - 1, 4)) +MAX_POPULATION_SIZE = 1_000 +MAX_GENERATIONS = 1_000 +MAX_DURATION_HOURS = 168 class LeakageIdentifyRequest(BaseModel): """漏损识别请求模型""" + + model_config = ConfigDict(extra="forbid") + network: str = Field(..., description="管网名称（或数据库名称）") - observed_pressure_data: str | dict[str, list[Any]] | list[dict[str, Any]] | None = Field( - None, description="观测的压力数据" + observed_pressure_data: dict[str, list[Any]] | list[dict[str, Any]] | None = ( + Field(None, description="观测的压力数据；文件路径不属于公共 API 输入") ) - start_time: float = Field(0, description="起始时间（小时）") - duration: float = Field(24, description="持续时间（小时）") - timestep: float = Field(5, description="时间步长（分钟）") - q_sum: float = Field(0.2, description="总流量（m3/s）") + start_time: float = Field(0, ge=0, description="起始时间（小时）") + duration: float = Field( + 24, gt=0, le=MAX_DURATION_HOURS, description="持续时间（小时）" + ) + timestep: float = Field(5, gt=0, le=1440, description="时间步长（分钟）") + q_sum: float = Field(0.2, ge=0, description="总流量（m3/s）") q_sum_unit: str = Field("m3/s", description="流量单位") - output_dir: str = Field("db_inp", description="输出目录") - pop_size: int = Field(50, description="种群大小") - max_gen: int = Field(100, description="最大代数") - n_workers: int = Field(DEFAULT_N_WORKERS, description="工作线程数") + pop_size: int = Field( + 50, ge=2, le=MAX_POPULATION_SIZE, description="种群大小" + ) + max_gen: int = Field(100, ge=1, le=MAX_GENERATIONS, description="最大代数") + n_workers: int = Field( + DEFAULT_N_WORKERS, + ge=1, + le=DEFAULT_N_WORKERS, + description="工作进程数", + ) output_flow_unit: str = Field("m3/s", description="输出流量单位") - dma_count: int | None = Field(None, description="DMA区域数量") + dma_count: int | None = Field(None, ge=1, description="DMA区域数量") scada_start: datetime | None = Field(None, description="SCADA数据起始时间") scada_end: datetime | None = Field(None, description="SCADA数据结束时间") sensor_nodes: list[str] | None = Field(None, description="传感器节点列表") @@ -63,6 +78,10 @@ async def identify_leakage( HTTPException: 当处理过程中发生错误时 """ try: - return run_leakage_identification(**data.model_dump(), username=username) + return await run_in_threadpool( + run_leakage_identification, + **data.model_dump(), + username=username, + ) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) diff --git a/app/api/v1/endpoints/sensor_placement.py b/app/api/v1/endpoints/sensor_placement.py index cc5d0a5..c65ef6f 100644 --- a/app/api/v1/endpoints/sensor_placement.py +++ b/app/api/v1/endpoints/sensor_placement.py @@ -7,10 +7,6 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query, status from fastapi.responses import StreamingResponse from starlette.concurrency import run_in_threadpool -from app.algorithms.sensor import ( - pressure_sensor_placement_kmeans, - pressure_sensor_placement_sensitivity, -) from app.auth.metadata_dependencies import get_current_metadata_user from app.auth.project_dependencies import ( ProjectContext, @@ -34,6 +30,8 @@ from app.services.sensor_placement import ( get_sensor_placement_candidate, get_sensor_placement_run, list_sensor_placement_runs, + optimize_sensor_placement_by_kmeans, + optimize_sensor_placement_by_sensitivity, update_sensor_placement_run, ) @@ -136,18 +134,18 @@ async def optimize_sensor_placement_scheme( network = _project_network(payload.network, project_context) _require_project_write(project_context) optimizer = ( - pressure_sensor_placement_sensitivity + optimize_sensor_placement_by_sensitivity if payload.method == "sensitivity" - else pressure_sensor_placement_kmeans + else optimize_sensor_placement_by_kmeans ) try: created = await run_in_threadpool( optimizer, - name=network, - scheme_name=payload.run_name, - sensor_number=payload.sensor_count, + project_code=network, + run_name=payload.run_name, + sensor_count=payload.sensor_count, min_diameter=payload.min_diameter, - username=current_user.username, + created_by=current_user.username, ) run = get_sensor_placement_run(network, created["run_id"]) return {**run, "can_edit": True} diff --git a/app/api/v1/endpoints/simulation.py b/app/api/v1/endpoints/simulation.py index 53f2e2b..59ce57e 100644 --- a/app/api/v1/endpoints/simulation.py +++ b/app/api/v1/endpoints/simulation.py @@ -1,33 +1,22 @@ from typing import Any, List, Literal, Optional from datetime import datetime, timedelta -import json -import threading -from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body +from fastapi import APIRouter, Body, Depends, HTTPException, Query from fastapi.responses import PlainTextResponse from app.auth.keycloak_dependencies import get_current_keycloak_username import app.services.simulation as simulation from app.services.tjnetwork import ( run_project, run_project_return_dict, - run_inp, - dump_output, ) -from app.algorithms.simulation.scenarios import ( +from app.services.simulation_scenarios import ( burst_analysis, valve_close_analysis, flushing_analysis, contaminant_simulation, - age_analysis, - # scheduling_analysis, pressure_regulation, ) -from app.services.simulation_ops import ( - project_management, - scheduling_simulation, - daily_scheduling_simulation, -) from app.services.valve_isolation import analyze_valve_isolation -from app.services.time_api import ( +from app.domain.time import ( parse_aware_time, parse_clock_duration_seconds, parse_utc_time, @@ -49,57 +38,13 @@ class RunSimulationManuallyByDate(BaseModel): return value -class BurstAnalysis(BaseModel): - name: str = Field(..., description="管网名称（或数据库名称）") - modify_pattern_start_time: str = Field(..., description="模式修改开始时间 (ISO 8601)") - burst_ID: List[str] | str | None = Field(None, description="爆管节点/管段ID列表") - burst_size: List[float] | float | int | None = Field(None, description="爆管流量大小") - modify_total_duration: int = Field(900, description="模拟总时长 (秒)") - modify_fixed_pump_pattern: Optional[dict[str, list]] = Field(None, description="定速泵模式修改") - modify_variable_pump_pattern: Optional[dict[str, list]] = Field(None, description="变速泵模式修改") - modify_valve_opening: Optional[dict[str, float]] = Field(None, description="阀门开度修改") - scheme_name: Optional[str] = Field(None, description="方案名称") - - -class SchedulingAnalysis(BaseModel): - network: str = Field(..., description="管网名称（或数据库名称）") - start_time: str = Field(..., description="开始时间") - pump_control: dict = Field(..., description="泵控制策略") - tank_id: str = Field(..., description="水箱ID") - water_plant_output_id: str = Field(..., description="水厂出水ID") - time_delta: Optional[int] = Field(300, description="时间步长 (秒)") - - class PressureRegulation(BaseModel): network: str = Field(..., description="管网名称（或数据库名称）") start_time: str = Field(..., description="开始时间") pump_control: dict = Field(..., description="泵控制策略") tank_init_level: Optional[dict] = Field(None, description="水箱初始水位") duration: Optional[int] = Field(900, description="持续时间 (秒)") - scheme_name: Optional[str] = Field(None, description="方案名称") - - -class ProjectManagement(BaseModel): - network: str = Field(..., description="管网名称（或数据库名称）") - start_time: str = Field(..., description="开始时间") - pump_control: dict = Field(..., description="泵控制策略") - tank_init_level: Optional[dict] = Field(None, description="水箱初始水位") - region_demand: Optional[dict] = Field(None, description="区域需水量控制") - - -class DailySchedulingAnalysis(BaseModel): - network: str = Field(..., description="管网名称（或数据库名称）") - start_time: str = Field(..., description="开始时间") - pump_control: dict = Field(..., description="泵控制策略") - reservoir_id: str = Field(..., description="水库ID") - tank_id: str = Field(..., description="水箱ID") - water_plant_output_id: str = Field(..., description="水厂出水ID") - time_delta: Optional[int] = Field(300, description="时间步长 (秒)") - - -class PumpFailureState(BaseModel): - time: str = Field(..., description="故障发生时间") - pump_status: dict = Field(..., description="泵状态字典") + scheme_name: str = Field(..., min_length=1, description="方案名称") def run_simulation_manually_by_date( @@ -161,41 +106,15 @@ def run_project_return_dict_endpoint(network: str = Query(..., description="管 return run_project_return_dict(network) -# put in inp folder, name without extension -@router.post("/inp-runs", summary="运行INP文件", description="运行指定INP文件格式的管网模型进行水力模拟。INP文件应该放在inp文件夹中，参数为文件名不含扩展名。") -def run_inp_endpoint(network: str = Query(..., description="inp文件名（不含扩展名）")) -> str: - """ - 运行INP文件 - - - **network**: inp文件名（不含扩展名） - - 从inp文件夹中读取指定的INP文件并运行模拟。 - """ - return run_inp(network) - - -# path is absolute path -@router.get("/outputs", summary="导出模拟输出", description="导出指定路径的模拟输出文件内容。参数应为绝对路径。") -def dump_output_endpoint(output: str = Query(..., description="模拟输出文件的绝对路径")) -> str: - """ - 导出模拟输出 - - - **output**: 模拟输出文件的绝对路径 - - 读取并返回指定路径的模拟输出内容。 - """ - return dump_output(output) - - # Analysis Endpoints @router.post("/burst-analyses", summary="爆管分析（高级）", description="高级版本的爆管分析，支持在指定时间点修改泵控制模式和阀门开度，以分析这些改变对爆管影响的作用。支持固定泵和变速泵的独立控制。") def fastapi_burst_analysis( network: str = Query(..., description="管网名称（或数据库名称）"), modify_pattern_start_time: str = Query(..., description="模式修改开始时间（ISO 8601格式）"), - burst_ID: list[str] = Query(..., description="爆管节点/管段ID列表"), - burst_size: list[float] = Query(..., description="对应各爆管点的爆管流量大小列表（L/s）"), - modify_total_duration: int = Query(..., description="模拟总时长（秒）"), - scheme_name: str = Query(..., description="分析方案名称"), + burst_ID: list[str] = Query(..., min_length=1, description="爆管节点/管段ID列表"), + burst_size: list[float] = Query(..., min_length=1, description="对应各爆管点的爆管流量大小列表（L/s）"), + modify_total_duration: int = Query(..., gt=0, description="模拟总时长（秒）"), + scheme_name: str = Query(..., min_length=1, description="分析方案名称"), username: str = Depends(get_current_keycloak_username), ) -> str: """ @@ -210,6 +129,11 @@ def fastapi_burst_analysis( 支持在指定时间修改泵控制模式和阀门开度。 """ + if len(burst_ID) != len(burst_size): + raise HTTPException( + status_code=422, + detail="burst_id 与 burst_size 的数量必须一致", + ) burst_analysis( name=network, modify_pattern_start_time=modify_pattern_start_time, @@ -442,48 +366,6 @@ def fastapi_contaminant_simulation( return result or "success" -@router.post("/water-age-analyses", response_class=PlainTextResponse, summary="水龄分析（高级）", description="高级版本的水龄分析，在指定时间点进行分析，支持自定义模拟持续时间。返回纯文本格式的分析结果。") -def fastapi_age_analysis( - network: str = Query(..., description="管网名称（或数据库名称）"), - start_time: str = Query(..., description="分析开始时间（ISO 8601格式）"), - duration: int = Query(..., description="模拟持续时间（秒）"), -) -> str: - """ - 水龄分析（高级版本） - - - **network**: 管网名称（或数据库名称） - - **start_time**: 分析开始时间 - - **duration**: 模拟持续时间（秒） - - 分析指定时间段内管网中各节点的水体停留时间。 - """ - result = age_analysis(network, start_time, duration) - return result or "success" - - -# @router.get("/schedulinganalysis/") -# async def scheduling_analysis_endpoint(network: str): -# return scheduling_analysis(network) - - -@router.post("/pressure-regulation-calculations", summary="压力调节（基础）", description="对管网的压力进行调节分析，通过控制泵的运行来维持目标节点的目标压力。此为基础版本。") -def pressure_regulation_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - target_node: str = Query(..., description="目标节点ID"), - target_pressure: float = Query(..., description="目标压力值（kPa）"), -): - """ - 压力调节（基础版本） - - - **network**: 管网名称（或数据库名称） - - **target_node**: 目标节点ID - - **target_pressure**: 目标压力值（kPa） - - 通过泵控制维持目标节点的压力。 - """ - return pressure_regulation(network, target_node, target_pressure) - - @router.post("/pressure-regulation-analyses", summary="压力调节（高级）", description="高级版本的压力调节分析，通过JSON请求体提供详细的控制参数，包括固定泵和变速泵的独立控制、水箱初始水位等。") def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description="压力调节控制参数")) -> str: """ @@ -495,7 +377,7 @@ def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description - **pump_control**: 泵控制策略字典 - **tank_init_level**: 水箱初始水位字典（可选） - **duration**: 模拟持续时间（秒，可选，默认900） - - **scheme_name**: 控制方案名称（可选） + - **scheme_name**: 控制方案名称 支持固定泵和变速泵的独立控制。 """ @@ -525,137 +407,6 @@ def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description return "success" -@router.post("/project-managements", summary="项目管理（高级）", description="高级版本的项目管理，通过JSON请求体提供详细的控制参数，包括泵控制策略、水箱初始水位和区域需水量控制。") -def fastapi_project_management(data: ProjectManagement = Body(..., description="项目管理控制参数")) -> str: - """ - 项目管理（高级版本） - - 请求体参数： - - **network**: 管网名称（或数据库名称） - - **start_time**: 管理开始时间 - - **pump_control**: 泵控制策略字典 - - **tank_init_level**: 水箱初始水位字典（可选） - - **region_demand**: 区域需水量控制字典（可选） - - 支持多维度的项目管理。 - """ - item = data.model_dump() - return project_management( - prj_name=item["network"], - start_datetime=item["start_time"], - pump_control=item["pump_control"], - tank_initial_level_control=item["tank_init_level"], - region_demand_control=item["region_demand"], - ) - - -# @router.get("/dailyschedulinganalysis/") -# async def daily_scheduling_analysis_endpoint(network: str): -# return daily_scheduling_analysis(network) - - -@router.post("/scheduling-analyses", summary="排程分析", description="对管网的供水排程进行分析，优化泵的运行时间和出水流量，平衡水厂出水、水箱进出水，满足用户需求。") -def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., description="排程分析参数")) -> str: - """ - 排程分析 - - 请求体参数： - - **network**: 管网名称（或数据库名称） - - **start_time**: 分析开始时间 - - **pump_control**: 泵控制策略字典 - - **tank_id**: 水箱ID - - **water_plant_output_id**: 水厂出水ID - - **time_delta**: 时间步长（秒，可选，默认300） - - 用于优化供水排程。 - """ - item = data.model_dump() - return scheduling_simulation( - item["network"], - item["start_time"], - item["pump_control"], - item["tank_id"], - item["water_plant_output_id"], - item["time_delta"], - ) - - -@router.post("/daily-scheduling-analyses", summary="日排程分析", description="对管网的每日供水排程进行分析，优化水库、水厂、水箱和用户需求的协调，制定合理的每日排程方案。") -def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body(..., description="日排程分析参数")) -> str: - """ - 日排程分析 - - 请求体参数： - - **network**: 管网名称（或数据库名称） - - **start_time**: 分析开始时间 - - **pump_control**: 泵控制策略字典 - - **reservoir_id**: 水库ID - - **tank_id**: 水箱ID - - **water_plant_output_id**: 水厂出水ID - - **time_delta**: 时间步长（秒，可选，默认300） - - 用于制定每日供水排程方案。 - """ - item = data.model_dump() - return daily_scheduling_simulation( - item["network"], - item["start_time"], - item["pump_control"], - item["reservoir_id"], - item["tank_id"], - item["water_plant_output_id"], - ) - - -# @router.get("/pumpfailure/") -# async def pump_failure_endpoint(network: str, pump_id: str, time: str): -# return pump_failure(network, pump_id, time) - - -@router.post("/pump-failure-events", summary="泵故障管理", description="记录和管理泵的故障状态，包括故障发生时间和受影响的泵列表。系统将记录故障日志并更新泵状态。") -def fastapi_pump_failure(data: PumpFailureState = Body(..., description="泵故障状态信息")) -> str: - """ - 泵故障管理 - - 请求体参数： - - **time**: 故障发生时间 - - **pump_status**: 泵状态字典，包含第一阶段和第二阶段泵的故障状态 - - 系统将验证泵信息的有效性并更新故障状态文件。 - """ - item = data.model_dump() - with open("./pump_failure_message.txt", "a", encoding="utf-8-sig") as f1: - f1.write("[{}] {}\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), item)) - with open("./pump_failure_status.txt", "r", encoding="utf-8-sig") as f2: - lines = f2.readlines() - first_stage_pump_status_dict = json.loads(json.dumps(eval(lines[0]))) - second_stage_pump_status_dict = json.loads(json.dumps(eval(lines[-1]))) - pump_status_dict = { - "first": first_stage_pump_status_dict, - "second": second_stage_pump_status_dict, - } - status_info = item.copy() - for pump_type in status_info["pump_status"].keys(): - if pump_type in pump_status_dict.keys(): - if all( - pump_id in pump_status_dict[pump_type].keys() - for pump_id in status_info["pump_status"][pump_type].keys() - ): - for pump_id in status_info["pump_status"][pump_type].keys(): - pump_status_dict[pump_type][pump_id] = int( - status_info["pump_status"][pump_type][pump_id] - ) - else: - return json.dumps("ERROR: Wrong Pump ID") - else: - return json.dumps("ERROR: Wrong Pump Type") - with open("./pump_failure_status.txt", "w", encoding="utf-8-sig") as f2_: - f2_.write( - "{}\n{}".format(pump_status_dict["first"], pump_status_dict["second"]) - ) - return json.dumps("SUCCESS") - - @router.post("/simulation-runs", summary="手动运行日期指定模拟", description="根据指定的开始时间和持续时间，手动运行水力模拟。开始时间必须是显式带时区的 ISO 8601 / RFC3339 时间。") def fastapi_run_simulation_manually_by_date( data: RunSimulationManuallyByDate = Body(..., description="模拟运行参数"), diff --git a/app/api/v1/endpoints/timeseries/composite.py b/app/api/v1/endpoints/timeseries/composite.py index 7b1e469..9170a65 100644 --- a/app/api/v1/endpoints/timeseries/composite.py +++ b/app/api/v1/endpoints/timeseries/composite.py @@ -3,7 +3,7 @@ from datetime import datetime from psycopg import AsyncConnection from uuid import UUID -from app.infra.db.timescaledb.composite_queries import CompositeQueries +from app.services.timeseries_analysis import TimeseriesAnalysisService from .dependencies import get_timescale_connection, get_postgres_connection router = APIRouter() @@ -46,7 +46,7 @@ async def get_scada_associated_simulation_data( ) if run_id is not None: - result = await CompositeQueries.get_scada_associated_analysis_simulation_data( + result = await TimeseriesAnalysisService.get_scada_associated_analysis_simulation_data( timescale_conn, postgres_conn, device_ids_list, @@ -56,7 +56,7 @@ async def get_scada_associated_simulation_data( ) else: result = ( - await CompositeQueries.get_scada_associated_realtime_simulation_data( + await TimeseriesAnalysisService.get_scada_associated_realtime_simulation_data( timescale_conn, postgres_conn, device_ids_list, @@ -116,7 +116,7 @@ async def get_feature_simulation_data( raise HTTPException(status_code=400, detail="feature_infos cannot be empty") if run_id is not None: - result = await CompositeQueries.get_analysis_simulation_data( + result = await TimeseriesAnalysisService.get_analysis_simulation_data( timescale_conn, feature_infos_list, start_time, @@ -124,7 +124,7 @@ async def get_feature_simulation_data( run_id, ) else: - result = await CompositeQueries.get_realtime_simulation_data( + result = await TimeseriesAnalysisService.get_realtime_simulation_data( timescale_conn, feature_infos_list, start_time, @@ -168,7 +168,7 @@ async def get_element_associated_scada_data( HTTPException: 当查询参数无效时返回400错误，未找到关联数据返回404错误 """ try: - result = await CompositeQueries.get_element_associated_scada_data( + result = await TimeseriesAnalysisService.get_element_associated_scada_data( timescale_conn, postgres_conn, element_id, start_time, end_time, use_cleaned ) if result is None: @@ -216,7 +216,7 @@ async def clean_scada_data( if device_ids else [] ) - return await CompositeQueries.clean_scada_data( + return await TimeseriesAnalysisService.clean_scada_data( timescale_conn, postgres_conn, device_ids_list, start_time, end_time ) except ValueError as e: @@ -246,7 +246,7 @@ async def predict_pipeline_health( HTTPException: 当模型文件不存在返回404错误，其他错误返回400或500错误 """ try: - return await CompositeQueries.predict_pipeline_health( + return await TimeseriesAnalysisService.predict_pipeline_health( timescale_conn, postgres_conn, query_time ) except ValueError as e: diff --git a/app/services/time_api.py b/app/domain/time.py similarity index 100% rename from app/services/time_api.py rename to app/domain/time.py diff --git a/app/infra/db/postgresql/network_assets.py b/app/infra/db/postgresql/network_assets.py new file mode 100644 index 0000000..f377602 --- /dev/null +++ b/app/infra/db/postgresql/network_assets.py @@ -0,0 +1,27 @@ +"""Read repositories for project network assets.""" + +from typing import Any + +from psycopg import AsyncConnection + + +class NetworkAssetRepository: + @staticmethod + async def get_pipes_by_ids( + conn: AsyncConnection, + pipe_ids: list[str], + ) -> list[dict[str, Any]]: + if not pipe_ids: + return [] + async with conn.cursor() as cur: + await cur.execute( + """ + SELECT id, diameter, start_node_id AS node1, + end_node_id AS node2 + FROM gis.pipes + WHERE id = ANY(%s) + ORDER BY id + """, + (pipe_ids,), + ) + return await cur.fetchall() diff --git a/app/infra/db/timescaledb/__init__.py b/app/infra/db/timescaledb/__init__.py index 3261953..58f4d7d 100644 --- a/app/infra/db/timescaledb/__init__.py +++ b/app/infra/db/timescaledb/__init__.py @@ -1 +1 @@ -from .composite_queries import CompositeQueries +"""TimescaleDB repositories and connection infrastructure.""" diff --git a/app/infra/db/timescaledb/internal_queries.py b/app/infra/db/timescaledb/internal_queries.py index 318427b..69cf9c5 100644 --- a/app/infra/db/timescaledb/internal_queries.py +++ b/app/infra/db/timescaledb/internal_queries.py @@ -10,7 +10,7 @@ from app.infra.db.timescaledb.sync_pool import timescale_connection from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository -from app.services.time_api import parse_utc_time +from app.domain.time import parse_utc_time class InternalStorage: diff --git a/app/infra/db/timescaledb/repositories/analysis.py b/app/infra/db/timescaledb/repositories/analysis.py index 4de9bcc..cda4e51 100644 --- a/app/infra/db/timescaledb/repositories/analysis.py +++ b/app/infra/db/timescaledb/repositories/analysis.py @@ -4,7 +4,7 @@ from uuid import UUID from psycopg import AsyncConnection, Connection, sql -from app.services.time_api import parse_utc_time +from app.domain.time import parse_utc_time class AnalysisResultsRepository: diff --git a/app/infra/db/timescaledb/repositories/realtime.py b/app/infra/db/timescaledb/repositories/realtime.py index 3a80f96..df53ea3 100644 --- a/app/infra/db/timescaledb/repositories/realtime.py +++ b/app/infra/db/timescaledb/repositories/realtime.py @@ -2,7 +2,7 @@ from typing import List, Any, Dict from datetime import datetime, timedelta from collections import defaultdict from psycopg import AsyncConnection, Connection, sql -from app.services.time_api import parse_utc_time +from app.domain.time import parse_utc_time class RealtimeRepository: diff --git a/app/services/burst_detection.py b/app/services/burst_detection.py index 2a21d5d..f1caa1e 100644 --- a/app/services/burst_detection.py +++ b/app/services/burst_detection.py @@ -8,13 +8,13 @@ from uuid import UUID import numpy as np import pandas as pd -from app.algorithms.burst_detection.burst_detector import BurstDetector +from app.algorithms.burst_detection.pressure_anomaly import PressureAnomalyDetector +from app.infra.db.postgresql.scada import get_all_scada_info from app.infra.db.timescaledb.internal_queries import InternalQueries from app.services.scheme_management import ( store_scheme_info, ) -from app.services.tjnetwork import get_all_scada_info -from app.services.time_api import parse_utc_time, utc_now +from app.domain.time import parse_utc_time, utc_now TARGET_DAY_COUNT = 15 @@ -178,7 +178,7 @@ def run_burst_detection( observed_input = observed_pressure_data observed_source = "request_payload" - detector = BurstDetector( + detector = PressureAnomalyDetector( mu=mu, points_per_day=points_per_day, iforest_params=iforest_params, diff --git a/app/services/burst_location.py b/app/services/burst_location.py index 1fe556b..cab29f4 100644 --- a/app/services/burst_location.py +++ b/app/services/burst_location.py @@ -7,14 +7,15 @@ from uuid import UUID import pandas as pd -from app.algorithms.burst_location import run_burst_location +from app.algorithms.burst_localization import run_burst_location +from app.infra.db.postgresql.scada import get_all_scada_info from app.infra.db.timescaledb.internal_queries import InternalQueries +from app.native.wndb.inp.exporter import dump_inp from app.services.scheme_management import ( get_analysis_run, store_scheme_info, ) -from app.services.tjnetwork import dump_inp, get_all_scada_info -from app.services.time_api import parse_utc_time, utc_now +from app.domain.time import parse_utc_time, utc_now SeriesInput = pd.Series | dict[str, Any] | list[dict[str, Any]] FLOW_SCADA_TYPES = {"pipe_flow", "flow", "demand"} diff --git a/app/services/demand_allocation.py b/app/services/demand_allocation.py new file mode 100644 index 0000000..658d478 --- /dev/null +++ b/app/services/demand_allocation.py @@ -0,0 +1,38 @@ +"""Application workflows for demand allocation.""" + +from app.algorithms.demand_allocation import allocate_demand_by_pipe_length +from app.native.wndb.gis.region_geometry import Topology, get_nodes_in_region +from app.native.wndb.model.elements import get_nodes + + +def calculate_demand_to_nodes( + project_code: str, + demand: float, + node_ids: list[str], +) -> dict[str, float]: + if not node_ids or demand == 0.0: + return {} + + topology = Topology(project_code, node_ids) + return allocate_demand_by_pipe_length( + demand, + topology.nodes(), + topology.links(), + ) + + +def calculate_demand_to_region( + project_code: str, + demand: float, + region_id: str, +) -> dict[str, float]: + node_ids = get_nodes_in_region(project_code, region_id) + return calculate_demand_to_nodes(project_code, demand, node_ids) + + +def calculate_demand_to_network( + project_code: str, + demand: float, +) -> dict[str, float]: + node_ids = get_nodes(project_code) + return calculate_demand_to_nodes(project_code, demand, node_ids) diff --git a/app/services/leakage_identifier.py b/app/services/dma_leakage_estimation.py similarity index 63% rename from app/services/leakage_identifier.py rename to app/services/dma_leakage_estimation.py index f5c374f..8110208 100644 --- a/app/services/leakage_identifier.py +++ b/app/services/dma_leakage_estimation.py @@ -1,23 +1,22 @@ -import math import os -from collections import deque from datetime import datetime from typing import Any -import numpy as np import pandas as pd -import wntr -from app.algorithms.leakage.identifier import LeakageIdentifier +from app.algorithms.dma_leakage_estimation.genetic_optimizer import DmaLeakageOptimizer +from app.algorithms.dma_leakage_estimation.topology_partitioning import ( + build_dma_partitions, +) from app.infra.db.timescaledb.internal_queries import InternalQueries -from app.services.scheme_management import store_analysis_run_with_result -from app.services.tjnetwork import ( - dump_inp, - get_all_scada_info, +from app.infra.db.postgresql.scada import get_all_scada_info +from app.native.wndb.gis.network_views import ( get_network_link_nodes, get_network_node_coords, ) -from app.services.time_api import parse_utc_time, utc_now +from app.native.wndb.inp.exporter import dump_inp +from app.services.scheme_management import store_analysis_run_with_result +from app.domain.time import parse_utc_time, utc_now DEFAULT_N_WORKERS = max(1, min((os.cpu_count() or 1) - 1, 4)) @@ -26,14 +25,13 @@ def run_leakage_identification( network: str, username: str, observed_pressure_data: ( - str | pd.DataFrame | dict[str, list[Any]] | list[dict[str, Any]] | None + pd.DataFrame | dict[str, list[Any]] | list[dict[str, Any]] | None ) = None, start_time: float = 0, duration: float = 24, timestep: float = 5, q_sum: float = 0.2, q_sum_unit: str = "m3/s", - output_dir: str = "db_inp", pop_size: int = 50, max_gen: int = 100, n_workers: int = DEFAULT_N_WORKERS, @@ -44,7 +42,6 @@ def run_leakage_identification( sensor_nodes: list[str] | None = None, scheme_name: str | None = None, ) -> dict[str, Any]: - os.makedirs(output_dir, exist_ok=True) inp_path = _prepare_leakage_inp(network) selected_sensor_nodes = ( @@ -75,8 +72,8 @@ def run_leakage_identification( ) observed_df = observed_pressure_data - q_sum_m3s = LeakageIdentifier._flow_to_m3s(q_sum, q_sum_unit) - identifier = LeakageIdentifier( + q_sum_m3s = DmaLeakageOptimizer._flow_to_m3s(q_sum, q_sum_unit) + identifier = DmaLeakageOptimizer( inp_path=inp_path, sensor_nodes=selected_sensor_nodes, area_map=area_map, @@ -87,7 +84,6 @@ def run_leakage_identification( ) result_df = identifier.run_identification( observed_pressure_data=observed_df, - output_dir=output_dir, pop_size=pop_size, max_gen=max_gen, n_workers=n_workers, @@ -188,171 +184,15 @@ def _build_area_map_by_topology( network: str, sensor_nodes: list[str], dma_count: int | None ) -> tuple[dict[str, str], list[dict[str, Any]], dict[str, dict[str, float]]]: node_coords = get_network_node_coords(network) - all_nodes = list(node_coords.keys()) - if not all_nodes: - raise ValueError("管网中未获取到可分区节点。") - - available_sensors = [node for node in sensor_nodes if node in node_coords] - if not available_sensors: - raise ValueError("无可用压力传感器，无法生成虚拟分区。") - area_count = _resolve_dma_count(dma_count, available_sensors, all_nodes) - sensor_area_map = _cluster_sensors_to_areas( - available_sensors, node_coords, area_count + area_map, areas = build_dma_partitions( + sensor_nodes, + node_coords, + get_network_link_nodes(network), + dma_count, ) - adjacency = _build_adjacency(network, all_nodes) - distance_by_sensor = { - sensor: _bfs_distances(adjacency, sensor) for sensor in available_sensors - } - - assignment_count = {sensor: 0 for sensor in available_sensors} - area_map: dict[str, str] = {} - for node_id in sorted(all_nodes): - sensor = _choose_sensor_for_node( - node_id=node_id, - sensors=available_sensors, - node_coords=node_coords, - distance_by_sensor=distance_by_sensor, - assignment_count=assignment_count, - ) - assignment_count[sensor] += 1 - area_map[node_id] = sensor_area_map[sensor] - - if not area_map: - raise ValueError("虚拟分区结果为空，无法生成节点区域映射。") - - areas = _build_area_meta(area_map, sensor_area_map) return area_map, areas, node_coords -def _resolve_dma_count( - dma_count: int | None, sensor_nodes: list[str], all_nodes: list[str] -) -> int: - if dma_count is None: - return min(len(sensor_nodes), len(all_nodes)) - if dma_count <= 0: - raise ValueError("dma_count 必须大于 0。") - if dma_count > len(all_nodes): - raise ValueError("dma_count 不能大于可分区节点数量。") - if dma_count > len(sensor_nodes): - raise ValueError("dma_count 不能大于可用传感器数量。") - return dma_count - - -def _cluster_sensors_to_areas( - sensor_nodes: list[str], node_coords: dict[str, dict[str, float]], area_count: int -) -> dict[str, str]: - if area_count >= len(sensor_nodes): - return {sensor: str(i + 1) for i, sensor in enumerate(sensor_nodes)} - - points = np.array( - [ - [float(node_coords[s]["x"]), float(node_coords[s]["y"])] - for s in sensor_nodes - ], - dtype=float, - ) - centers = points[:area_count].copy() - labels = np.zeros(points.shape[0], dtype=int) - for _ in range(20): - d2 = ((points[:, None, :] - centers[None, :, :]) ** 2).sum(axis=2) - new_labels = d2.argmin(axis=1) - if np.array_equal(labels, new_labels): - break - labels = new_labels - for i in range(area_count): - cluster_points = points[labels == i] - if cluster_points.size > 0: - centers[i] = cluster_points.mean(axis=0) - return { - sensor: str(int(labels[idx]) + 1) for idx, sensor in enumerate(sensor_nodes) - } - - -def _build_adjacency(network: str, all_nodes: list[str]) -> dict[str, set[str]]: - adjacency: dict[str, set[str]] = {node: set() for node in all_nodes} - for link in get_network_link_nodes(network): - parts = str(link).split(":") - if len(parts) < 4: - continue - node1, node2 = parts[-2], parts[-1] - if node1 in adjacency and node2 in adjacency: - adjacency[node1].add(node2) - adjacency[node2].add(node1) - return adjacency - - -def _bfs_distances(adjacency: dict[str, set[str]], start: str) -> dict[str, int]: - distances: dict[str, int] = {start: 0} - queue: deque[str] = deque([start]) - while queue: - node = queue.popleft() - for neighbor in adjacency.get(node, set()): - if neighbor in distances: - continue - distances[neighbor] = distances[node] + 1 - queue.append(neighbor) - return distances - - -def _choose_sensor_for_node( - node_id: str, - sensors: list[str], - node_coords: dict[str, dict[str, float]], - distance_by_sensor: dict[str, dict[str, int]], - assignment_count: dict[str, int], -) -> str: - min_distance = None - candidates: list[str] = [] - for sensor in sensors: - d = distance_by_sensor.get(sensor, {}).get(node_id) - if d is None: - continue - if min_distance is None or d < min_distance: - min_distance = d - candidates = [sensor] - elif d == min_distance: - candidates.append(sensor) - if not candidates: - node_coord = node_coords[node_id] - return min( - sensors, - key=lambda sensor: _euclidean_distance( - node_coord, node_coords.get(sensor, node_coord) - ), - ) - return min(candidates, key=lambda sensor: (assignment_count[sensor], sensor)) - - -def _euclidean_distance(a: dict[str, float], b: dict[str, float]) -> float: - return math.hypot(float(a["x"]) - float(b["x"]), float(a["y"]) - float(b["y"])) - - -def _build_area_meta( - area_map: dict[str, str], sensor_area_map: dict[str, str] -) -> list[dict[str, Any]]: - nodes_by_area: dict[str, list[str]] = {} - for node_id, area_id in area_map.items(): - nodes_by_area.setdefault(area_id, []).append(node_id) - - sensors_by_area: dict[str, list[str]] = {} - for sensor, area_id in sensor_area_map.items(): - sensors_by_area.setdefault(area_id, []).append(sensor) - - areas: list[dict[str, Any]] = [] - for area_id in sorted(nodes_by_area.keys(), key=lambda x: int(x)): - node_ids = sorted(nodes_by_area.get(area_id, [])) - sensor_nodes = sorted(sensors_by_area.get(area_id, [])) - areas.append( - { - "area_id": area_id, - "sensor_nodes": sensor_nodes, - "node_ids": node_ids, - "node_count": len(node_ids), - } - ) - return areas - - def _build_area_node_map(area_map: dict[str, str]) -> dict[str, list[str]]: area_node_map: dict[str, list[str]] = {} for node_id, area_id in area_map.items(): diff --git a/app/services/network_import.py b/app/services/network_import.py index 7a98c0a..960c107 100644 --- a/app/services/network_import.py +++ b/app/services/network_import.py @@ -1,4 +1,4 @@ -from app.services.tjnetwork import read_inp +from app.native.wndb.inp.importer import read_inp def network_update(file_path: str, project_code: str) -> None: diff --git a/app/services/scheme_management.py b/app/services/scheme_management.py index f02daa3..dd956d5 100644 --- a/app/services/scheme_management.py +++ b/app/services/scheme_management.py @@ -4,7 +4,7 @@ from uuid import UUID, uuid4 from app.infra.db.postgresql.analysis import AnalysisRepository from app.native.wndb.core.connection import project_connection, project_transaction -from app.services.time_api import parse_utc_time +from app.domain.time import parse_utc_time def store_scheme_info( diff --git a/app/services/sensor_placement.py b/app/services/sensor_placement.py index 90291c0..e2046f5 100644 --- a/app/services/sensor_placement.py +++ b/app/services/sensor_placement.py @@ -1,5 +1,8 @@ +from contextlib import contextmanager from datetime import datetime +import fcntl from io import BytesIO +from pathlib import Path from typing import Any from uuid import UUID @@ -8,8 +11,12 @@ from openpyxl.styles import Alignment, Font, PatternFill from openpyxl.worksheet.worksheet import Worksheet from openpyxl.utils import get_column_letter from pyproj import Transformer +import wntr +from app.algorithms.pressure_sensor_placement import kmeans_placement +from app.algorithms.pressure_sensor_placement import sensitivity_placement from app.infra.db.postgresql import sensor_placement as sensor_placement_repository +from app.native.wndb.inp.exporter import dump_inp class SensorPlacementNotFoundError(LookupError): @@ -24,6 +31,107 @@ class SensorPlacementConflictError(RuntimeError): pass +def _sensor_inp_path(project_code: str) -> Path: + if ( + not project_code + or project_code in {".", ".."} + or "/" in project_code + or "\\" in project_code + or "\x00" in project_code + ): + raise SensorPlacementValidationError("管网名称不是有效的项目标识") + return Path("db_inp") / f"{project_code}.db.inp" + + +@contextmanager +def _sensor_inp_lock(project_code: str): + inp_path = _sensor_inp_path(project_code) + inp_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = inp_path.with_suffix(".sensor.lock") + with lock_path.open("w", encoding="utf-8") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise SensorPlacementConflictError( + "当前项目已有监测点优化任务正在运行，请稍后重试" + ) from exc + try: + yield inp_path + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _create_validated_placement( + project_code: str, + *, + run_name: str, + min_diameter: int, + created_by: str, + sensor_locations: list[str], +) -> dict[str, Any]: + validate_sensor_placement_nodes(project_code, sensor_locations) + return sensor_placement_repository.create_sensor_placement( + project_code, + run_name=run_name, + min_diameter=min_diameter, + created_by=created_by, + sensor_locations=sensor_locations, + ) + + +def optimize_sensor_placement_by_sensitivity( + project_code: str, + run_name: str, + sensor_count: int, + min_diameter: int, + created_by: str, +) -> dict[str, Any]: + """Run sensitivity placement and persist the validated result.""" + + with _sensor_inp_lock(project_code): + inp_path = _sensor_inp_path(project_code) + dump_inp(project_code, str(inp_path), "2") + network_model = wntr.network.WaterNetworkModel(str(inp_path)) + sensor_locations = sensitivity_placement.optimize_sensor_placement( + network_model, + sensor_num=sensor_count, + min_diameter=min_diameter, + ) + return _create_validated_placement( + project_code, + run_name=run_name, + min_diameter=min_diameter, + created_by=created_by, + sensor_locations=sensor_locations, + ) + + +def optimize_sensor_placement_by_kmeans( + project_code: str, + run_name: str, + sensor_count: int, + min_diameter: int, + created_by: str, +) -> dict[str, Any]: + """Export the model, run K-means placement, and persist the result.""" + + with _sensor_inp_lock(project_code) as inp_path: + dump_inp(project_code, str(inp_path), "2") + network_model = wntr.network.WaterNetworkModel(str(inp_path)) + sensor_locations = kmeans_placement.optimize_sensor_placement( + network_model, + sensor_count=sensor_count, + min_diameter_mm=min_diameter, + ) + return _create_validated_placement( + project_code, + run_name=run_name, + min_diameter=min_diameter, + created_by=created_by, + sensor_locations=sensor_locations, + ) + + _to_wgs84 = Transformer.from_crs("EPSG:3857", "EPSG:4326", always_xy=True) _STATUS_LABELS = { "current": "当前方案", diff --git a/app/services/simulation.py b/app/services/simulation.py index 5a96d51..a5863bf 100644 --- a/app/services/simulation.py +++ b/app/services/simulation.py @@ -1,41 +1,28 @@ import numpy as np -from app.services.tjnetwork import ( - ChangeSet, - get_demand, - get_option, - get_pattern, - get_pump, - get_reservoir, - get_status, - get_tank, - get_time, - read_all, - run_project, - set_demand, - set_pattern, - set_status, - set_tank, - set_time, -) +from app.infra.epanet import run_project +from app.native.wndb.core.database import ChangeSet +from app.native.wndb.model.demands import get_demand, set_demand +from app.native.wndb.model.options import get_option +from app.native.wndb.model.patterns import get_pattern, set_pattern +from app.native.wndb.model.pumps import get_pump +from app.native.wndb.model.reservoirs import get_reservoir +from app.native.wndb.model.status import get_status, set_status +from app.native.wndb.model.tanks import get_tank, set_tank +from app.native.wndb.model.times import get_time, set_time # from get_real_status import * -from datetime import datetime, timedelta +from datetime import datetime from math import modf -import os import json import pytz -import requests import time -from typing import Optional, Tuple from uuid import UUID -import typing import logging -import app.services.project_info as project_info from app.infra.db.postgresql.scada import ( ScadaElementMappings, load_realtime_element_mappings, ) -from app.services.time_api import parse_beijing_time, parse_clock_duration_seconds +from app.domain.time import parse_beijing_time, parse_clock_duration_seconds from app.native.wndb.core.connection import project_transaction from app.native.wndb.core.database import refresh_materialized_views_after_commit from app.infra.db.timescaledb.internal_queries import ( @@ -479,7 +466,7 @@ def run_simulation( cs = ChangeSet() cs.append(pump_pattern) set_pattern(name_c, cs) - # 显式阀门控制沿用 run_simulation_ex 的处理顺序和覆盖规则。 + # 显式阀门控制优先于旧的开度参数。 if valve_control is not None: _apply_valve_control(name_c, valve_control) # 保留原开度参数逻辑，兼容现有方案调用。 diff --git a/app/services/simulation_ops.py b/app/services/simulation_ops.py deleted file mode 100644 index 73f7d27..0000000 --- a/app/services/simulation_ops.py +++ /dev/null @@ -1,230 +0,0 @@ -import json -from datetime import datetime -from functools import wraps -from math import pi - -import pytz - -from app.algorithms.simulation.runner import run_simulation_ex -from app.native.wndb.core.projects import temporary_project_database -from app.services.tjnetwork import ( - get_pipe, - get_tank, -) - - -def _isolated_operation(purpose: str): - def decorator(func): - @wraps(func) - def wrapper(prj_name: str, *args, **kwargs): - with temporary_project_database(prj_name, purpose) as temporary: - kwargs["_temporary_project"] = temporary - return func(prj_name, *args, **kwargs) - - return wrapper - - return decorator - - -############################################################ -# project management 07 ***暂时不使用，与业务需求无关*** -############################################################ - - -@_isolated_operation("project_management") -def project_management( - prj_name, - start_datetime, - pump_control, - tank_initial_level_control=None, - region_demand_control=None, - _temporary_project=None, -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - if _temporary_project is None: - raise RuntimeError("Project-management isolation was not prepared") - new_name = _temporary_project - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - result = run_simulation_ex( - name=new_name, - simulation_type="realtime", - start_datetime=start_datetime, - duration=86400, - pump_control=pump_control, - tank_initial_level_control=tank_initial_level_control, - region_demand_control=region_demand_control, - downloading_prohibition=True, - ) - return result - - -############################################################ -# scheduling analysis 08 ***暂时不使用，与业务需求无关*** -############################################################ - - -@_isolated_operation("scheduling") -def scheduling_simulation( - prj_name, - start_time, - pump_control, - tank_id, - water_plant_output_id, - time_delta=300, - _temporary_project=None, -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - if _temporary_project is None: - raise RuntimeError("Scheduling isolation was not prepared") - new_name = _temporary_project - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - - simulation_result = json.loads( - run_simulation_ex( - new_name, "realtime", start_time, duration=0, pump_control=pump_control - ) - ) - - output_data = simulation_result.get("output") - if not isinstance(output_data, dict): - raise RuntimeError("run_simulation_ex did not return JSON output content") - - tank = get_tank(new_name, tank_id) # 水塔信息 - tank_floor_space = pi * pow(tank["diameter"] / 2, 2) # 水塔底面积(m^2) - tank_init_level = tank["init_level"] # 水塔初始水位(m) - tank_pipes_id = tank["links"] # pipes list - - tank_pipe_flow_direction = ( - {} - ) # 管道流向修正系数, 水塔为下游节点时为1, 水塔为上游节点时为-1 - for pipe_id in tank_pipes_id: - if get_pipe(new_name, pipe_id)["node2"] == tank_id: # 水塔为下游节点 - tank_pipe_flow_direction[pipe_id] = 1 - else: - tank_pipe_flow_direction[pipe_id] = -1 - - node_results = output_data.get("node_results") or [] # [{'node': str, 'result': [{'pressure': float}]}] - water_plant_output_pressure = 0 - for node_result in node_results: - if node_result["node"] == water_plant_output_id: # 水厂出水压力(m) - water_plant_output_pressure = node_result["result"][-1]["pressure"] - water_plant_output_pressure /= 100 # 预计水厂出水压力(Mpa) - - pipe_results = output_data.get("link_results") or [] # [{'link': str, 'result': [{'flow': float}]}] - tank_inflow = 0 - for pipe_result in pipe_results: - for pipe_id in tank_pipes_id: # 遍历与水塔相连的管道 - if pipe_result["link"] == pipe_id: # 水塔入流流量(L/s) - tank_inflow += ( - pipe_result["result"][-1]["flow"] - * tank_pipe_flow_direction[pipe_id] - ) - tank_inflow /= 1000 # 水塔入流流量(m^3/s) - tank_level_delta = tank_inflow * time_delta / tank_floor_space # 水塔水位改变值(m) - tank_level = tank_init_level + tank_level_delta # 预计水塔水位(m) - - simulation_results = { - "water_plant_output_pressure": water_plant_output_pressure, - "tank_init_level": tank_init_level, - "tank_level": tank_level, - } - - return json.dumps(simulation_results) - - -@_isolated_operation("daily_scheduling") -def daily_scheduling_simulation( - prj_name, - start_time, - pump_control, - reservoir_id, - tank_id, - water_plant_output_id, - _temporary_project=None, -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - if _temporary_project is None: - raise RuntimeError("Daily-scheduling isolation was not prepared") - new_name = _temporary_project - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - - simulation_result = json.loads( - run_simulation_ex( - new_name, - "realtime", - start_time, - duration=86400, - pump_control=pump_control, - ) - ) - - output_data = simulation_result.get("output") - if not isinstance(output_data, dict): - raise RuntimeError("run_simulation_ex did not return JSON output content") - - node_results = output_data.get("node_results") or [] # [{'node': str, 'result': [{'pressure': float, 'head': float}]}] - water_plant_output_pressure = [] - reservoir_level = [] - tank_level = [] - for node_result in node_results: - if node_result["node"] == water_plant_output_id: - for result in node_result["result"]: - water_plant_output_pressure.append( - result["pressure"] / 100 - ) # 水厂出水压力(Mpa) - elif node_result["node"] == reservoir_id: - for result in node_result["result"]: - reservoir_level.append(result["head"] - 250.35) # 清水池液位(m) - elif node_result["node"] == tank_id: - for result in node_result["result"]: - tank_level.append(result["pressure"]) # 调节池液位(m) - - simulation_results = { - "water_plant_output_pressure": water_plant_output_pressure, - "reservoir_level": reservoir_level, - "tank_level": tank_level, - } - - return json.dumps(simulation_results) diff --git a/app/algorithms/simulation/scenarios.py b/app/services/simulation_scenarios.py similarity index 91% rename from app/algorithms/simulation/scenarios.py rename to app/services/simulation_scenarios.py index 794f027..20a2f5e 100644 --- a/app/algorithms/simulation/scenarios.py +++ b/app/services/simulation_scenarios.py @@ -8,33 +8,27 @@ from math import pi, sqrt import pytz import app.services.simulation as simulation -from app.algorithms.simulation.runner import ( - run_simulation_ex, - from_clock_to_seconds_2, -) from app.native.wndb.core.projects import temporary_project_database -from app.services.tjnetwork import ( - ChangeSet, +from app.native.wndb.core.database import ChangeSet +from app.native.wndb.model.demands import get_demand, set_demand +from app.native.wndb.model.elements import get_node_links, is_junction +from app.native.wndb.model.emitters import get_emitter, set_emitter +from app.native.wndb.model.options import ( OPTION_DEMAND_MODEL_PDA, OPTION_QUALITY_CHEMICAL, - SOURCE_TYPE_SETPOINT, - add_pattern, - add_source, - get_demand, - get_emitter, - get_node_links, get_option, - get_pattern, - get_pipe, - get_source, - get_time, - is_junction, - set_demand, - set_emitter, set_option, - set_source, - set_time, ) +from app.native.wndb.model.patterns import add_pattern, get_pattern +from app.native.wndb.model.pipes import get_pipe +from app.native.wndb.model.sources import ( + SOURCE_TYPE_SETPOINT, + add_source, + get_source, + set_source, +) +from app.native.wndb.model.times import get_time, set_time +from app.domain.time import parse_clock_duration_seconds def _isolated_analysis(purpose: str): @@ -283,7 +277,7 @@ def valve_close_analysis( modify_pattern_start_time=modify_pattern_start_time, modify_total_duration=modify_total_duration, modify_valve_opening=modify_valve_opening, - scheme_type="valve_close_Analysis", + scheme_type="valve_close_analysis", scheme_name=scheme_name, result_db_name=name, ) @@ -371,7 +365,7 @@ def flushing_analysis( # 新建 pattern time_option = get_time(new_name) hydraulic_step = time_option["HYDRAULIC TIMESTEP"] - secs = from_clock_to_seconds_2(hydraulic_step) + secs = parse_clock_duration_seconds(hydraulic_step) cs_pattern = ChangeSet() pt = {} factors = [] @@ -511,7 +505,7 @@ def contaminant_simulation( set_time(new_name, cs) # set QUALITY TIMESTEP time_option = get_time(new_name) hydraulic_step = time_option["HYDRAULIC TIMESTEP"] - secs = from_clock_to_seconds_2(hydraulic_step) + secs = parse_clock_duration_seconds(hydraulic_step) operation_step = 0 # step 1. set duration if modify_total_duration == None: @@ -590,51 +584,6 @@ def contaminant_simulation( # execute_undo(name) -############################################################ -# age analysis 05 ***水龄模拟目前还没和实时模拟打通，不确定是否需要，先不要使用*** -############################################################ - - -def age_analysis( - name: str, modify_pattern_start_time: str, modify_total_duration: int = 900 -) -> None: - """ - 水龄模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - with temporary_project_database(name, "age_analysis") as new_name: - result = run_simulation_ex( - new_name, - "realtime", - modify_pattern_start_time, - duration=modify_total_duration, - downloading_prohibition=True, - ) - simulation_result = json.loads(result) - output_data = simulation_result.get("output") - if not isinstance(output_data, dict): - raise RuntimeError("run_simulation_ex did not return JSON output content") - - nodes_age = [] - node_result = output_data.get("node_results") or [] - for node in node_result: - nodes_age.append(node["result"][-1]["quality"]) - links_age = [] - link_result = output_data.get("link_results") or [] - for link in link_result: - links_age.append(link["result"][-1]["quality"]) - age_result = {"nodes": nodes_age, "links": links_age} - # age_result = {'nodes': nodes_age, 'links': links_age, 'nodeIDs': node_name, 'linkIDs': link_name} - return json.dumps(age_result) - - ############################################################ # pressure regulation 06 ############################################################ diff --git a/app/infra/db/timescaledb/composite_queries.py b/app/services/timeseries_analysis.py similarity index 95% rename from app/infra/db/timescaledb/composite_queries.py rename to app/services/timeseries_analysis.py index bf3a1ad..5070938 100644 --- a/app/infra/db/timescaledb/composite_queries.py +++ b/app/services/timeseries_analysis.py @@ -7,16 +7,19 @@ import pandas as pd from psycopg import AsyncConnection from uuid import UUID -from app.algorithms.cleaning.flow import clean_flow_data_df_kf -from app.algorithms.cleaning.pressure import clean_pressure_data_df_km -from app.algorithms.health.analyzer import PipelineHealthAnalyzer +from app.algorithms.pipe_health_prediction.survival_predictor import ( + PipeHealthSurvivalPredictor, +) +from app.algorithms.scada_cleaning.flow_series import clean_flow_data_df_kf +from app.algorithms.scada_cleaning.pressure_series import clean_pressure_data_df_km +from app.infra.db.postgresql.network_assets import NetworkAssetRepository from app.infra.db.postgresql.scada import ScadaInfoRepository from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository -class CompositeQueries: +class TimeseriesAnalysisService: """ 复合查询类，提供跨表查询功能 """ @@ -55,7 +58,7 @@ class CompositeQueries: Raises: ValueError: 当 SCADA 设备未找到或字段无效时 """ - scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) + scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn) link_devices: dict[str, str] = {} node_devices: dict[str, str] = {} for device_id in device_ids: @@ -118,7 +121,7 @@ class CompositeQueries: Raises: ValueError: 当 SCADA 设备未找到或字段无效时 """ - scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) + scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn) link_devices: dict[str, str] = {} node_devices: dict[str, str] = {} for device_id in device_ids: @@ -293,7 +296,7 @@ class CompositeQueries: ValueError: 当元素类型无效时 """ - scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) + scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn) associated_scada = next( ( scada @@ -342,7 +345,7 @@ class CompositeQueries: Raises: ValueError: 当前项目没有可清洗设备或指定时间范围内没有监测数据 """ - scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) + scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn) supported_types = {"pressure", "pipe_flow", "flow"} if device_ids: @@ -509,17 +512,9 @@ class CompositeQueries: valid_link_ids = list(velocity_data.keys()) # GIS 物化视图是低频更新管网的查询面；只读取本次有结果的管道。 - async with postgres_conn.cursor() as cur: - await cur.execute( - """ - SELECT id, diameter, start_node_id AS node1, - end_node_id AS node2 - FROM gis.pipes - WHERE id = ANY(%s) - """, - (valid_link_ids,), - ) - all_links = await cur.fetchall() + all_links = await NetworkAssetRepository.get_pipes_by_ids( + postgres_conn, valid_link_ids + ) # 转换为字典以快速查找 links_dict = {str(link["id"]): link for link in all_links} @@ -597,8 +592,8 @@ class CompositeQueries: } ) - # 7. 使用PipelineHealthAnalyzer进行预测 - analyzer = PipelineHealthAnalyzer() + # 7. 使用生存模型进行预测 + analyzer = PipeHealthSurvivalPredictor() survival_functions = analyzer.predict_survival(data) # 8. 组合结果 results = [] diff --git a/app/services/tjnetwork.py b/app/services/tjnetwork.py index de77ad5..0925993 100644 --- a/app/services/tjnetwork.py +++ b/app/services/tjnetwork.py @@ -1,14 +1,14 @@ """Application-facing water-network service boundary. -The native WNDB package is organized by responsibility. This module only -combines operations that need more than one native module and re-exports the -small set of operations used by HTTP and algorithm services. +The native WNDB package is organized by responsibility. This transitional +facade re-exports the operations still consumed by HTTP endpoints. Application +services and algorithms depend on their explicit native/repository modules. """ from typing import Any import app.infra.epanet as epanet -from app.algorithms.water_demand import ( +from app.services.demand_allocation import ( calculate_demand_to_network, calculate_demand_to_nodes, calculate_demand_to_region, @@ -234,10 +234,6 @@ def run_inp(name: str) -> str: return epanet.run_inp(name) -def dump_output(path: str) -> str: - return epanet.dump_output(path) - - def get_node_properties(name: str, node_id: str) -> dict[str, Any]: if is_junction(name, node_id): return get_junction(name, node_id) diff --git a/app/services/valve_isolation.py b/app/services/valve_isolation.py index 993b23e..6f88db8 100644 --- a/app/services/valve_isolation.py +++ b/app/services/valve_isolation.py @@ -1,6 +1,7 @@ from typing import Any -from app.algorithms.isolation.valve import valve_isolation_analysis +from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis +from app.native.wndb.gis.network_views import get_network_link_nodes def analyze_valve_isolation( @@ -8,4 +9,5 @@ def analyze_valve_isolation( accident_element: str | list[str], disabled_valves: list[str] = None, ) -> dict[str, Any]: - return valve_isolation_analysis(network, accident_element, disabled_valves) + link_entries = get_network_link_nodes(network) + return valve_isolation_analysis(link_entries, accident_element, disabled_valves) diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 04f21c8..ccb9024 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -2,7 +2,7 @@ This module is reserved for general utility functions and helpers. Currently, specific utilities are often implemented within their respective service modules -(e.g., time handling in `app.services.time_api`) or core modules. +(e.g., time handling in `app.domain.time`) or core modules. Future expansion may include: - Date and time manipulation helpers diff --git a/contracts/manifest.json b/contracts/manifest.json index 70684fc..5d6f350 100644 --- a/contracts/manifest.json +++ b/contracts/manifest.json @@ -3,7 +3,7 @@ "contracts": { "server": { "file": "server-v1.openapi.json", - "sha256": "d0364fb08c6f18fac2ea9c9980ef21115fc01f110cd10f25f90d97d0bc0e6367" + "sha256": "b6640fa87909c9a8a8747e9003ace35ea6a209f99f6e1dc15cb0bd5c313dc7c3" } } } diff --git a/contracts/server-v1.openapi.json b/contracts/server-v1.openapi.json index 0e9bc10..eb55f7d 100644 --- a/contracts/server-v1.openapi.json +++ b/contracts/server-v1.openapi.json @@ -923,63 +923,13 @@ "title": "BurstLocationRequestRest", "type": "object" }, - "DailySchedulingAnalysisRest": { - "properties": { - "pump_control": { - "description": "泵控制策略", - "title": "Pump Control", - "type": "object" - }, - "reservoir_id": { - "description": "水库ID", - "title": "Reservoir Id", - "type": "string" - }, - "start_time": { - "description": "开始时间", - "title": "Start Time", - "type": "string" - }, - "tank_id": { - "description": "水箱ID", - "title": "Tank Id", - "type": "string" - }, - "time_delta": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": 300, - "description": "时间步长 (秒)", - "title": "Time Delta" - }, - "water_plant_output_id": { - "description": "水厂出水ID", - "title": "Water Plant Output Id", - "type": "string" - } - }, - "required": [ - "start_time", - "pump_control", - "reservoir_id", - "tank_id", - "water_plant_output_id" - ], - "title": "DailySchedulingAnalysisRest", - "type": "object" - }, "JsonValue": {}, "LeakageIdentifyRequestRest": { "properties": { "dma_count": { "anyOf": [ { + "minimum": 1.0, "type": "integer" }, { @@ -992,26 +942,29 @@ "duration": { "default": 24, "description": "持续时间（小时）", + "exclusiveMinimum": 0.0, + "maximum": 168.0, "title": "Duration", "type": "number" }, "max_gen": { "default": 100, "description": "最大代数", + "maximum": 1000.0, + "minimum": 1.0, "title": "Max Gen", "type": "integer" }, "n_workers": { "default": 4, - "description": "工作线程数", + "description": "工作进程数", + "maximum": 4.0, + "minimum": 1.0, "title": "N Workers", "type": "integer" }, "observed_pressure_data": { "anyOf": [ - { - "type": "string" - }, { "additionalProperties": { "items": {}, @@ -1029,15 +982,9 @@ "type": "null" } ], - "description": "观测的压力数据", + "description": "观测的压力数据；文件路径不属于公共 API 输入", "title": "Observed Pressure Data" }, - "output_dir": { - "default": "db_inp", - "description": "输出目录", - "title": "Output Dir", - "type": "string" - }, "output_flow_unit": { "default": "m3/s", "description": "输出流量单位", @@ -1047,12 +994,15 @@ "pop_size": { "default": 50, "description": "种群大小", + "maximum": 1000.0, + "minimum": 2.0, "title": "Pop Size", "type": "integer" }, "q_sum": { "default": 0.2, "description": "总流量（m3/s）", + "minimum": 0.0, "title": "Q Sum", "type": "number" }, @@ -1118,12 +1068,15 @@ "start_time": { "default": 0, "description": "起始时间（小时）", + "minimum": 0.0, "title": "Start Time", "type": "number" }, "timestep": { "default": 5, "description": "时间步长（分钟）", + "exclusiveMinimum": 0.0, + "maximum": 1440.0, "title": "Timestep", "type": "number" } @@ -1663,16 +1616,10 @@ "type": "object" }, "scheme_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], "description": "方案名称", - "title": "Scheme Name" + "minLength": 1, + "title": "Scheme Name", + "type": "string" }, "start_time": { "description": "开始时间", @@ -1694,7 +1641,8 @@ }, "required": [ "start_time", - "pump_control" + "pump_control", + "scheme_name" ], "title": "PressureRegulationRest", "type": "object" @@ -1888,50 +1836,6 @@ "title": "ProjectDatabaseUpsertRequest", "type": "object" }, - "ProjectManagementRest": { - "properties": { - "pump_control": { - "description": "泵控制策略", - "title": "Pump Control", - "type": "object" - }, - "region_demand": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "description": "区域需水量控制", - "title": "Region Demand" - }, - "start_time": { - "description": "开始时间", - "title": "Start Time", - "type": "string" - }, - "tank_init_level": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "description": "水箱初始水位", - "title": "Tank Init Level" - } - }, - "required": [ - "start_time", - "pump_control" - ], - "title": "ProjectManagementRest", - "type": "object" - }, "ProjectMemberCreateRequest": { "properties": { "project_role": { @@ -2129,26 +2033,6 @@ "title": "ProjectSummaryResponse", "type": "object" }, - "PumpFailureState": { - "properties": { - "pump_status": { - "description": "泵状态字典", - "title": "Pump Status", - "type": "object" - }, - "time": { - "description": "故障发生时间", - "title": "Time", - "type": "string" - } - }, - "required": [ - "time", - "pump_status" - ], - "title": "PumpFailureState", - "type": "object" - }, "RealtimeLinkBatchItem": { "properties": { "flow": { @@ -2382,51 +2266,6 @@ "title": "ScadaReadingBatchItem", "type": "object" }, - "SchedulingAnalysisRest": { - "properties": { - "pump_control": { - "description": "泵控制策略", - "title": "Pump Control", - "type": "object" - }, - "start_time": { - "description": "开始时间", - "title": "Start Time", - "type": "string" - }, - "tank_id": { - "description": "水箱ID", - "title": "Tank Id", - "type": "string" - }, - "time_delta": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": 300, - "description": "时间步长 (秒)", - "title": "Time Delta" - }, - "water_plant_output_id": { - "description": "水厂出水ID", - "title": "Water Plant Output Id", - "type": "string" - } - }, - "required": [ - "start_time", - "pump_control", - "tank_id", - "water_plant_output_id" - ], - "title": "SchedulingAnalysisRest", - "type": "object" - }, "SensorPlacementExportRequest": { "properties": { "adjustment_status": { @@ -6536,6 +6375,7 @@ "items": { "type": "string" }, + "minItems": 1, "title": "Burst Id", "type": "array" } @@ -6550,6 +6390,7 @@ "items": { "type": "number" }, + "minItems": 1, "title": "Burst Size", "type": "array" } @@ -6561,6 +6402,7 @@ "required": true, "schema": { "description": "模拟总时长（秒）", + "exclusiveMinimum": 0, "title": "Modify Total Duration", "type": "integer" } @@ -6572,6 +6414,7 @@ "required": true, "schema": { "description": "分析方案名称", + "minLength": 1, "title": "Scheme Name", "type": "string" } @@ -7944,116 +7787,6 @@ ] } }, - "/api/v1/daily-scheduling-analyses": { - "post": { - "description": "对管网的每日供水排程进行分析，优化水库、水厂、水箱和用户需求的协调，制定合理的每日排程方案。", - "operationId": "post_daily_scheduling_analyses", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DailySchedulingAnalysisRest", - "description": "日排程分析参数" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Daily Scheduling Analyses", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "日排程分析", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/demands/properties": { "get": { "description": "获取指定水网中节点的需水量属性信息", @@ -9769,105 +9502,6 @@ ] } }, - "/api/v1/inp-runs": { - "post": { - "description": "运行指定INP文件格式的管网模型进行水力模拟。INP文件应该放在inp文件夹中，参数为文件名不含扩展名。", - "operationId": "post_inp_runs", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Inp Runs", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "运行INP文件", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/junctions": { "delete": { "description": "从供水网络中删除指定的节点。", @@ -18150,116 +17784,6 @@ ] } }, - "/api/v1/outputs": { - "get": { - "description": "导出指定路径的模拟输出文件内容。参数应为绝对路径。", - "operationId": "get_outputs", - "parameters": [ - { - "description": "模拟输出文件的绝对路径", - "in": "query", - "name": "output", - "required": true, - "schema": { - "description": "模拟输出文件的绝对路径", - "title": "Output", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Get Outputs", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "导出模拟输出", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/patterns": { "delete": { "description": "从网络中删除指定的模式", @@ -21769,236 +21293,6 @@ ] } }, - "/api/v1/pressure-regulation-calculations": { - "post": { - "description": "对管网的压力进行调节分析，通过控制泵的运行来维持目标节点的目标压力。此为基础版本。", - "operationId": "post_pressure_regulation_calculations", - "parameters": [ - { - "description": "目标节点ID", - "in": "query", - "name": "target_node", - "required": true, - "schema": { - "description": "目标节点ID", - "title": "Target Node", - "type": "string" - } - }, - { - "description": "目标压力值（kPa）", - "in": "query", - "name": "target_pressure", - "required": true, - "schema": { - "description": "目标压力值（kPa）", - "title": "Target Pressure", - "type": "number" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "压力调节（基础）", - "tags": [ - "Simulation Control" - ] - } - }, - "/api/v1/project-managements": { - "post": { - "description": "高级版本的项目管理，通过JSON请求体提供详细的控制参数，包括泵控制策略、水箱初始水位和区域需水量控制。", - "operationId": "post_project_managements", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProjectManagementRest", - "description": "项目管理控制参数" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Project Managements", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "项目管理（高级）", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/project-return-dict-runs": { "post": { "description": "基于指定的管网项目运行标准水力模拟，返回JSON格式的字典，包含输出数据和报告文本。", @@ -22711,116 +22005,6 @@ ] } }, - "/api/v1/pump-failure-events": { - "post": { - "description": "记录和管理泵的故障状态，包括故障发生时间和受影响的泵列表。系统将记录故障日志并更新泵状态。", - "operationId": "post_pump_failure_events", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PumpFailureState", - "description": "泵故障状态信息" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Pump Failure Events", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "泵故障管理", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/pumps": { "delete": { "description": "从网络中删除指定的水泵", @@ -27726,116 +26910,6 @@ ] } }, - "/api/v1/scheduling-analyses": { - "post": { - "description": "对管网的供水排程进行分析，优化泵的运行时间和出水流量，平衡水厂出水、水箱进出水，满足用户需求。", - "operationId": "post_scheduling_analyses", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SchedulingAnalysisRest", - "description": "排程分析参数" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Scheduling Analyses", - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "排程分析", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/sensor-placement-candidates/{node_id}": { "get": { "operationId": "get_sensor_placement_candidates_node_id", @@ -39439,126 +38513,6 @@ ] } }, - "/api/v1/water-age-analyses": { - "post": { - "description": "高级版本的水龄分析，在指定时间点进行分析，支持自定义模拟持续时间。返回纯文本格式的分析结果。", - "operationId": "post_water_age_analyses", - "parameters": [ - { - "description": "分析开始时间（ISO 8601格式）", - "in": "query", - "name": "start_time", - "required": true, - "schema": { - "description": "分析开始时间（ISO 8601格式）", - "title": "Start Time", - "type": "string" - } - }, - { - "description": "模拟持续时间（秒）", - "in": "query", - "name": "duration", - "required": true, - "schema": { - "description": "模拟持续时间（秒）", - "title": "Duration", - "type": "integer" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "text/plain": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "水龄分析（高级）", - "tags": [ - "Simulation Control" - ] - } - }, "/api/v1/web-searches": { "post": { "description": "调用 Bocha Web Search API 获取实时网页搜索结果", diff --git a/docs/sensor-sensitivity-optimization.md b/docs/sensor-sensitivity-optimization.md index 85aecc6..5859bce 100644 --- a/docs/sensor-sensitivity-optimization.md +++ b/docs/sensor-sensitivity-optimization.md @@ -1,6 +1,6 @@ # 监测点灵敏度算法优化说明 -本文记录压力监测点布置算法的改造方案、数学含义、复杂度变化和本地验证结果。实现位于 `app/algorithms/sensor/sensitivity.py`，对外兼容入口仍为 `get_ID(name, sensor_num, min_diameter)`。 +本文记录压力监测点布置算法的改造方案、数学含义、复杂度变化和本地验证结果。纯计算实现位于 `app/algorithms/pressure_sensor_placement/sensitivity_placement.py`；`app/services/sensor_placement.py` 负责从池化的项目数据库导出 INP、加载模型、校验并保存结果。 ## 改造目标 diff --git a/resources/db_v2/WNDB_STRUCTURE.md b/resources/db_v2/WNDB_STRUCTURE.md index b011516..55b2ebd 100644 --- a/resources/db_v2/WNDB_STRUCTURE.md +++ b/resources/db_v2/WNDB_STRUCTURE.md @@ -135,12 +135,12 @@ flowchart LR | 职责 | 当前路径 | 原因 | | --- | --- | --- | -| 用水量分配计算 | `app/algorithms/water_demand/` | 属于业务算法，WNDB 只提供节点、需求和区域查询 | -| SCADA 资产查询 | `app/infra/db/postgresql/scada_assets.py` | 对应 `asset.scada_devices` 的 PostgreSQL 仓储 | +| 用水量分配计算 | `app/algorithms/demand_allocation/` | 只接收普通拓扑数据；数据库装载位于 `app/services/demand_allocation.py` | +| SCADA 资产查询 | `app/infra/db/postgresql/scada.py` | 对应 `asset.scada_devices` 的 PostgreSQL 仓储 | | 测压点选址结果 | `app/infra/db/postgresql/sensor_placement.py` | 对应 `analysis.runs` 和 `analysis.results` 的仓储 | -| 服务组合入口 | `app/services/tjnetwork.py` | 组合 WNDB、EPANET 和业务仓储，供接口与算法层调用 | +| 网络 API 门面 | `app/services/tjnetwork.py` | 暂时组合 WNDB 与 EPANET 供 HTTP 接口调用；算法层不再依赖该门面 | -`tjnetwork.py` 从约 995 行缩减到约 320 行。它不再通过 WNDB 根包获得全部函数，只显式导入当前服务使用的能力。过时的 `scripts/test_tjnetwork.py` 依赖已移除的 operation、snapshot、DMA 和旧 SCADA API，已经一并删除。 +`tjnetwork.py` 从约 995 行缩减到约 285 行。它不再通过 WNDB 根包获得全部函数，只显式导入当前接口使用的能力。过时的 `scripts/test_tjnetwork.py` 依赖已移除的 operation、snapshot、DMA 和旧 SCADA API，已经一并删除。 ### HTTP 执行边界 @@ -152,8 +152,8 @@ WNDB 当前使用同步 `psycopg` 连接池。`network/`、`components/` 和同 ```mermaid flowchart TD - API[HTTP 接口与业务服务] - SERVICE[services.tjnetwork] + API[HTTP 接口] + SERVICE[应用服务] ALGORITHM[业务算法] REPOSITORY[PostgreSQL 业务仓储] COMMANDS[wndb.commands] @@ -164,8 +164,8 @@ flowchart TD DB[(项目业务数据库)] API --> SERVICE - API --> ALGORITHM API --> REPOSITORY + SERVICE --> ALGORITHM SERVICE --> COMMANDS SERVICE --> INP SERVICE --> MODEL @@ -187,13 +187,15 @@ flowchart TD WNDB 根包不再作为依赖汇聚点。上层若只需要管道查询，应直接依赖 `model.pipes`；需要区域几何时依赖 `gis.region_geometry`；需要项目连接时依赖 `core.connection`。 +`app/algorithms` 已按业务能力改为 `burst_detection`、`burst_localization`、`scada_cleaning`、`pipe_health_prediction`、`valve_isolation`、`dma_leakage_estimation`、`pressure_sensor_placement` 和 `demand_allocation`。自动化架构测试禁止算法层反向依赖 service、infra、native 或 API，也禁止 infra 依赖 service/algorithm。少量只做仓储转发的 HTTP 接口仍直接依赖 repository，后续按业务聚合需要迁入应用服务，不为机械包装而增加空壳层。 + ## 仍需留意的文件规模 `importer.py` 和 `region_geometry.py` 行数较多，但函数仍围绕单一职责。只有在继续增加 INP 格式或区域算法时，才需要分别拆出版本转换器或边界算法模块。目前没有必要为了控制文件行数继续分层。 ## 验证结果 -- 本地 conda 环境单元、鉴权和 API 测试：286 项通过，2 项按条件跳过。 +- 本地 conda 环境单元、鉴权和 API 测试：313 项通过，2 项按条件跳过。 - 一次性实库从 `tjwater_v2_template` 创建后，通过 INP 暂存解析和事务替换得到 11 个节点、13 条连接、11 条坐标及 9 条 junction 物化视图记录，验证后已完整删除。 - `tjwater_v2` 统一视图覆盖 87,907 个节点和 91,054 条链路，与六个来源物化视图的合计数量一致。实测完整节点读取约 0.17 秒、完整链路读取约 0.10 秒、完整拓扑两次批量查询约 1.12 秒；耗时仅作为当前环境基线，不作为固定性能承诺。 - `tjwater_v2` 真实数据库测试：11 项通过，覆盖业务库和时序库并发借用、失效连接自动重建、临时库模型/SCADA/视图完整克隆与清理、嵌套事务回滚、分析运行生命周期、恶意标识符转义、明细表复合主键、统一 GIS 查询视图，以及 WNDB pattern 增删改、级联解除需求关联和整体回滚。 diff --git a/tests/api/test_algorithm_execution_endpoints.py b/tests/api/test_algorithm_execution_endpoints.py new file mode 100644 index 0000000..ac78141 --- /dev/null +++ b/tests/api/test_algorithm_execution_endpoints.py @@ -0,0 +1,54 @@ +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.v1.endpoints import burst_detection, burst_location + + +def _client(module) -> TestClient: + app = FastAPI() + app.include_router(module.router, prefix="/api/v1") + app.dependency_overrides[module.get_current_keycloak_username] = lambda: "tester" + return TestClient(app) + + +def test_burst_detection_runs_service_outside_event_loop(monkeypatch): + captured = {} + + async def fake_threadpool(func, **kwargs): + captured["func"] = func + captured["kwargs"] = kwargs + return {"status": "completed"} + + monkeypatch.setattr(burst_detection, "run_in_threadpool", fake_threadpool) + response = _client(burst_detection).post( + "/api/v1/burst-detections", + json={"network": "demo", "observed_pressure_data": {"S1": [1.0]}}, + ) + + assert response.status_code == 200 + assert captured["func"] is burst_detection.run_burst_detection + assert captured["kwargs"]["username"] == "tester" + + +def test_burst_location_runs_service_outside_event_loop(monkeypatch): + captured = {} + + async def fake_threadpool(func, **kwargs): + captured["func"] = func + captured["kwargs"] = kwargs + return {"status": "completed"} + + monkeypatch.setattr(burst_location, "run_in_threadpool", fake_threadpool) + response = _client(burst_location).post( + "/api/v1/burst-locations", + json={ + "network": "demo", + "burst_leakage": 0.1, + "burst_pressure": {"S1": 1.0}, + "normal_pressure": {"S1": 1.1}, + }, + ) + + assert response.status_code == 200 + assert captured["func"] is burst_location.run_burst_location_by_network + assert captured["kwargs"]["username"] == "tester" diff --git a/tests/api/test_leakage_endpoints.py b/tests/api/test_leakage_endpoints.py index ecd01d2..7d217b8 100644 --- a/tests/api/test_leakage_endpoints.py +++ b/tests/api/test_leakage_endpoints.py @@ -33,3 +33,58 @@ def test_identify_leakage_success(monkeypatch): ) assert response.status_code == 200 assert response.json()["area_count"] == 0 + + +def test_identify_leakage_rejects_server_file_paths(): + client = _build_client() + + response = client.post( + "/api/v1/leakage-identifications", + json={ + "network": "demo", + "observed_pressure_data": "/etc/passwd", + "output_dir": "/tmp/leakage-results", + }, + ) + + assert response.status_code == 422 + + +def test_identify_leakage_rejects_unbounded_workload(): + client = _build_client() + + response = client.post( + "/api/v1/leakage-identifications", + json={ + "network": "demo", + "observed_pressure_data": {"S1": [1.0]}, + "pop_size": 1_000_000, + "max_gen": 1_000_000, + "n_workers": 10_000, + }, + ) + + assert response.status_code == 422 + + +def test_identify_leakage_runs_service_outside_event_loop(monkeypatch): + captured = {} + + async def fake_threadpool(func, **kwargs): + captured["func"] = func + captured["kwargs"] = kwargs + return {"rows": [], "area_count": 0} + + monkeypatch.setattr(leakage_endpoint, "run_in_threadpool", fake_threadpool) + client = _build_client() + response = client.post( + "/api/v1/leakage-identifications", + json={ + "network": "demo", + "observed_pressure_data": {"S1": [1.0]}, + }, + ) + + assert response.status_code == 200 + assert captured["func"] is leakage_endpoint.run_leakage_identification + assert captured["kwargs"]["username"] == "tester" diff --git a/tests/api/test_sensor_placement_endpoints.py b/tests/api/test_sensor_placement_endpoints.py index f4bc828..a1741ce 100644 --- a/tests/api/test_sensor_placement_endpoints.py +++ b/tests/api/test_sensor_placement_endpoints.py @@ -55,7 +55,7 @@ def test_optimize_returns_analysis_run(monkeypatch): captured = {} monkeypatch.setattr( endpoint, - "pressure_sensor_placement_kmeans", + "optimize_sensor_placement_by_kmeans", lambda **kwargs: captured.update(kwargs) or {"run_id": RUN_ID}, ) monkeypatch.setattr(endpoint, "get_sensor_placement_run", lambda *_: _run()) @@ -74,7 +74,7 @@ def test_optimize_returns_analysis_run(monkeypatch): assert response.status_code == 200 assert response.json()["run_id"] == str(RUN_ID) - assert captured["username"] == "alice" + assert captured["created_by"] == "alice" def test_optimize_rejects_project_mismatch(monkeypatch): diff --git a/tests/api/test_simulation_endpoints.py b/tests/api/test_simulation_endpoints.py index 0ced024..92e40bc 100644 --- a/tests/api/test_simulation_endpoints.py +++ b/tests/api/test_simulation_endpoints.py @@ -25,15 +25,6 @@ def _load_simulation_module(monkeypatch): seconds = parts[2] if len(parts) == 3 else 0 return hours * 3600 + minutes * 60 + seconds - install_stub( - monkeypatch, - "app.services.time_api", - { - "parse_aware_time": parse_aware_time, - "parse_clock_duration_seconds": parse_clock_duration_seconds, - "parse_utc_time": parse_utc_time, - }, - ) install_stub( monkeypatch, "app.services.simulation", @@ -62,42 +53,22 @@ def _load_simulation_module(monkeypatch): "dump_output": lambda output: f"dump::{output}", }, ) - install_stub(monkeypatch, "app.algorithms", package=True) - install_stub(monkeypatch, "app.algorithms.simulation", package=True) install_stub( monkeypatch, - "app.algorithms.simulation.scenarios", + "app.services.simulation_scenarios", { "burst_analysis": lambda *args, **kwargs: "burst", "valve_close_analysis": lambda *args, **kwargs: "valve", "flushing_analysis": lambda *args, **kwargs: "flush", "contaminant_simulation": lambda *args, **kwargs: "contaminant", - "age_analysis": lambda *args, **kwargs: "age", "pressure_regulation": lambda *args, **kwargs: "pressure", }, ) - install_stub( - monkeypatch, - "app.algorithms.sensor", - { - "pressure_sensor_placement_sensitivity": lambda *args, **kwargs: [], - "pressure_sensor_placement_kmeans": lambda *args, **kwargs: [], - }, - ) install_stub( monkeypatch, "app.services.network_import", {"network_update": lambda *args, **kwargs: "updated"}, ) - install_stub( - monkeypatch, - "app.services.simulation_ops", - { - "project_management": lambda *args, **kwargs: "managed", - "scheduling_simulation": lambda *args, **kwargs: "scheduled", - "daily_scheduling_simulation": lambda *args, **kwargs: "daily", - }, - ) install_stub( monkeypatch, "app.services.valve_isolation", @@ -126,78 +97,52 @@ def test_run_project_endpoint_returns_plain_text(monkeypatch): assert response.text == "report::demo" -def test_scheduling_analysis_maps_request_body(monkeypatch): +def test_removed_legacy_simulation_workflows_are_not_exposed(monkeypatch): module = _load_simulation_module(monkeypatch) - captured = {} - - def fake_schedule(network, start_time, pump_control, tank_id, water_plant_output_id, time_delta): - captured["args"] = ( - network, - start_time, - pump_control, - tank_id, - water_plant_output_id, - time_delta, - ) - return "scheduled" - - monkeypatch.setattr(module, "scheduling_simulation", fake_schedule) client = TestClient(build_test_app(module.router, "/api/v1")) - response = client.post( - "/api/v1/scheduling-analyses", - json={ - "network": "demo", - "start_time": "2025-01-01T08:00:00+08:00", - "pump_control": {"P1": [1, 0, 1]}, - "tank_id": "T1", - "water_plant_output_id": "R1", - }, - ) - - assert response.status_code == 200 - assert response.json() == "scheduled" - assert captured["args"] == ( - "demo", - "2025-01-01T08:00:00+08:00", - {"P1": [1, 0, 1]}, - "T1", - "R1", - 300, - ) - - -def test_project_management_maps_named_arguments(monkeypatch): - module = _load_simulation_module(monkeypatch) - captured = {} - - def fake_project_management(**kwargs): - captured.update(kwargs) - return "managed" - - monkeypatch.setattr(module, "project_management", fake_project_management) - client = TestClient(build_test_app(module.router, "/api/v1")) - - response = client.post( + for path in ( "/api/v1/project-managements", - json={ + "/api/v1/scheduling-analyses", + "/api/v1/daily-scheduling-analyses", + "/api/v1/water-age-analyses", + "/api/v1/inp-runs", + "/api/v1/outputs", + "/api/v1/pump-failure-events", + ): + assert client.post(path).status_code == 404 + + +def test_removed_basic_pressure_regulation_is_not_exposed(monkeypatch): + module = _load_simulation_module(monkeypatch) + client = TestClient(build_test_app(module.router, "/api/v1")) + + response = client.post( + "/api/v1/pressure-regulation-calculations", + params={ "network": "demo", - "start_time": "2025-01-01T08:00:00+08:00", - "pump_control": {"P1": [1]}, - "tank_init_level": {"T1": 10.0}, - "region_demand": {"R1": 20.0}, + "target_node": "J1", + "target_pressure": 50, }, ) - assert response.status_code == 200 - assert response.json() == "managed" - assert captured == { - "prj_name": "demo", - "start_datetime": "2025-01-01T08:00:00+08:00", - "pump_control": {"P1": [1]}, - "tank_initial_level_control": {"T1": 10.0}, - "region_demand_control": {"R1": 20.0}, - } + assert response.status_code == 404 + + +def test_pressure_regulation_requires_scheme_name(monkeypatch): + module = _load_simulation_module(monkeypatch) + client = TestClient(build_test_app(module.router, "/api/v1")) + + response = client.post( + "/api/v1/pressure-regulation-analyses", + json={ + "network": "demo", + "start_time": "2025-01-02T03:04:05+08:00", + "pump_control": {}, + }, + ) + + assert response.status_code == 422 def test_run_simulation_manually_by_date_uses_utc_aware_timestamps(monkeypatch): @@ -353,6 +298,34 @@ def test_burst_endpoint_passes_current_username(monkeypatch): assert captured["username"] == "alice" +def test_burst_endpoint_rejects_mismatched_pipe_and_size_counts(monkeypatch): + module = _load_simulation_module(monkeypatch) + called = False + + def fake_burst_analysis(**kwargs): + nonlocal called + called = True + + monkeypatch.setattr(module, "burst_analysis", fake_burst_analysis) + client = _build_authenticated_client(module) + + response = client.post( + "/api/v1/burst-analyses", + params=[ + ("network", "demo"), + ("modify_pattern_start_time", "2025-01-02T03:04:05+08:00"), + ("burst_ID", "P1"), + ("burst_ID", "P2"), + ("burst_size", "10.0"), + ("modify_total_duration", "900"), + ("scheme_name", "burst_case_01"), + ], + ) + + assert response.status_code == 422 + assert called is False + + def test_flushing_endpoint_passes_required_scheme_name(monkeypatch): module = _load_simulation_module(monkeypatch) captured = {} diff --git a/tests/unit/test_age_analysis.py b/tests/unit/test_age_analysis.py deleted file mode 100644 index 6bbcd8a..0000000 --- a/tests/unit/test_age_analysis.py +++ /dev/null @@ -1,111 +0,0 @@ -import json -from contextlib import contextmanager - -from tests.conftest import install_stub, load_module_from_path - - -def _load_scenarios_module(monkeypatch): - install_stub(monkeypatch, "app.services", package=True) - install_stub(monkeypatch, "app.algorithms", package=True) - install_stub(monkeypatch, "app.algorithms.simulation", package=True) - install_stub(monkeypatch, "app.services.simulation", {}) - install_stub( - monkeypatch, - "app.algorithms.simulation.runner", - { - "run_simulation_ex": lambda *args, **kwargs: json.dumps( - {"output": {"node_results": [], "link_results": []}} - ), - "from_clock_to_seconds_2": lambda value: value, - }, - ) - install_stub(monkeypatch, "app.services.scheme_management", {"store_scheme_info": lambda *args, **kwargs: None}) - install_stub( - monkeypatch, - "app.services.tjnetwork", - { - "ChangeSet": type("ChangeSet", (), {}), - "OPTION_DEMAND_MODEL_PDA": "OPTION_DEMAND_MODEL_PDA", - "OPTION_QUALITY_CHEMICAL": "OPTION_QUALITY_CHEMICAL", - "SOURCE_TYPE_SETPOINT": "SOURCE_TYPE_SETPOINT", - "add_pattern": lambda *args, **kwargs: None, - "add_source": lambda *args, **kwargs: None, - "copy_project": lambda *args, **kwargs: None, - "delete_project": lambda *args, **kwargs: None, - "get_demand": lambda *args, **kwargs: None, - "get_emitter": lambda *args, **kwargs: None, - "get_node_links": lambda *args, **kwargs: None, - "get_option": lambda *args, **kwargs: None, - "get_pattern": lambda *args, **kwargs: None, - "get_pipe": lambda *args, **kwargs: None, - "get_source": lambda *args, **kwargs: None, - "get_time": lambda *args, **kwargs: None, - "have_project": lambda *args, **kwargs: False, - "is_junction": lambda *args, **kwargs: False, - "set_demand": lambda *args, **kwargs: None, - "set_emitter": lambda *args, **kwargs: None, - "set_option": lambda *args, **kwargs: None, - "set_source": lambda *args, **kwargs: None, - "set_time": lambda *args, **kwargs: None, - }, - ) - return load_module_from_path( - "tests_age_analysis_scenarios_module", - "app/algorithms/simulation/scenarios.py", - ) - - -def test_age_analysis_passes_duration_by_keyword(monkeypatch): - module = _load_scenarios_module(monkeypatch) - captured = {} - - @contextmanager - def fake_temporary_project(project, purpose): - yield f"{purpose}_{project}_run" - - monkeypatch.setattr(module, "temporary_project_database", fake_temporary_project) - - def fake_run_simulation_ex(*args, **kwargs): - captured["args"] = args - captured["kwargs"] = kwargs - return json.dumps({"output": {"node_results": [], "link_results": []}}) - - monkeypatch.setattr(module, "run_simulation_ex", fake_run_simulation_ex) - - module.age_analysis("demo", "2026-06-03T07:00:00+08:00", 300) - - assert captured["args"] == ( - "age_analysis_demo_run", - "realtime", - "2026-06-03T07:00:00+08:00", - ) - assert captured["kwargs"] == { - "duration": 300, - "downloading_prohibition": True, - } - - -def test_isolated_analysis_cleans_database_after_early_return(monkeypatch): - module = _load_scenarios_module(monkeypatch) - lifecycle: list[tuple[str, str]] = [] - - @contextmanager - def fake_temporary_project(project, purpose): - lifecycle.append(("create", purpose)) - try: - yield "isolated_run" - finally: - lifecycle.append(("delete", purpose)) - - monkeypatch.setattr( - module, "temporary_project_database", fake_temporary_project - ) - - @module._isolated_analysis("probe") - def return_early(name, *, _temporary_project=None): - assert name == "demo" - assert _temporary_project == "isolated_run" - return "done" - - assert return_early("demo") == "done" - assert lifecycle == [("create", "probe"), ("delete", "probe")] diff --git a/tests/unit/test_analysis_simulation.py b/tests/unit/test_analysis_simulation.py index 1e47b58..e6042cf 100644 --- a/tests/unit/test_analysis_simulation.py +++ b/tests/unit/test_analysis_simulation.py @@ -19,38 +19,38 @@ def _empty_scada_mappings(simulation): ) -def test_run_simulation_exposes_explicit_valve_control(): +def test_run_simulation_accepts_explicit_valve_control(): from app.services import simulation assert "valve_control" in inspect.signature(simulation.run_simulation).parameters -def test_extended_runner_cleans_temporary_database_after_failure(monkeypatch): - from app.algorithms.simulation import runner +def test_valve_close_analysis_uses_normalized_scheme_type(monkeypatch): + from app.services import simulation_scenarios - lifecycle: list[tuple[str, str]] = [] + captured = {} + monkeypatch.setattr(simulation_scenarios, "get_option", lambda _name: {}) + monkeypatch.setattr( + simulation_scenarios, + "set_option", + lambda _name, _changes: None, + ) + monkeypatch.setattr( + simulation_scenarios.simulation, + "run_simulation", + lambda **kwargs: captured.update(kwargs), + ) - @contextmanager - def temporary_project(project: str, purpose: str): - lifecycle.append(("create", project)) - try: - yield "isolated_project" - finally: - lifecycle.append(("delete", project)) + simulation_scenarios.valve_close_analysis.__wrapped__( + name="demo", + modify_pattern_start_time="2026-01-01T00:00:00+08:00", + modify_valve_opening={"V1": 0.0}, + scheme_name="valve_case", + _temporary_project="temporary_demo", + ) - monkeypatch.setattr(runner, "temporary_project_database", temporary_project) - - @runner._clean_extended_simulation - def fail(name, simulation_type, *, _temporary_project=None): - assert name == "demo" - assert simulation_type == "extended" - assert _temporary_project == "isolated_project" - raise RuntimeError("simulation failed") - - with pytest.raises(RuntimeError, match="simulation failed"): - fail("demo", "extended") - - assert lifecycle == [("create", "demo"), ("delete", "demo")] + assert captured["scheme_type"] == "valve_close_analysis" + assert captured["result_db_name"] == "demo" def test_apply_valve_control_matches_runner_semantics(monkeypatch): diff --git a/tests/unit/test_architecture_boundaries.py b/tests/unit/test_architecture_boundaries.py new file mode 100644 index 0000000..c791eac --- /dev/null +++ b/tests/unit/test_architecture_boundaries.py @@ -0,0 +1,109 @@ +"""Executable dependency rules for the service/algorithm/database layers.""" + +import ast +from pathlib import Path + + +APP_ROOT = Path(__file__).parents[2] / "app" + + +def _app_imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imports: set[str] = set() + package_parts = ("app", *path.relative_to(APP_ROOT).parent.parts) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name for alias in node.names if alias.name.startswith("app.")) + elif isinstance(node, ast.ImportFrom): + if node.level: + parent_count = node.level - 1 + if parent_count >= len(package_parts): + continue + base_parts = package_parts[: len(package_parts) - parent_count] + if node.module: + imports.add(".".join((*base_parts, *node.module.split(".")))) + else: + imports.update( + ".".join((*base_parts, alias.name.split(".")[0])) + for alias in node.names + ) + elif node.module and node.module.startswith("app."): + imports.add(node.module) + return imports + + +def _forbidden_imports(root: Path, prefixes: tuple[str, ...]) -> list[str]: + violations: list[str] = [] + for path in sorted(root.rglob("*.py")): + for module in sorted(_app_imports(path)): + if module.startswith(prefixes): + violations.append(f"{path.relative_to(APP_ROOT)} -> {module}") + return violations + + +def test_algorithms_are_independent_of_application_and_io_layers(): + violations = _forbidden_imports( + APP_ROOT / "algorithms", + ("app.api", "app.auth", "app.infra", "app.native", "app.services"), + ) + + assert violations == [] + + +def test_infrastructure_does_not_depend_on_application_or_algorithms(): + violations = _forbidden_imports( + APP_ROOT / "infra", + ("app.api", "app.algorithms", "app.services"), + ) + + assert violations == [] + + +def test_services_do_not_execute_sql_directly(): + violations = [] + for path in sorted((APP_ROOT / "services").rglob("*.py")): + source = path.read_text(encoding="utf-8") + if ".cursor(" in source or ".execute(" in source: + violations.append(str(path.relative_to(APP_ROOT))) + + assert violations == [] + + +def test_services_do_not_depend_on_the_http_network_facade(): + violations = _forbidden_imports( + APP_ROOT / "services", + ("app.services.tjnetwork",), + ) + + assert violations == [] + + +def test_algorithm_packages_use_explicit_business_names(): + legacy_names = { + "burst_location", + "cleaning", + "health", + "isolation", + "leakage", + "sensor", + "simulation", + "water_demand", + } + present_names = { + path.name + for path in (APP_ROOT / "algorithms").iterdir() + if path.is_dir() and any(path.glob("*.py")) + } + + assert present_names.isdisjoint(legacy_names) + + +def test_application_code_does_not_open_direct_psycopg_connections(): + violations = [] + direct_calls = ("psycopg.connect(", "Connection.connect(", "AsyncConnection.connect(") + for path in sorted(APP_ROOT.rglob("*.py")): + source = path.read_text(encoding="utf-8") + if any(call in source for call in direct_calls): + violations.append(str(path.relative_to(APP_ROOT))) + + assert violations == [] diff --git a/tests/unit/test_demand_allocation.py b/tests/unit/test_demand_allocation.py new file mode 100644 index 0000000..68e49f6 --- /dev/null +++ b/tests/unit/test_demand_allocation.py @@ -0,0 +1,26 @@ +import pytest + +from app.algorithms.demand_allocation import allocate_demand_by_pipe_length + + +def test_allocate_demand_by_incident_pipe_length(): + nodes = { + "J1": {"type": "junction", "links": ["P1"]}, + "J2": {"type": "junction", "links": ["P1", "P2"]}, + "R1": {"type": "reservoir", "links": ["P2"]}, + } + links = {"P1": {"length": 100.0}, "P2": {"length": 300.0}} + + result = allocate_demand_by_pipe_length(80.0, nodes, links) + + assert result == {"J1": pytest.approx(10.0), "J2": pytest.approx(40.0)} + + +def test_allocate_demand_returns_empty_for_zero_length_topology(): + result = allocate_demand_by_pipe_length( + 80.0, + {"J1": {"type": "junction", "links": ["P1"]}}, + {"P1": {"length": 0.0}}, + ) + + assert result == {} diff --git a/tests/unit/test_dma_leakage_flow_units.py b/tests/unit/test_dma_leakage_flow_units.py new file mode 100644 index 0000000..82d3855 --- /dev/null +++ b/tests/unit/test_dma_leakage_flow_units.py @@ -0,0 +1,20 @@ +import pytest + +from app.algorithms.dma_leakage_estimation.genetic_optimizer import DmaLeakageOptimizer + + +@pytest.mark.parametrize( + ("unit", "expected"), + [ + ("m3/s", 1.0), + ("m³/s", 1.0), + ("m3/h", 3600.0), + ("m³/h", 3600.0), + ], +) +def test_dma_leakage_optimizer_accepts_display_flow_units(unit, expected): + assert DmaLeakageOptimizer._flow_from_m3s(1.0, unit) == expected + + +def test_dma_leakage_optimizer_accepts_display_flow_units_for_input(): + assert DmaLeakageOptimizer._flow_to_m3s(3600.0, "m³/h") == 1.0 diff --git a/tests/unit/test_dma_topology_partitioning.py b/tests/unit/test_dma_topology_partitioning.py new file mode 100644 index 0000000..025b04e --- /dev/null +++ b/tests/unit/test_dma_topology_partitioning.py @@ -0,0 +1,50 @@ +import pytest + +from app.algorithms.dma_leakage_estimation.topology_partitioning import ( + build_dma_partitions, +) + + +def test_partition_assigns_nodes_to_nearest_connected_sensor(): + coordinates = { + "A": {"x": 0.0, "y": 0.0}, + "B": {"x": 1.0, "y": 0.0}, + "C": {"x": 2.0, "y": 0.0}, + "D": {"x": 3.0, "y": 0.0}, + } + links = ["P1:pipe:A:B", "P2:pipe:B:C", "P3:pipe:C:D"] + + area_map, areas = build_dma_partitions( + ["A", "D"], coordinates, links, dma_count=2 + ) + + assert area_map == {"A": "1", "B": "1", "C": "2", "D": "2"} + assert [area["node_count"] for area in areas] == [2, 2] + + +def test_partition_rejects_more_areas_than_sensors(): + with pytest.raises(ValueError, match="传感器数量"): + build_dma_partitions( + ["A"], + {"A": {"x": 0.0, "y": 0.0}, "B": {"x": 1.0, "y": 0.0}}, + ["P1:pipe:A:B"], + dma_count=2, + ) + + +def test_partition_preserves_requested_area_count_for_coincident_sensors(): + coordinates = { + "A": {"x": 0.0, "y": 0.0}, + "B": {"x": 0.0, "y": 0.0}, + "C": {"x": 0.0, "y": 0.0}, + } + + area_map, areas = build_dma_partitions( + ["A", "B", "C"], + coordinates, + ["P1:pipe:A:B", "P2:pipe:B:C"], + dma_count=2, + ) + + assert set(area_map.values()) == {"1", "2"} + assert [area["area_id"] for area in areas] == ["1", "2"] diff --git a/tests/unit/test_time_api.py b/tests/unit/test_domain_time.py similarity index 77% rename from tests/unit/test_time_api.py rename to tests/unit/test_domain_time.py index 319c316..09a26f2 100644 --- a/tests/unit/test_time_api.py +++ b/tests/unit/test_domain_time.py @@ -5,11 +5,9 @@ from pathlib import Path import pytest -def _load_time_api_module(): - module_path = ( - Path(__file__).resolve().parents[2] / "app" / "services" / "time_api.py" - ) - spec = importlib.util.spec_from_file_location("tests_time_api_under_test", module_path) +def _load_time_module(): + module_path = Path(__file__).resolve().parents[2] / "app" / "domain" / "time.py" + spec = importlib.util.spec_from_file_location("tests_domain_time", module_path) module = importlib.util.module_from_spec(spec) assert spec and spec.loader spec.loader.exec_module(module) @@ -17,28 +15,28 @@ def _load_time_api_module(): def test_parse_utc_time_rejects_naive_datetimes(): - module = _load_time_api_module() + module = _load_time_module() with pytest.raises(ValueError, match="timezone information"): module.parse_utc_time("2025-01-01T08:00:00") def test_parse_utc_time_normalizes_offset_datetime_to_utc(): - module = _load_time_api_module() + module = _load_time_module() result = module.parse_utc_time("2025-01-01T08:00:00+08:00") assert result == datetime(2025, 1, 1, 0, 0, tzinfo=timezone.utc) def test_extract_date_keeps_original_offset_calendar_day(): - module = _load_time_api_module() + module = _load_time_module() result = module.extract_date("2025-01-01T00:30:00+08:00") assert result == date(2025, 1, 1) def test_utc_now_returns_timezone_aware_utc_datetime(): - module = _load_time_api_module() + module = _load_time_module() result = module.utc_now() assert result.tzinfo == timezone.utc @@ -58,14 +56,14 @@ def test_utc_now_returns_timezone_aware_utc_datetime(): def test_parse_clock_duration_seconds_accepts_epanet_clock_formats( clock, expected_seconds ): - module = _load_time_api_module() + module = _load_time_module() assert module.parse_clock_duration_seconds(clock) == expected_seconds @pytest.mark.parametrize("clock", ["bad", "1:60", "1:00:60", "-1:00"]) def test_parse_clock_duration_seconds_rejects_invalid_clock_formats(clock): - module = _load_time_api_module() + module = _load_time_module() with pytest.raises(ValueError): module.parse_clock_duration_seconds(clock) diff --git a/tests/unit/test_leakage_flow_units.py b/tests/unit/test_leakage_flow_units.py deleted file mode 100644 index bbad2c8..0000000 --- a/tests/unit/test_leakage_flow_units.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest - -from app.algorithms.leakage.identifier import LeakageIdentifier - - -@pytest.mark.parametrize( - ("unit", "expected"), - [ - ("m3/s", 1.0), - ("m³/s", 1.0), - ("m3/h", 3600.0), - ("m³/h", 3600.0), - ], -) -def test_leakage_identifier_accepts_display_flow_units(unit, expected): - assert LeakageIdentifier._flow_from_m3s(1.0, unit) == expected - - -def test_leakage_identifier_accepts_display_flow_units_for_input(): - assert LeakageIdentifier._flow_to_m3s(3600.0, "m³/h") == 1.0 diff --git a/tests/unit/test_pipeline_health_analyzer.py b/tests/unit/test_pipe_health_survival_predictor.py similarity index 87% rename from tests/unit/test_pipeline_health_analyzer.py rename to tests/unit/test_pipe_health_survival_predictor.py index 6342295..50aeb1b 100644 --- a/tests/unit/test_pipeline_health_analyzer.py +++ b/tests/unit/test_pipe_health_survival_predictor.py @@ -1,13 +1,13 @@ -""" -tests.unit.test_pipeline_health_analyzer 的 Docstring -""" +"""Pipe health survival predictor tests.""" -def test_pipeline_health_analyzer(): - from app.algorithms.health.analyzer import PipelineHealthAnalyzer +def test_pipe_health_survival_predictor(): + from app.algorithms.pipe_health_prediction.survival_predictor import ( + PipeHealthSurvivalPredictor, + ) # 初始化分析器，假设模型文件路径为'models/rsf_model.joblib' - analyzer = PipelineHealthAnalyzer() + analyzer = PipeHealthSurvivalPredictor() # 创建示例输入数据（9个样本） import pandas as pd import time diff --git a/tests/unit/test_pressure_cleaning.py b/tests/unit/test_pressure_cleaning.py index 8023590..74829f7 100644 --- a/tests/unit/test_pressure_cleaning.py +++ b/tests/unit/test_pressure_cleaning.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd import pytest -from app.algorithms.cleaning import pressure as pressure_cleaning +from app.algorithms.scada_cleaning import pressure_series as pressure_cleaning DATA_DIR = Path(__file__).resolve().parents[3] / "data" diff --git a/tests/unit/test_project_scada_metadata.py b/tests/unit/test_project_scada_metadata.py index 4c06041..bf03748 100644 --- a/tests/unit/test_project_scada_metadata.py +++ b/tests/unit/test_project_scada_metadata.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock from uuid import uuid4 from app.api.v1.endpoints import project_data -from app.infra.db.timescaledb import composite_queries +from app.services import timeseries_analysis as composite_queries PROJECT_SCADA = { @@ -48,7 +48,7 @@ def test_realtime_scada_simulation_uses_current_project_metadata(monkeypatch): ) result = asyncio.run( - composite_queries.CompositeQueries.get_scada_associated_realtime_simulation_data( + composite_queries.TimeseriesAnalysisService.get_scada_associated_realtime_simulation_data( object(), object(), [PROJECT_SCADA["device_id"]], @@ -82,7 +82,7 @@ def test_analysis_scada_simulation_uses_current_project_metadata(monkeypatch): ) result = asyncio.run( - composite_queries.CompositeQueries.get_scada_associated_analysis_simulation_data( + composite_queries.TimeseriesAnalysisService.get_scada_associated_analysis_simulation_data( object(), object(), [PROJECT_SCADA["device_id"]], @@ -114,7 +114,7 @@ def test_element_scada_query_uses_current_project_metadata(monkeypatch): ) result = asyncio.run( - composite_queries.CompositeQueries.get_element_associated_scada_data( + composite_queries.TimeseriesAnalysisService.get_element_associated_scada_data( object(), object(), "J1", diff --git a/tests/unit/test_scada_cleaning.py b/tests/unit/test_scada_cleaning.py index 2a8d37d..14b2c39 100644 --- a/tests/unit/test_scada_cleaning.py +++ b/tests/unit/test_scada_cleaning.py @@ -8,7 +8,7 @@ import pytest from fastapi import HTTPException from app.api.v1.endpoints.timeseries import composite as composite_endpoint -from app.infra.db.timescaledb import composite_queries +from app.services import timeseries_analysis as composite_queries class _FakeTimescaleConnection: @@ -57,7 +57,7 @@ def test_clean_scada_uses_current_project_metadata(monkeypatch): ) result = asyncio.run( - composite_queries.CompositeQueries.clean_scada_data( + composite_queries.TimeseriesAnalysisService.clean_scada_data( _FakeTimescaleConnection(), object(), ["fengyang-pressure-1"], @@ -85,7 +85,7 @@ def test_clean_scada_rejects_devices_missing_from_project_metadata(monkeypatch): with pytest.raises(ValueError, match="缺少元数据"): asyncio.run( - composite_queries.CompositeQueries.clean_scada_data( + composite_queries.TimeseriesAnalysisService.clean_scada_data( _FakeTimescaleConnection(), _FakeTimescaleConnection(), ["fengyang-pressure-1"], @@ -132,7 +132,7 @@ def test_clean_scada_rejects_zero_database_updates(monkeypatch): with pytest.raises(ValueError, match="未产生任何数据库更新"): asyncio.run( - composite_queries.CompositeQueries.clean_scada_data( + composite_queries.TimeseriesAnalysisService.clean_scada_data( _FakeTimescaleConnection(), object(), ["fengyang-pressure-1"], @@ -176,7 +176,7 @@ def test_clean_scada_propagates_write_failures(monkeypatch): with pytest.raises(RuntimeError, match="database write failed"): asyncio.run( - composite_queries.CompositeQueries.clean_scada_data( + composite_queries.TimeseriesAnalysisService.clean_scada_data( _FakeTimescaleConnection(), object(), ["fengyang-pressure-1"], @@ -188,7 +188,7 @@ def test_clean_scada_propagates_write_failures(monkeypatch): def test_clean_scada_endpoint_returns_http_400_for_validation_error(monkeypatch): monkeypatch.setattr( - composite_endpoint.CompositeQueries, + composite_endpoint.TimeseriesAnalysisService, "clean_scada_data", AsyncMock(side_effect=ValueError("当前项目没有可清洗的 SCADA 设备")), ) diff --git a/tests/unit/test_sensor_kmeans.py b/tests/unit/test_sensor_kmeans.py new file mode 100644 index 0000000..91121c8 --- /dev/null +++ b/tests/unit/test_sensor_kmeans.py @@ -0,0 +1,95 @@ +from types import SimpleNamespace + +import pytest + +from app.algorithms.pressure_sensor_placement.kmeans_placement import ( + optimize_sensor_placement, +) + + +class FakeNetwork: + junction_name_list = ["J1", "J2", "J3"] + pipe_name_list = ["P1", "P2"] + + def __init__(self): + self._nodes = { + "J1": SimpleNamespace(coordinates=(0.0, 0.0)), + "J2": SimpleNamespace(coordinates=(1.0, 0.0)), + "J3": SimpleNamespace(coordinates=(2.0, 0.0)), + } + self._links = { + "P1": SimpleNamespace( + diameter=0.4, start_node_name="J1", end_node_name="J2" + ), + "P2": SimpleNamespace( + diameter=0.1, start_node_name="J2", end_node_name="J3" + ), + } + + def get_node(self, node_id): + return self._nodes[node_id] + + def get_link(self, link_id): + return self._links[link_id] + + +class DuplicateNearestNodeNetwork: + junction_name_list = ["J1", "J2", "J3", "J4"] + pipe_name_list = ["P1", "P2", "P3"] + + def __init__(self): + self._nodes = { + "J1": SimpleNamespace( + coordinates=(-0.2967811321994103, 0.4867937179527957) + ), + "J2": SimpleNamespace( + coordinates=(1.8589649622870763, -1.0070372175769569) + ), + "J3": SimpleNamespace( + coordinates=(-1.4882734126175363, -1.6220731365802619) + ), + "J4": SimpleNamespace( + coordinates=(1.203441836600899, 2.2320704059758474) + ), + } + self._links = { + "P1": SimpleNamespace( + diameter=0.4, start_node_name="J1", end_node_name="J2" + ), + "P2": SimpleNamespace( + diameter=0.4, start_node_name="J2", end_node_name="J3" + ), + "P3": SimpleNamespace( + diameter=0.4, start_node_name="J3", end_node_name="J4" + ), + } + + def get_node(self, node_id): + return self._nodes[node_id] + + def get_link(self, link_id): + return self._links[link_id] + + +def test_kmeans_filters_candidates_by_minimum_pipe_diameter(): + selected = optimize_sensor_placement( + FakeNetwork(), sensor_count=2, min_diameter_mm=300 + ) + + assert set(selected) == {"J1", "J2"} + + +def test_kmeans_rejects_sensor_count_above_eligible_candidates(): + with pytest.raises(ValueError, match="候选节点数量"): + optimize_sensor_placement( + FakeNetwork(), sensor_count=3, min_diameter_mm=300 + ) + + +def test_kmeans_returns_unique_monitoring_nodes(): + selected = optimize_sensor_placement( + DuplicateNearestNodeNetwork(), sensor_count=2, min_diameter_mm=300 + ) + + assert len(selected) == 2 + assert len(set(selected)) == 2 diff --git a/tests/unit/test_sensor_sensitivity.py b/tests/unit/test_sensor_sensitivity.py index 42a8a50..32befe8 100644 --- a/tests/unit/test_sensor_sensitivity.py +++ b/tests/unit/test_sensor_sensitivity.py @@ -6,7 +6,7 @@ import wntr from scipy.sparse import csr_matrix, isspmatrix_csr from scipy.sparse.csgraph import dijkstra -from app.algorithms.sensor import sensitivity +from app.algorithms.pressure_sensor_placement import sensitivity_placement as sensitivity def _build_test_network() -> wntr.network.WaterNetworkModel: diff --git a/tests/unit/test_valve_isolation.py b/tests/unit/test_valve_isolation.py index f628297..34af18e 100644 --- a/tests/unit/test_valve_isolation.py +++ b/tests/unit/test_valve_isolation.py @@ -1,26 +1,10 @@ -from collections import defaultdict - -from app.algorithms.isolation import valve +from app.algorithms.valve_isolation import topology_search as valve -def test_non_isolatable_omits_affected_node_ids_but_keeps_count(monkeypatch): - pipe_adj = defaultdict( - set, - { - "A": {"B"}, - "B": {"A", "C"}, - "C": {"B"}, - }, - ) - topology = ( - pipe_adj, - {"V-optional": ("A", "C")}, - {"P-1": ("A", "B", "pipe")}, - {"A", "B", "C"}, - ) - monkeypatch.setattr(valve, "_get_network_topology", lambda _network: topology) +def test_non_isolatable_omits_affected_node_ids_but_keeps_count(): + links = ["P-1:pipe:A:B", "P-2:pipe:B:C", "V-optional:valve:A:C"] - result = valve.valve_isolation_analysis("demo", "P-1") + result = valve.valve_isolation_analysis(links, "P-1") assert result["isolatable"] is False assert result["affected_node_count"] == 3 @@ -28,17 +12,10 @@ def test_non_isolatable_omits_affected_node_ids_but_keeps_count(monkeypatch): assert result["optional_valves"] == ["V-optional"] -def test_isolatable_keeps_affected_node_ids_and_count(monkeypatch): - pipe_adj = defaultdict(set, {"A": {"B"}, "B": {"A"}}) - topology = ( - pipe_adj, - {"V-close": ("B", "C")}, - {"P-1": ("A", "B", "pipe")}, - {"A", "B", "C"}, - ) - monkeypatch.setattr(valve, "_get_network_topology", lambda _network: topology) +def test_isolatable_keeps_affected_node_ids_and_count(): + links = ["P-1:pipe:A:B", "V-close:valve:B:C"] - result = valve.valve_isolation_analysis("demo", "P-1") + result = valve.valve_isolation_analysis(links, "P-1") assert result["isolatable"] is True assert result["affected_node_count"] == 2 @@ -46,21 +23,14 @@ def test_isolatable_keeps_affected_node_ids_and_count(monkeypatch): assert result["must_close_valves"] == ["V-close"] -def test_disabled_valve_expands_affected_area_before_counting(monkeypatch): - pipe_adj = defaultdict(set, {"A": {"B"}, "B": {"A"}}) - topology = ( - pipe_adj, - { - "V-disabled": ("B", "C"), - "V-close": ("C", "D"), - }, - {"P-1": ("A", "B", "pipe")}, - {"A", "B", "C", "D"}, - ) - monkeypatch.setattr(valve, "_get_network_topology", lambda _network: topology) - +def test_disabled_valve_expands_affected_area_before_counting(): + links = [ + "P-1:pipe:A:B", + "V-disabled:valve:B:C", + "V-close:valve:C:D", + ] result = valve.valve_isolation_analysis( - "demo", + links, "P-1", disabled_valves=["V-disabled"], )