From b74799a39d9aa4d64a3447c11348c0d59e2ac43a Mon Sep 17 00:00:00 2001 From: Jiang Date: Thu, 27 Aug 2026 17:26:22 +0800 Subject: [PATCH] refactor(db)!: finalize pooled WNDB v2 migration --- .env.example | 12 +- README.md | 4 +- app/algorithms/simulation/runner.py | 44 +- app/algorithms/simulation/scenarios.py | 153 +- app/algorithms/water_demand/service.py | 28 +- app/api/problem_details.py | 20 + app/api/v1/endpoints/components/controls.py | 25 +- app/api/v1/endpoints/components/curves.py | 27 +- app/api/v1/endpoints/components/options.py | 45 +- app/api/v1/endpoints/components/patterns.py | 27 +- app/api/v1/endpoints/components/quality.py | 95 +- app/api/v1/endpoints/components/visuals.py | 63 +- app/api/v1/endpoints/meta.py | 13 +- app/api/v1/endpoints/model_import.py | 20 +- app/api/v1/endpoints/network/demands.py | 29 +- app/api/v1/endpoints/network/general.py | 69 +- app/api/v1/endpoints/network/geometry.py | 12 +- app/api/v1/endpoints/network/junctions.py | 45 +- app/api/v1/endpoints/network/pipes.py | 49 +- app/api/v1/endpoints/network/pumps.py | 29 +- app/api/v1/endpoints/network/regions.py | 27 +- app/api/v1/endpoints/network/reservoirs.py | 41 +- app/api/v1/endpoints/network/tags.py | 17 +- app/api/v1/endpoints/network/tanks.py | 65 +- app/api/v1/endpoints/network/valves.py | 43 +- app/api/v1/endpoints/project.py | 402 +-- app/api/v1/endpoints/sensor_placement.py | 9 +- app/api/v1/endpoints/simulation.py | 44 +- app/api/v1/endpoints/timeseries/realtime.py | 35 +- app/api/v1/rest_router.py | 24 +- app/auth/project_dependencies.py | 69 +- app/core/config.py | 12 +- app/infra/db/dynamic_manager.py | 431 ++-- app/infra/db/project_routing.py | 32 +- app/infra/db/timescaledb/composite_queries.py | 194 +- app/infra/db/timescaledb/internal_queries.py | 5 +- .../db/timescaledb/repositories/analysis.py | 47 + .../db/timescaledb/repositories/realtime.py | 140 +- .../db/timescaledb/repositories/scada.py | 35 +- app/infra/db/timescaledb/sync_pool.py | 2 + app/main.py | 6 - app/native/wndb/commands/cascade.py | 7 +- app/native/wndb/commands/executor.py | 11 +- app/native/wndb/core/connection.py | 20 + app/native/wndb/core/database.py | 109 +- app/native/wndb/core/model_replace.py | 284 +++ app/native/wndb/core/projects.py | 284 ++- app/native/wndb/gis/coordinates.py | 70 +- app/native/wndb/gis/network_views.py | 191 ++ app/native/wndb/gis/region_geometry.py | 80 +- app/native/wndb/inp/exporter.py | 25 +- app/native/wndb/inp/importer.py | 136 +- app/native/wndb/model/demands.py | 13 - app/native/wndb/model/elements.py | 38 +- app/native/wndb/model/junctions.py | 21 +- .../{options_legacy.py => options_v2.py} | 12 +- app/native/wndb/model/options_v3.py | 2 - app/native/wndb/model/patterns.py | 5 +- app/native/wndb/model/pipes.py | 60 +- app/native/wndb/model/pumps.py | 21 +- app/native/wndb/model/reservoirs.py | 21 +- app/native/wndb/model/tanks.py | 21 +- app/native/wndb/model/valves.py | 21 +- app/services/__init__.py | 2 +- app/services/simulation.py | 555 ++-- app/services/simulation_ops.py | 96 +- app/services/tjnetwork.py | 47 +- contracts/manifest.json | 2 +- contracts/server-v1.openapi.json | 2255 +++++------------ resources/db_v2/DATABASE_ARCHITECTURE.md | 42 +- resources/db_v2/WNDB_STRUCTURE.md | 50 +- scripts/build_pyd.py | 3 - scripts/clean_projects.py | 29 +- scripts/create_template.py | 136 - scripts/demo.py | 2 - scripts/dev.py | 21 - scripts/get_current_status.py | 66 - scripts/online_Analysis.py | 1548 ----------- scripts/open_szh.py | 8 - scripts/restore_project.py | 15 - scripts/restore_projects.py | 11 - scripts/run_simlation.py | 8 - tests/api/test_meta_endpoints.py | 14 +- tests/api/test_model_import_endpoints.py | 47 + tests/api/test_openapi_contract.py | 54 +- tests/api/test_project_endpoints.py | 50 +- tests/api/test_wndb_endpoint_threading.py | 68 + .../integration/test_database_pooling_live.py | 124 +- tests/unit/test_age_analysis.py | 43 +- tests/unit/test_analysis_simulation.py | 92 +- tests/unit/test_clean_projects_cli.py | 17 + tests/unit/test_dynamic_manager.py | 106 +- tests/unit/test_pool_connection_checks.py | 104 + tests/unit/test_project_routing.py | 37 +- tests/unit/test_project_scada_metadata.py | 28 +- tests/unit/test_realtime_repository.py | 75 +- tests/unit/test_scada_cleaning.py | 7 +- tests/unit/test_scada_repository.py | 28 + tests/unit/test_wndb_batch_transactions.py | 216 +- tests/unit/test_wndb_endpoint_boundaries.py | 54 + tests/unit/test_wndb_importer.py | 66 + tests/unit/test_wndb_materialized_refresh.py | 23 +- tests/unit/test_wndb_network_views.py | 163 ++ tests/unit/test_wndb_projects.py | 279 ++ tests/unit/test_wndb_query_safety.py | 21 + 105 files changed, 4988 insertions(+), 5565 deletions(-) create mode 100644 app/native/wndb/core/model_replace.py create mode 100644 app/native/wndb/gis/network_views.py rename app/native/wndb/model/{options_legacy.py => options_v2.py} (90%) delete mode 100644 scripts/create_template.py delete mode 100644 scripts/dev.py delete mode 100644 scripts/get_current_status.py delete mode 100644 scripts/online_Analysis.py delete mode 100644 scripts/open_szh.py delete mode 100644 scripts/restore_project.py delete mode 100644 scripts/restore_projects.py delete mode 100644 scripts/run_simlation.py create mode 100644 tests/api/test_wndb_endpoint_threading.py create mode 100644 tests/unit/test_clean_projects_cli.py create mode 100644 tests/unit/test_pool_connection_checks.py create mode 100644 tests/unit/test_wndb_endpoint_boundaries.py create mode 100644 tests/unit/test_wndb_importer.py create mode 100644 tests/unit/test_wndb_network_views.py create mode 100644 tests/unit/test_wndb_projects.py diff --git a/.env.example b/.env.example index 75e9e4d..c46ebf3 100644 --- a/.env.example +++ b/.env.example @@ -40,13 +40,15 @@ METADATA_DB_USER="tjwater" METADATA_DB_PASSWORD="password" # Per-project synchronous connection pools -PROJECT_PG_CACHE_SIZE="50" -PROJECT_TS_CACHE_SIZE="50" +PROJECT_PG_CACHE_SIZE="16" +PROJECT_TS_CACHE_SIZE="16" PROJECT_PG_POOL_MIN_SIZE="0" -PROJECT_PG_POOL_SIZE="5" -PROJECT_PG_MAX_OVERFLOW="10" +PROJECT_PG_POOL_SIZE="4" +PROJECT_PG_MAX_OVERFLOW="2" PROJECT_TS_POOL_MIN_SIZE="0" -PROJECT_TS_POOL_MAX_SIZE="10" +PROJECT_TS_POOL_MAX_SIZE="4" +WNDB_TEMPLATE_DB_NAME="tjwater_v2_template" +WNDB_TEMP_DB_MAX_COUNT="8" # ============================================ # Keycloak JWT (可选) diff --git a/README.md b/README.md index c852278..9f552e4 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,11 @@ docker compose -f infra/docker/docker-compose.yml config 项目级 REST 请求通过 `X-Project-Id` 解析元数据中的数据库配置： -- `biz_data` DSN 用于管网业务数据；`{project_code}_template` 和模拟临时库沿用该 DSN 的主机、端口与凭据，仅替换数据库名。 +- `biz_data` DSN 用于管网业务数据；版本模板固定由 `WNDB_TEMPLATE_DB_NAME` 配置（当前为 `tjwater_v2_template`），模拟临时库沿用该 DSN 的主机、端口与凭据，仅替换数据库名。 - `iot_data` DSN 用于 TimescaleDB，始终使用元数据配置的完整 DSN，不再从项目代码推导数据库名。 - 元数据、业务库和 TimescaleDB 可以部署在同一主机，也可以分别部署。 -使用模板复制或临时方案库的模拟功能时，`biz_data` 账号必须具备现有数据库创建、删除和连接终止操作所需的 PostgreSQL 权限。 +使用模板复制或临时方案库的模拟功能时，`biz_data` 账号必须具备数据库创建和删除权限；只有显式删除项目时才会终止该项目的现有数据库会话，普通复制不会主动中断复制源会话。 ## 测试与发布 diff --git a/app/algorithms/simulation/runner.py b/app/algorithms/simulation/runner.py index f25acd4..d2fe1a9 100644 --- a/app/algorithms/simulation/runner.py +++ b/app/algorithms/simulation/runner.py @@ -1,9 +1,7 @@ import numpy as np +from functools import wraps from app.services.tjnetwork import ( ChangeSet, - close_project, - copy_project, - delete_project, get_pattern, get_patterns, get_pump, @@ -11,9 +9,6 @@ from app.services.tjnetwork import ( get_status, get_tank, get_time, - have_project, - is_project_open, - open_project, read_all, run_project, set_pattern, @@ -29,6 +24,7 @@ import pytz import requests import time import app.services.project_info as project_info +from app.native.wndb.core.projects import temporary_project_database from app.services.time_api import parse_clock_duration_seconds url_path = 'http://10.101.15.16:9000/loong' # 内网 @@ -572,9 +568,6 @@ def trim_time_flag(url_date_time:str)->str: # 单时间步长模拟 def run_simulation(name:str,start_datetime:str,end_datetime:str=None, duration:int=900)->str: - if(is_project_open(name)): - close_project(name) - open_project(name) #get_current_data(cur_datetime) #extract the patternindex from datetime #e.g. 0: the first time step for 00:00-00:14; 1: the second step for 00:15-00:30 @@ -654,32 +647,38 @@ def run_simulation(name:str,start_datetime:str,end_datetime:str=None, duration:i # 在线模拟 +def _clean_extended_simulation(func): + @wraps(func) + def wrapper(name: str, simulation_type: str, *args, **kwargs): + if simulation_type.upper() != "EXTENDED": + return func(name, simulation_type, *args, **kwargs) + with temporary_project_database(name, "extended_simulation") as temporary: + kwargs["_temporary_project"] = temporary + return func(name, simulation_type, *args, **kwargs) + + return wrapper + + +@_clean_extended_simulation def run_simulation_ex(name: str, simulation_type: str, start_datetime: str, end_datetime: str = None, duration: int = 0, pump_control: dict[str, list] = None, tank_initial_level_control: dict[str, float] = None, region_demand_control: dict[str, float] = None, valve_control: dict[str, dict] = None, - downloading_prohibition: bool = False) -> str: + downloading_prohibition: bool = False, + _temporary_project: str | None = None) -> str: time_cost_start = time.perf_counter() print('{} -- Hydraulic simulation started.'.format( datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S'))) - if is_project_open(name): - close_project(name) - if simulation_type.upper() == 'REALTIME': # 实时模拟(修改原数据库) name_c = name elif simulation_type.upper() == 'EXTENDED': # 扩展模拟(复制数据库) - name_c = '_'.join([name, 'c']) - if have_project(name_c): - if is_project_open(name_c): - close_project(name_c) - delete_project(name_c) - copy_project(name, name_c) # 备份项目 + if _temporary_project is None: + raise RuntimeError("Extended simulation isolation was not prepared") + name_c = _temporary_project else: raise Exception('Incorrect simulation type, choose in (realtime, extended)') - open_project(name_c) - # 时间处理 # extract the pattern index from datetime # e.g. 0: the first time step for 00:00-00:14; 1: the second step for 00:15-00:30 @@ -848,8 +847,6 @@ def run_simulation_ex(name: str, simulation_type: str, start_datetime: str, datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S'), time_cost_end - time_cost_start)) - close_project(name_c) - return result @@ -858,7 +855,6 @@ if __name__ == '__main__': # tQ=get_current_total_Q() # print(f"the current tQ is {tQ}\n") # data=get_hist_data(ids,conver_beingtime_to_ucttime('2024-04-10 15:05:00'),conver_beingtime_to_ucttime('2024-04-10 15:10:00')) - # open_project("beibeizone") # read_inp("beibeizone","beibeizone-export_nochinese.inp") # run_simulation("beibeizone","2024-04-01T08:00:00Z") # read_inp('bb_server', 'model20_en.inp') diff --git a/app/algorithms/simulation/scenarios.py b/app/algorithms/simulation/scenarios.py index 021696f..579ca86 100644 --- a/app/algorithms/simulation/scenarios.py +++ b/app/algorithms/simulation/scenarios.py @@ -1,5 +1,6 @@ import json from datetime import datetime +from functools import wraps from math import pi, sqrt import pytz @@ -9,6 +10,7 @@ from app.algorithms.simulation.runner import ( run_simulation_ex, from_clock_to_seconds_2, ) +from app.native.wndb.core.projects import temporary_project_database from app.services.tjnetwork import ( ChangeSet, OPTION_DEMAND_MODEL_PDA, @@ -16,9 +18,6 @@ from app.services.tjnetwork import ( SOURCE_TYPE_SETPOINT, add_pattern, add_source, - close_project, - copy_project, - delete_project, get_demand, get_emitter, get_node_links, @@ -27,10 +26,7 @@ from app.services.tjnetwork import ( get_pipe, get_source, get_time, - have_project, is_junction, - is_project_open, - open_project, set_demand, set_emitter, set_option, @@ -39,11 +35,23 @@ from app.services.tjnetwork import ( ) +def _isolated_analysis(purpose: str): + def decorator(func): + @wraps(func) + def wrapper(name: str, *args, **kwargs): + with temporary_project_database(name, purpose) as temporary: + kwargs["_temporary_project"] = temporary + return func(name, *args, **kwargs) + + return wrapper + + return decorator + + ############################################################ # burst analysis 01 ############################################################ def convert_to_local_unit(proj: str, emitters: float) -> float: - open_project(proj) proj_opt = get_option(proj) str_unit = proj_opt.get("UNITS") @@ -61,6 +69,7 @@ def convert_to_local_unit(proj: str, emitters: float) -> float: return emitters +@_isolated_analysis("burst_analysis") def burst_analysis( name: str, modify_pattern_start_time: str, @@ -72,6 +81,7 @@ def burst_analysis( modify_valve_opening: dict[str, float] = None, scheme_name: str = None, username: str | None = None, + _temporary_project: str | None = None, ) -> None: """ 爆管模拟 @@ -101,21 +111,17 @@ def burst_analysis( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"burst_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) + if _temporary_project is None: + raise RuntimeError("Burst analysis isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) simulation.run_simulation( name=new_name, simulation_type="manually_temporary", @@ -207,20 +213,19 @@ def burst_analysis( ) # step 3. restore the base model status # execute_undo(name) #有疑惑 - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) ############################################################ # valve closing analysis 02 ############################################################ +@_isolated_analysis("valve_close_analysis") def valve_close_analysis( name: str, modify_pattern_start_time: str, modify_total_duration: int = 900, modify_valve_opening: dict[str, float] = None, scheme_name: str = None, + _temporary_project: str | None = None, ) -> None: """ 关阀模拟 @@ -235,21 +240,17 @@ def valve_close_analysis( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"valve_close_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) + if _temporary_project is None: + raise RuntimeError("Valve-close analysis isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -287,9 +288,6 @@ def valve_close_analysis( # step 3. restore the base model # for valve in valves: # execute_undo(name) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) # return result @@ -297,6 +295,7 @@ def valve_close_analysis( # flushing analysis 03 # Pipe_Flushing_Analysis(prj_name,date_time, Valve_id_list, Drainage_Node_Id, Flushing_flow[opt], Flushing_duration[opt])->out_file:string ############################################################ +@_isolated_analysis("flushing_analysis") def flushing_analysis( name: str, modify_pattern_start_time: str, @@ -307,6 +306,7 @@ def flushing_analysis( scheme_name: str = None, username: str | None = None, valve_control: dict[str, dict] = None, + _temporary_project: str | None = None, ) -> None: """ 管道冲洗模拟 @@ -334,23 +334,17 @@ def flushing_analysis( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"flushing_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) + if _temporary_project is None: + raise RuntimeError("Flushing analysis isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -449,9 +443,6 @@ def flushing_analysis( scheme_detail=scheme_detail, ) # step 4. restore the base model - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) # return result @@ -459,6 +450,7 @@ def flushing_analysis( # Contaminant simulation 04 # ############################################################ +@_isolated_analysis("contaminant_simulation") def contaminant_simulation( name: str, modify_pattern_start_time: str, # 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' @@ -468,6 +460,7 @@ def contaminant_simulation( scheme_name: str = None, source_pattern: str = None, # 污染源时间变化模式名称 username: str | None = None, + _temporary_project: str | None = None, ) -> None: """ 污染模拟 @@ -494,23 +487,17 @@ def contaminant_simulation( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"contaminant_Sim_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) + if _temporary_project is None: + raise RuntimeError("Contaminant simulation isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -599,9 +586,6 @@ def contaminant_simulation( # for i in range(1,operation_step): # execute_undo(name) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) ############################################################ @@ -623,45 +607,19 @@ def age_analysis( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"age_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - # step 1. run simulation + with temporary_project_database(name, "age_analysis") as new_name: + result = run_simulation_ex( + new_name, + "realtime", + modify_pattern_start_time, + duration=modify_total_duration, + downloading_prohibition=True, + ) + simulation_result = json.loads(result) + output_data = simulation_result.get("output") + if not isinstance(output_data, dict): + raise RuntimeError("run_simulation_ex did not return JSON output content") - result = run_simulation_ex( - new_name, - "realtime", - modify_pattern_start_time, - duration=modify_total_duration, - downloading_prohibition=True, - ) - simulation_result = json.loads(result) - output_data = simulation_result.get("output") - if not isinstance(output_data, dict): - raise RuntimeError("run_simulation_ex did not return JSON output content") - # step 2. restore the base model status - # execute_undo(name) #有疑惑 - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) nodes_age = [] node_result = output_data.get("node_results") or [] for node in node_result: @@ -680,6 +638,7 @@ def age_analysis( ############################################################ +@_isolated_analysis("pressure_regulation") def pressure_regulation( name: str, modify_pattern_start_time: str, @@ -688,6 +647,7 @@ def pressure_regulation( modify_fixed_pump_pattern: dict[str, list] = None, modify_variable_pump_pattern: dict[str, list] = None, scheme_name: str = None, + _temporary_project: str | None = None, ) -> None: """ 区域调压模拟，用来模拟未来15分钟内，开关水泵对区域压力的影响 @@ -704,23 +664,17 @@ def pressure_regulation( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"pressure_regulation_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) + if _temporary_project is None: + raise RuntimeError("Pressure-regulation isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -751,7 +705,4 @@ def pressure_regulation( scheme_name=scheme_name, result_db_name=name, ) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) # return result diff --git a/app/algorithms/water_demand/service.py b/app/algorithms/water_demand/service.py index f6c4521..3a1a8af 100644 --- a/app/algorithms/water_demand/service.py +++ b/app/algorithms/water_demand/service.py @@ -1,8 +1,11 @@ from app.native.wndb.commands.executor import execute_batch_command from app.native.wndb.core.database import ChangeSet from app.native.wndb.gis.region_geometry import Topology, get_nodes_in_region -from app.native.wndb.model.demands import get_demand -from app.native.wndb.model.elements import get_nodes, is_junction +from app.native.wndb.gis.network_views import ( + get_junction_demands, + sum_junction_base_demand, +) +from app.native.wndb.model.elements import get_nodes DISTRIBUTION_TYPE_ADD = 'ADD' @@ -28,7 +31,7 @@ def calculate_demand_to_nodes(name: str, demand: float, nodes: list[str]) -> dic result: dict[str, float] = {} for node, value in t_nodes.items(): - if not is_junction(name, node): + if value["type"] != "junction": continue demand_per_node = 0.0 for link in value['links']: @@ -68,15 +71,19 @@ def distribute_demand_to_nodes(name: str, demand: float, nodes: list[str], type: demand_per_length = demand / length_sum cs = ChangeSet() + demands_by_junction = get_junction_demands( + name, + [node for node, value in t_nodes.items() if value["type"] == "junction"], + ) for node, value in t_nodes.items(): - if not is_junction(name, node): + if value["type"] != "junction": continue demand_per_node = 0.0 for link in value['links']: demand_per_node += abs(t_links[link]['length']) * demand_per_length * 0.5 - ds = get_demand(name, node)['demands'] + ds = demands_by_junction.get(node, []) if len(ds) == 0: ds = [{'demand': demand_per_node, 'pattern': None, 'category': None}] elif type == DISTRIBUTION_TYPE_ADD: @@ -92,13 +99,6 @@ def distribute_demand_to_region(name: str, demand: float, region: str, type: str nodes = get_nodes_in_region(name, region) return distribute_demand_to_nodes(name, demand, nodes, type) -def get_total_base_demand(name:str,region:str)->float: +def get_total_base_demand(name: str, region: str) -> float: nodes = get_nodes_in_region(name, region) - t_demands=0.0 - for node in nodes: - if not is_junction(name, node): - continue - ds = get_demand(name, node)['demands'] - t_demands= t_demands+ds[0]['demand'] - - return t_demands + return sum_junction_base_demand(name, nodes) diff --git a/app/api/problem_details.py b/app/api/problem_details.py index 018bf93..2fd1fb1 100644 --- a/app/api/problem_details.py +++ b/app/api/problem_details.py @@ -8,6 +8,8 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from pydantic import BaseModel, Field +from app.native.wndb.core.database import MaterializedViewRefreshAfterCommitError + class ProblemDetails(BaseModel): """RFC 9457 compatible error response used by the REST contract.""" @@ -53,6 +55,24 @@ def _problem_response( def install_problem_details_handlers(app: FastAPI) -> None: + @app.exception_handler(MaterializedViewRefreshAfterCommitError) + async def materialized_view_refresh_error_handler( + request: Request, + exc: MaterializedViewRefreshAfterCommitError, + ) -> JSONResponse: + response = _problem_response( + request, + status_code=503, + title="Materialized view refresh failed", + detail=( + f"Project {exc.project!r} changes were committed, but GIS query " + "views could not be refreshed. Do not repeat the write blindly." + ), + code="materialized_view_refresh_failed_after_commit", + ) + response.headers["X-TJWater-Changes-Committed"] = "true" + return response + @app.exception_handler(RequestValidationError) async def validation_error_handler( request: Request, diff --git a/app/api/v1/endpoints/components/controls.py b/app/api/v1/endpoints/components/controls.py index fd3b653..e4f8bec 100644 --- a/app/api/v1/endpoints/components/controls.py +++ b/app/api/v1/endpoints/components/controls.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, get_control, @@ -13,7 +14,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/control", summary="获取控制架构", description="获取网络中控制对象的架构定义") -async def fastapi_get_control_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_control_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取控制架构。 返回指定网络中控制对象的属性架构定义。 @@ -21,7 +22,7 @@ async def fastapi_get_control_schema(network: str = Query(..., description="管 return get_control_schema(network) @router.get("/controls/properties", summary="获取控制属性", description="获取指定网络中的控制属性信息") -async def fastapi_get_control_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_control_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取控制属性。 返回指定网络中的控制对象属性信息。 @@ -29,19 +30,19 @@ async def fastapi_get_control_properties(network: str = Query(..., description=" return get_control(network) @router.patch("/controls/properties", response_model=None, summary="设置控制属性", description="更新指定网络中的控制属性") -async def fastapi_set_control_properties( +def fastapi_set_control_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置控制属性。 更新指定网络中的控制属性值。 """ - props = await req.json() + props = payload return set_control(network, ChangeSet(props)) @router.get("/rule-schemas", summary="获取规则架构", description="获取网络中规则对象的架构定义") -async def fastapi_get_rule_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_rule_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取规则架构。 返回指定网络中规则对象的属性架构定义。 @@ -49,7 +50,7 @@ async def fastapi_get_rule_schema(network: str = Query(..., description="管网 return get_rule_schema(network) @router.get("/rule-properties", summary="获取规则属性", description="获取指定网络中的规则属性信息") -async def fastapi_get_rule_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_rule_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取规则属性。 返回指定网络中的规则对象属性信息。 @@ -57,13 +58,13 @@ async def fastapi_get_rule_properties(network: str = Query(..., description="管 return get_rule(network) @router.patch("/rule-properties", response_model=None, summary="设置规则属性", description="更新指定网络中的规则属性") -async def fastapi_set_rule_properties( +def fastapi_set_rule_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置规则属性。 更新指定网络中的规则属性值。 """ - props = await req.json() + props = payload return set_rule(network, ChangeSet(props)) diff --git a/app/api/v1/endpoints/components/curves.py b/app/api/v1/endpoints/components/curves.py index 81358f6..c7872d5 100644 --- a/app/api/v1/endpoints/components/curves.py +++ b/app/api/v1/endpoints/components/curves.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_curve, @@ -14,7 +15,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/curve", summary="获取曲线架构", description="获取网络中曲线对象的架构定义") -async def fastapi_get_curve_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_curve_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取曲线架构。 返回指定网络中曲线对象的属性架构定义。 @@ -22,23 +23,23 @@ async def fastapi_get_curve_schema(network: str = Query(..., description="管网 return get_curve_schema(network) @router.post("/curves", response_model=None, summary="添加曲线", description="在网络中添加一条新的曲线") -async def fastapi_add_curve( +def fastapi_add_curve( network: str = Query(..., description="管网名称（或数据库名称）"), curve: str = Query(..., description="曲线ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加曲线。 在指定网络中创建一条新的曲线，并设置其初始属性。 """ - props = await req.json() + props = payload ps = { "id": curve, } | props return add_curve(network, ChangeSet(ps)) @router.delete("/curves", response_model=None, summary="删除曲线", description="从网络中删除指定的曲线") -async def fastapi_delete_curve( +def fastapi_delete_curve( network: str = Query(..., description="管网名称（或数据库名称）"), curve: str = Query(..., description="曲线ID") ) -> ChangeSet: @@ -50,7 +51,7 @@ async def fastapi_delete_curve( return delete_curve(network, ChangeSet(ps)) @router.get("/curves/properties", summary="获取曲线属性", description="获取指定曲线的属性信息") -async def fastapi_get_curve_properties( +def fastapi_get_curve_properties( network: str = Query(..., description="管网名称（或数据库名称）"), curve: str = Query(..., description="曲线ID") ) -> dict[str, Any]: @@ -61,21 +62,21 @@ async def fastapi_get_curve_properties( return get_curve(network, curve) @router.patch("/curves/properties", response_model=None, summary="设置曲线属性", description="更新指定曲线的属性") -async def fastapi_set_curve_properties( +def fastapi_set_curve_properties( network: str = Query(..., description="管网名称（或数据库名称）"), curve: str = Query(..., description="曲线ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置曲线属性。 更新指定曲线的属性值。 """ - props = await req.json() + props = payload ps = {"id": curve} | props return set_curve(network, ChangeSet(ps)) @router.get("/curves", summary="获取所有曲线", description="获取网络中的所有曲线列表") -async def fastapi_get_curves(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: +def fastapi_get_curves(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: """获取所有曲线。 返回指定网络中的所有曲线ID列表。 @@ -83,7 +84,7 @@ async def fastapi_get_curves(network: str = Query(..., description="管网名称 return get_curves(network) @router.get("/curves/existence", summary="检查曲线存在性", description="检查指定的曲线是否存在") -async def fastapi_is_curve( +def fastapi_is_curve( network: str = Query(..., description="管网名称（或数据库名称）"), curve: str = Query(..., description="曲线ID") ) -> bool: diff --git a/app/api/v1/endpoints/components/options.py b/app/api/v1/endpoints/components/options.py index d89d422..a6b23ac 100644 --- a/app/api/v1/endpoints/components/options.py +++ b/app/api/v1/endpoints/components/options.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, get_energy, @@ -19,7 +20,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/time", summary="获取时间选项架构", description="获取网络中时间选项的架构定义") -async def fastapi_get_time_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_time_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取时间选项架构。 返回指定网络中时间相关选项的属性架构定义。 @@ -27,7 +28,7 @@ async def fastapi_get_time_schema(network: str = Query(..., description="管网 return get_time_schema(network) @router.get("/network-options/time", summary="获取时间选项属性", description="获取指定网络中的时间选项属性信息") -async def fastapi_get_time_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_time_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取时间选项属性。 返回指定网络中的时间相关选项属性。 @@ -35,19 +36,19 @@ async def fastapi_get_time_properties(network: str = Query(..., description="管 return get_time(network) @router.patch("/time-properties", response_model=None, summary="设置时间选项属性", description="更新指定网络中的时间选项属性") -async def fastapi_set_time_properties( +def fastapi_set_time_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置时间选项属性。 更新指定网络中的时间相关选项属性值。 """ - props = await req.json() + props = payload return set_time(network, ChangeSet(props)) @router.get("/network-schemas/energy", summary="获取能耗选项架构", description="获取网络中能耗选项的架构定义") -async def fastapi_get_energy_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_energy_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取能耗选项架构。 返回指定网络中能耗相关选项的属性架构定义。 @@ -55,7 +56,7 @@ async def fastapi_get_energy_schema(network: str = Query(..., description="管 return get_energy_schema(network) @router.get("/network-options/energy", summary="获取能耗选项属性", description="获取指定网络中的能耗选项属性信息") -async def fastapi_get_energy_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_energy_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取能耗选项属性。 返回指定网络中的能耗相关选项属性。 @@ -63,19 +64,19 @@ async def fastapi_get_energy_properties(network: str = Query(..., description=" return get_energy(network) @router.patch("/energy-properties", response_model=None, summary="设置能耗选项属性", description="更新指定网络中的能耗选项属性") -async def fastapi_set_energy_properties( +def fastapi_set_energy_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置能耗选项属性。 更新指定网络中的能耗相关选项属性值。 """ - props = await req.json() + props = payload return set_energy(network, ChangeSet(props)) @router.get("/network-schemas/pump-energy", summary="获取泵能耗选项架构", description="获取网络中泵能耗选项的架构定义") -async def fastapi_get_pump_energy_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_pump_energy_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取泵能耗选项架构。 返回指定网络中泵能耗相关选项的属性架构定义。 @@ -83,7 +84,7 @@ async def fastapi_get_pump_energy_schema(network: str = Query(..., description=" return get_pump_energy_schema(network) @router.get("/network-options/pump-energy", summary="获取泵能耗属性", description="获取指定泵的能耗属性信息") -async def fastapi_get_pump_energy_proeprties( +def fastapi_get_pump_energy_proeprties( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="泵ID") ) -> dict[str, Any]: @@ -94,21 +95,21 @@ async def fastapi_get_pump_energy_proeprties( return get_pump_energy(network, pump) @router.patch("/network-options/pump-energy", response_model=None, summary="设置泵能耗属性", description="更新指定泵的能耗属性") -async def fastapi_set_pump_energy_properties( +def fastapi_set_pump_energy_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="泵ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置泵能耗属性。 更新指定泵的能耗相关属性值。 """ - props = await req.json() + props = payload ps = {"id": pump} | props return set_pump_energy(network, ChangeSet(ps)) @router.get("/network-schemas/option", summary="获取选项架构", description="获取网络中选项对象的架构定义") -async def fastapi_get_option_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_option_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取选项架构。 返回指定网络中选项对象的属性架构定义。 @@ -116,7 +117,7 @@ async def fastapi_get_option_schema(network: str = Query(..., description="管 return get_option_v3_schema(network) @router.get("/network-options", summary="获取选项属性", description="获取指定网络中的选项属性信息") -async def fastapi_get_option_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_option_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取选项属性。 返回指定网络中的选项对象属性信息。 @@ -124,13 +125,13 @@ async def fastapi_get_option_properties(network: str = Query(..., description=" return get_option_v3(network) @router.patch("/network-options", response_model=None, summary="设置选项属性", description="更新指定网络中的选项属性") -async def fastapi_set_option_properties( +def fastapi_set_option_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置选项属性。 更新指定网络中的选项属性值。 """ - props = await req.json() + props = payload return set_option_v3(network, ChangeSet(props)) diff --git a/app/api/v1/endpoints/components/patterns.py b/app/api/v1/endpoints/components/patterns.py index 81edfd3..0e79fcb 100644 --- a/app/api/v1/endpoints/components/patterns.py +++ b/app/api/v1/endpoints/components/patterns.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_pattern, @@ -14,7 +15,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/pattern", summary="获取模式架构", description="获取网络中模式对象的架构定义") -async def fastapi_get_pattern_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_pattern_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取模式架构。 返回指定网络中模式对象的属性架构定义。 @@ -22,23 +23,23 @@ async def fastapi_get_pattern_schema(network: str = Query(..., description="管 return get_pattern_schema(network) @router.post("/patterns", response_model=None, summary="添加模式", description="在网络中添加一个新的模式") -async def fastapi_add_pattern( +def fastapi_add_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), pattern: str = Query(..., description="模式ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加模式。 在指定网络中创建一个新的模式，并设置其初始属性。 """ - props = await req.json() + props = payload ps = { "id": pattern, } | props return add_pattern(network, ChangeSet(ps)) @router.delete("/patterns", response_model=None, summary="删除模式", description="从网络中删除指定的模式") -async def fastapi_delete_pattern( +def fastapi_delete_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), pattern: str = Query(..., description="模式ID") ) -> ChangeSet: @@ -50,7 +51,7 @@ async def fastapi_delete_pattern( return delete_pattern(network, ChangeSet(ps)) @router.get("/patterns/properties", summary="获取模式属性", description="获取指定模式的属性信息") -async def fastapi_get_pattern_properties( +def fastapi_get_pattern_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pattern: str = Query(..., description="模式ID") ) -> dict[str, Any]: @@ -61,21 +62,21 @@ async def fastapi_get_pattern_properties( return get_pattern(network, pattern) @router.patch("/patterns/properties", response_model=None, summary="设置模式属性", description="更新指定模式的属性") -async def fastapi_set_pattern_properties( +def fastapi_set_pattern_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pattern: str = Query(..., description="模式ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置模式属性。 更新指定模式的属性值。 """ - props = await req.json() + props = payload ps = {"id": pattern} | props return set_pattern(network, ChangeSet(ps)) @router.get("/patterns/existence", summary="检查模式存在性", description="检查指定的模式是否存在") -async def fastapi_is_pattern( +def fastapi_is_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), pattern: str = Query(..., description="模式ID") ) -> bool: @@ -86,7 +87,7 @@ async def fastapi_is_pattern( return is_pattern(network, pattern) @router.get("/patterns", summary="获取所有模式", description="获取网络中的所有模式列表") -async def fastapi_get_patterns(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: +def fastapi_get_patterns(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: """获取所有模式。 返回指定网络中的所有模式ID列表。 diff --git a/app/api/v1/endpoints/components/quality.py b/app/api/v1/endpoints/components/quality.py index 2f5e940..a229e36 100644 --- a/app/api/v1/endpoints/components/quality.py +++ b/app/api/v1/endpoints/components/quality.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_mixing, @@ -32,7 +33,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/quality", summary="获取水质架构", description="获取网络中水质对象的架构定义") -async def fastapi_get_quality_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_quality_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取水质架构。 返回指定网络中水质对象的属性架构定义。 @@ -40,7 +41,7 @@ async def fastapi_get_quality_schema(network: str = Query(..., description="管 return get_quality_schema(network) @router.get("/quality-configurations/properties", summary="获取水质属性", description="获取指定节点的水质属性信息") -async def fastapi_get_quality_properties( +def fastapi_get_quality_properties( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> dict[str, Any]: @@ -51,19 +52,19 @@ async def fastapi_get_quality_properties( return get_quality(network, node) @router.patch("/quality-configurations/properties", response_model=None, summary="设置水质属性", description="更新指定节点的水质属性") -async def fastapi_set_quality_properties( +def fastapi_set_quality_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置水质属性。 更新指定节点的水质属性值。 """ - props = await req.json() + props = payload return set_quality(network, ChangeSet(props)) @router.get("/network-schemas/emitter", summary="获取发射器架构", description="获取网络中发射器对象的架构定义") -async def fastapi_get_emitter_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_emitter_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取发射器架构。 返回指定网络中发射器对象的属性架构定义。 @@ -71,7 +72,7 @@ async def fastapi_get_emitter_schema(network: str = Query(..., description="管 return get_emitter_schema(network) @router.get("/emitters/properties", summary="获取发射器属性", description="获取指定连接点的发射器属性信息") -async def fastapi_get_emitter_properties( +def fastapi_get_emitter_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="连接点ID") ) -> dict[str, Any]: @@ -82,21 +83,21 @@ async def fastapi_get_emitter_properties( return get_emitter(network, junction) @router.patch("/emitters/properties", response_model=None, summary="设置发射器属性", description="更新指定连接点的发射器属性") -async def fastapi_set_emitter_properties( +def fastapi_set_emitter_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="连接点ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置发射器属性。 更新指定连接点的发射器属性值。 """ - props = await req.json() + props = payload ps = {"junction": junction} | props return set_emitter(network, ChangeSet(ps)) @router.get("/network-schemas/source", summary="获取水源架构", description="获取网络中水源对象的架构定义") -async def fastapi_get_source_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_source_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取水源架构。 返回指定网络中水源对象的属性架构定义。 @@ -104,7 +105,7 @@ async def fastapi_get_source_schema(network: str = Query(..., description="管 return get_source_schema(network) @router.get("/sources/detail", summary="获取水源属性", description="获取指定节点的水源属性信息") -async def fastapi_get_source( +def fastapi_get_source( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> dict[str, Any]: @@ -115,31 +116,31 @@ async def fastapi_get_source( return get_source(network, node) @router.patch("/sources", response_model=None, summary="设置水源属性", description="更新指定节点的水源属性") -async def fastapi_set_source( +def fastapi_set_source( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置水源属性。 更新指定节点的水源属性值。 """ - props = await req.json() + props = payload return set_source(network, ChangeSet(props)) @router.post("/sources", response_model=None, summary="添加水源", description="在网络中添加一个新的水源") -async def fastapi_add_source( +def fastapi_add_source( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加水源。 在指定网络中创建一个新的水源，并设置其初始属性。 """ - props = await req.json() + props = payload return add_source(network, ChangeSet(props)) @router.delete("/sources", response_model=None, summary="删除水源", description="从网络中删除指定节点的水源") -async def fastapi_delete_source( +def fastapi_delete_source( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> ChangeSet: @@ -151,7 +152,7 @@ async def fastapi_delete_source( return delete_source(network, ChangeSet(props)) @router.get("/network-schemas/reaction", summary="获取反应架构", description="获取网络中反应对象的架构定义") -async def fastapi_get_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取反应架构。 返回指定网络中反应对象的属性架构定义。 @@ -159,7 +160,7 @@ async def fastapi_get_reaction_schema(network: str = Query(..., description="管 return get_reaction_schema(network) @router.get("/reactions/detail", summary="获取反应属性", description="获取指定网络中的反应属性信息") -async def fastapi_get_reaction(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_reaction(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取反应属性。 返回指定网络中的反应属性信息。 @@ -167,19 +168,19 @@ async def fastapi_get_reaction(network: str = Query(..., description="管网名 return get_reaction(network) @router.patch("/reactions", response_model=None, summary="设置反应属性", description="更新指定网络中的反应属性") -async def fastapi_set_reaction( +def fastapi_set_reaction( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置反应属性。 更新指定网络中的反应属性值。 """ - props = await req.json() + props = payload return set_reaction(network, ChangeSet(props)) @router.get("/network-schemas/pipe-reaction", summary="获取管道反应架构", description="获取网络中管道反应对象的架构定义") -async def fastapi_get_pipe_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_pipe_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取管道反应架构。 返回指定网络中管道反应对象的属性架构定义。 @@ -187,7 +188,7 @@ async def fastapi_get_pipe_reaction_schema(network: str = Query(..., description return get_pipe_reaction_schema(network) @router.get("/pipe-reactions/detail", summary="获取管道反应属性", description="获取指定管道的反应属性信息") -async def fastapi_get_pipe_reaction( +def fastapi_get_pipe_reaction( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> dict[str, Any]: @@ -198,19 +199,19 @@ async def fastapi_get_pipe_reaction( return get_pipe_reaction(network, pipe) @router.patch("/pipe-reactions", response_model=None, summary="设置管道反应属性", description="更新指定管道的反应属性") -async def fastapi_set_pipe_reaction( +def fastapi_set_pipe_reaction( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置管道反应属性。 更新指定管道的反应属性值。 """ - props = await req.json() + props = payload return set_pipe_reaction(network, ChangeSet(props)) @router.get("/network-schemas/tank-reaction", summary="获取水池反应架构", description="获取网络中水池反应对象的架构定义") -async def fastapi_get_tank_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_tank_reaction_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取水池反应架构。 返回指定网络中水池反应对象的属性架构定义。 @@ -218,7 +219,7 @@ async def fastapi_get_tank_reaction_schema(network: str = Query(..., description return get_tank_reaction_schema(network) @router.get("/tank-reactions/detail", summary="获取水池反应属性", description="获取指定水池的反应属性信息") -async def fastapi_get_tank_reaction( +def fastapi_get_tank_reaction( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水池ID") ) -> dict[str, Any]: @@ -229,19 +230,19 @@ async def fastapi_get_tank_reaction( return get_tank_reaction(network, tank) @router.patch("/tank-reactions", response_model=None, summary="设置水池反应属性", description="更新指定水池的反应属性") -async def fastapi_set_tank_reaction( +def fastapi_set_tank_reaction( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置水池反应属性。 更新指定水池的反应属性值。 """ - props = await req.json() + props = payload return set_tank_reaction(network, ChangeSet(props)) @router.get("/network-schemas/mixing", summary="获取混合架构", description="获取网络中混合对象的架构定义") -async def fastapi_get_mixing_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_mixing_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取混合架构。 返回指定网络中混合对象的属性架构定义。 @@ -249,7 +250,7 @@ async def fastapi_get_mixing_schema(network: str = Query(..., description="管 return get_mixing_schema(network) @router.get("/mixing-configurations/detail", summary="获取混合属性", description="获取指定水池的混合属性信息") -async def fastapi_get_mixing( +def fastapi_get_mixing( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水池ID") ) -> dict[str, Any]: @@ -260,37 +261,37 @@ async def fastapi_get_mixing( return get_mixing(network, tank) @router.patch("/mixing-configurations", response_model=None, summary="设置混合属性", description="更新指定水池的混合属性") -async def fastapi_set_mixing( +def fastapi_set_mixing( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置混合属性。 更新指定水池的混合属性值。 """ - props = await req.json() + props = payload return set_mixing(network, ChangeSet(props)) @router.post("/mixing-configurations", response_model=None, summary="添加混合", description="在网络中添加一个新的混合") -async def fastapi_add_mixing( +def fastapi_add_mixing( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加混合。 在指定网络中创建一个新的混合，并设置其初始属性。 """ - props = await req.json() + props = payload return add_mixing(network, ChangeSet(props)) @router.delete("/mixing-configurations", response_model=None, summary="删除混合", description="从网络中删除指定的混合") -async def fastapi_delete_mixing( +def fastapi_delete_mixing( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """删除混合。 从指定网络中删除指定的混合及其相关数据。 """ - props = await req.json() + props = payload return delete_mixing(network, ChangeSet(props)) diff --git a/app/api/v1/endpoints/components/visuals.py b/app/api/v1/endpoints/components/visuals.py index 571fac8..425d045 100644 --- a/app/api/v1/endpoints/components/visuals.py +++ b/app/api/v1/endpoints/components/visuals.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body, Response -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_label, @@ -24,7 +25,7 @@ import json router = APIRouter() @router.get("/network-schemas/vertex", summary="获取图形元素架构", description="获取网络中图形元素对象的架构定义") -async def fastapi_get_vertex_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_vertex_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取图形元素架构。 返回指定网络中图形元素对象的属性架构定义。 @@ -32,7 +33,7 @@ async def fastapi_get_vertex_schema(network: str = Query(..., description="管 return get_vertex_schema(network) @router.get("/visual-elements/properties", summary="获取图形元素属性", description="获取指定图形元素的属性信息") -async def fastapi_get_vertex_properties( +def fastapi_get_vertex_properties( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="图形元素链接") ) -> dict[str, Any]: @@ -43,43 +44,43 @@ async def fastapi_get_vertex_properties( return get_vertex(network, link) @router.patch("/visual-elements/properties", response_model=None, summary="设置图形元素属性", description="更新指定图形元素的属性") -async def fastapi_set_vertex_properties( +def fastapi_set_vertex_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置图形元素属性。 更新指定图形元素的属性值。 """ - props = await req.json() + props = payload return set_vertex(network, ChangeSet(props)) @router.post("/visual-elements", response_model=None, summary="添加图形元素", description="在网络中添加一个新的图形元素") -async def fastapi_add_vertex( +def fastapi_add_vertex( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加图形元素。 在指定网络中创建一个新的图形元素，并设置其初始属性。 """ - props = await req.json() + props = payload return add_vertex(network, ChangeSet(props)) @router.delete("/visual-elements", response_model=None, summary="删除图形元素", description="从网络中删除指定的图形元素") -async def fastapi_delete_vertex( +def fastapi_delete_vertex( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """删除图形元素。 从指定网络中删除指定的图形元素及其相关数据。 """ - props = await req.json() + props = payload return delete_vertex(network, ChangeSet(props)) @router.get("/visual-elements/links", response_class=PlainTextResponse, summary="获取所有图形元素链接", description="获取网络中的所有图形元素链接列表") -async def fastapi_get_all_vertex_links(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: +def fastapi_get_all_vertex_links(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: """获取所有图形元素链接。 返回指定网络中的所有图形元素链接列表。 @@ -87,7 +88,7 @@ async def fastapi_get_all_vertex_links(network: str = Query(..., description=" return json.dumps(get_all_vertex_links(network)) @router.get("/all-vertices", response_class=PlainTextResponse, summary="获取所有图形元素", description="获取网络中的所有图形元素详细信息") -async def fastapi_get_all_vertices(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[dict[str, Any]]: +def fastapi_get_all_vertices(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[dict[str, Any]]: """获取所有图形元素。 返回指定网络中的所有图形元素详细信息。 @@ -95,7 +96,7 @@ async def fastapi_get_all_vertices(network: str = Query(..., description="管网 return json.dumps(get_all_vertices(network)) @router.get("/network-schemas/label", summary="获取标签架构", description="获取网络中标签对象的架构定义") -async def fastapi_get_label_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_label_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取标签架构。 返回指定网络中标签对象的属性架构定义。 @@ -103,7 +104,7 @@ async def fastapi_get_label_schema(network: str = Query(..., description="管网 return get_label_schema(network) @router.get("/labels/properties", summary="获取标签属性", description="获取指定坐标处的标签属性信息") -async def fastapi_get_label_properties( +def fastapi_get_label_properties( network: str = Query(..., description="管网名称（或数据库名称）"), x: float = Query(..., description="X坐标"), y: float = Query(..., description="Y坐标") @@ -115,43 +116,43 @@ async def fastapi_get_label_properties( return get_label(network, x, y) @router.patch("/labels/properties", response_model=None, summary="设置标签属性", description="更新指定标签的属性") -async def fastapi_set_label_properties( +def fastapi_set_label_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置标签属性。 更新指定标签的属性值。 """ - props = await req.json() + props = payload return set_label(network, ChangeSet(props)) @router.post("/labels", response_model=None, summary="添加标签", description="在网络中添加一个新的标签") -async def fastapi_add_label( +def fastapi_add_label( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """添加标签。 在指定网络中创建一个新的标签，并设置其初始属性。 """ - props = await req.json() + props = payload return add_label(network, ChangeSet(props)) @router.delete("/labels", response_model=None, summary="删除标签", description="从网络中删除指定的标签") -async def fastapi_delete_label( +def fastapi_delete_label( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """删除标签。 从指定网络中删除指定的标签及其相关数据。 """ - props = await req.json() + props = payload return delete_label(network, ChangeSet(props)) @router.get("/network-schemas/backdrop", summary="获取背景架构", description="获取网络中背景对象的架构定义") -async def fastapi_get_backdrop_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_backdrop_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """获取背景架构。 返回指定网络中背景对象的属性架构定义。 @@ -159,7 +160,7 @@ async def fastapi_get_backdrop_schema(network: str = Query(..., description="管 return get_backdrop_schema(network) @router.get("/backdrops/properties", summary="获取背景属性", description="获取指定网络的背景属性信息") -async def fastapi_get_backdrop_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fastapi_get_backdrop_properties(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取背景属性。 返回指定网络的背景属性信息。 @@ -167,13 +168,13 @@ async def fastapi_get_backdrop_properties(network: str = Query(..., description= return get_backdrop(network) @router.patch("/backdrops/properties", response_model=None, summary="设置背景属性", description="更新指定网络的背景属性") -async def fastapi_set_backdrop_properties( +def fastapi_set_backdrop_properties( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置背景属性。 更新指定网络的背景属性值。 """ - props = await req.json() + props = payload return set_backdrop(network, ChangeSet(props)) diff --git a/app/api/v1/endpoints/meta.py b/app/api/v1/endpoints/meta.py index a84b197..59a0cc1 100644 --- a/app/api/v1/endpoints/meta.py +++ b/app/api/v1/endpoints/meta.py @@ -2,14 +2,11 @@ import logging from fastapi import APIRouter, Depends, HTTPException, status, Query, Path import psycopg from psycopg import AsyncConnection -from sqlalchemy import text -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.ext.asyncio import AsyncSession from app.auth.project_dependencies import ( ProjectContext, get_project_context, - get_project_pg_session, + get_project_pg_connection, get_project_timescale_connection, get_metadata_repository, ) @@ -90,7 +87,7 @@ async def list_user_projects( @router.get("/projects/current/database-health", summary="检查数据库健康状态", description="检查项目数据库连接的健康状况") async def project_db_health( - pg_session: AsyncSession = Depends(get_project_pg_session), + pg_conn: AsyncConnection = Depends(get_project_pg_connection), ts_conn: AsyncConnection = Depends(get_project_timescale_connection), ): """ @@ -99,8 +96,10 @@ async def project_db_health( 检查PostgreSQL和TimescaleDB数据库的连接状态 """ try: - await pg_session.execute(text("SELECT 1")) - except SQLAlchemyError as exc: + async with pg_conn.cursor() as cur: + await cur.execute("SELECT 1") + await cur.fetchone() + except psycopg.Error as exc: logger.error("Project PostgreSQL health check failed", exc_info=True) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/app/api/v1/endpoints/model_import.py b/app/api/v1/endpoints/model_import.py index 9eb28ae..b62e6a8 100644 --- a/app/api/v1/endpoints/model_import.py +++ b/app/api/v1/endpoints/model_import.py @@ -12,6 +12,7 @@ from fastapi import ( UploadFile, status, ) +from starlette.concurrency import run_in_threadpool from app.auth.metadata_dependencies import ( get_current_metadata_admin, @@ -24,6 +25,7 @@ from app.auth.project_dependencies import ( from app.core.audit import AuditAction, log_audit_event from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository from app.infra.db.project_routing import activate_project_routing +from app.native.wndb.core.database import MaterializedViewRefreshAfterCommitError from app.services.network_import import network_update from app.services.tjnetwork import run_inp @@ -87,8 +89,8 @@ def _validate_inp_bytes(content: bytes, filename: str) -> str: async def _read_upload(file: UploadFile) -> tuple[bytes, str]: filename = Path(file.filename or "").name content = await file.read(MAX_INP_FILE_BYTES + 1) - _validate_inp_bytes(content, filename) - return content, filename + normalized = _validate_inp_bytes(content, filename).encode("utf-8") + return normalized, filename async def _audit_model_change( @@ -114,7 +116,7 @@ async def _audit_model_change( ) -async def _run_uploaded_inp(content: bytes) -> str: +def _run_uploaded_inp_sync(content: bytes) -> str: target_dir = Path("inp") target_dir.mkdir(parents=True, exist_ok=True) model_name = f"admin_model_{uuid4().hex}" @@ -123,7 +125,11 @@ async def _run_uploaded_inp(content: bytes) -> str: return run_inp(model_name) -async def _update_from_inp(content: bytes, project_code: str) -> None: +async def _run_uploaded_inp(content: bytes) -> str: + return await run_in_threadpool(_run_uploaded_inp_sync, content) + + +def _update_from_inp_sync(content: bytes, project_code: str) -> None: temp_path: Path | None = None try: with NamedTemporaryFile(suffix=".inp", delete=False) as temp_file: @@ -135,9 +141,15 @@ async def _update_from_inp(content: bytes, project_code: str) -> None: temp_path.unlink(missing_ok=True) +async def _update_from_inp(content: bytes, project_code: str) -> None: + await run_in_threadpool(_update_from_inp_sync, content, project_code) + + async def _apply_model_update(content: bytes, project_code: str) -> None: try: await _update_from_inp(content, project_code) + except MaterializedViewRefreshAfterCommitError: + raise except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/app/api/v1/endpoints/network/demands.py b/app/api/v1/endpoints/network/demands.py index 16a169f..cac2288 100644 --- a/app/api/v1/endpoints/network/demands.py +++ b/app/api/v1/endpoints/network/demands.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, calculate_demand_to_network, @@ -21,7 +22,7 @@ router = APIRouter() summary="获取需水量属性架构", description="获取指定水网中需水量(Demand)的属性架构定义" ) -async def fastapi_get_demand_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fastapi_get_demand_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """ 获取需水量属性架构。 @@ -35,7 +36,7 @@ async def fastapi_get_demand_schema(network: str = Query(..., description="管 summary="获取需水量属性", description="获取指定水网中节点的需水量属性信息" ) -async def fastapi_get_demand_properties( +def fastapi_get_demand_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点ID") ) -> dict[str, Any]: @@ -54,17 +55,17 @@ async def fastapi_get_demand_properties( summary="设置需水量属性", description="设置指定水网中节点的需水量属性信息" ) -async def fastapi_set_demand_properties( +def fastapi_set_demand_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 设置节点的需水量属性。 修改指定节点的需水量信息。请求体应包含需水量值、水压等级等属性。 """ - props = await req.json() + props = payload ps = {"junction": junction} | props return set_demand(network, ChangeSet(ps)) @@ -76,9 +77,9 @@ async def fastapi_set_demand_properties( summary="计算需水量到节点分配", description="将总需水量按指定方式分配到多个节点" ) -async def fastapi_calculate_demand_to_nodes( +def fastapi_calculate_demand_to_nodes( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> dict[str, float]: """ 计算需水量到节点分配。 @@ -91,7 +92,7 @@ async def fastapi_calculate_demand_to_nodes( "nodes": 节点ID列表(list[str]) } """ - props = await req.json() + props = payload demand = props["demand"] nodes = props["nodes"] return calculate_demand_to_nodes(network, demand, nodes) @@ -101,9 +102,9 @@ async def fastapi_calculate_demand_to_nodes( summary="计算需水量到区域分配", description="将总需水量按区域特征分配到该区域内的节点" ) -async def fastapi_calculate_demand_to_region( +def fastapi_calculate_demand_to_region( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> dict[str, float]: """ 计算需水量到区域分配。 @@ -116,7 +117,7 @@ async def fastapi_calculate_demand_to_region( "region": 区域ID(str) } """ - props = await req.json() + props = payload demand = props["demand"] region = props["region"] return calculate_demand_to_region(network, demand, region) @@ -126,7 +127,7 @@ async def fastapi_calculate_demand_to_region( summary="计算需水量到整网分配", description="将需水量均匀分配到整个水网的所有需水节点" ) -async def fastapi_calculate_demand_to_network( +def fastapi_calculate_demand_to_network( network: str = Query(..., description="管网名称（或数据库名称）"), demand: float = Query(..., description="总需水量(m³/h)", gt=0) ) -> dict[str, float]: diff --git a/app/api/v1/endpoints/network/general.py b/app/api/v1/endpoints/network/general.py index c87cfed..c0f7533 100644 --- a/app/api/v1/endpoints/network/general.py +++ b/app/api/v1/endpoints/network/general.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, delete_junction, @@ -48,7 +49,7 @@ router = APIRouter() summary="检查节点有效性", description="检查指定ID是否为水网中的有效节点" ) -async def fastapi_is_node( +def fastapi_is_node( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> bool: @@ -60,7 +61,7 @@ async def fastapi_is_node( summary="检查是否为接点", description="检查指定ID是否为水网中的接点(需求点)" ) -async def fastapi_is_junction( +def fastapi_is_junction( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> bool: @@ -72,7 +73,7 @@ async def fastapi_is_junction( summary="检查是否为水源", description="检查指定ID是否为水网中的水源(水库/河流)" ) -async def fastapi_is_reservoir( +def fastapi_is_reservoir( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> bool: @@ -84,7 +85,7 @@ async def fastapi_is_reservoir( summary="检查是否为蓄水池", description="检查指定ID是否为水网中的蓄水池" ) -async def fastapi_is_tank( +def fastapi_is_tank( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> bool: @@ -96,7 +97,7 @@ async def fastapi_is_tank( summary="检查管线有效性", description="检查指定ID是否为水网中的有效管线" ) -async def fastapi_is_link( +def fastapi_is_link( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> bool: @@ -108,7 +109,7 @@ async def fastapi_is_link( summary="检查是否为管道", description="检查指定ID是否为水网中的管道" ) -async def fastapi_is_pipe( +def fastapi_is_pipe( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> bool: @@ -120,7 +121,7 @@ async def fastapi_is_pipe( summary="检查是否为泵", description="检查指定ID是否为水网中的泵" ) -async def fastapi_is_pump( +def fastapi_is_pump( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> bool: @@ -132,7 +133,7 @@ async def fastapi_is_pump( summary="检查是否为阀门", description="检查指定ID是否为水网中的阀门" ) -async def fastapi_is_valve( +def fastapi_is_valve( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> bool: @@ -144,7 +145,7 @@ async def fastapi_is_valve( summary="获取节点类型", description="获取指定节点的类型(接点/水源/蓄水池)" ) -async def fastapi_get_node_type( +def fastapi_get_node_type( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> str: @@ -156,7 +157,7 @@ async def fastapi_get_node_type( summary="获取管线类型", description="获取指定管线的类型(管道/泵/阀门)" ) -async def fastapi_get_link_type( +def fastapi_get_link_type( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> str: @@ -168,7 +169,7 @@ async def fastapi_get_link_type( summary="获取元素类型", description="获取指定元素的类型(节点或管线)" ) -async def fastapi_get_element_type( +def fastapi_get_element_type( network: str = Query(..., description="管网名称（或数据库名称）"), element: str = Query(..., description="元素ID") ) -> str: @@ -180,7 +181,7 @@ async def fastapi_get_element_type( summary="获取元素类型值", description="获取指定元素的类型数值标识" ) -async def fastapi_get_element_type_value( +def fastapi_get_element_type_value( network: str = Query(..., description="管网名称（或数据库名称）"), element: str = Query(..., description="元素ID") ) -> int: @@ -192,7 +193,7 @@ async def fastapi_get_element_type_value( summary="获取所有节点", description="获取指定水网中的所有节点ID列表" ) -async def fastapi_get_nodes(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: +def fastapi_get_nodes(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: """获取水网中所有节点的ID列表。""" return get_nodes(network) @@ -201,7 +202,7 @@ async def fastapi_get_nodes(network: str = Query(..., description="管网名称 summary="获取所有管线", description="获取指定水网中的所有管线ID列表" ) -async def fastapi_get_links(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: +def fastapi_get_links(network: str = Query(..., description="管网名称（或数据库名称）")) -> list[str]: """获取水网中所有管线的ID列表。""" return get_links(network) @@ -226,7 +227,7 @@ def get_node_links_endpoint( summary="获取节点属性", description="获取指定节点的所有属性信息" ) -async def fast_get_node_properties( +def fast_get_node_properties( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> dict[str, Any]: @@ -238,7 +239,7 @@ async def fast_get_node_properties( summary="获取管线属性", description="获取指定管线的所有属性信息" ) -async def fast_get_link_properties( +def fast_get_link_properties( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> dict[str, Any]: @@ -250,7 +251,7 @@ async def fast_get_link_properties( summary="获取SCADA点属性", description="获取指定SCADA点的属性信息" ) -async def fast_get_scada_properties( +def fast_get_scada_properties( network: str = Query(..., description="管网名称（或数据库名称）"), scada: str = Query(..., description="SCADA点ID") ) -> dict[str, Any]: @@ -262,7 +263,7 @@ async def fast_get_scada_properties( summary="获取所有SCADA点属性", description="获取指定水网中所有SCADA点的属性信息" ) -async def fast_get_all_scada_properties( +def fast_get_all_scada_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """获取水网中所有SCADA点的属性列表。""" @@ -273,7 +274,7 @@ async def fast_get_all_scada_properties( summary="获取指定类型元素属性", description="获取指定类型的元素属性信息" ) -async def fast_get_element_properties_with_type( +def fast_get_element_properties_with_type( network: str = Query(..., description="管网名称（或数据库名称）"), elementtype: str = Query(..., description="元素类型"), element: str = Query(..., description="元素ID") @@ -286,7 +287,7 @@ async def fast_get_element_properties_with_type( summary="获取元素属性", description="获取指定元素的属性信息" ) -async def fast_get_element_properties( +def fast_get_element_properties( network: str = Query(..., description="管网名称（或数据库名称）"), element: str = Query(..., description="元素ID") ) -> dict[str, Any]: @@ -302,7 +303,7 @@ async def fast_get_element_properties( summary="获取标题属性架构", description="获取指定水网的标题(标题)属性架构定义" ) -async def fast_get_title_schema( +def fast_get_title_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """获取水网标题的属性架构。""" @@ -313,7 +314,7 @@ async def fast_get_title_schema( summary="获取水网标题属性", description="获取指定水网的标题(Title)信息" ) -async def fast_get_title(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def fast_get_title(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """获取水网的标题属性。""" return get_title(network) @@ -323,12 +324,12 @@ async def fast_get_title(network: str = Query(..., description="管网名称（ summary="设置水网标题属性", description="设置指定水网的标题(Title)信息" ) -async def fastapi_set_title( +def fastapi_set_title( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置水网的标题属性。""" - props = await req.json() + props = payload return set_title(network, ChangeSet(props)) ############################################################ @@ -340,7 +341,7 @@ async def fastapi_set_title( summary="获取状态属性架构", description="获取指定水网的状态(Status)属性架构定义" ) -async def fastapi_get_status_schema( +def fastapi_get_status_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """获取水网状态的属性架构。""" @@ -351,7 +352,7 @@ async def fastapi_get_status_schema( summary="获取管线状态", description="获取指定管线的状态信息" ) -async def fastapi_get_status( +def fastapi_get_status( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> dict[str, Any]: @@ -364,13 +365,13 @@ async def fastapi_get_status( summary="设置管线状态", description="设置指定管线的状态信息" ) -async def fastapi_set_status_properties( +def fastapi_set_status_properties( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置管线的状态属性。""" - props = await req.json() + props = payload ps = {"link": link} | props return set_status(network, ChangeSet(ps)) @@ -384,7 +385,7 @@ async def fastapi_set_status_properties( summary="删除节点", description="删除指定的节点(接点/水源/蓄水池)" ) -async def fastapi_delete_node( +def fastapi_delete_node( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> ChangeSet: @@ -404,7 +405,7 @@ async def fastapi_delete_node( summary="删除管线", description="删除指定的管线(管道/泵/阀门)" ) -async def fastapi_delete_link( +def fastapi_delete_link( network: str = Query(..., description="管网名称（或数据库名称）"), link: str = Query(..., description="管线ID") ) -> ChangeSet: diff --git a/app/api/v1/endpoints/network/geometry.py b/app/api/v1/endpoints/network/geometry.py index 3b7a90f..d14f6db 100644 --- a/app/api/v1/endpoints/network/geometry.py +++ b/app/api/v1/endpoints/network/geometry.py @@ -27,7 +27,7 @@ router = APIRouter() # # example: set_coord(p, ChangeSet({'node': 'j1', 'x': 1.0, 'y': 2.0})) # @router.post("/setcoord/", response_model=None) # async def fastapi_set_coord(network: str, req: Request) -> ChangeSet: -# props = await req.json() +# props = payload # return set_coord(network, ChangeSet(props)) @router.get( @@ -35,7 +35,7 @@ router = APIRouter() summary="获取节点坐标", description="获取指定节点的地理坐标(X, Y)" ) -async def fastapi_get_node_coord( +def fastapi_get_node_coord( network: str = Query(..., description="管网名称（或数据库名称）"), node: str = Query(..., description="节点ID") ) -> dict[str, float] | None: @@ -48,7 +48,7 @@ async def fastapi_get_node_coord( summary="获取范围内的网络元素", description="获取指定地理范围内的网络节点和管线" ) -async def fastapi_get_network_in_extent( +def fastapi_get_network_in_extent( network: str = Query(..., description="管网名称（或数据库名称）"), x1: float = Query(..., description="范围左下角X坐标", alias="x1"), y1: float = Query(..., description="范围左下角Y坐标", alias="y1"), @@ -63,7 +63,7 @@ async def fastapi_get_network_in_extent( summary="获取主要节点坐标", description="获取直径大于等于指定值的节点坐标" ) -async def fastapi_get_majornode_coords( +def fastapi_get_majornode_coords( network: str = Query(..., description="管网名称（或数据库名称）"), diameter: int = Query(..., description="最小直径(mm)", gt=0) ) -> dict[str, dict[str, float]]: @@ -75,7 +75,7 @@ async def fastapi_get_majornode_coords( summary="获取主要管道节点", description="获取直径大于等于指定值的管道的节点ID" ) -async def fastapi_get_major_pipe_nodes( +def fastapi_get_major_pipe_nodes( network: str = Query(..., description="管网名称（或数据库名称）"), diameter: int = Query(..., description="最小直径(mm)", gt=0) ) -> list[str] | None: @@ -87,7 +87,7 @@ async def fastapi_get_major_pipe_nodes( summary="获取网络管线节点", description="获取指定水网所有管线的起点和终点节点" ) -async def fastapi_get_network_link_nodes( +def fastapi_get_network_link_nodes( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[str] | None: """获取网络中所有管线的连接节点。""" diff --git a/app/api/v1/endpoints/network/junctions.py b/app/api/v1/endpoints/network/junctions.py index 7a6c06c..fed6316 100644 --- a/app/api/v1/endpoints/network/junctions.py +++ b/app/api/v1/endpoints/network/junctions.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_junction, @@ -13,7 +14,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/junction", summary="获取节点架构", description="获取指定项目的节点属性架构和数据类型定义。") -async def fast_get_junction_schema( +def fast_get_junction_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """ @@ -27,7 +28,7 @@ async def fast_get_junction_schema( return get_junction_schema(network) @router.post("/junctions", response_model=None, summary="添加节点", description="在供水网络中添加新的节点，指定节点ID和空间坐标。") -async def fastapi_add_junction( +def fastapi_add_junction( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), x: float = Query(..., description="X 坐标"), @@ -51,7 +52,7 @@ async def fastapi_add_junction( return add_junction(network, ChangeSet(ps)) @router.delete("/junctions", response_model=None, summary="删除节点", description="从供水网络中删除指定的节点。") -async def fastapi_delete_junction( +def fastapi_delete_junction( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> ChangeSet: @@ -69,7 +70,7 @@ async def fastapi_delete_junction( return delete_junction(network, ChangeSet(ps)) @router.get("/junctions/elevation", summary="获取节点标高", description="获取指定节点的标高（海拔高度）。") -async def fastapi_get_junction_elevation( +def fastapi_get_junction_elevation( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> float: @@ -87,7 +88,7 @@ async def fastapi_get_junction_elevation( return ps["elevation"] @router.get("/junctions/x", summary="获取节点 X 坐标", description="获取指定节点的 X 坐标值。") -async def fastapi_get_junction_x( +def fastapi_get_junction_x( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> float: @@ -105,7 +106,7 @@ async def fastapi_get_junction_x( return ps["x"] @router.get("/junctions/y", summary="获取节点 Y 坐标", description="获取指定节点的 Y 坐标值。") -async def fastapi_get_junction_y( +def fastapi_get_junction_y( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> float: @@ -123,7 +124,7 @@ async def fastapi_get_junction_y( return ps["y"] @router.get("/junctions/coord", summary="获取节点坐标", description="获取指定节点的 X 和 Y 坐标。") -async def fastapi_get_junction_coord( +def fastapi_get_junction_coord( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> dict[str, float]: @@ -142,7 +143,7 @@ async def fastapi_get_junction_coord( return coord @router.get("/junctions/demand", summary="获取节点需水量", description="获取指定节点的需水量。") -async def fastapi_get_junction_demand( +def fastapi_get_junction_demand( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> float: @@ -160,7 +161,7 @@ async def fastapi_get_junction_demand( return ps["demand"] @router.get("/junctions/pattern", summary="获取节点需水模式", description="获取指定节点的需水模式标识。") -async def fastapi_get_junction_pattern( +def fastapi_get_junction_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> str: @@ -178,7 +179,7 @@ async def fastapi_get_junction_pattern( return ps["pattern"] @router.patch("/junctions/elevation", response_model=None, summary="设置节点标高", description="设置指定节点的标高值。") -async def fastapi_set_junction_elevation( +def fastapi_set_junction_elevation( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), elevation: float = Query(..., description="标高（海拔高度）") @@ -198,7 +199,7 @@ async def fastapi_set_junction_elevation( return set_junction(network, ChangeSet(ps)) @router.patch("/junctions/x", response_model=None, summary="设置节点 X 坐标", description="设置指定节点的 X 坐标值。") -async def fastapi_set_junction_x( +def fastapi_set_junction_x( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), x: float = Query(..., description="X 坐标值") @@ -218,7 +219,7 @@ async def fastapi_set_junction_x( return set_junction(network, ChangeSet(ps)) @router.patch("/junctions/y", response_model=None, summary="设置节点 Y 坐标", description="设置指定节点的 Y 坐标值。") -async def fastapi_set_junction_y( +def fastapi_set_junction_y( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), y: float = Query(..., description="Y 坐标值") @@ -238,7 +239,7 @@ async def fastapi_set_junction_y( return set_junction(network, ChangeSet(ps)) @router.patch("/junctions/coord", response_model=None, summary="设置节点坐标", description="设置指定节点的 X 和 Y 坐标。") -async def fastapi_set_junction_coord( +def fastapi_set_junction_coord( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), x: float = Query(..., description="X 坐标值"), @@ -260,7 +261,7 @@ async def fastapi_set_junction_coord( return set_junction(network, ChangeSet(ps)) @router.patch("/junctions/demand", response_model=None, summary="设置节点需水量", description="设置指定节点的需水量。") -async def fastapi_set_junction_demand( +def fastapi_set_junction_demand( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), demand: float = Query(..., description="需水量值") @@ -280,7 +281,7 @@ async def fastapi_set_junction_demand( return set_junction(network, ChangeSet(ps)) @router.patch("/junctions/pattern", response_model=None, summary="设置节点需水模式", description="设置指定节点的需水模式标识。") -async def fastapi_set_junction_pattern( +def fastapi_set_junction_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), pattern: str = Query(..., description="需水模式标识") @@ -300,7 +301,7 @@ async def fastapi_set_junction_pattern( return set_junction(network, ChangeSet(ps)) @router.get("/junctions/properties", summary="获取节点属性", description="获取指定节点的所有属性信息。") -async def fastapi_get_junction_properties( +def fastapi_get_junction_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID") ) -> dict[str, Any]: @@ -317,7 +318,7 @@ async def fastapi_get_junction_properties( return get_junction(network, junction) @router.get("/junctions", summary="获取所有节点属性", description="获取指定项目中所有节点的属性信息。") -async def fastapi_get_all_junction_properties( +def fastapi_get_all_junction_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -336,10 +337,10 @@ async def fastapi_get_all_junction_properties( return results @router.patch("/junctions/properties", response_model=None, summary="批量设置节点属性", description="批量设置指定节点的多个属性。") -async def fastapi_set_junction_properties( +def fastapi_set_junction_properties( network: str = Query(..., description="管网名称（或数据库名称）"), junction: str = Query(..., description="节点 ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 批量设置节点属性。 @@ -354,6 +355,6 @@ async def fastapi_set_junction_properties( Returns: ChangeSet: 包含变更信息的结果 """ - props = await req.json() + props = payload ps = {"id": junction} | props return set_junction(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/network/pipes.py b/app/api/v1/endpoints/network/pipes.py index a500180..657ce57 100644 --- a/app/api/v1/endpoints/network/pipes.py +++ b/app/api/v1/endpoints/network/pipes.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, PIPE_STATUS_OPEN, @@ -14,7 +15,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/pipe", summary="获取管道模式", description="获取管道对象的模式定义，包含所有可用字段及其类型") -async def fastapi_get_pipe_schema( +def fastapi_get_pipe_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """ @@ -29,7 +30,7 @@ async def fastapi_get_pipe_schema( return get_pipe_schema(network) @router.post("/pipes", response_model=None, summary="添加管道", description="向网络中添加新的管道，需要提供管道的基本参数如长度、管径、粗糙度等") -async def fastapi_add_pipe( +def fastapi_add_pipe( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道标识符"), node1: str = Query(..., description="管道起始节点ID"), @@ -70,7 +71,7 @@ async def fastapi_add_pipe( return add_pipe(network, ChangeSet(ps)) @router.delete("/pipes", response_model=None, summary="删除管道", description="从网络中删除指定的管道") -async def fastapi_delete_pipe( +def fastapi_delete_pipe( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="要删除的管道ID") ) -> ChangeSet: @@ -88,7 +89,7 @@ async def fastapi_delete_pipe( return delete_pipe(network, ChangeSet(ps)) @router.get("/pipes/node1", summary="获取管道起始节点", description="获取指定管道的起始节点ID") -async def fastapi_get_pipe_node1( +def fastapi_get_pipe_node1( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> str | None: @@ -106,7 +107,7 @@ async def fastapi_get_pipe_node1( return ps["node1"] @router.get("/pipes/node2", summary="获取管道终止节点", description="获取指定管道的终止节点ID") -async def fastapi_get_pipe_node2( +def fastapi_get_pipe_node2( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> str | None: @@ -124,7 +125,7 @@ async def fastapi_get_pipe_node2( return ps["node2"] @router.get("/pipes/length", summary="获取管道长度", description="获取指定管道的长度") -async def fastapi_get_pipe_length( +def fastapi_get_pipe_length( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> float | None: @@ -142,7 +143,7 @@ async def fastapi_get_pipe_length( return ps["length"] @router.get("/pipes/diameter", summary="获取管道管径", description="获取指定管道的管径") -async def fastapi_get_pipe_diameter( +def fastapi_get_pipe_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> float | None: @@ -160,7 +161,7 @@ async def fastapi_get_pipe_diameter( return ps["diameter"] @router.get("/pipes/roughness", summary="获取管道粗糙度", description="获取指定管道的粗糙度") -async def fastapi_get_pipe_roughness( +def fastapi_get_pipe_roughness( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> float | None: @@ -178,7 +179,7 @@ async def fastapi_get_pipe_roughness( return ps["roughness"] @router.get("/pipes/minor-loss", summary="获取管道局部阻力系数", description="获取指定管道的局部阻力系数") -async def fastapi_get_pipe_minor_loss( +def fastapi_get_pipe_minor_loss( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> float | None: @@ -196,7 +197,7 @@ async def fastapi_get_pipe_minor_loss( return ps["minor_loss"] @router.get("/pipes/status", summary="获取管道状态", description="获取指定管道的状态（开启或关闭）") -async def fastapi_get_pipe_status( +def fastapi_get_pipe_status( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> str | None: @@ -214,7 +215,7 @@ async def fastapi_get_pipe_status( return ps["status"] @router.patch("/pipes/node1", response_model=None, summary="设置管道起始节点", description="设置指定管道的起始节点") -async def fastapi_set_pipe_node1( +def fastapi_set_pipe_node1( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), node1: str = Query(..., description="新的起始节点ID") @@ -234,7 +235,7 @@ async def fastapi_set_pipe_node1( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/node2", response_model=None, summary="设置管道终止节点", description="设置指定管道的终止节点") -async def fastapi_set_pipe_node2( +def fastapi_set_pipe_node2( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), node2: str = Query(..., description="新的终止节点ID") @@ -254,7 +255,7 @@ async def fastapi_set_pipe_node2( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/length", response_model=None, summary="设置管道长度", description="设置指定管道的长度") -async def fastapi_set_pipe_length( +def fastapi_set_pipe_length( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), length: float = Query(..., description="新的管道长度（单位：米）") @@ -274,7 +275,7 @@ async def fastapi_set_pipe_length( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/diameter", response_model=None, summary="设置管道管径", description="设置指定管道的管径") -async def fastapi_set_pipe_diameter( +def fastapi_set_pipe_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), diameter: float = Query(..., description="新的管道管径（单位：毫米）") @@ -294,7 +295,7 @@ async def fastapi_set_pipe_diameter( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/roughness", response_model=None, summary="设置管道粗糙度", description="设置指定管道的粗糙度") -async def fastapi_set_pipe_roughness( +def fastapi_set_pipe_roughness( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), roughness: float = Query(..., description="新的管道粗糙度值") @@ -314,7 +315,7 @@ async def fastapi_set_pipe_roughness( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/minor-loss", response_model=None, summary="设置管道局部阻力系数", description="设置指定管道的局部阻力系数") -async def fastapi_set_pipe_minor_loss( +def fastapi_set_pipe_minor_loss( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), minor_loss: float = Query(..., description="新的局部阻力系数值") @@ -334,7 +335,7 @@ async def fastapi_set_pipe_minor_loss( return set_pipe(network, ChangeSet(ps)) @router.patch("/pipes/status", response_model=None, summary="设置管道状态", description="设置指定管道的状态（开启或关闭）") -async def fastapi_set_pipe_status( +def fastapi_set_pipe_status( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), status: str = Query(..., description="新的管道状态（开启/关闭）") @@ -354,7 +355,7 @@ async def fastapi_set_pipe_status( return set_pipe(network, ChangeSet(ps)) @router.get("/pipes/properties", summary="获取管道属性", description="获取指定管道的所有属性信息") -async def fastapi_get_pipe_properties( +def fastapi_get_pipe_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID") ) -> dict[str, Any]: @@ -371,7 +372,7 @@ async def fastapi_get_pipe_properties( return get_pipe(network, pipe) @router.get("/pipes", summary="获取所有管道属性", description="获取网络中所有管道的属性信息列表") -async def fastapi_get_all_pipe_properties( +def fastapi_get_all_pipe_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -388,10 +389,10 @@ async def fastapi_get_all_pipe_properties( return results @router.patch("/pipes/properties", response_model=None, summary="设置管道属性", description="批量设置指定管道的多个属性") -async def fastapi_set_pipe_properties( +def fastapi_set_pipe_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pipe: str = Query(..., description="管道ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 批量设置管道属性。 @@ -404,6 +405,6 @@ async def fastapi_set_pipe_properties( Returns: ChangeSet对象，包含本次修改的变更信息 """ - props = await req.json() + props = payload ps = {"id": pipe} | props return set_pipe(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/network/pumps.py b/app/api/v1/endpoints/network/pumps.py index 3e6a85a..152aea2 100644 --- a/app/api/v1/endpoints/network/pumps.py +++ b/app/api/v1/endpoints/network/pumps.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_pump, @@ -13,7 +14,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/pump", summary="获取水泵模式", description="获取水泵对象的模式定义，包含所有可用字段及其类型") -async def fastapi_get_pump_schema( +def fastapi_get_pump_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """ @@ -28,7 +29,7 @@ async def fastapi_get_pump_schema( return get_pump_schema(network) @router.post("/pumps", response_model=None, summary="添加水泵", description="向网络中添加新的水泵，需要提供水泵的基本参数如功率等") -async def fastapi_add_pump( +def fastapi_add_pump( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵标识符"), node1: str = Query(..., description="水泵起始节点ID"), @@ -52,7 +53,7 @@ async def fastapi_add_pump( return add_pump(network, ChangeSet(ps)) @router.delete("/pumps", response_model=None, summary="删除水泵", description="从网络中删除指定的水泵") -async def fastapi_delete_pump( +def fastapi_delete_pump( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="要删除的水泵ID") ) -> ChangeSet: @@ -70,7 +71,7 @@ async def fastapi_delete_pump( return delete_pump(network, ChangeSet(ps)) @router.get("/pumps/node1", summary="获取水泵起始节点", description="获取指定水泵的起始节点ID") -async def fastapi_get_pump_node1( +def fastapi_get_pump_node1( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID") ) -> str | None: @@ -88,7 +89,7 @@ async def fastapi_get_pump_node1( return ps["node1"] @router.get("/pumps/node2", summary="获取水泵终止节点", description="获取指定水泵的终止节点ID") -async def fastapi_get_pump_node2( +def fastapi_get_pump_node2( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID") ) -> str | None: @@ -106,7 +107,7 @@ async def fastapi_get_pump_node2( return ps["node2"] @router.patch("/pumps/node1", response_model=None, summary="设置水泵起始节点", description="设置指定水泵的起始节点") -async def fastapi_set_pump_node1( +def fastapi_set_pump_node1( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID"), node1: str = Query(..., description="新的起始节点ID") @@ -126,7 +127,7 @@ async def fastapi_set_pump_node1( return set_pump(network, ChangeSet(ps)) @router.patch("/pumps/node2", response_model=None, summary="设置水泵终止节点", description="设置指定水泵的终止节点") -async def fastapi_set_pump_node2( +def fastapi_set_pump_node2( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID"), node2: str = Query(..., description="新的终止节点ID") @@ -146,7 +147,7 @@ async def fastapi_set_pump_node2( return set_pump(network, ChangeSet(ps)) @router.get("/pumps/properties", summary="获取水泵属性", description="获取指定水泵的所有属性信息") -async def fastapi_get_pump_properties( +def fastapi_get_pump_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID") ) -> dict[str, Any]: @@ -163,7 +164,7 @@ async def fastapi_get_pump_properties( return get_pump(network, pump) @router.get("/pumps", summary="获取所有水泵属性", description="获取网络中所有水泵的属性信息列表") -async def fastapi_get_all_pump_properties( +def fastapi_get_all_pump_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -180,10 +181,10 @@ async def fastapi_get_all_pump_properties( return results @router.patch("/pumps/properties", response_model=None, summary="设置水泵属性", description="批量设置指定水泵的多个属性") -async def fastapi_set_pump_properties( +def fastapi_set_pump_properties( network: str = Query(..., description="管网名称（或数据库名称）"), pump: str = Query(..., description="水泵ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 批量设置水泵属性。 @@ -196,6 +197,6 @@ async def fastapi_set_pump_properties( Returns: ChangeSet对象，包含本次修改的变更信息 """ - props = await req.json() + props = payload ps = {"id": pump} | props return set_pump(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/network/regions.py b/app/api/v1/endpoints/network/regions.py index 52ef2ee..f6cad07 100644 --- a/app/api/v1/endpoints/network/regions.py +++ b/app/api/v1/endpoints/network/regions.py @@ -1,6 +1,6 @@ from typing import Any -from fastapi import APIRouter, Query, Request +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, @@ -18,21 +18,21 @@ router = APIRouter() @router.get("/network-schemas/region", summary="获取区域属性架构") -async def get_region_schema_endpoint( +def get_region_schema_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), ) -> dict[str, dict[str, Any]]: return get_region_schema(network) @router.get("/regions", summary="获取区域列表") -async def get_regions_endpoint( +def get_regions_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), ) -> list[dict[str, Any]]: return [get_region(network, region_id) for region_id in get_regions(network)] @router.get("/regions/detail", summary="获取区域信息") -async def get_region_endpoint( +def get_region_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), id: str = Query(..., description="区域 ID"), ) -> dict[str, Any]: @@ -40,7 +40,7 @@ async def get_region_endpoint( @router.get("/regions/nodes", summary="获取区域节点") -async def get_region_nodes_endpoint( +def get_region_nodes_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), id: str = Query(..., description="区域 ID"), ) -> list[str]: @@ -48,26 +48,25 @@ async def get_region_nodes_endpoint( @router.patch("/regions", summary="修改区域", response_model=None) -async def set_region_endpoint( +def set_region_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), - request: Request = None, + payload: dict[str, Any] = Body(...), ) -> ChangeSet: - return set_region(network, ChangeSet(await request.json())) + return set_region(network, ChangeSet(payload)) @router.post("/regions", summary="添加区域", response_model=None) -async def add_region_endpoint( +def add_region_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), - request: Request = None, + payload: dict[str, Any] = Body(...), ) -> ChangeSet: - payload = await request.json() payload["boundary"] = [tuple(point[:2]) for point in payload.get("boundary", [])] return add_region(network, ChangeSet(payload)) @router.delete("/regions", summary="删除区域", response_model=None) -async def delete_region_endpoint( +def delete_region_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), - request: Request = None, + payload: dict[str, Any] = Body(...), ) -> ChangeSet: - return delete_region(network, ChangeSet(await request.json())) + return delete_region(network, ChangeSet(payload)) diff --git a/app/api/v1/endpoints/network/reservoirs.py b/app/api/v1/endpoints/network/reservoirs.py index 2b1d018..dcc905e 100644 --- a/app/api/v1/endpoints/network/reservoirs.py +++ b/app/api/v1/endpoints/network/reservoirs.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_reservoir, @@ -17,7 +18,7 @@ router = APIRouter() summary="获取水库模式", description="获取指定供水网络中所有水库的模式/属性字段定义" ) -async def fast_get_reservoir_schema( +def fast_get_reservoir_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """ @@ -39,7 +40,7 @@ async def fast_get_reservoir_schema( summary="添加水库", description="在指定供水网络中添加新的水库/水源节点" ) -async def fastapi_add_reservoir( +def fastapi_add_reservoir( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), x: float = Query(..., description="水库的X坐标"), @@ -70,7 +71,7 @@ async def fastapi_add_reservoir( summary="删除水库", description="从指定供水网络中删除指定的水库/水源节点" ) -async def fastapi_delete_reservoir( +def fastapi_delete_reservoir( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="要删除的水库的唯一标识符") ) -> ChangeSet: @@ -94,7 +95,7 @@ async def fastapi_delete_reservoir( summary="获取水库水头", description="获取指定水库的供水水头/总水头值" ) -async def fastapi_get_reservoir_head( +def fastapi_get_reservoir_head( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> float | None: @@ -118,7 +119,7 @@ async def fastapi_get_reservoir_head( summary="获取水库模式", description="获取指定水库的运行模式/供水模式" ) -async def fastapi_get_reservoir_pattern( +def fastapi_get_reservoir_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> str | None: @@ -142,7 +143,7 @@ async def fastapi_get_reservoir_pattern( summary="获取水库X坐标", description="获取指定水库的X坐标位置" ) -async def fastapi_get_reservoir_x( +def fastapi_get_reservoir_x( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> dict[str, float] | None: @@ -166,7 +167,7 @@ async def fastapi_get_reservoir_x( summary="获取水库Y坐标", description="获取指定水库的Y坐标位置" ) -async def fastapi_get_reservoir_y( +def fastapi_get_reservoir_y( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> dict[str, float] | None: @@ -190,7 +191,7 @@ async def fastapi_get_reservoir_y( summary="获取水库坐标", description="获取指定水库的平面坐标（X和Y坐标）" ) -async def fastapi_get_reservoir_coord( +def fastapi_get_reservoir_coord( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> dict[str, float] | None: @@ -216,7 +217,7 @@ async def fastapi_get_reservoir_coord( summary="设置水库水头", description="更新指定水库的供水水头/总水头值" ) -async def fastapi_set_reservoir_head( +def fastapi_set_reservoir_head( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), head: float = Query(..., description="新的水头值（米）") @@ -243,7 +244,7 @@ async def fastapi_set_reservoir_head( summary="设置水库模式", description="更新指定水库的运行模式/供水模式" ) -async def fastapi_set_reservoir_pattern( +def fastapi_set_reservoir_pattern( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), pattern: str = Query(..., description="新的运行模式") @@ -270,7 +271,7 @@ async def fastapi_set_reservoir_pattern( summary="设置水库X坐标", description="更新指定水库的X坐标位置" ) -async def fastapi_set_reservoir_x( +def fastapi_set_reservoir_x( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), x: float = Query(..., description="新的X坐标值") @@ -297,7 +298,7 @@ async def fastapi_set_reservoir_x( summary="设置水库Y坐标", description="更新指定水库的Y坐标位置" ) -async def fastapi_set_reservoir_y( +def fastapi_set_reservoir_y( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), y: float = Query(..., description="新的Y坐标值") @@ -324,7 +325,7 @@ async def fastapi_set_reservoir_y( summary="设置水库坐标", description="更新指定水库的平面坐标（X和Y坐标）" ) -async def fastapi_set_reservoir_coord( +def fastapi_set_reservoir_coord( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), x: float = Query(..., description="新的X坐标值"), @@ -352,7 +353,7 @@ async def fastapi_set_reservoir_coord( summary="获取水库属性", description="获取指定水库的所有属性" ) -async def fastapi_get_reservoir_properties( +def fastapi_get_reservoir_properties( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符") ) -> dict[str, Any]: @@ -375,7 +376,7 @@ async def fastapi_get_reservoir_properties( summary="获取所有水库属性", description="获取指定供水网络中所有水库的属性" ) -async def fastapi_get_all_reservoir_properties( +def fastapi_get_all_reservoir_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -398,10 +399,10 @@ async def fastapi_get_all_reservoir_properties( summary="设置水库属性", description="批量更新指定水库的多个属性" ) -async def fastapi_set_reservoir_properties( +def fastapi_set_reservoir_properties( network: str = Query(..., description="管网名称（或数据库名称）"), reservoir: str = Query(..., description="水库的唯一标识符"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 设置水库的多个属性。 @@ -416,6 +417,6 @@ async def fastapi_set_reservoir_properties( Returns: 包含操作变更集的ChangeSet对象 """ - props = await req.json() + props = payload ps = {"id": reservoir} | props return set_reservoir(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/network/tags.py b/app/api/v1/endpoints/network/tags.py index 83be913..6d110ba 100644 --- a/app/api/v1/endpoints/network/tags.py +++ b/app/api/v1/endpoints/network/tags.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, get_tag, @@ -19,7 +20,7 @@ router = APIRouter() summary="获取标签属性架构", description="获取指定水网的标签(Tag)属性架构定义" ) -async def fastapi_get_tag_schema( +def fastapi_get_tag_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """获取标签的属性架构。""" @@ -30,7 +31,7 @@ async def fastapi_get_tag_schema( summary="获取标签信息", description="获取指定类型和ID的标签信息" ) -async def fastapi_get_tag( +def fastapi_get_tag( network: str = Query(..., description="管网名称（或数据库名称）"), t_type: str = Query(..., description="标签类型"), id: str = Query(..., description="元素ID") @@ -43,7 +44,7 @@ async def fastapi_get_tag( summary="获取所有标签", description="获取指定水网中的所有标签信息" ) -async def fastapi_get_tags( +def fastapi_get_tags( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """获取水网中所有标签的列表。""" @@ -56,10 +57,10 @@ async def fastapi_get_tags( summary="设置标签", description="为指定元素设置或修改标签信息" ) -async def fastapi_set_tag( +def fastapi_set_tag( network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """设置标签信息。""" - props = await req.json() + props = payload return set_tag(network, ChangeSet(props)) diff --git a/app/api/v1/endpoints/network/tanks.py b/app/api/v1/endpoints/network/tanks.py index bc08fc6..dada464 100644 --- a/app/api/v1/endpoints/network/tanks.py +++ b/app/api/v1/endpoints/network/tanks.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, add_tank, @@ -13,7 +14,7 @@ from app.services.tjnetwork import ( router = APIRouter() @router.get("/network-schemas/tank", summary="获取水箱模式", description="获取指定网络的水箱数据结构模式定义") -async def fast_get_tank_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: +def fast_get_tank_schema(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, dict[str, Any]]: """ 获取水箱的数据结构模式。 @@ -26,7 +27,7 @@ async def fast_get_tank_schema(network: str = Query(..., description="管网名 return get_tank_schema(network) @router.post("/tanks", summary="新增水箱", description="向指定网络中新增一个水箱", response_model=None) -async def fastapi_add_tank( +def fastapi_add_tank( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), x: float = Query(..., description="X坐标"), @@ -70,7 +71,7 @@ async def fastapi_add_tank( return add_tank(network, ChangeSet(ps)) @router.delete("/tanks", summary="删除水箱", description="删除指定网络中的水箱", response_model=None) -async def fastapi_delete_tank( +def fastapi_delete_tank( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> ChangeSet: @@ -88,7 +89,7 @@ async def fastapi_delete_tank( return delete_tank(network, ChangeSet(ps)) @router.get("/tanks/elevation", summary="获取水箱标高", description="获取指定水箱的标高值") -async def fastapi_get_tank_elevation( +def fastapi_get_tank_elevation( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -106,7 +107,7 @@ async def fastapi_get_tank_elevation( return ps["elevation"] @router.get("/tanks/init-level", summary="获取水箱初始水位", description="获取指定水箱的初始水位值") -async def fastapi_get_tank_init_level( +def fastapi_get_tank_init_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -124,7 +125,7 @@ async def fastapi_get_tank_init_level( return ps["init_level"] @router.get("/tanks/min-level", summary="获取水箱最小水位", description="获取指定水箱的最小水位值") -async def fastapi_get_tank_min_level( +def fastapi_get_tank_min_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -142,7 +143,7 @@ async def fastapi_get_tank_min_level( return ps["min_level"] @router.get("/tanks/max-level", summary="获取水箱最大水位", description="获取指定水箱的最大水位值") -async def fastapi_get_tank_max_level( +def fastapi_get_tank_max_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -160,7 +161,7 @@ async def fastapi_get_tank_max_level( return ps["max_level"] @router.get("/tanks/diameter", summary="获取水箱直径", description="获取指定水箱的直径值") -async def fastapi_get_tank_diameter( +def fastapi_get_tank_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -178,7 +179,7 @@ async def fastapi_get_tank_diameter( return ps["diameter"] @router.get("/tanks/min-vol", summary="获取水箱最小体积", description="获取指定水箱的最小体积值") -async def fastapi_get_tank_min_vol( +def fastapi_get_tank_min_vol( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float | None: @@ -196,7 +197,7 @@ async def fastapi_get_tank_min_vol( return ps["min_vol"] @router.get("/tanks/vol-curve", summary="获取水箱容积曲线", description="获取指定水箱的容积曲线标识") -async def fastapi_get_tank_vol_curve( +def fastapi_get_tank_vol_curve( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> str | None: @@ -214,7 +215,7 @@ async def fastapi_get_tank_vol_curve( return ps["vol_curve"] @router.get("/tanks/overflow", summary="获取水箱溢流口", description="获取指定水箱的溢流口配置") -async def fastapi_get_tank_overflow( +def fastapi_get_tank_overflow( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> str | None: @@ -232,7 +233,7 @@ async def fastapi_get_tank_overflow( return ps["overflow"] @router.get("/tanks/x", summary="获取水箱X坐标", description="获取指定水箱的X坐标值") -async def fastapi_get_tank_x( +def fastapi_get_tank_x( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float: @@ -250,7 +251,7 @@ async def fastapi_get_tank_x( return ps["x"] @router.get("/tanks/y", summary="获取水箱Y坐标", description="获取指定水箱的Y坐标值") -async def fastapi_get_tank_y( +def fastapi_get_tank_y( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> float: @@ -268,7 +269,7 @@ async def fastapi_get_tank_y( return ps["y"] @router.get("/tanks/coord", summary="获取水箱坐标", description="获取指定水箱的X和Y坐标") -async def fastapi_get_tank_coord( +def fastapi_get_tank_coord( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> dict[str, float]: @@ -287,7 +288,7 @@ async def fastapi_get_tank_coord( return coord @router.patch("/tanks/elevation", summary="设置水箱标高", description="设置指定水箱的标高值", response_model=None) -async def fastapi_set_tank_elevation( +def fastapi_set_tank_elevation( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), elevation: float = Query(..., description="新的标高值") @@ -307,7 +308,7 @@ async def fastapi_set_tank_elevation( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/init-level", summary="设置水箱初始水位", description="设置指定水箱的初始水位值", response_model=None) -async def fastapi_set_tank_init_level( +def fastapi_set_tank_init_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), init_level: float = Query(..., description="新的初始水位值") @@ -327,7 +328,7 @@ async def fastapi_set_tank_init_level( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/min-level", summary="设置水箱最小水位", description="设置指定水箱的最小水位值", response_model=None) -async def fastapi_set_tank_min_level( +def fastapi_set_tank_min_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), min_level: float = Query(..., description="新的最小水位值") @@ -347,7 +348,7 @@ async def fastapi_set_tank_min_level( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/max-level", summary="设置水箱最大水位", description="设置指定水箱的最大水位值", response_model=None) -async def fastapi_set_tank_max_level( +def fastapi_set_tank_max_level( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), max_level: float = Query(..., description="新的最大水位值") @@ -367,7 +368,7 @@ async def fastapi_set_tank_max_level( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/diameter", summary="设置水箱直径", description="设置指定水箱的直径值", response_model=None) -async def fastapi_set_tank_diameter( +def fastapi_set_tank_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), diameter: float = Query(..., description="新的直径值") @@ -387,7 +388,7 @@ async def fastapi_set_tank_diameter( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/min-vol", summary="设置水箱最小体积", description="设置指定水箱的最小体积值", response_model=None) -async def fastapi_set_tank_min_vol( +def fastapi_set_tank_min_vol( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), min_vol: float = Query(..., description="新的最小体积值") @@ -407,7 +408,7 @@ async def fastapi_set_tank_min_vol( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/vol-curve", summary="设置水箱容积曲线", description="设置指定水箱的容积曲线标识", response_model=None) -async def fastapi_set_tank_vol_curve( +def fastapi_set_tank_vol_curve( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), vol_curve: str = Query(..., description="新的容积曲线标识") @@ -427,7 +428,7 @@ async def fastapi_set_tank_vol_curve( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/overflow", summary="设置水箱溢流口", description="设置指定水箱的溢流口配置", response_model=None) -async def fastapi_set_tank_overflow( +def fastapi_set_tank_overflow( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), overflow: str = Query(..., description="新的溢流口配置") @@ -447,7 +448,7 @@ async def fastapi_set_tank_overflow( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/x", summary="设置水箱X坐标", description="设置指定水箱的X坐标值", response_model=None) -async def fastapi_set_tank_x( +def fastapi_set_tank_x( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), x: float = Query(..., description="新的X坐标值") @@ -467,7 +468,7 @@ async def fastapi_set_tank_x( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/y", summary="设置水箱Y坐标", description="设置指定水箱的Y坐标值", response_model=None) -async def fastapi_set_tank_y( +def fastapi_set_tank_y( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), y: float = Query(..., description="新的Y坐标值") @@ -487,7 +488,7 @@ async def fastapi_set_tank_y( return set_tank(network, ChangeSet(ps)) @router.patch("/tanks/coord", summary="设置水箱坐标", description="设置指定水箱的X和Y坐标", response_model=None) -async def fastapi_set_tank_coord( +def fastapi_set_tank_coord( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), x: float = Query(..., description="新的X坐标值"), @@ -509,7 +510,7 @@ async def fastapi_set_tank_coord( return set_tank(network, ChangeSet(ps)) @router.get("/tanks/properties", summary="获取水箱属性", description="获取指定水箱的所有属性") -async def fastapi_get_tank_properties( +def fastapi_get_tank_properties( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID") ) -> dict[str, Any]: @@ -526,7 +527,7 @@ async def fastapi_get_tank_properties( return get_tank(network, tank) @router.get("/tanks", summary="获取所有水箱属性", description="获取指定网络中所有水箱的属性") -async def fastapi_get_all_tank_properties( +def fastapi_get_all_tank_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -543,10 +544,10 @@ async def fastapi_get_all_tank_properties( return results @router.patch("/tanks/properties", summary="设置水箱属性", description="批量设置指定水箱的多个属性", response_model=None) -async def fastapi_set_tank_properties( +def fastapi_set_tank_properties( network: str = Query(..., description="管网名称（或数据库名称）"), tank: str = Query(..., description="水箱ID"), - req: Request = None + payload: dict[str, Any] = Body(...) ) -> ChangeSet: """ 批量设置水箱的属性。 @@ -559,6 +560,6 @@ async def fastapi_set_tank_properties( Returns: 包含变更信息的ChangeSet对象 """ - props = await req.json() + props = payload ps = {"id": tank} | props return set_tank(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/network/valves.py b/app/api/v1/endpoints/network/valves.py index 550b46a..8a615cb 100644 --- a/app/api/v1/endpoints/network/valves.py +++ b/app/api/v1/endpoints/network/valves.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, Request, Query, Path, Body -from typing import Any, List, Dict, Union +from typing import Any + +from fastapi import APIRouter, Body, Query from app.services.tjnetwork import ( ChangeSet, VALVES_TYPE_PRV, @@ -18,7 +19,7 @@ router = APIRouter() summary="获取阀门架构", description="获取指定水网中所有阀门的架构和字段定义", ) -async def fastapi_get_valve_schema( +def fastapi_get_valve_schema( network: str = Query(..., description="管网名称（或数据库名称）") ) -> dict[str, dict[str, Any]]: """ @@ -34,7 +35,7 @@ async def fastapi_get_valve_schema( summary="添加阀门", description="在指定的水网中添加新的阀门", ) -async def fastapi_add_valve( +def fastapi_add_valve( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), node1: str = Query(..., description="起点节点ID"), @@ -67,7 +68,7 @@ async def fastapi_add_valve( summary="删除阀门", description="从指定的水网中删除指定的阀门", ) -async def fastapi_delete_valve( +def fastapi_delete_valve( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> ChangeSet: @@ -84,7 +85,7 @@ async def fastapi_delete_valve( summary="获取阀门起点节点", description="获取指定阀门连接的起点节点ID", ) -async def fastapi_get_valve_node1( +def fastapi_get_valve_node1( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> str | None: @@ -101,7 +102,7 @@ async def fastapi_get_valve_node1( summary="获取阀门终点节点", description="获取指定阀门连接的终点节点ID", ) -async def fastapi_get_valve_node2( +def fastapi_get_valve_node2( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> str | None: @@ -118,7 +119,7 @@ async def fastapi_get_valve_node2( summary="获取阀门直径", description="获取指定阀门的直径", ) -async def fastapi_get_valve_diameter( +def fastapi_get_valve_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> float | None: @@ -135,7 +136,7 @@ async def fastapi_get_valve_diameter( summary="获取阀门类型", description="获取指定阀门的类型", ) -async def fastapi_get_valve_type( +def fastapi_get_valve_type( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> str | None: @@ -152,7 +153,7 @@ async def fastapi_get_valve_type( summary="获取阀门开度", description="获取指定阀门的开度/设置值", ) -async def fastapi_get_valve_setting( +def fastapi_get_valve_setting( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> float | None: @@ -169,7 +170,7 @@ async def fastapi_get_valve_setting( summary="获取阀门损失系数", description="获取指定阀门的损失系数", ) -async def fastapi_get_valve_minor_loss( +def fastapi_get_valve_minor_loss( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> float | None: @@ -187,7 +188,7 @@ async def fastapi_get_valve_minor_loss( summary="设置阀门起点节点", description="设置指定阀门的起点节点", ) -async def fastapi_set_valve_node1( +def fastapi_set_valve_node1( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), node1: str = Query(..., description="新的起点节点ID"), @@ -206,7 +207,7 @@ async def fastapi_set_valve_node1( summary="设置阀门终点节点", description="设置指定阀门的终点节点", ) -async def fastapi_set_valve_node2( +def fastapi_set_valve_node2( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), node2: str = Query(..., description="新的终点节点ID"), @@ -225,7 +226,7 @@ async def fastapi_set_valve_node2( summary="设置阀门直径", description="设置指定阀门的直径", ) -async def fastapi_set_valve_diameter( +def fastapi_set_valve_diameter( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), diameter: float = Query(..., description="新的直径值（mm）"), @@ -244,7 +245,7 @@ async def fastapi_set_valve_diameter( summary="设置阀门类型", description="设置指定阀门的类型", ) -async def fastapi_set_valve_type( +def fastapi_set_valve_type( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), type: str = Query(..., description="新的阀门类型"), @@ -263,7 +264,7 @@ async def fastapi_set_valve_type( summary="设置阀门开度", description="设置指定阀门的开度/设置值", ) -async def fastapi_set_valve_setting( +def fastapi_set_valve_setting( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), setting: float = Query(..., description="新的开度值"), @@ -281,7 +282,7 @@ async def fastapi_set_valve_setting( summary="获取阀门所有属性", description="获取指定阀门的所有属性", ) -async def fastapi_get_valve_properties( +def fastapi_get_valve_properties( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), ) -> dict[str, Any]: @@ -297,7 +298,7 @@ async def fastapi_get_valve_properties( summary="获取所有阀门属性", description="获取指定水网中所有阀门的属性", ) -async def fastapi_get_all_valve_properties( +def fastapi_get_all_valve_properties( network: str = Query(..., description="管网名称（或数据库名称）") ) -> list[dict[str, Any]]: """ @@ -315,16 +316,16 @@ async def fastapi_get_all_valve_properties( summary="批量设置阀门属性", description="批量设置指定阀门的多个属性", ) -async def fastapi_set_valve_properties( +def fastapi_set_valve_properties( network: str = Query(..., description="管网名称（或数据库名称）"), valve: str = Query(..., description="阀门ID"), - req: Request = None, + payload: dict[str, Any] = Body(...), ) -> ChangeSet: """ 批量设置阀门的属性。 更新指定阀门的一个或多个属性，通过JSON请求体传递要更新的属性。 """ - props = await req.json() + props = payload ps = {"id": valve} | props return set_valve(network, ChangeSet(ps)) diff --git a/app/api/v1/endpoints/project.py b/app/api/v1/endpoints/project.py index 4e5ce40..24bf2f5 100644 --- a/app/api/v1/endpoints/project.py +++ b/app/api/v1/endpoints/project.py @@ -1,43 +1,18 @@ import json -from fastapi import APIRouter, Request, HTTPException, Query, Path, Depends -from fastapi.responses import PlainTextResponse -from typing import Any, Dict, List +from fastapi import APIRouter, HTTPException, Query, Depends from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository -from app.auth.project_dependencies import get_metadata_repository -from app.auth.permissions import ( - ENVIRONMENT_MANAGE, - require_permission, +from app.auth.project_dependencies import ( + get_metadata_repository, ) from app.domain.schemas.metadata import ProjectMetaResponse -import app.services.project_info as project_info from app.services.tjnetwork import ( ChangeSet, - list_project, - have_project, - create_project, - delete_project, - is_project_open, - open_project, - close_project, - copy_project, export_inp, - read_inp, - dump_inp, get_all_vertices, get_all_scada_info, - convert_inp_v3_to_v2, ) -# For inp file upload/download -import os -from fastapi import Response, status -from fastapi.responses import FileResponse -inpDir = "data/" # Assuming data directory exists or is defined somewhere. -# In main.py it was likely global. For safety, let's use a relative path or get from config. -# But let's stick to what main.py probably used or a default. - router = APIRouter() -lockedPrjs: Dict[str, str] = {} @router.get("/projects/current", summary="获取项目信息", description="从数据库获取项目的详细信息，包括地图范围等。", response_model=ProjectMetaResponse) async def get_project_info_endpoint( @@ -63,105 +38,8 @@ async def get_project_info_endpoint( project_role="viewer", # Default role for public access ) -@router.get("/project-codes", summary="获取项目列表", description="获取服务器上所有可用的供水管网项目名称列表。") -async def list_projects_endpoint() -> list[str]: - """ - 获取项目列表 - - 返回所有已创建项目的名称列表。 - """ - return list_project() - -@router.get("/projects/existence", summary="检查项目是否存在", description="检查指定名称的项目是否存在。") -async def have_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）") -): - """ - 检查项目是否存在 - - - **network**: 管网名称（或数据库名称） - """ - return have_project(network) - -@router.post("/projects", summary="创建新项目", description="创建一个新的供水管网项目。如果项目已存在，可能会覆盖或报错（取决于底层实现）。") -async def create_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - _=Depends(require_permission(ENVIRONMENT_MANAGE)), -): - """ - 创建新项目 - - - **network**: 管网名称（或数据库名称） - """ - create_project(network) - return network - -@router.delete("/projects", summary="删除项目", description="永久删除指定的供水管网项目。此操作不可恢复。") -async def delete_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - _=Depends(require_permission(ENVIRONMENT_MANAGE)), -): - """ - 删除项目 - - - **network**: 管网名称（或数据库名称） - """ - delete_project(network) - return True - -@router.get("/projects/current/status", summary="检查项目是否已打开", description="检查指定项目是否已被加载到内存中。") -async def is_project_open_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）") -): - """ - 检查项目是否已打开 - - - **network**: 管网名称（或数据库名称） - """ - return is_project_open(network) - -@router.post("/projects/current", summary="打开项目", description="将指定项目加载到内存中，并初始化数据库连接池。") -async def open_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）") -): - """ - 打开项目 - - - **network**: 管网名称（或数据库名称） - """ - open_project(network) - - return network - -@router.delete("/projects/current", summary="关闭项目", description="将指定项目从内存中卸载，释放资源。") -async def close_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）") -): - """ - 关闭项目 - - - **network**: 管网名称（或数据库名称） - """ - close_project(network) - return True - -@router.post("/project-copies", summary="复制项目", description="将现有项目复制为新项目。") -async def copy_project_endpoint( - source: str = Query(..., description="管网名称（或数据库名称）"), - target: str = Query(..., description="管网名称（或数据库名称）"), - _=Depends(require_permission(ENVIRONMENT_MANAGE)), -): - """ - 复制项目 - - - **source**: 管网名称（或数据库名称） - - **target**: 管网名称（或数据库名称） - """ - copy_project(source, target) - return True - @router.get("/projects/current/exports/change-set", response_model=None, summary="导出项目为 ChangeSet", description="导出项目的变更集 (ChangeSet)，包含顶点、SCADA 元素、DMA、SA、VD 等信息。") -async def export_inp_endpoint( +def export_inp_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), version: str = Query(..., description="版本号 (通常用于增量更新)") ) -> ChangeSet: @@ -173,279 +51,7 @@ async def export_inp_endpoint( """ cs = export_inp(network, version) op = cs.operations[0] - open_project(network) op["vertex"] = json.dumps(get_all_vertices(network)) op["scada"] = json.dumps(get_all_scada_info(network)) - close_project(network) - - return cs -@router.post("/projects/current/imports", summary="读取 INP 文件到项目", description="从服务器文件系统中读取指定的 INP 文件并加载到项目中。") -async def read_inp_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - inp: str = Query(..., description="INP 文件名 (不包含路径)") -) -> bool: - """ - 读取 INP 文件到项目 - - - **network**: 管网名称（或数据库名称） - - **inp**: INP 文件名 - """ - read_inp(network, inp) - return True - -@router.post("/projects/current/exports/inp", summary="导出项目到 INP 文件", description="将项目当前状态保存为 INP 文件到服务器文件系统。") -async def dump_inp_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - inp: str = Query(..., description="目标文件名") -) -> bool: - """ - 导出项目到 INP 文件 - - - **network**: 管网名称（或数据库名称） - - **inp**: 目标文件名 - """ - dump_inp(network, inp) - return True - -@router.get("/projects/current/lock", summary="检查项目是否被锁定", description="检查指定项目是否处于锁定状态。") -async def is_project_locked_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 检查项目是否被锁定 - - - **network**: 管网名称（或数据库名称） - """ - return network in lockedPrjs.keys() - -@router.get("/projects/current/lock/ownership", summary="检查项目是否被当前用户锁定", description="检查指定项目是否被当前访问地址 (IP) 锁定。") -async def is_project_locked_by_me_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 检查项目是否被当前用户锁定 - - - **network**: 管网名称（或数据库名称） - """ - client_host = req.client.host - return lockedPrjs.get(network) == client_host - -# 0 successfully locked -# 1 already locked by you -# 2 locked by others -@router.post("/projects/current/lock", summary="锁定项目", description="锁定指定项目以防止并发修改。") -async def lock_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 锁定项目 - - 返回值: - - **0**: 锁定成功 - - **1**: 已被当前用户锁定 - - **2**: 已被其他用户锁定 - """ - client_host = req.client.host - if not network in lockedPrjs.keys(): - lockedPrjs[network] = client_host - return 0 - else: - if lockedPrjs.get(network) == client_host: - return 1 - else: - return 2 - -@router.delete("/projects/current/lock", summary="解锁项目", description="释放对项目的锁定。") -def unlock_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 解锁项目 - - 只有锁定者才能解锁。 - """ - client_host = req.client.host - if lockedPrjs.get(network) == client_host: - print("delete key") - del lockedPrjs[network] - return True - - return False - -@router.get("/projects/current/files/inp", status_code=status.HTTP_200_OK, summary="下载 INP 文件", description="从服务器数据目录下载指定的 INP 文件。") -async def fastapi_download_inp( - name: str = Query(..., description="文件名"), - response: Response = None -): - """ - 下载 INP 文件 - - - **name**: 文件名 - """ - filePath = inpDir + name - if os.path.exists(filePath): - return FileResponse( - filePath, media_type="application/octet-stream", filename="inp.inp" - ) - else: - response.status_code = status.HTTP_400_BAD_REQUEST - return True - -# DingZQ, 2024-12-28, convert v3 to v2 -@router.post("/project-conversions", response_model=None, summary="转换 INP V3 为 V2", description="将 EPANET 3.0 格式的 INP 内容转换为 2.x 格式。") -async def fastapi_convert_v3_to_v2( - req: Request -) -> ChangeSet: - """ - 转换 INP V3 为 V2 - - - **req**: 请求体，需包含 `{"inp": "..."}` 结构 - """ - network = "v3Tov2" - jo_root = await req.json() - inp = jo_root["inp"] - cs = convert_inp_v3_to_v2(inp) - op = cs.operations[0] - open_project(network) - op["vertex"] = json.dumps(get_all_vertices(network)) - op["scada"] = json.dumps(get_all_scada_info(network)) - - close_project(network) - - return cs - -async def read_inp_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - inp: str = Query(..., description="INP 文件名 (不包含路径)") -) -> bool: - """ - 读取 INP 文件到项目 - - - **network**: 管网名称（或数据库名称） - - **inp**: INP 文件名 - """ - read_inp(network, inp) - return True - -async def dump_inp_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - inp: str = Query(..., description="目标文件名") -) -> bool: - """ - 导出项目到 INP 文件 - - - **network**: 管网名称（或数据库名称） - - **inp**: 目标文件名 - """ - dump_inp(network, inp) - return True - -async def is_project_locked_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 检查项目是否被锁定 - - - **network**: 管网名称（或数据库名称） - """ - return network in lockedPrjs.keys() - -async def is_project_locked_by_me_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 检查项目是否被当前用户锁定 - - - **network**: 管网名称（或数据库名称） - """ - client_host = req.client.host - return lockedPrjs.get(network) == client_host - -# 0 successfully locked -# 1 already locked by you -# 2 locked by others -async def lock_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 锁定项目 - - 返回值: - - **0**: 锁定成功 - - **1**: 已被当前用户锁定 - - **2**: 已被其他用户锁定 - """ - client_host = req.client.host - if not network in lockedPrjs.keys(): - lockedPrjs[network] = client_host - return 0 - else: - if lockedPrjs.get(network) == client_host: - return 1 - else: - return 2 - -def unlock_project_endpoint( - network: str = Query(..., description="管网名称（或数据库名称）"), - req: Request = None -): - """ - 解锁项目 - - 只有锁定者才能解锁。 - """ - client_host = req.client.host - if lockedPrjs.get(network) == client_host: - print("delete key") - del lockedPrjs[network] - return True - - return False - -async def fastapi_download_inp( - name: str = Query(..., description="文件名"), - response: Response = None -): - """ - 下载 INP 文件 - - - **name**: 文件名 - """ - filePath = inpDir + name - if os.path.exists(filePath): - return FileResponse( - filePath, media_type="application/octet-stream", filename="inp.inp" - ) - else: - response.status_code = status.HTTP_400_BAD_REQUEST - return True - -# DingZQ, 2024-12-28, convert v3 to v2 -async def fastapi_convert_v3_to_v2( - req: Request -) -> ChangeSet: - """ - 转换 INP V3 为 V2 - - - **req**: 请求体，需包含 `{"inp": "..."}` 结构 - """ - network = "v3Tov2" - jo_root = await req.json() - inp = jo_root["inp"] - cs = convert_inp_v3_to_v2(inp) - op = cs.operations[0] - open_project(network) - op["vertex"] = json.dumps(get_all_vertices(network)) - op["scada"] = json.dumps(get_all_scada_info(network)) - - close_project(network) - return cs diff --git a/app/api/v1/endpoints/sensor_placement.py b/app/api/v1/endpoints/sensor_placement.py index ba3bf16..e2f0861 100644 --- a/app/api/v1/endpoints/sensor_placement.py +++ b/app/api/v1/endpoints/sensor_placement.py @@ -12,7 +12,12 @@ from app.algorithms.sensor import ( pressure_sensor_placement_sensitivity, ) from app.auth.metadata_dependencies import get_current_metadata_user -from app.auth.project_dependencies import ProjectContext, get_project_context +from app.auth.project_dependencies import ( + ProjectContext, + get_project_context, + use_project_business_routing, +) +from app.infra.db.project_routing import ActiveProjectRouting from app.domain.schemas.sensor_placement import ( SensorPointResponse, SensorPlacementExportRequest, @@ -106,6 +111,7 @@ def _get_run_response( async def get_sensor_placement_candidate_detail( node_id: str = Path(..., min_length=1, max_length=32), project_context: ProjectContext = Depends(get_project_context), + _routing: ActiveProjectRouting = Depends(use_project_business_routing), ) -> dict[str, Any]: try: return await run_in_threadpool( @@ -166,6 +172,7 @@ async def optimize_sensor_placement_scheme( ) async def get_sensor_placement_runs( project_context: ProjectContext = Depends(get_project_context), + _routing: ActiveProjectRouting = Depends(use_project_business_routing), ) -> list[dict[str, Any]]: return await run_in_threadpool( list_sensor_placement_runs, diff --git a/app/api/v1/endpoints/simulation.py b/app/api/v1/endpoints/simulation.py index 17da470..ff200ba 100644 --- a/app/api/v1/endpoints/simulation.py +++ b/app/api/v1/endpoints/simulation.py @@ -127,7 +127,7 @@ def run_simulation_manually_by_date( # 必须用这个PlainTextResponse，不然每个key都有引号 @router.post("/project-runs", response_class=PlainTextResponse, summary="运行项目模拟", description="基于指定的管网项目运行标准水力模拟，返回纯文本格式的模拟报告。") -async def run_project_endpoint(network: str = Query(..., description="管网名称（或数据库名称）")) -> str: +def run_project_endpoint(network: str = Query(..., description="管网名称（或数据库名称）")) -> str: """ 运行项目模拟 @@ -143,7 +143,7 @@ async def run_project_endpoint(network: str = Query(..., description="管网名 # output 是 json # report 是 text @router.post("/project-return-dict-runs", summary="运行项目模拟（返回字典）", description="基于指定的管网项目运行标准水力模拟，返回JSON格式的字典，包含输出数据和报告文本。") -async def run_project_return_dict_endpoint(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: +def run_project_return_dict_endpoint(network: str = Query(..., description="管网名称（或数据库名称）")) -> dict[str, Any]: """ 运行项目模拟（返回字典） @@ -160,7 +160,7 @@ async def run_project_return_dict_endpoint(network: str = Query(..., description # put in inp folder, name without extension @router.post("/inp-runs", summary="运行INP文件", description="运行指定INP文件格式的管网模型进行水力模拟。INP文件应该放在inp文件夹中，参数为文件名不含扩展名。") -async def run_inp_endpoint(network: str = Query(..., description="inp文件名（不含扩展名）")) -> str: +def run_inp_endpoint(network: str = Query(..., description="inp文件名（不含扩展名）")) -> str: """ 运行INP文件 @@ -173,7 +173,7 @@ async def run_inp_endpoint(network: str = Query(..., description="inp文件名 # path is absolute path @router.get("/outputs", summary="导出模拟输出", description="导出指定路径的模拟输出文件内容。参数应为绝对路径。") -async def dump_output_endpoint(output: str = Query(..., description="模拟输出文件的绝对路径")) -> str: +def dump_output_endpoint(output: str = Query(..., description="模拟输出文件的绝对路径")) -> str: """ 导出模拟输出 @@ -186,7 +186,7 @@ async def dump_output_endpoint(output: str = Query(..., description="模拟输 # Analysis Endpoints @router.post("/burst-analyses", summary="爆管分析（高级）", description="高级版本的爆管分析，支持在指定时间点修改泵控制模式和阀门开度，以分析这些改变对爆管影响的作用。支持固定泵和变速泵的独立控制。") -async def fastapi_burst_analysis( +def fastapi_burst_analysis( network: str = Query(..., description="管网名称（或数据库名称）"), modify_pattern_start_time: str = Query(..., description="模式修改开始时间（ISO 8601格式）"), burst_ID: list[str] = Query(..., description="爆管节点/管段ID列表"), @@ -220,7 +220,7 @@ async def fastapi_burst_analysis( @router.post("/valve-closure-analyses", response_class=PlainTextResponse, summary="阀门关闭分析（高级）", description="高级版本的阀门关闭分析，支持同时关闭多个阀门，并在指定持续时间内进行模拟。返回纯文本格式的分析结果。") -async def fastapi_valve_close_analysis( +def fastapi_valve_close_analysis( network: str = Query(..., description="管网名称（或数据库名称）"), start_time: str = Query(..., description="阀门关闭开始时间（ISO 8601格式）"), valves: List[str] = Query(..., description="要关闭的阀门ID列表"), @@ -249,7 +249,7 @@ async def fastapi_valve_close_analysis( @router.post("/valve-isolation-analyses", summary="阀门隔离分析", description="分析当发生突发事件时，通过关闭指定阀门进行隔离，确定哪些阀门必须关闭、哪些可选关闭，以及隔离的可行性。") -async def valve_isolation_endpoint( +def valve_isolation_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), accident_element: List[str] = Query(..., description="发生事故的管段/节点ID列表"), disabled_valves: List[str] = Query(None, description="已故障的阀门ID列表（可选）"), @@ -290,7 +290,7 @@ async def valve_isolation_endpoint( @router.post("/flushing-analyses", response_class=PlainTextResponse, summary="冲洗分析（高级）", description="高级版本的冲洗分析，支持按状态和设置值控制多个可选阀门，指定排污节点，并设置固定的冲洗流量。返回纯文本格式的分析结果。") -async def fastapi_flushing_analysis( +def fastapi_flushing_analysis( network: str = Query(..., description="管网名称（或数据库名称）"), start_time: str = Query(..., description="冲洗开始时间（ISO 8601格式）"), valves: List[str] | None = Query(None, description="参与控制的阀门ID列表（可选）"), @@ -403,7 +403,7 @@ async def fastapi_flushing_analysis( @router.post("/contaminant-simulations", response_class=PlainTextResponse, summary="污染物模拟", description="对管网中的污染物扩散进行模拟，评估污染源对管网的影响范围和浓度分布。支持指定污染源位置、污染浓度和扩散模式。") -async def fastapi_contaminant_simulation( +def fastapi_contaminant_simulation( network: str = Query(..., description="管网名称（或数据库名称）"), start_time: str = Query(..., description="污染开始时间（ISO 8601格式）"), source: str = Query(..., description="污染源节点ID"), @@ -440,7 +440,7 @@ async def fastapi_contaminant_simulation( @router.post("/water-age-analyses", response_class=PlainTextResponse, summary="水龄分析（高级）", description="高级版本的水龄分析，在指定时间点进行分析，支持自定义模拟持续时间。返回纯文本格式的分析结果。") -async def fastapi_age_analysis( +def fastapi_age_analysis( network: str = Query(..., description="管网名称（或数据库名称）"), start_time: str = Query(..., description="分析开始时间（ISO 8601格式）"), duration: int = Query(..., description="模拟持续时间（秒）"), @@ -464,7 +464,7 @@ async def fastapi_age_analysis( @router.post("/pressure-regulation-calculations", summary="压力调节（基础）", description="对管网的压力进行调节分析，通过控制泵的运行来维持目标节点的目标压力。此为基础版本。") -async def pressure_regulation_endpoint( +def pressure_regulation_endpoint( network: str = Query(..., description="管网名称（或数据库名称）"), target_node: str = Query(..., description="目标节点ID"), target_pressure: float = Query(..., description="目标压力值（kPa）"), @@ -482,7 +482,7 @@ async def pressure_regulation_endpoint( @router.post("/pressure-regulation-analyses", summary="压力调节（高级）", description="高级版本的压力调节分析，通过JSON请求体提供详细的控制参数，包括固定泵和变速泵的独立控制、水箱初始水位等。") -async def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description="压力调节控制参数")) -> str: +def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description="压力调节控制参数")) -> str: """ 压力调节（高级版本） @@ -496,7 +496,7 @@ async def fastapi_pressure_regulation(data: PressureRegulation = Body(..., descr 支持固定泵和变速泵的独立控制。 """ - item = data.dict() + item = data.model_dump() simulation.query_corresponding_element_id_and_query_id(item["network"]) fixed_pumps = set(globals.fixed_pumps_id.keys()) variable_pumps = set(globals.variable_pumps_id.keys()) @@ -520,7 +520,7 @@ async def fastapi_pressure_regulation(data: PressureRegulation = Body(..., descr @router.post("/project-managements", summary="项目管理（高级）", description="高级版本的项目管理，通过JSON请求体提供详细的控制参数，包括泵控制策略、水箱初始水位和区域需水量控制。") -async def fastapi_project_management(data: ProjectManagement = Body(..., description="项目管理控制参数")) -> str: +def fastapi_project_management(data: ProjectManagement = Body(..., description="项目管理控制参数")) -> str: """ 项目管理（高级版本） @@ -533,7 +533,7 @@ async def fastapi_project_management(data: ProjectManagement = Body(..., descrip 支持多维度的项目管理。 """ - item = data.dict() + item = data.model_dump() return project_management( prj_name=item["network"], start_datetime=item["start_time"], @@ -549,7 +549,7 @@ async def fastapi_project_management(data: ProjectManagement = Body(..., descrip @router.post("/scheduling-analyses", summary="排程分析", description="对管网的供水排程进行分析，优化泵的运行时间和出水流量，平衡水厂出水、水箱进出水，满足用户需求。") -async def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., description="排程分析参数")) -> str: +def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., description="排程分析参数")) -> str: """ 排程分析 @@ -563,7 +563,7 @@ async def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., descr 用于优化供水排程。 """ - item = data.dict() + item = data.model_dump() return scheduling_simulation( item["network"], item["start_time"], @@ -575,7 +575,7 @@ async def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., descr @router.post("/daily-scheduling-analyses", summary="日排程分析", description="对管网的每日供水排程进行分析，优化水库、水厂、水箱和用户需求的协调，制定合理的每日排程方案。") -async def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body(..., description="日排程分析参数")) -> str: +def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body(..., description="日排程分析参数")) -> str: """ 日排程分析 @@ -590,7 +590,7 @@ async def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body 用于制定每日供水排程方案。 """ - item = data.dict() + item = data.model_dump() return daily_scheduling_simulation( item["network"], item["start_time"], @@ -607,7 +607,7 @@ async def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body @router.post("/pump-failure-events", summary="泵故障管理", description="记录和管理泵的故障状态，包括故障发生时间和受影响的泵列表。系统将记录故障日志并更新泵状态。") -async def fastapi_pump_failure(data: PumpFailureState = Body(..., description="泵故障状态信息")) -> str: +def fastapi_pump_failure(data: PumpFailureState = Body(..., description="泵故障状态信息")) -> str: """ 泵故障管理 @@ -617,7 +617,7 @@ async def fastapi_pump_failure(data: PumpFailureState = Body(..., description=" 系统将验证泵信息的有效性并更新故障状态文件。 """ - item = data.dict() + item = data.model_dump() with open("./pump_failure_message.txt", "a", encoding="utf-8-sig") as f1: f1.write("[{}] {}\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), item)) with open("./pump_failure_status.txt", "r", encoding="utf-8-sig") as f2: @@ -651,7 +651,7 @@ async def fastapi_pump_failure(data: PumpFailureState = Body(..., description=" @router.post("/simulation-runs", summary="手动运行日期指定模拟", description="根据指定的开始时间和持续时间，手动运行水力模拟。开始时间必须是显式带时区的 ISO 8601 / RFC3339 时间。") -async def fastapi_run_simulation_manually_by_date( +def fastapi_run_simulation_manually_by_date( data: RunSimulationManuallyByDate = Body(..., description="模拟运行参数"), ) -> dict[str, str]: """ diff --git a/app/api/v1/endpoints/timeseries/realtime.py b/app/api/v1/endpoints/timeseries/realtime.py index 705d27a..62d8277 100644 --- a/app/api/v1/endpoints/timeseries/realtime.py +++ b/app/api/v1/endpoints/timeseries/realtime.py @@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body from typing import List from datetime import datetime from psycopg import AsyncConnection +from pydantic import BaseModel from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository from .dependencies import get_timescale_connection @@ -13,9 +14,31 @@ TIME_RANGE_START_DESC = f"时间范围开始时间。{TIME_WITH_TZ_DESC}" TIME_RANGE_END_DESC = f"时间范围结束时间。{TIME_WITH_TZ_DESC}" +class RealtimeLinkBatchItem(BaseModel): + time: datetime + id: str + flow: float | None = None + friction: float | None = None + headloss: float | None = None + quality: float | None = None + reaction: float | None = None + setting: float | None = None + status: float | None = None + velocity: float | None = None + + +class RealtimeNodeBatchItem(BaseModel): + time: datetime + id: str + actual_demand: float | None = None + total_head: float | None = None + pressure: float | None = None + quality: float | None = None + + @router.post("/timeseries/realtime/links/batches", status_code=201, summary="批量插入实时管道数据") async def insert_realtime_links( - data: List[dict] = Body(..., description="管道数据列表，每项包含管道ID、时间戳等信息"), + data: List[RealtimeLinkBatchItem] = Body(..., description="同一时间点的管道快照数据"), conn: AsyncConnection = Depends(get_timescale_connection) ): """ @@ -29,7 +52,9 @@ async def insert_realtime_links( Returns: 插入成功的记录数 """ - await RealtimeRepository.insert_links_batch(conn, data) + await RealtimeRepository.insert_links_batch( + conn, [item.model_dump() for item in data] + ) return {"message": f"Inserted {len(data)} records"} @@ -119,7 +144,7 @@ async def update_realtime_link_field( @router.post("/timeseries/realtime/nodes/batches", status_code=201, summary="批量插入实时节点数据") async def insert_realtime_nodes( - data: List[dict] = Body(..., description="节点数据列表，每项包含节点ID、时间戳等信息"), + data: List[RealtimeNodeBatchItem] = Body(..., description="同一时间点的节点快照数据"), conn: AsyncConnection = Depends(get_timescale_connection) ): """ @@ -133,7 +158,9 @@ async def insert_realtime_nodes( Returns: 插入成功的记录数 """ - await RealtimeRepository.insert_nodes_batch(conn, data) + await RealtimeRepository.insert_nodes_batch( + conn, [item.model_dump() for item in data] + ) return {"message": f"Inserted {len(data)} records"} diff --git a/app/api/v1/rest_router.py b/app/api/v1/rest_router.py index 941523a..d5bfa12 100644 --- a/app/api/v1/rest_router.py +++ b/app/api/v1/rest_router.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, Query from fastapi.encoders import jsonable_encoder from fastapi.routing import APIRoute from pydantic import BaseModel, JsonValue, create_model +from starlette.concurrency import run_in_threadpool from starlette.responses import Response from app.api.problem_details import ProblemDetails @@ -38,6 +39,14 @@ class Page(BaseModel, Generic[T]): offset: int +async def _call_endpoint(endpoint, *args, **kwargs): + """Call async handlers directly and offload synchronous handlers.""" + if inspect.iscoroutinefunction(endpoint): + return await endpoint(*args, **kwargs) + result = await run_in_threadpool(endpoint, *args, **kwargs) + return await result if inspect.isawaitable(result) else result + + _NAME_IS_NETWORK = { "pressure_sensor_placement_sensitivity_endpoint", "pressure_sensor_placement_kmeans_endpoint", @@ -59,7 +68,6 @@ _TIMESCALE_ROUTED_ENDPOINT_MODULES = { "app.api.v1.endpoints.leakage", "app.api.v1.endpoints.simulation", } -_TIMESCALE_ROUTED_ENDPOINT_NAMES = {"open_project_endpoint"} def _clean_name(name: str) -> str: @@ -183,10 +191,7 @@ def _with_header_project_context(endpoint, route_name: str): if model_has_username: kwargs.pop(injected_user_name, None) with activate_project_routing(project_routing): - result = endpoint(*args, **kwargs) - if inspect.isawaitable(result): - return await result - return result + return await _call_endpoint(endpoint, *args, **kwargs) parameters = [] for name, parameter in signature.parameters.items(): @@ -206,7 +211,6 @@ def _with_header_project_context(endpoint, route_name: str): get_project_simulation_routing if ( endpoint.__module__ in _TIMESCALE_ROUTED_ENDPOINT_MODULES - or endpoint.__name__ in _TIMESCALE_ROUTED_ENDPOINT_NAMES ) else get_project_business_routing ) @@ -262,9 +266,7 @@ def _with_pagination(endpoint): else: limit = kwargs.pop("_rest_limit") offset = kwargs.pop("_rest_offset") - result = endpoint(*args, **kwargs) - if inspect.isawaitable(result): - result = await result + result = await _call_endpoint(endpoint, *args, **kwargs) if not isinstance(result, list): return result if handler_handles_pagination: @@ -313,9 +315,7 @@ def _with_jsonable_response(endpoint): @wraps(endpoint) async def wrapper(*args, **kwargs): - result = endpoint(*args, **kwargs) - if inspect.isawaitable(result): - result = await result + result = await _call_endpoint(endpoint, *args, **kwargs) if isinstance(result, Response): return result return jsonable_encoder(result) diff --git a/app/auth/project_dependencies.py b/app/auth/project_dependencies.py index 85895ff..23ca5c7 100644 --- a/app/auth/project_dependencies.py +++ b/app/auth/project_dependencies.py @@ -16,7 +16,7 @@ from app.infra.db.metadb.repositories.metadata_repository import ( MetadataRepository, ProjectDbRouting, ) -from app.infra.db.project_routing import ActiveProjectRouting +from app.infra.db.project_routing import ActiveProjectRouting, activate_project_routing DB_ROLE_BIZ_DATA = "biz_data" DB_ROLE_IOT_DATA = "iot_data" @@ -107,6 +107,14 @@ async def get_project_business_routing( return await resolve_project_business_routing(ctx, metadata_repo) +async def use_project_business_routing( + routing: ActiveProjectRouting = Depends(get_project_business_routing), +) -> AsyncGenerator[ActiveProjectRouting, None]: + """Keep the routed BizDB active for an entire endpoint invocation.""" + with activate_project_routing(routing): + yield routing + + async def resolve_project_business_routing( ctx: ProjectContext, metadata_repo: MetadataRepository, @@ -189,31 +197,6 @@ async def _get_project_routing( return routing -async def get_project_pg_session( - ctx: ProjectContext = Depends(get_project_context), - metadata_repo: MetadataRepository = Depends(get_metadata_repository), -) -> AsyncGenerator[AsyncSession, None]: - routing = await _get_project_routing( - metadata_repo, - ctx.project_id, - DB_ROLE_BIZ_DATA, - DB_TYPE_POSTGRES, - "PostgreSQL", - ) - - pool_min_size = routing.pool_min_size or settings.PROJECT_PG_POOL_SIZE - pool_max_size = routing.pool_max_size or settings.PROJECT_PG_POOL_SIZE - sessionmaker = await project_connection_manager.get_pg_sessionmaker( - ctx.project_id, - DB_ROLE_BIZ_DATA, - routing.dsn, - pool_min_size, - pool_max_size, - ) - async with sessionmaker() as session: - yield session - - async def get_project_pg_connection( ctx: ProjectContext = Depends(get_project_context), metadata_repo: MetadataRepository = Depends(get_metadata_repository), @@ -226,16 +209,23 @@ async def get_project_pg_connection( "PostgreSQL", ) - pool_min_size = routing.pool_min_size or settings.PROJECT_PG_POOL_SIZE - pool_max_size = routing.pool_max_size or settings.PROJECT_PG_POOL_SIZE - pool = await project_connection_manager.get_pg_pool( + pool_min_size = ( + routing.pool_min_size + if routing.pool_min_size is not None + else settings.PROJECT_PG_POOL_MIN_SIZE + ) + pool_max_size = ( + routing.pool_max_size + if routing.pool_max_size is not None + else settings.PROJECT_PG_POOL_SIZE + ) + async with project_connection_manager.pg_connection( ctx.project_id, DB_ROLE_BIZ_DATA, routing.dsn, pool_min_size, pool_max_size, - ) - async with pool.connection() as conn: + ) as conn: yield conn @@ -251,14 +241,21 @@ async def get_project_timescale_connection( "TimescaleDB", ) - pool_min_size = routing.pool_min_size or settings.PROJECT_TS_POOL_MIN_SIZE - pool_max_size = routing.pool_max_size or settings.PROJECT_TS_POOL_MAX_SIZE - pool = await project_connection_manager.get_timescale_pool( + pool_min_size = ( + routing.pool_min_size + if routing.pool_min_size is not None + else settings.PROJECT_TS_POOL_MIN_SIZE + ) + pool_max_size = ( + routing.pool_max_size + if routing.pool_max_size is not None + else settings.PROJECT_TS_POOL_MAX_SIZE + ) + async with project_connection_manager.timescale_connection( ctx.project_id, DB_ROLE_IOT_DATA, routing.dsn, pool_min_size, pool_max_size, - ) - async with pool.connection() as conn: + ) as conn: yield conn diff --git a/app/core/config.py b/app/core/config.py index c8ea03d..8a00719 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -36,13 +36,15 @@ class Settings(BaseSettings): METADATA_DB_POOL_SIZE: int = 5 METADATA_DB_MAX_OVERFLOW: int = 10 - PROJECT_PG_CACHE_SIZE: int = 50 - PROJECT_TS_CACHE_SIZE: int = 50 + PROJECT_PG_CACHE_SIZE: int = 16 + PROJECT_TS_CACHE_SIZE: int = 16 PROJECT_PG_POOL_MIN_SIZE: int = 0 - PROJECT_PG_POOL_SIZE: int = 5 - PROJECT_PG_MAX_OVERFLOW: int = 10 + PROJECT_PG_POOL_SIZE: int = 4 + PROJECT_PG_MAX_OVERFLOW: int = 2 PROJECT_TS_POOL_MIN_SIZE: int = 0 - PROJECT_TS_POOL_MAX_SIZE: int = 10 + PROJECT_TS_POOL_MAX_SIZE: int = 4 + WNDB_TEMPLATE_DB_NAME: str = "tjwater_v2_template" + WNDB_TEMP_DB_MAX_COUNT: int = 8 # Keycloak access token verification KEYCLOAK_PUBLIC_KEY: str = "" diff --git a/app/infra/db/dynamic_manager.py b/app/infra/db/dynamic_manager.py index c78a2e5..bd8de7b 100644 --- a/app/infra/db/dynamic_manager.py +++ b/app/infra/db/dynamic_manager.py @@ -1,40 +1,29 @@ import asyncio import logging from collections import OrderedDict +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from dataclasses import dataclass from typing import Dict from uuid import UUID +from psycopg import AsyncConnection from psycopg_pool import AsyncConnectionPool from psycopg.rows import dict_row -from sqlalchemy.engine.url import make_url -from sqlalchemy.ext.asyncio import ( - AsyncEngine, - AsyncSession, - async_sessionmaker, - create_async_engine, -) from app.core.config import settings logger = logging.getLogger(__name__) +_check_async_connection = AsyncConnectionPool.check_connection -@dataclass(frozen=True) -class PgEngineEntry: - engine: AsyncEngine - sessionmaker: async_sessionmaker[AsyncSession] - connection_url: str - pool_min_size: int - pool_max_size: int - - -@dataclass(frozen=True) +@dataclass class PoolEntry: pool: AsyncConnectionPool connection_url: str pool_min_size: int pool_max_size: int + borrow_count: int = 0 @dataclass(frozen=True) @@ -45,186 +34,203 @@ class CacheKey: class ProjectConnectionManager: def __init__(self) -> None: - self._pg_cache: Dict[CacheKey, PgEngineEntry] = OrderedDict() self._ts_cache: Dict[CacheKey, PoolEntry] = OrderedDict() self._pg_raw_cache: Dict[CacheKey, PoolEntry] = OrderedDict() - self._pg_lock = asyncio.Lock() + self._retired_ts: list[tuple[CacheKey, PoolEntry]] = [] + self._retired_pg: list[tuple[CacheKey, PoolEntry]] = [] self._ts_lock = asyncio.Lock() self._pg_raw_lock = asyncio.Lock() - def _normalize_pg_url(self, url: str) -> str: - parsed = make_url(url) - if parsed.drivername in {"postgresql", "postgres"}: - parsed = parsed.set(drivername="postgresql+psycopg") - return parsed.render_as_string(hide_password=False) - - async def get_pg_sessionmaker( + async def _get_timescale_pool_locked( self, project_id: UUID, db_role: str, connection_url: str, pool_min_size: int, pool_max_size: int, - ) -> async_sessionmaker[AsyncSession]: - async with self._pg_lock: - normalized_url = self._normalize_pg_url(connection_url) - pool_min_size = max(1, pool_min_size) - pool_max_size = max(pool_min_size, pool_max_size) - - key = CacheKey(project_id=project_id, db_role=db_role) - entry = self._pg_cache.get(key) - if entry: - if ( - entry.connection_url == normalized_url - and entry.pool_min_size == pool_min_size - and entry.pool_max_size == pool_max_size - ): - self._pg_cache.move_to_end(key) - return entry.sessionmaker - - await entry.engine.dispose() - logger.info( - "Rebuilding PostgreSQL engine for project %s (%s) due to config change", - project_id, - db_role, - ) - self._pg_cache.pop(key, None) - - engine = create_async_engine( - normalized_url, - pool_size=pool_min_size, - max_overflow=max(0, pool_max_size - pool_min_size), - pool_pre_ping=True, - ) - sessionmaker = async_sessionmaker(engine, expire_on_commit=False) - self._pg_cache[key] = PgEngineEntry( - engine=engine, - sessionmaker=sessionmaker, - connection_url=normalized_url, - pool_min_size=pool_min_size, - pool_max_size=pool_max_size, - ) - await self._evict_pg_if_needed() + ) -> tuple[CacheKey, AsyncConnectionPool]: + pool_min_size = max(0, pool_min_size) + pool_max_size = max(1, pool_min_size, pool_max_size) + key = CacheKey(project_id=project_id, db_role=db_role) + entry = self._ts_cache.get(key) + if entry: + if ( + entry.connection_url == connection_url + and entry.pool_min_size == pool_min_size + and entry.pool_max_size == pool_max_size + ): + self._ts_cache.move_to_end(key) + return key, entry.pool logger.info( - "Created PostgreSQL engine for project %s (%s)", project_id, db_role + "Rebuilding TimescaleDB pool for project %s (%s) due to config change", + project_id, + db_role, ) - return sessionmaker - - async def get_timescale_pool( - self, - project_id: UUID, - db_role: str, - connection_url: str, - pool_min_size: int, - pool_max_size: int, - ) -> AsyncConnectionPool: - async with self._ts_lock: - pool_min_size = max(1, pool_min_size) - pool_max_size = max(pool_min_size, pool_max_size) - - key = CacheKey(project_id=project_id, db_role=db_role) - entry = self._ts_cache.get(key) - if entry: - if ( - entry.connection_url == connection_url - and entry.pool_min_size == pool_min_size - and entry.pool_max_size == pool_max_size - ): - self._ts_cache.move_to_end(key) - return entry.pool + pool = AsyncConnectionPool( + conninfo=connection_url, + min_size=pool_min_size, + max_size=pool_max_size, + open=False, + kwargs={"row_factory": dict_row}, + check=_check_async_connection, + ) + await pool.open() + if entry is not None: + if entry.borrow_count: + self._retired_ts.append((key, entry)) + else: await entry.pool.close() - logger.info( - "Rebuilding TimescaleDB pool for project %s (%s) due to config change", - project_id, - db_role, - ) - self._ts_cache.pop(key, None) + self._ts_cache[key] = PoolEntry( + pool=pool, + connection_url=connection_url, + pool_min_size=pool_min_size, + pool_max_size=pool_max_size, + ) + logger.info("Created TimescaleDB pool for project %s (%s)", project_id, db_role) + return key, pool - pool = AsyncConnectionPool( - conninfo=connection_url, - min_size=pool_min_size, - max_size=pool_max_size, - open=False, - kwargs={"row_factory": dict_row}, - ) - await pool.open() - self._ts_cache[key] = PoolEntry( - pool=pool, - connection_url=connection_url, - pool_min_size=pool_min_size, - pool_max_size=pool_max_size, - ) - await self._evict_ts_if_needed() - logger.info( - "Created TimescaleDB pool for project %s (%s)", project_id, db_role - ) - return pool - - async def get_pg_pool( + async def _get_pg_pool_locked( self, project_id: UUID, db_role: str, connection_url: str, pool_min_size: int, pool_max_size: int, - ) -> AsyncConnectionPool: + ) -> tuple[CacheKey, AsyncConnectionPool]: + pool_min_size = max(0, pool_min_size) + pool_max_size = max(1, pool_min_size, pool_max_size) + key = CacheKey(project_id=project_id, db_role=db_role) + entry = self._pg_raw_cache.get(key) + if entry: + if ( + entry.connection_url == connection_url + and entry.pool_min_size == pool_min_size + and entry.pool_max_size == pool_max_size + ): + self._pg_raw_cache.move_to_end(key) + return key, entry.pool + logger.info( + "Rebuilding PostgreSQL pool for project %s (%s) due to config change", + project_id, + db_role, + ) + + pool = AsyncConnectionPool( + conninfo=connection_url, + min_size=pool_min_size, + max_size=pool_max_size, + open=False, + kwargs={"row_factory": dict_row}, + check=_check_async_connection, + ) + await pool.open() + if entry is not None: + if entry.borrow_count: + self._retired_pg.append((key, entry)) + else: + await entry.pool.close() + self._pg_raw_cache[key] = PoolEntry( + pool=pool, + connection_url=connection_url, + pool_min_size=pool_min_size, + pool_max_size=pool_max_size, + ) + logger.info("Created PostgreSQL pool for project %s (%s)", project_id, db_role) + return key, pool + + @asynccontextmanager + async def pg_connection( + self, + project_id: UUID, + db_role: str, + connection_url: str, + pool_min_size: int, + pool_max_size: int, + ) -> AsyncIterator[AsyncConnection]: async with self._pg_raw_lock: - pool_min_size = max(1, pool_min_size) - pool_max_size = max(pool_min_size, pool_max_size) - - key = CacheKey(project_id=project_id, db_role=db_role) - entry = self._pg_raw_cache.get(key) - if entry: - if ( - entry.connection_url == connection_url - and entry.pool_min_size == pool_min_size - and entry.pool_max_size == pool_max_size - ): - self._pg_raw_cache.move_to_end(key) - return entry.pool - - await entry.pool.close() - logger.info( - "Rebuilding PostgreSQL pool for project %s (%s) due to config change", - project_id, - db_role, - ) - self._pg_raw_cache.pop(key, None) - - pool = AsyncConnectionPool( - conninfo=connection_url, - min_size=pool_min_size, - max_size=pool_max_size, - open=False, - kwargs={"row_factory": dict_row}, - ) - await pool.open() - self._pg_raw_cache[key] = PoolEntry( - pool=pool, - connection_url=connection_url, - pool_min_size=pool_min_size, - pool_max_size=pool_max_size, + key, pool = await self._get_pg_pool_locked( + project_id, + db_role, + connection_url, + pool_min_size, + pool_max_size, ) + borrowed_entry = self._pg_raw_cache[key] + borrowed_entry.borrow_count += 1 await self._evict_pg_raw_if_needed() - logger.info( - "Created PostgreSQL pool for project %s (%s)", project_id, db_role - ) - return pool - async def _evict_pg_if_needed(self) -> None: - while len(self._pg_cache) > settings.PROJECT_PG_CACHE_SIZE: - key, entry = self._pg_cache.popitem(last=False) - await entry.engine.dispose() - logger.info( - "Evicted PostgreSQL engine for project %s (%s)", - key.project_id, - key.db_role, - ) + try: + async with pool.connection() as conn: + yield conn + finally: + async with self._pg_raw_lock: + borrowed_entry.borrow_count -= 1 + if borrowed_entry.borrow_count == 0 and borrowed_entry.pool is not ( + self._pg_raw_cache.get(key).pool + if key in self._pg_raw_cache + else None + ): + self._retired_pg = [ + item for item in self._retired_pg if item[1] is not borrowed_entry + ] + await borrowed_entry.pool.close() + await self._evict_pg_raw_if_needed() - async def _evict_ts_if_needed(self) -> None: + @asynccontextmanager + async def timescale_connection( + self, + project_id: UUID, + db_role: str, + connection_url: str, + pool_min_size: int, + pool_max_size: int, + ) -> AsyncIterator[AsyncConnection]: + async with self._ts_lock: + key, pool = await self._get_timescale_pool_locked( + project_id, + db_role, + connection_url, + pool_min_size, + pool_max_size, + ) + borrowed_entry = self._ts_cache[key] + borrowed_entry.borrow_count += 1 + await self._evict_ts_if_needed() + + try: + async with pool.connection() as conn: + yield conn + finally: + async with self._ts_lock: + borrowed_entry.borrow_count -= 1 + if borrowed_entry.borrow_count == 0 and borrowed_entry.pool is not ( + self._ts_cache.get(key).pool + if key in self._ts_cache + else None + ): + self._retired_ts = [ + item for item in self._retired_ts if item[1] is not borrowed_entry + ] + await borrowed_entry.pool.close() + await self._evict_ts_if_needed() + + async def _evict_ts_if_needed( + self, protected_key: CacheKey | None = None + ) -> None: while len(self._ts_cache) > settings.PROJECT_TS_CACHE_SIZE: - key, entry = self._ts_cache.popitem(last=False) + idle = next( + ( + (key, entry) + for key, entry in self._ts_cache.items() + if entry.borrow_count == 0 and key != protected_key + ), + None, + ) + if idle is None: + return + key, entry = idle + self._ts_cache.pop(key) await entry.pool.close() logger.info( "Evicted TimescaleDB pool for project %s (%s)", @@ -232,9 +238,22 @@ class ProjectConnectionManager: key.db_role, ) - async def _evict_pg_raw_if_needed(self) -> None: + async def _evict_pg_raw_if_needed( + self, protected_key: CacheKey | None = None + ) -> None: while len(self._pg_raw_cache) > settings.PROJECT_PG_CACHE_SIZE: - key, entry = self._pg_raw_cache.popitem(last=False) + idle = next( + ( + (key, entry) + for key, entry in self._pg_raw_cache.items() + if entry.borrow_count == 0 and key != protected_key + ), + None, + ) + if idle is None: + return + key, entry = idle + self._pg_raw_cache.pop(key) await entry.pool.close() logger.info( "Evicted PostgreSQL pool for project %s (%s)", @@ -242,17 +261,61 @@ class ProjectConnectionManager: key.db_role, ) - async def close_all(self) -> None: - async with self._pg_lock: - for key, entry in list(self._pg_cache.items()): - await entry.engine.dispose() - logger.info( - "Closed PostgreSQL engine for project %s (%s)", - key.project_id, - key.db_role, - ) - self._pg_cache.clear() + async def close_project( + self, project_id: UUID, db_role: str | None = None + ) -> bool: + """Close this worker's idle pools for a project. + Returns ``False`` without interrupting requests when any matching pool is + currently borrowed. Callers may retry after those requests complete. + """ + closed = True + for cache, lock, label in ( + (self._ts_cache, self._ts_lock, "TimescaleDB"), + (self._pg_raw_cache, self._pg_raw_lock, "PostgreSQL"), + ): + async with lock: + keys = [ + key + for key in cache + if key.project_id == project_id + and (db_role is None or key.db_role == db_role) + ] + for key in keys: + entry = cache[key] + if entry.borrow_count: + closed = False + continue + cache.pop(key) + await entry.pool.close() + logger.info( + "Closed %s pool for project %s (%s)", + label, + key.project_id, + key.db_role, + ) + for retired, lock in ( + (self._retired_ts, self._ts_lock), + (self._retired_pg, self._pg_raw_lock), + ): + async with lock: + matches = [ + item + for item in retired + if item[0].project_id == project_id + and (db_role is None or item[0].db_role == db_role) + ] + if any(entry.borrow_count for _key, entry in matches): + closed = False + for item in matches: + key, entry = item + if entry.borrow_count: + continue + retired.remove(item) + await entry.pool.close() + return closed + + async def close_all(self) -> None: async with self._ts_lock: for key, entry in list(self._ts_cache.items()): await entry.pool.close() @@ -262,6 +325,9 @@ class ProjectConnectionManager: key.db_role, ) self._ts_cache.clear() + for _key, entry in self._retired_ts: + await entry.pool.close() + self._retired_ts.clear() async with self._pg_raw_lock: for key, entry in list(self._pg_raw_cache.items()): @@ -272,6 +338,9 @@ class ProjectConnectionManager: key.db_role, ) self._pg_raw_cache.clear() + for _key, entry in self._retired_pg: + await entry.pool.close() + self._retired_pg.clear() project_connection_manager = ProjectConnectionManager() diff --git a/app/infra/db/project_routing.py b/app/infra/db/project_routing.py index 47e417e..9054fde 100644 --- a/app/infra/db/project_routing.py +++ b/app/infra/db/project_routing.py @@ -5,9 +5,9 @@ from contextvars import ContextVar, Token from dataclasses import dataclass from typing import Iterator -from psycopg.conninfo import make_conninfo +from psycopg.conninfo import conninfo_to_dict, make_conninfo -from app.core.config import get_pgconn_string, get_timescaledb_pgconn_string +from app.core.config import get_pgconn_string, get_timescaledb_pgconn_string, settings @dataclass(frozen=True) @@ -16,6 +16,17 @@ class ActiveProjectRouting: business_dsn: str timescale_dsn: str | None = None + @property + def business_database_name(self) -> str: + """Return the physical BizDB name selected by metadata routing.""" + database_name = conninfo_to_dict(self.business_dsn).get("dbname") + if not database_name: + raise RuntimeError( + f"Business database routing for project {self.project_code!r} " + "does not contain a database name" + ) + return database_name + _active_project_routing: ContextVar[ActiveProjectRouting | None] = ContextVar( "active_project_routing", @@ -42,6 +53,23 @@ def _dsn_for_database(dsn: str, database_name: str) -> str: return make_conninfo(dsn, dbname=database_name) +def get_project_database_name(name: str) -> str: + """Resolve a logical project code to its routed physical BizDB name.""" + routing = get_active_project_routing() + if routing is not None and name == routing.project_code: + return routing.business_database_name + return name + + +def get_project_template_database_name(name: str | None = None) -> str: + """Return the configured immutable template for the WNDB schema version. + + The template belongs to the database schema version, not to an individual + logical project or its temporary physical database name. + """ + return settings.WNDB_TEMPLATE_DB_NAME + + def get_project_pgconn_string(db_name: str | None = None) -> str: routing = get_active_project_routing() if routing is None: diff --git a/app/infra/db/timescaledb/composite_queries.py b/app/infra/db/timescaledb/composite_queries.py index 9b83074..aec980f 100644 --- a/app/infra/db/timescaledb/composite_queries.py +++ b/app/infra/db/timescaledb/composite_queries.py @@ -56,38 +56,40 @@ class CompositeQueries: Raises: ValueError: 当 SCADA 设备未找到或字段无效时 """ - result = {} scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) - + link_devices: dict[str, str] = {} + node_devices: dict[str, str] = {} for device_id in device_ids: target_scada = scada_by_id.get(device_id) if not target_scada: raise ValueError(f"SCADA device {device_id} not found") - scada_type = target_scada["device_type"] - element_id = ( - target_scada["link_id"] - if scada_type in {"pipe_flow", "flow"} - else target_scada["node_id"] - ) - - if scada_type == "pipe_flow": - # 查询 link 模拟数据 - res = await RealtimeRepository.get_link_field_by_time_range( - timescale_conn, start_time, end_time, element_id, "flow" - ) + if scada_type in {"pipe_flow", "flow"}: + link_devices[device_id] = target_scada["link_id"] elif scada_type == "pressure": - # 查询 node 模拟数据 - res = await RealtimeRepository.get_node_field_by_time_range( - timescale_conn, start_time, end_time, element_id, "pressure" - ) + node_devices[device_id] = target_scada["node_id"] else: raise ValueError(f"Unknown SCADA type: {scada_type}") - # 添加 scada_id 到每个数据项 - for item in res: - item["scada_id"] = device_id - result[device_id] = res - return result + + link_series = await RealtimeRepository.get_link_fields_by_ids_time_range( + timescale_conn, start_time, end_time, + list(dict.fromkeys(link_devices.values())), "flow", + ) + node_series = await RealtimeRepository.get_node_fields_by_ids_time_range( + timescale_conn, start_time, end_time, + list(dict.fromkeys(node_devices.values())), "pressure", + ) + return { + device_id: [ + {**item, "scada_id": device_id} + for item in ( + link_series.get(element_id, []) + if device_id in link_devices + else node_series.get(element_id, []) + ) + ] + for device_id, element_id in (link_devices | node_devices).items() + } @staticmethod async def get_scada_associated_analysis_simulation_data( @@ -117,38 +119,41 @@ class CompositeQueries: Raises: ValueError: 当 SCADA 设备未找到或字段无效时 """ - result = {} scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn) - + link_devices: dict[str, str] = {} + node_devices: dict[str, str] = {} for device_id in device_ids: target_scada = scada_by_id.get(device_id) if not target_scada: raise ValueError(f"SCADA device {device_id} not found") scada_type = target_scada["device_type"] - element_id = ( - target_scada["link_id"] - if scada_type in {"pipe_flow", "flow"} - else target_scada["node_id"] - ) - - if scada_type == "pipe_flow": - # 查询 link 模拟数据 - res = await AnalysisResultsRepository.get_link_series( - timescale_conn, run_id, element_id, start_time, end_time, "flow" - ) + if scada_type in {"pipe_flow", "flow"}: + link_devices[device_id] = target_scada["link_id"] elif scada_type == "pressure": - # 查询 node 模拟数据 - res = await AnalysisResultsRepository.get_node_series( - timescale_conn, run_id, element_id, start_time, end_time, "pressure" - ) + node_devices[device_id] = target_scada["node_id"] else: raise ValueError(f"Unknown SCADA type: {scada_type}") - # 添加 scada_id 到每个数据项 - for item in res: - item["scada_id"] = device_id - result[device_id] = res - return result + + link_series = await AnalysisResultsRepository.get_series_by_ids( + timescale_conn, run_id, "link", + list(dict.fromkeys(link_devices.values())), start_time, end_time, "flow", + ) + node_series = await AnalysisResultsRepository.get_series_by_ids( + timescale_conn, run_id, "node", + list(dict.fromkeys(node_devices.values())), start_time, end_time, "pressure", + ) + return { + device_id: [ + {**item, "scada_id": device_id} + for item in ( + link_series.get(element_id, []) + if device_id in link_devices + else node_series.get(element_id, []) + ) + ] + for device_id, element_id in (link_devices | node_devices).items() + } @staticmethod async def get_realtime_simulation_data( @@ -175,26 +180,33 @@ class CompositeQueries: Raises: ValueError: 当 SCADA 设备未找到或字段无效时 """ - result = {} + pipe_ids: list[str] = [] + junction_ids: list[str] = [] for feature_id, feature_type in feature_infos: - if feature_type.lower() == "pipe": - # 查询 link 模拟数据 - res = await RealtimeRepository.get_link_field_by_time_range( - timescale_conn, start_time, end_time, feature_id, "flow" - ) + pipe_ids.append(feature_id) elif feature_type.lower() == "junction": - # 查询 node 模拟数据 - res = await RealtimeRepository.get_node_field_by_time_range( - timescale_conn, start_time, end_time, feature_id, "pressure" - ) + junction_ids.append(feature_id) else: raise ValueError(f"Unknown type: {feature_type}") - # 添加 scada_id 到每个数据项 - for item in res: - item["feature_id"] = feature_id - result[feature_id] = res - return result + link_series = await RealtimeRepository.get_link_fields_by_ids_time_range( + timescale_conn, start_time, end_time, list(dict.fromkeys(pipe_ids)), "flow" + ) + node_series = await RealtimeRepository.get_node_fields_by_ids_time_range( + timescale_conn, start_time, end_time, + list(dict.fromkeys(junction_ids)), "pressure", + ) + return { + feature_id: [ + {**item, "feature_id": feature_id} + for item in ( + link_series.get(feature_id, []) + if feature_type.lower() == "pipe" + else node_series.get(feature_id, []) + ) + ] + for feature_id, feature_type in feature_infos + } @staticmethod async def get_analysis_simulation_data( @@ -223,25 +235,34 @@ class CompositeQueries: Raises: ValueError: 当类型无效时 """ - result = {} + pipe_ids: list[str] = [] + junction_ids: list[str] = [] for feature_id, feature_type in feature_infos: if feature_type.lower() == "pipe": - # 查询 link 模拟数据 - res = await AnalysisResultsRepository.get_link_series( - timescale_conn, run_id, feature_id, start_time, end_time, "flow" - ) + pipe_ids.append(feature_id) elif feature_type.lower() == "junction": - # 查询 node 模拟数据 - res = await AnalysisResultsRepository.get_node_series( - timescale_conn, run_id, feature_id, start_time, end_time, "pressure" - ) + junction_ids.append(feature_id) else: raise ValueError(f"Unknown type: {feature_type}") - # 添加 feature_id 到每个数据项 - for item in res: - item["feature_id"] = feature_id - result[feature_id] = res - return result + link_series = await AnalysisResultsRepository.get_series_by_ids( + timescale_conn, run_id, "link", list(dict.fromkeys(pipe_ids)), + start_time, end_time, "flow", + ) + node_series = await AnalysisResultsRepository.get_series_by_ids( + timescale_conn, run_id, "node", list(dict.fromkeys(junction_ids)), + start_time, end_time, "pressure", + ) + return { + feature_id: [ + {**item, "feature_id": feature_id} + for item in ( + link_series.get(feature_id, []) + if feature_type.lower() == "pipe" + else node_series.get(feature_id, []) + ) + ] + for feature_id, feature_type in feature_infos + } @staticmethod async def get_element_associated_scada_data( @@ -399,7 +420,7 @@ class CompositeQueries: if scada_by_id[device_id]["device_type"] in {"pipe_flow", "flow"} ] - updated_rows = 0 + cleaned_rows: list[tuple[datetime, str, float | None]] = [] for grouped_ids, cleaning_function in ( (pressure_ids, clean_pressure_data_df_km), (flow_ids, clean_flow_data_df_kf), @@ -422,18 +443,25 @@ class CompositeQueries: if isinstance(time_value, datetime) else datetime.fromisoformat(str(time_value)) ) - await ScadaRepository.update_scada_field( - timescale_conn, - time_dt, - device_id, - "cleaned_value", - value, + cleaned_rows.append( + ( + time_dt, + device_id, + None if pd.isna(value) else float(value), + ) ) - updated_rows += 1 - if updated_rows == 0: + if not cleaned_rows: raise ValueError("SCADA 数据清洗未产生任何数据库更新") + updated_rows = await ScadaRepository.update_scada_field_batch( + timescale_conn, + cleaned_rows, + "cleaned_value", + ) + if updated_rows == 0: + raise ValueError("SCADA 清洗结果未匹配任何已有监测数据") + return "success" @staticmethod diff --git a/app/infra/db/timescaledb/internal_queries.py b/app/infra/db/timescaledb/internal_queries.py index 2c69db7..a9ee386 100644 --- a/app/infra/db/timescaledb/internal_queries.py +++ b/app/infra/db/timescaledb/internal_queries.py @@ -259,7 +259,7 @@ class InternalQueries: query = sql.SQL( "SELECT btrim({}::text) AS id, time, {} FROM {}.{} " "WHERE run_id = %s AND time >= %s AND time <= %s " - "AND btrim({}::text) = ANY(%s)" + "AND btrim({}::text) = ANY(%s) ORDER BY id, time" ).format( sql.Identifier(id_column), sql.Identifier(field), @@ -274,7 +274,8 @@ class InternalQueries: else: query = sql.SQL( "SELECT btrim({}::text) AS id, time, {} FROM {}.{} " - "WHERE time >= %s AND time <= %s AND btrim({}::text) = ANY(%s)" + "WHERE time >= %s AND time <= %s " + "AND btrim({}::text) = ANY(%s) ORDER BY id, time" ).format( sql.Identifier(id_column), sql.Identifier(field), diff --git a/app/infra/db/timescaledb/repositories/analysis.py b/app/infra/db/timescaledb/repositories/analysis.py index b2799d5..4de9bcc 100644 --- a/app/infra/db/timescaledb/repositories/analysis.py +++ b/app/infra/db/timescaledb/repositories/analysis.py @@ -175,6 +175,53 @@ class AnalysisResultsRepository: await cur.execute(query, (run_id, link_id, start_time, end_time)) return await cur.fetchall() + @staticmethod + async def get_series_by_ids( + conn: AsyncConnection, + run_id: UUID, + element_type: str, + element_ids: list[str], + start_time: datetime, + end_time: datetime, + field: str, + ) -> dict[str, list[dict[str, Any]]]: + if element_type == "node": + table_name, id_column, valid_fields = ( + "node_results", "node_id", AnalysisResultsRepository.NODE_FIELDS + ) + elif element_type == "link": + table_name, id_column, valid_fields = ( + "link_results", "link_id", AnalysisResultsRepository.LINK_FIELDS + ) + else: + raise ValueError(f"invalid analysis element type: {element_type}") + if field not in valid_fields: + raise ValueError(f"invalid {element_type} result field: {field}") + + result: dict[str, list[dict[str, Any]]] = { + element_id: [] for element_id in element_ids + } + if not element_ids: + return result + query = sql.SQL( + "SELECT {} AS element_id, time, {} AS value FROM analysis.{} " + "WHERE run_id = %s AND {} = ANY(%s) AND time BETWEEN %s AND %s " + "ORDER BY {}, time" + ).format( + sql.Identifier(id_column), + sql.Identifier(field), + sql.Identifier(table_name), + sql.Identifier(id_column), + sql.Identifier(id_column), + ) + async with conn.cursor() as cur: + await cur.execute(query, (run_id, element_ids, start_time, end_time)) + for row in await cur.fetchall(): + result.setdefault(str(row["element_id"]), []).append( + {"time": row["time"], "value": row["value"]} + ) + return result + @staticmethod async def get_values_at_time( conn: AsyncConnection, diff --git a/app/infra/db/timescaledb/repositories/realtime.py b/app/infra/db/timescaledb/repositories/realtime.py index 10fe5bc..91c4928 100644 --- a/app/infra/db/timescaledb/repositories/realtime.py +++ b/app/infra/db/timescaledb/repositories/realtime.py @@ -7,6 +7,19 @@ from app.services.time_api import parse_utc_time class RealtimeRepository: + @staticmethod + def _batch_time(data: List[dict]) -> datetime: + """Return one normalized timestamp shared by every row in a snapshot.""" + if not data: + raise ValueError("Realtime batch must not be empty") + times = { + parse_utc_time(item["time"], field_name="time") + for item in data + } + if len(times) != 1: + raise ValueError("Realtime batch must contain exactly one timestamp") + return times.pop() + # --- Link Simulation --- @staticmethod @@ -15,8 +28,7 @@ class RealtimeRepository: if not data: return - # 假设同一批次的数据时间是相同的 - target_time = data[0]["time"] + target_time = RealtimeRepository._batch_time(data) # 使用事务确保原子性 async with conn.transaction(): @@ -38,7 +50,7 @@ class RealtimeRepository: for item in data: await copy.write_row( ( - item["time"], + target_time, item["id"], item.get("flow"), item.get("friction"), @@ -57,8 +69,7 @@ class RealtimeRepository: if not data: return - # 假设同一批次的数据时间是相同的 - target_time = data[0]["time"] + target_time = RealtimeRepository._batch_time(data) # 使用事务确保原子性 with conn.transaction(): @@ -80,7 +91,7 @@ class RealtimeRepository: for item in data: copy.write_row( ( - item["time"], + target_time, item["id"], item.get("flow"), item.get("friction"), @@ -99,7 +110,8 @@ class RealtimeRepository: ) -> List[dict]: async with conn.cursor() as cur: await cur.execute( - "SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s AND link_id = %s", + "SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s " + "AND link_id = %s ORDER BY time", (start_time, end_time, link_id), ) return await cur.fetchall() @@ -112,7 +124,8 @@ class RealtimeRepository: normalized_end_time = parse_utc_time(end_time, field_name="end_time") async with conn.cursor() as cur: await cur.execute( - "SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s", + "SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s " + "ORDER BY time, link_id", (normalized_start_time, normalized_end_time), ) return await cur.fetchall() @@ -140,7 +153,8 @@ class RealtimeRepository: raise ValueError(f"Invalid field: {field}") query = sql.SQL( - "SELECT time, {} FROM realtime.link_results WHERE time >= %s AND time <= %s AND link_id = %s" + "SELECT time, {} FROM realtime.link_results WHERE time >= %s " + "AND time <= %s AND link_id = %s ORDER BY time" ).format(sql.Identifier(field)) async with conn.cursor() as cur: @@ -150,6 +164,36 @@ class RealtimeRepository: {"time": row["time"].isoformat(), "value": row[field]} for row in rows ] + @staticmethod + async def get_link_fields_by_ids_time_range( + conn: AsyncConnection, + start_time: datetime, + end_time: datetime, + link_ids: list[str], + field: str, + ) -> dict[str, list[dict[str, Any]]]: + valid_fields = { + "flow", "friction", "headloss", "quality", "reaction", + "setting", "status", "velocity", + } + if field not in valid_fields: + raise ValueError(f"Invalid field: {field}") + result = {link_id: [] for link_id in link_ids} + if not link_ids: + return result + query = sql.SQL( + "SELECT link_id, time, {} FROM realtime.link_results " + "WHERE time BETWEEN %s AND %s AND link_id = ANY(%s) " + "ORDER BY link_id, time" + ).format(sql.Identifier(field)) + async with conn.cursor() as cur: + await cur.execute(query, (start_time, end_time, link_ids)) + for row in await cur.fetchall(): + result.setdefault(str(row["link_id"]), []).append( + {"time": row["time"].isoformat(), "value": row[field]} + ) + return result + @staticmethod async def get_links_field_by_time_range( conn: AsyncConnection, @@ -172,7 +216,8 @@ class RealtimeRepository: raise ValueError(f"Invalid field: {field}") query = sql.SQL( - "SELECT link_id, time, {} FROM realtime.link_results WHERE time >= %s AND time <= %s" + "SELECT link_id, time, {} FROM realtime.link_results " + "WHERE time >= %s AND time <= %s ORDER BY link_id, time" ).format(sql.Identifier(field)) async with conn.cursor() as cur: @@ -230,8 +275,7 @@ class RealtimeRepository: if not data: return - # 假设同一批次的数据时间是相同的 - target_time = data[0]["time"] + target_time = RealtimeRepository._batch_time(data) # 使用事务确保原子性 async with conn.transaction(): @@ -253,7 +297,7 @@ class RealtimeRepository: for item in data: await copy.write_row( ( - item["time"], + target_time, item["id"], item.get("actual_demand"), item.get("total_head"), @@ -267,8 +311,7 @@ class RealtimeRepository: if not data: return - # 假设同一批次的数据时间是相同的 - target_time = data[0]["time"] + target_time = RealtimeRepository._batch_time(data) # 使用事务确保原子性 with conn.transaction(): @@ -290,7 +333,7 @@ class RealtimeRepository: for item in data: copy.write_row( ( - item["time"], + target_time, item["id"], item.get("actual_demand"), item.get("total_head"), @@ -305,7 +348,8 @@ class RealtimeRepository: ) -> List[dict]: async with conn.cursor() as cur: await cur.execute( - "SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s AND node_id = %s", + "SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s " + "AND node_id = %s ORDER BY time", (start_time, end_time, node_id), ) return await cur.fetchall() @@ -318,7 +362,8 @@ class RealtimeRepository: normalized_end_time = parse_utc_time(end_time, field_name="end_time") async with conn.cursor() as cur: await cur.execute( - "SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s", + "SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s " + "ORDER BY time, node_id", (normalized_start_time, normalized_end_time), ) return await cur.fetchall() @@ -336,7 +381,8 @@ class RealtimeRepository: raise ValueError(f"Invalid field: {field}") query = sql.SQL( - "SELECT time, {} FROM realtime.node_results WHERE time >= %s AND time <= %s AND node_id = %s" + "SELECT time, {} FROM realtime.node_results WHERE time >= %s " + "AND time <= %s AND node_id = %s ORDER BY time" ).format(sql.Identifier(field)) async with conn.cursor() as cur: @@ -346,6 +392,33 @@ class RealtimeRepository: {"time": row["time"].isoformat(), "value": row[field]} for row in rows ] + @staticmethod + async def get_node_fields_by_ids_time_range( + conn: AsyncConnection, + start_time: datetime, + end_time: datetime, + node_ids: list[str], + field: str, + ) -> dict[str, list[dict[str, Any]]]: + valid_fields = {"actual_demand", "total_head", "pressure", "quality"} + if field not in valid_fields: + raise ValueError(f"Invalid field: {field}") + result = {node_id: [] for node_id in node_ids} + if not node_ids: + return result + query = sql.SQL( + "SELECT node_id, time, {} FROM realtime.node_results " + "WHERE time BETWEEN %s AND %s AND node_id = ANY(%s) " + "ORDER BY node_id, time" + ).format(sql.Identifier(field)) + async with conn.cursor() as cur: + await cur.execute(query, (start_time, end_time, node_ids)) + for row in await cur.fetchall(): + result.setdefault(str(row["node_id"]), []).append( + {"time": row["time"].isoformat(), "value": row[field]} + ) + return result + @staticmethod async def get_nodes_field_by_time_range( conn: AsyncConnection, start_time: datetime, end_time: datetime, field: str @@ -355,7 +428,8 @@ class RealtimeRepository: raise ValueError(f"Invalid field: {field}") query = sql.SQL( - "SELECT node_id, time, {} FROM realtime.node_results WHERE time >= %s AND time <= %s" + "SELECT node_id, time, {} FROM realtime.node_results " + "WHERE time >= %s AND time <= %s ORDER BY node_id, time" ).format(sql.Identifier(field)) async with conn.cursor() as cur: @@ -459,6 +533,19 @@ class RealtimeRepository: # transactions (savepoints), while this outer transaction guarantees # that a link write failure also rolls back the node replacement. async with conn.transaction(): + async with conn.cursor() as cur: + await cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))", + (simulation_time,), + ) + await cur.execute( + "DELETE FROM realtime.node_results WHERE time = %s", + (simulation_time,), + ) + await cur.execute( + "DELETE FROM realtime.link_results WHERE time = %s", + (simulation_time,), + ) if node_data: await RealtimeRepository.insert_nodes_batch(conn, node_data) @@ -525,6 +612,19 @@ class RealtimeRepository: # transactions (savepoints), while this outer transaction guarantees # that a link write failure also rolls back the node replacement. with conn.transaction(): + with conn.cursor() as cur: + cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))", + (simulation_time,), + ) + cur.execute( + "DELETE FROM realtime.node_results WHERE time = %s", + (simulation_time,), + ) + cur.execute( + "DELETE FROM realtime.link_results WHERE time = %s", + (simulation_time,), + ) if node_data: RealtimeRepository.insert_nodes_batch_sync(conn, node_data) diff --git a/app/infra/db/timescaledb/repositories/scada.py b/app/infra/db/timescaledb/repositories/scada.py index 09ef7e5..0c19654 100644 --- a/app/infra/db/timescaledb/repositories/scada.py +++ b/app/infra/db/timescaledb/repositories/scada.py @@ -35,7 +35,8 @@ class ScadaRepository: ) -> List[dict]: async with conn.cursor() as cur: await cur.execute( - "SELECT * FROM scada.measurements WHERE device_id = ANY(%s) AND time >= %s AND time <= %s", + "SELECT * FROM scada.measurements WHERE device_id = ANY(%s) " + "AND time >= %s AND time <= %s ORDER BY device_id, time", (device_ids, start_time, end_time), ) return await cur.fetchall() @@ -49,7 +50,8 @@ class ScadaRepository: ) -> List[dict]: with conn.cursor(row_factory=dict_row) as cur: cur.execute( - "SELECT * FROM scada.measurements WHERE device_id = ANY(%s) AND time >= %s AND time <= %s", + "SELECT * FROM scada.measurements WHERE device_id = ANY(%s) " + "AND time >= %s AND time <= %s ORDER BY device_id, time", (device_ids, start_time, end_time), ) return cur.fetchall() @@ -88,7 +90,9 @@ class ScadaRepository: raise ValueError(f"Invalid field: {field}") query = sql.SQL( - "SELECT device_id, time, {} FROM scada.measurements WHERE time >= %s AND time <= %s AND device_id = ANY(%s)" + "SELECT device_id, time, {} FROM scada.measurements " + "WHERE time >= %s AND time <= %s AND device_id = ANY(%s) " + "ORDER BY device_id, time" ).format(sql.Identifier(field)) async with conn.cursor() as cur: @@ -122,6 +126,31 @@ class ScadaRepository: if cur.rowcount == 0: await cur.execute(insert_query, (time, device_id, value)) + @staticmethod + async def update_scada_field_batch( + conn: AsyncConnection, + rows: list[tuple[datetime, str, float | None]], + field: str, + ) -> int: + """Update existing SCADA samples in one set-based statement.""" + valid_fields = {"monitored_value", "cleaned_value"} + if field not in valid_fields: + raise ValueError(f"Invalid field: {field}") + if not rows: + return 0 + + query = sql.SQL( + "UPDATE scada.measurements AS measurement SET {} = batch.value " + "FROM unnest(%s::timestamptz[], %s::text[], %s::double precision[]) " + "AS batch(time, device_id, value) " + "WHERE measurement.time = batch.time " + "AND measurement.device_id = batch.device_id" + ).format(sql.Identifier(field)) + times, device_ids, values = zip(*rows) + async with conn.cursor() as cur: + await cur.execute(query, (list(times), list(device_ids), list(values))) + return cur.rowcount + @staticmethod async def delete_scada_by_id_time_range( conn: AsyncConnection, device_id: str, start_time: datetime, end_time: datetime diff --git a/app/infra/db/timescaledb/sync_pool.py b/app/infra/db/timescaledb/sync_pool.py index 6d703d7..8553487 100644 --- a/app/infra/db/timescaledb/sync_pool.py +++ b/app/infra/db/timescaledb/sync_pool.py @@ -11,6 +11,7 @@ from app.core.config import settings from app.infra.db.project_routing import get_project_timescale_pgconn_string +_check_connection = ConnectionPool.check_connection _pools: OrderedDict[str, ConnectionPool] = OrderedDict() _pool_conninfo: dict[str, str] = {} _pool_borrows: dict[str, int] = {} @@ -49,6 +50,7 @@ def get_timescale_pool(db_name: str) -> ConnectionPool: min_size=settings.PROJECT_TS_POOL_MIN_SIZE, max_size=settings.PROJECT_TS_POOL_MAX_SIZE, kwargs={"row_factory": dict_row}, + check=_check_connection, open=True, ) _pools[db_name] = pool diff --git a/app/main.py b/app/main.py index 5896c3e..a71bd87 100644 --- a/app/main.py +++ b/app/main.py @@ -5,14 +5,12 @@ from fastapi.middleware.cors import CORSMiddleware import logging from datetime import datetime -import app.services.project_info as project_info from app.api.problem_details import install_problem_details_handlers from app.api.v1.rest_router import api_router from app.infra.db.dynamic_manager import project_connection_manager from app.infra.db.metadb.database import close_metadata_engine from app.infra.db.timescaledb.sync_pool import close_all_timescale_pools from app.native.wndb.core.connection import close_all_project_pools -from app.services.tjnetwork import open_project from app.core.config import settings # 导入审计中间件 @@ -30,10 +28,6 @@ async def lifespan(app: FastAPI): logger.info("TJWater CloudService is starting...") logger.info("**********************************************************") - if project_info.name: - print(project_info.name) - open_project(project_info.name) - yield # 清理资源 await project_connection_manager.close_all() diff --git a/app/native/wndb/commands/cascade.py b/app/native/wndb/commands/cascade.py index b0669a9..8f24456 100644 --- a/app/native/wndb/commands/cascade.py +++ b/app/native/wndb/commands/cascade.py @@ -7,7 +7,7 @@ from ..model.reservoirs import unset_reservoir_by_pattern from ..model.tanks import unset_tank_by_curve from ..model.pumps import unset_pump_by_curve, unset_pump_by_pattern from ..model.tags import delete_tag_by_node, delete_tag_by_link -from ..model.demands import delete_demand_by_junction, unset_demand_by_pattern +from ..model.demands import delete_demand_by_junction from ..model.status import delete_status_by_link from ..model.energy import delete_pump_energy_by_pump, unset_pump_energy_by_pattern, unset_pump_energy_by_curve from ..model.emitters import delete_emitter_by_junction @@ -167,7 +167,6 @@ def expand_pattern_delete(name: str, cs: ChangeSet) -> ChangeSet: result.merge(unset_reservoir_by_pattern(name, id)) result.merge(unset_pump_by_pattern(name, id)) - result.merge(unset_demand_by_pattern(name, id)) result.merge(unset_pump_energy_by_pattern(name, id)) result.merge(unset_source_by_pattern(name, id)) result.merge(cs) @@ -191,7 +190,7 @@ def expand_curve_delete(name: str, cs: ChangeSet) -> ChangeSet: return result -def expand_legacy_options_update(cs: ChangeSet) -> ChangeSet: +def expand_v2_options_update(cs: ChangeSet) -> ChangeSet: cs.operations[0]['operation'] = API_UPDATE cs.operations[0]['type'] = 'option' new_cs = cs @@ -239,6 +238,6 @@ _DELETE_REWRITERS: dict[str, DeleteRewriter] = { } _UPDATE_REWRITERS: dict[str, UpdateRewriter] = { - "option": expand_legacy_options_update, + "option": expand_v2_options_update, "option_v3": expand_v3_options_update, } diff --git a/app/native/wndb/commands/executor.py b/app/native/wndb/commands/executor.py index 7eea13e..5a29b00 100644 --- a/app/native/wndb/commands/executor.py +++ b/app/native/wndb/commands/executor.py @@ -2,13 +2,14 @@ from collections.abc import Callable -from ..core.connection import project_transaction from ..core.database import ( API_ADD, API_DELETE, API_UPDATE, ChangeSet, - refresh_materialized_views, + changes_affect_materialized_views, + model_mutation_transaction, + refresh_materialized_views_after_commit, ) from ..gis.backdrop import set_backdrop from ..gis.labels import add_label, delete_label, set_label @@ -131,7 +132,7 @@ def _execute_delete_command(name: str, change_set: ChangeSet) -> ChangeSet: def execute_batch_commands(name: str, change_set: ChangeSet) -> ChangeSet: - with project_transaction(name): + with model_mutation_transaction(name): rewritten = ChangeSet() for operation in change_set.operations: rewritten.merge(expand_command(name, ChangeSet(operation))) @@ -146,8 +147,8 @@ def execute_batch_commands(name: str, change_set: ChangeSet) -> ChangeSet: elif operation_type == API_DELETE: result.merge(_execute_delete_command(name, ChangeSet(operation))) - if rewritten.operations: - refresh_materialized_views(name) + if changes_affect_materialized_views(rewritten): + refresh_materialized_views_after_commit(name) return result diff --git a/app/native/wndb/core/connection.py b/app/native/wndb/core/connection.py index d3e6d82..10c4d0e 100644 --- a/app/native/wndb/core/connection.py +++ b/app/native/wndb/core/connection.py @@ -11,6 +11,7 @@ from psycopg_pool import ConnectionPool from app.core.config import settings from app.infra.db.project_routing import get_project_pgconn_string +_check_connection = ConnectionPool.check_connection _pools: OrderedDict[str, ConnectionPool] = OrderedDict() _pool_conninfo: dict[str, str] = {} _pool_borrows: dict[str, int] = {} @@ -21,6 +22,10 @@ _active_project_connection: ContextVar[tuple[str, Connection] | None] = ContextV "wndb_active_project_connection", default=None, ) +_active_model_mutation_locks: ContextVar[frozenset[str]] = ContextVar( + "wndb_active_model_mutation_locks", + default=frozenset(), +) def _close_pool(pool: ConnectionPool) -> None: @@ -78,6 +83,7 @@ def get_project_pool(name: str) -> ConnectionPool: min_size=settings.PROJECT_PG_POOL_MIN_SIZE, max_size=settings.PROJECT_PG_POOL_SIZE + settings.PROJECT_PG_MAX_OVERFLOW, kwargs={"autocommit": True, "row_factory": dict_row}, + check=_check_connection, open=True, ) _pools[name] = pool @@ -140,6 +146,7 @@ def get_admin_pool() -> ConnectionPool: min_size=settings.PROJECT_PG_POOL_MIN_SIZE, max_size=settings.PROJECT_PG_POOL_SIZE, kwargs={"autocommit": True, "row_factory": dict_row}, + check=_check_connection, open=True, ) _admin_pools[conninfo] = pool @@ -192,10 +199,12 @@ def project_transaction(name: str) -> Iterator[Connection]: try: with pool.connection() as conn: token = _active_project_connection.set((name, conn)) + lock_token = _active_model_mutation_locks.set(frozenset()) try: with conn.transaction(): yield conn finally: + _active_model_mutation_locks.reset(lock_token) _active_project_connection.reset(token) finally: with _registry_lock: @@ -208,6 +217,17 @@ def is_project_transaction_active(name: str) -> bool: return active is not None and active[0] == name +def is_model_mutation_lock_active(name: str) -> bool: + """Return whether the current project transaction already owns its model lock.""" + return name in _active_model_mutation_locks.get() + + +def mark_model_mutation_lock_active(name: str) -> None: + """Record a transaction-scoped advisory lock to avoid duplicate round trips.""" + locks = _active_model_mutation_locks.get() + _active_model_mutation_locks.set(locks | {name}) + + @contextmanager def admin_connection() -> Iterator[Connection]: """Borrow a PostgreSQL administration connection from its pool.""" diff --git a/app/native/wndb/core/database.py b/app/native/wndb/core/database.py index b71d7fb..171c55d 100644 --- a/app/native/wndb/core/database.py +++ b/app/native/wndb/core/database.py @@ -1,10 +1,19 @@ -from collections.abc import Mapping, Sequence -from typing import Any +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from typing import Any, Callable from psycopg import sql from psycopg.rows import Row, dict_row -from .connection import is_project_transaction_active, project_connection +from app.infra.db.project_routing import get_project_database_name + +from .connection import ( + is_model_mutation_lock_active, + is_project_transaction_active, + mark_model_mutation_lock_active, + project_connection, + project_transaction, +) API_ADD = "add" API_UPDATE = "update" @@ -61,6 +70,20 @@ class DatabaseCommand: self.sql = statement self.changes = changes + +class MaterializedViewRefreshAfterCommitError(RuntimeError): + """Report a failed view refresh without implying that the write rolled back.""" + + changes_committed = True + + def __init__(self, project: str) -> None: + self.project = project + super().__init__( + f"Project {project!r} changes were committed, but materialized view " + "refresh failed" + ) + + QueryParams = Sequence[Any] | Mapping[str, Any] @@ -78,6 +101,27 @@ def _execute(cur, query: str, params: QueryParams | None = None): return cur.execute(query, params) if params is not None else cur.execute(query) +def acquire_model_mutation_lock(conn, name: str) -> None: + """Serialize model replacement and ordinary WNDB mutations per database.""" + if is_model_mutation_lock_active(name): + return + physical_name = get_project_database_name(name) + with conn.cursor() as cur: + cur.execute( + "select pg_advisory_xact_lock(hashtextextended(%s, 0))", + (f"tjwater:wndb:model:{physical_name}",), + ) + mark_model_mutation_lock_active(name) + + +@contextmanager +def model_mutation_transaction(name: str) -> Iterator[Any]: + """Open a project transaction and acquire its model lock before reading.""" + with project_transaction(name) as conn: + acquire_model_mutation_lock(conn, name) + yield conn + + def read(name: str, query: str, params: QueryParams | None = None) -> Row: with project_connection(name) as conn, conn.cursor(row_factory=dict_row) as cur: _execute(cur, query, params) @@ -104,8 +148,15 @@ def try_read( def write(name: str, query: str, params: QueryParams | None = None) -> None: - with project_connection(name) as conn, conn.cursor() as cur: - _execute(cur, query, params) + connection_context = ( + project_connection(name) + if is_project_transaction_active(name) + else model_mutation_transaction(name) + ) + with connection_context as conn: + acquire_model_mutation_lock(conn, name) + with conn.cursor() as cur: + _execute(cur, query, params) def refresh_materialized_views(name: str, *, concurrently: bool = True) -> None: @@ -114,6 +165,13 @@ def refresh_materialized_views(name: str, *, concurrently: bool = True) -> None: cur.execute("CALL gis.refresh_all_materialized_views(%s)", (concurrently,)) +def refresh_materialized_views_after_commit(name: str) -> None: + try: + refresh_materialized_views(name) + except Exception as exc: + raise MaterializedViewRefreshAfterCommitError(name) from exc + + _MATERIALIZED_VIEW_SOURCES = ( "network.nodes", "network.junctions", @@ -135,9 +193,48 @@ def _affects_materialized_views(command: DatabaseCommand) -> bool: return any(source in statement for source in _MATERIALIZED_VIEW_SOURCES) +_MATERIALIZED_VIEW_ELEMENT_TYPES = frozenset( + { + "junction", + "reservoir", + "tank", + "pipe", + "pump", + "valve", + "demand", + "vertex", + } +) + + +def changes_affect_materialized_views(change_set: ChangeSet) -> bool: + """Return whether a dispatched WNDB batch changes a published GIS source.""" + return any( + operation.get("type") in _MATERIALIZED_VIEW_ELEMENT_TYPES + for operation in change_set.operations + ) + + def execute_command(name: str, command: DatabaseCommand) -> ChangeSet: """Apply a model mutation without the removed database undo/redo journal.""" write(name, command.sql) if _affects_materialized_views(command) and not is_project_transaction_active(name): - refresh_materialized_views(name) + refresh_materialized_views_after_commit(name) return ChangeSet.from_list(command.changes) + + +def execute_locked_command( + name: str, + builder: Callable[[], DatabaseCommand | None], +) -> ChangeSet: + """Build a read-modify-write command only after acquiring the model lock.""" + nested_transaction = is_project_transaction_active(name) + command: DatabaseCommand | None = None + with model_mutation_transaction(name): + command = builder() + if command is None: + return ChangeSet() + result = execute_command(name, command) + if not nested_transaction and _affects_materialized_views(command): + refresh_materialized_views_after_commit(name) + return result diff --git a/app/native/wndb/core/model_replace.py b/app/native/wndb/core/model_replace.py new file mode 100644 index 0000000..e69e740 --- /dev/null +++ b/app/native/wndb/core/model_replace.py @@ -0,0 +1,284 @@ +from psycopg import Connection, sql + +from .connection import project_connection, project_transaction +from .database import acquire_model_mutation_lock + +_MODEL_SCHEMAS = ("network", "gis") +_ALLOWED_EXTERNAL_REFERENCES = { + ("analysis", "results", "network", "nodes"), + ("analysis", "results", "network", "links"), + ("asset", "scada_devices", "network", "nodes"), + ("asset", "scada_devices", "network", "links"), +} + + +def _model_tables(conn: Connection) -> list[tuple[str, str]]: + with conn.cursor() as cur: + cur.execute( + """ + select n.nspname as schema_name, c.relname as table_name + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = any(%s) and c.relkind in ('r', 'p') + order by n.nspname, c.relname + """, + (list(_MODEL_SCHEMAS),), + ) + return [(row["schema_name"], row["table_name"]) for row in cur.fetchall()] + + +def _copy_order( + conn: Connection, tables: list[tuple[str, str]] +) -> list[tuple[str, str]]: + table_set = set(tables) + dependencies: dict[tuple[str, str], set[tuple[str, str]]] = { + table: set() for table in tables + } + with conn.cursor() as cur: + cur.execute( + """ + select source_ns.nspname as source_schema, + source.relname as source_table, + target_ns.nspname as target_schema, + target.relname as target_table + from pg_constraint constraint_row + join pg_class source on source.oid = constraint_row.conrelid + join pg_namespace source_ns on source_ns.oid = source.relnamespace + join pg_class target on target.oid = constraint_row.confrelid + join pg_namespace target_ns on target_ns.oid = target.relnamespace + where constraint_row.contype = 'f' + and source_ns.nspname = any(%s) + and target_ns.nspname = any(%s) + """, + (list(_MODEL_SCHEMAS), list(_MODEL_SCHEMAS)), + ) + for row in cur.fetchall(): + source = (row["source_schema"], row["source_table"]) + target = (row["target_schema"], row["target_table"]) + if source in table_set and target in table_set and source != target: + dependencies[source].add(target) + + ordered: list[tuple[str, str]] = [] + remaining = set(tables) + while remaining: + ready = sorted( + table for table in remaining if not (dependencies[table] & remaining) + ) + if not ready: + cycle = ", ".join(f"{schema}.{table}" for schema, table in sorted(remaining)) + raise RuntimeError(f"Model table foreign-key cycle detected: {cycle}") + ordered.extend(ready) + remaining.difference_update(ready) + return ordered + + +def _table_columns( + conn: Connection, schema_name: str, table_name: str +) -> list[str]: + with conn.cursor() as cur: + cur.execute( + """ + select column_name + from information_schema.columns + where table_schema = %s and table_name = %s + and is_generated = 'NEVER' + order by ordinal_position + """, + (schema_name, table_name), + ) + return [row["column_name"] for row in cur.fetchall()] + + +def _external_references(conn: Connection) -> set[tuple[str, str, str, str]]: + with conn.cursor() as cur: + cur.execute( + """ + select source_ns.nspname as source_schema, + source.relname as source_table, + target_ns.nspname as target_schema, + target.relname as target_table + from pg_constraint constraint_row + join pg_class source on source.oid = constraint_row.conrelid + join pg_namespace source_ns on source_ns.oid = source.relnamespace + join pg_class target on target.oid = constraint_row.confrelid + join pg_namespace target_ns on target_ns.oid = target.relnamespace + where constraint_row.contype = 'f' + and target_ns.nspname = any(%s) + and source_ns.nspname <> all(%s) + """, + (list(_MODEL_SCHEMAS), list(_MODEL_SCHEMAS)), + ) + return { + ( + row["source_schema"], + row["source_table"], + row["target_schema"], + row["target_table"], + ) + for row in cur.fetchall() + } + + +def _copy_table( + source_conn: Connection, + target_conn: Connection, + schema_name: str, + table_name: str, + columns: list[str], +) -> None: + relation = sql.Identifier(schema_name, table_name) + column_list = sql.SQL(", ").join(map(sql.Identifier, columns)) + copy_out = sql.SQL("copy {} ({}) to stdout").format(relation, column_list) + copy_in = sql.SQL("copy {} ({}) from stdin").format(relation, column_list) + with source_conn.cursor().copy(copy_out) as source_copy: + with target_conn.cursor().copy(copy_in) as target_copy: + for chunk in source_copy: + target_copy.write(chunk) + + +def replace_project_model( + target_project: str, + source_project: str, + *, + copy_source_scada: bool = False, +) -> None: + """Atomically replace WNDB/GIS model tables from a validated staging DB. + + Business and analysis tables remain in the target database. Historical + analysis rows keep references that still exist and become element-neutral + when an element disappeared. SCADA devices are retained only when their + bound node or link still exists in the replacement model. + + ``copy_source_scada`` is used only for temporary project clones. Normal INP + replacement keeps the target project's existing device mappings and drops + mappings whose model element no longer exists. + """ + with project_connection(source_project) as source_conn, source_conn.transaction(): + with source_conn.cursor() as cur: + cur.execute("set transaction isolation level repeatable read, read only") + + source_tables = _model_tables(source_conn) + source_columns = { + table: _table_columns(source_conn, *table) for table in source_tables + } + source_scada_columns = ( + _table_columns(source_conn, "asset", "scada_devices") + if copy_source_scada + else [] + ) + copy_order = _copy_order(source_conn, source_tables) + + with project_transaction(target_project) as target_conn: + acquire_model_mutation_lock(target_conn, target_project) + + target_tables = _model_tables(target_conn) + if set(target_tables) != set(source_tables): + missing = sorted(set(source_tables) - set(target_tables)) + extra = sorted(set(target_tables) - set(source_tables)) + raise RuntimeError( + f"Staging/target model schema mismatch; missing={missing}, extra={extra}" + ) + for table, columns in source_columns.items(): + if _table_columns(target_conn, *table) != columns: + raise RuntimeError( + f"Staging/target columns differ for {table[0]}.{table[1]}" + ) + if copy_source_scada and _table_columns( + target_conn, "asset", "scada_devices" + ) != source_scada_columns: + raise RuntimeError( + "Source/target columns differ for asset.scada_devices" + ) + + unexpected = _external_references(target_conn) - _ALLOWED_EXTERNAL_REFERENCES + if unexpected: + formatted = ", ".join( + f"{source_schema}.{source_table}->{target_schema}.{target_table}" + for source_schema, source_table, target_schema, target_table in sorted( + unexpected + ) + ) + raise RuntimeError( + f"Model replacement has unsupported external references: {formatted}" + ) + + with target_conn.cursor() as cur: + if not copy_source_scada: + cur.execute( + "create temporary table model_scada_snapshot on commit drop " + "as table asset.scada_devices" + ) + cur.execute("delete from asset.scada_devices") + cur.execute( + "create temporary table model_result_refs on commit drop as " + "select result_id, node_id, link_id from analysis.results" + ) + cur.execute("update analysis.results set node_id = null, link_id = null") + for schema_name, table_name in reversed(copy_order): + cur.execute( + sql.SQL("delete from {}").format( + sql.Identifier(schema_name, table_name) + ) + ) + + for schema_name, table_name in copy_order: + _copy_table( + source_conn, + target_conn, + schema_name, + table_name, + source_columns[(schema_name, table_name)], + ) + + if copy_source_scada: + _copy_table( + source_conn, + target_conn, + "asset", + "scada_devices", + source_scada_columns, + ) + + with target_conn.cursor() as cur: + if not copy_source_scada: + cur.execute( + """ + insert into asset.scada_devices + select snapshot.* + from model_scada_snapshot snapshot + where ( + snapshot.node_id is not null + and exists ( + select 1 from network.nodes node + where node.id = snapshot.node_id + ) + ) or ( + snapshot.link_id is not null + and exists ( + select 1 from network.links link + where link.id = snapshot.link_id + ) + ) + """ + ) + cur.execute( + """ + update analysis.results result + set node_id = case + when exists ( + select 1 from network.nodes node + where node.id = refs.node_id + ) then refs.node_id + else null + end, + link_id = case + when exists ( + select 1 from network.links link + where link.id = refs.link_id + ) then refs.link_id + else null + end + from model_result_refs refs + where refs.result_id = result.result_id + """ + ) diff --git a/app/native/wndb/core/projects.py b/app/native/wndb/core/projects.py index cd2ea0c..0526796 100644 --- a/app/native/wndb/core/projects.py +++ b/app/native/wndb/core/projects.py @@ -1,13 +1,46 @@ +from collections.abc import Iterable +from contextlib import contextmanager +import re +from uuid import uuid4 + from psycopg import sql from psycopg.rows import dict_row + +from app.core.config import settings +from app.infra.db.project_routing import ( + get_project_database_name, + get_project_template_database_name, +) + from .connection import ( admin_connection, close_project_pool, - get_project_pool, - is_project_pool_open, ) -_server_databases = ["template0", "template1", "postgres", "project"] +_SERVER_DATABASES = frozenset({"template0", "template1", "postgres", "project"}) +_TEMPORARY_DATABASE_PREFIX = "tjw_tmp_" + + +def _protected_databases() -> frozenset[str]: + return _SERVER_DATABASES | { + settings.METADATA_DB_NAME, + settings.WNDB_TEMPLATE_DB_NAME, + } + + +def _validate_project_database(name: str, *, allow_template_source: bool = False) -> None: + if not name: + raise ValueError("Project database name must not be empty") + + protected = {database.casefold() for database in _protected_databases()} + is_template = name.casefold().endswith("_template") + if ( + allow_template_source + and name.casefold() == settings.WNDB_TEMPLATE_DB_NAME.casefold() + ): + return + if name.casefold() in protected or is_template: + raise ValueError(f"Database {name!r} is protected and cannot be managed as a project") def list_project() -> list[str]: @@ -16,92 +49,219 @@ def list_project() -> list[str]: with conn.cursor(row_factory=dict_row) as cur: for p in cur.execute( "select datname from pg_database where datname <> all(%s) order by datname", - (_server_databases,), + (list(_protected_databases()),), ): - ps.append(p["datname"]) + if not str(p["datname"]).casefold().endswith("_template"): + ps.append(p["datname"]) return ps +@contextmanager +def _database_locks(cur, *database_names: str): + """Serialize physical database lifecycle operations across server workers.""" + lock_names = sorted(set(database_names), key=str.casefold) + for database_name in lock_names: + cur.execute( + "select pg_advisory_lock(hashtextextended(%s, 0))", + (f"tjwater:wndb:{database_name}",), + ) + try: + yield + finally: + for database_name in reversed(lock_names): + cur.execute( + "select pg_advisory_unlock(hashtextextended(%s, 0))", + (f"tjwater:wndb:{database_name}",), + ) + + +@contextmanager +def _temporary_database_capacity(cur, database_name: str): + """Serialize temporary creation and enforce a server-wide hard limit.""" + if not database_name.startswith(_TEMPORARY_DATABASE_PREFIX): + yield + return + + lock_name = "tjwater:wndb:temporary-database-capacity" + cur.execute("select pg_advisory_lock(hashtextextended(%s, 0))", (lock_name,)) + try: + cur.execute( + "select count(*) as count from pg_database where datname like %s", + (f"{_TEMPORARY_DATABASE_PREFIX}%",), + ) + row = cur.fetchone() + count = int(row["count"] if row is not None else 0) + limit = max(1, settings.WNDB_TEMP_DB_MAX_COUNT) + if count >= limit: + raise RuntimeError( + f"Temporary database limit reached ({count}/{limit}); " + "retry after an active analysis completes" + ) + yield + finally: + cur.execute("select pg_advisory_unlock(hashtextextended(%s, 0))", (lock_name,)) + + +def _database_allows_connections(cur, database_name: str) -> bool: + cur.execute( + "select datallowconn from pg_database where datname = %s", + (database_name,), + ) + row = cur.fetchone() + if row is None: + raise ValueError(f"Database {database_name!r} does not exist") + return bool(row["datallowconn"]) + + +def _set_database_connections(cur, database_name: str, *, allowed: bool) -> None: + cur.execute( + "update pg_database set datallowconn = %s where datname = %s", + (allowed, database_name), + ) + + +def temporary_project_name(project: str, purpose: str) -> str: + """Return a collision-resistant physical database name for one run.""" + physical_name = get_project_database_name(project) + safe_purpose = re.sub(r"[^a-z0-9_]+", "_", purpose.casefold()).strip("_") + safe_project = re.sub(r"[^a-z0-9_]+", "_", physical_name.casefold()).strip("_") + prefix = ( + f"{_TEMPORARY_DATABASE_PREFIX}{safe_purpose or 'run'}_" + f"{safe_project or 'project'}" + )[:29] + return f"{prefix}_{uuid4().hex}" + + +@contextmanager +def temporary_project_database(project: str, purpose: str): + """Clone one project's runnable model into an isolated temporary database.""" + temporary_name = temporary_project_name(project, purpose) + try: + copy_project(get_project_template_database_name(project), temporary_name) + # Import lazily to keep physical database lifecycle independent from + # model-copy implementation details at module import time. + from .database import refresh_materialized_views_after_commit + from .model_replace import replace_project_model + + replace_project_model( + temporary_name, + project, + copy_source_scada=True, + ) + refresh_materialized_views_after_commit(temporary_name) + yield temporary_name + finally: + if have_project(temporary_name): + delete_project(temporary_name) + + +@contextmanager +def temporary_template_database(name_hint: str, purpose: str): + """Create an empty schema-only temporary database from the fixed template.""" + temporary_name = temporary_project_name(name_hint, purpose) + try: + copy_project(get_project_template_database_name(name_hint), temporary_name) + yield temporary_name + finally: + if have_project(temporary_name): + delete_project(temporary_name) + + def have_project(name: str) -> bool: + database_name = get_project_database_name(name) with admin_connection() as conn: with conn.cursor() as cur: - cur.execute("select 1 from pg_database where datname = %s", (name,)) + cur.execute("select 1 from pg_database where datname = %s", (database_name,)) return cur.fetchone() is not None def copy_project(source: str, new: str) -> None: + physical_source = get_project_database_name(source) + physical_new = get_project_database_name(new) + _validate_project_database(physical_source, allow_template_source=True) + _validate_project_database(physical_new) close_project_pool(source) + close_project_pool(new) with admin_connection() as admin_conn: with admin_conn.cursor() as cur: - cur.execute( - "update pg_database set datallowconn = false where datname = %s", - (source,), - ) - try: - cur.execute( - "select pg_terminate_backend(pid) from pg_stat_activity where datname = %s and pid <> pg_backend_pid()", - (source,), - ) - cur.execute( - sql.SQL("create database {} with template = {}").format( - sql.Identifier(new), sql.Identifier(source) - ) - ) - finally: - cur.execute( - "update pg_database set datallowconn = true where datname = %s", - (source,), - ) + with _temporary_database_capacity(cur, physical_new): + with _database_locks(cur, physical_source, physical_new): + source_allowed = _database_allows_connections(cur, physical_source) + if source_allowed: + _set_database_connections( + cur, + physical_source, + allowed=False, + ) + try: + cur.execute( + "select pg_terminate_backend(pid) from pg_stat_activity " + "where datname = %s and pid <> pg_backend_pid()", + (physical_source,), + ) + cur.execute( + sql.SQL("create database {} with template = {}").format( + sql.Identifier(physical_new), + sql.Identifier(physical_source), + ) + ) + finally: + if source_allowed: + _set_database_connections( + cur, + physical_source, + allowed=True, + ) def create_project(name: str) -> None: - return copy_project("project", name) + return copy_project(get_project_template_database_name(name), name) def delete_project(name: str) -> None: + database_name = get_project_database_name(name) + _validate_project_database(database_name) close_project_pool(name) with admin_connection() as conn: with conn.cursor() as cur: - cur.execute( - "select pg_terminate_backend(pid) from pg_stat_activity " - "where datname = %s and pid <> pg_backend_pid()", - (name,), - ) - cur.execute( - sql.SQL("drop database {}").format(sql.Identifier(name)) - ) + with _database_locks(cur, database_name): + was_allowed = _database_allows_connections(cur, database_name) + if was_allowed: + _set_database_connections(cur, database_name, allowed=False) + try: + cur.execute( + "select pg_terminate_backend(pid) from pg_stat_activity " + "where datname = %s and pid <> pg_backend_pid()", + (database_name,), + ) + cur.execute( + sql.SQL("drop database {}").format( + sql.Identifier(database_name) + ) + ) + except Exception: + if was_allowed: + _set_database_connections(cur, database_name, allowed=True) + raise -def clean_project(excluded: list[str] = []) -> None: - projects = list_project() +def clean_project(projects: Iterable[str]) -> None: + """Delete only the explicitly supplied project databases.""" + targets = list(dict.fromkeys(projects)) + physical_targets = [get_project_database_name(project) for project in targets] + for database_name in physical_targets: + _validate_project_database(database_name) + + if not targets: + return + with admin_connection() as conn: with conn.cursor(row_factory=dict_row) as cur: row = cur.execute("select current_database()").fetchone() - if row != None: - current_db = row["current_database"] - if current_db in projects: - projects.remove(current_db) - for project in projects: - if project in _server_databases or project in excluded: - continue - cur.execute( - "select pg_terminate_backend(pid) from pg_stat_activity " - "where datname = %s and pid <> pg_backend_pid()", - (project,), - ) - cur.execute( - sql.SQL("drop database {}").format(sql.Identifier(project)) - ) + current_db = row["current_database"] if row is not None else None + if current_db in physical_targets: + raise ValueError(f"Cannot delete the current database {current_db!r}") - -def open_project(name: str) -> None: - get_project_pool(name) - - -def is_project_open(name: str) -> bool: - return is_project_pool_open(name) - - -def close_project(name: str) -> None: - close_project_pool(name) + for project in targets: + delete_project(project) diff --git a/app/native/wndb/gis/coordinates.py b/app/native/wndb/gis/coordinates.py index 1c53780..7a24322 100644 --- a/app/native/wndb/gis/coordinates.py +++ b/app/native/wndb/gis/coordinates.py @@ -1,10 +1,5 @@ -from psycopg.rows import dict_row +from ..core.database import read_all, sql_literal, try_read -from ..core.connection import project_connection -from ..core.database import read_all, sql_literal, try_read, write -from ..core.connection import project_connection -from ..model.elements import get_link_nodes -from psycopg.rows import dict_row def sql_update_coord(node: str, x: float, y: float) -> str: geom = f"st_setsrid(st_makepoint({sql_literal(x)}, {sql_literal(y)}), 900914)" @@ -21,8 +16,8 @@ def sql_delete_coord(node: str) -> str: def from_postgis_point(coord: str) -> dict[str, float]: - xy = coord.lower().removeprefix('point(').removesuffix(')').split(' ') - return { 'x': float(xy[0]), 'y': float(xy[1]) } + xy = coord.lower().removeprefix("point(").removesuffix(")").split(" ") + return {"x": float(xy[0]), "y": float(xy[1])} def get_node_coord(name: str, node: str) -> dict[str, float]: @@ -31,51 +26,15 @@ def get_node_coord(name: str, node: str) -> dict[str, float]: "select st_astext(geom) as coord_geom from gis.node_geometries where node_id = %s", (node,), ) - if row == None: - write(name, sql_insert_coord(node, 0.0, 0.0)) - return {'x': 0.0, 'y': 0.0} - return from_postgis_point(row['coord_geom']) - -# DingZQ 2025-01-03, get nodes in extent -# return node id list -# node_id:junction:x:y -def get_nodes_in_extent(name: str, x1: float, y1: float, x2: float, y2: float) -> list[str]: - nodes = [] - objs = read_all(name, 'select node_id, st_astext(geom) as coord_geom from gis.node_geometries') - for obj in objs: - node_id = obj['node_id'] - coord = from_postgis_point(obj['coord_geom']) - x = coord['x'] - y = coord['y'] - if x1 <= x <= x2 and y1 <= y <= y2: - nodes.append(f"{node_id}:junction:{x}:{y}") - return nodes - -# DingZQ 2025-01-03, get links in extent -# return link id list -# link_id:pipe:node_id1:node_id2 -def get_links_in_extent(name: str, x1: float, y1: float, x2: float, y2: float) -> list[str]: - node_ids = set([s.split(':')[0] for s in get_nodes_in_extent(name, x1, y1, x2, y2)]) - - all_link_ids = [] - with project_connection(name) as conn: - with conn.cursor(row_factory=dict_row) as cur: - cur.execute("select link_id from network.pipes") - for record in cur: - all_link_ids.append(record['link_id']) - - links = [] - for link_id in all_link_ids: - nodes = get_link_nodes(name, link_id) - if nodes[0] in node_ids and nodes[1] in node_ids: - links.append(f"{link_id}:pipe:{nodes[0]}:{nodes[1]}") - return links + if row is None: + return {"x": 0.0, "y": 0.0} + return from_postgis_point(row["coord_geom"]) def node_has_coord(name: str, node: str) -> bool: return try_read( name, "select node_id from gis.node_geometries where node_id = %s", (node,) - ) != None + ) is not None #-------------------------------------------------------------- @@ -94,11 +53,14 @@ def inp_in_coord(line: str) -> str: def inp_out_coord(name: str) -> list[str]: lines = [] - objs = read_all(name, 'select node_id, st_astext(geom) as coord_geom from gis.node_geometries') + objs = read_all( + name, + "select node_id, st_astext(geom) as coord_geom from gis.node_geometries", + ) for obj in objs: - node = obj['node_id'] - coord = from_postgis_point(obj['coord_geom']) - x = coord['x'] - y = coord['y'] - lines.append(f'{node} {x} {y}') + node = obj["node_id"] + coord = from_postgis_point(obj["coord_geom"]) + x = coord["x"] + y = coord["y"] + lines.append(f"{node} {x} {y}") return lines diff --git a/app/native/wndb/gis/network_views.py b/app/native/wndb/gis/network_views.py new file mode 100644 index 0000000..25b6e99 --- /dev/null +++ b/app/native/wndb/gis/network_views.py @@ -0,0 +1,191 @@ +"""Read-only projections backed by the GIS materialized-view query layer.""" + +from typing import Any + +from ..core.database import read, read_all + + +def _node_coord_rows( + rows: list[Any], +) -> dict[str, dict[str, Any]]: + return { + str(row["id"]): { + "x": float(row["x"]), + "y": float(row["y"]), + "type": str(row["node_type"]), + } + for row in rows + } + + +def get_network_node_coords(name: str) -> dict[str, dict[str, Any]]: + """Return every publishable node with one view-backed query.""" + rows = read_all( + name, + """ + SELECT id, x, y, node_type + FROM gis.network_nodes + ORDER BY id + """, + ) + return _node_coord_rows(rows) + + +def get_major_node_coords( + name: str, diameter: int +) -> dict[str, dict[str, Any]]: + """Return endpoints of pipes above the requested diameter.""" + rows = read_all( + name, + """ + SELECT n.id, n.x, n.y, n.node_type + FROM gis.pipes AS p + CROSS JOIN LATERAL ( + VALUES (p.start_node_id), (p.end_node_id) + ) AS endpoint(id) + JOIN gis.network_nodes AS n ON n.id = endpoint.id + WHERE p.diameter > %s + GROUP BY n.id, n.x, n.y, n.node_type + ORDER BY n.id + """, + (diameter,), + ) + return _node_coord_rows(rows) + + +def get_network_link_nodes(name: str) -> list[str]: + """Return every publishable link in the established API wire format.""" + rows = read_all( + name, + """ + SELECT id, link_type, start_node_id, end_node_id + FROM gis.network_links + ORDER BY id + """, + ) + return [ + f"{row['id']}:{row['link_type']}:{row['start_node_id']}:{row['end_node_id']}" + for row in rows + ] + + +def get_major_pipe_nodes(name: str, diameter: int) -> list[str]: + """Return large pipes in the established API wire format.""" + rows = read_all( + name, + """ + SELECT id, start_node_id, end_node_id + FROM gis.pipes + WHERE diameter > %s + ORDER BY id + """, + (diameter,), + ) + return [ + f"{row['id']}:pipe:{row['start_node_id']}:{row['end_node_id']}" + for row in rows + ] + + +def get_topology_rows( + name: str, node_ids: list[str] +) -> tuple[list[Any], list[Any]]: + """Load selected nodes and their internal links with two batch queries.""" + if not node_ids: + return [], [] + + nodes = read_all( + name, + """ + SELECT n.id, + ST_X(g.geom) AS x, + ST_Y(g.geom) AS y, + n.node_type::text AS node_type + FROM network.nodes AS n + JOIN gis.node_geometries AS g ON g.node_id = n.id + WHERE n.id = ANY(%s) + ORDER BY n.id + """, + (node_ids,), + ) + links = read_all( + name, + """ + SELECT l.id, + l.start_node_id, + l.end_node_id, + COALESCE(p.length, 0.0) AS length + FROM network.links AS l + LEFT JOIN network.pipes AS p ON p.link_id = l.id + WHERE l.start_node_id = ANY(%s) + AND l.end_node_id = ANY(%s) + ORDER BY l.id + """, + (node_ids, node_ids), + ) + return nodes, links + + +def get_boundary_link_ids(name: str, node_ids: list[str]) -> list[str]: + """Return links with exactly one endpoint inside the supplied node set.""" + if not node_ids: + return [] + rows = read_all( + name, + """ + SELECT id + FROM network.links + WHERE (start_node_id = ANY(%s)) <> (end_node_id = ANY(%s)) + ORDER BY id + """, + (node_ids, node_ids), + ) + return [str(row["id"]) for row in rows] + + +def get_junction_demands( + name: str, node_ids: list[str] +) -> dict[str, list[dict[str, Any]]]: + """Read authoritative demands for selected junctions from model tables.""" + if not node_ids: + return {} + rows = read_all( + name, + """ + SELECT junction_id, + sequence_no, + base_demand, + pattern_id, + category + FROM network.demands + WHERE junction_id = ANY(%s) + ORDER BY junction_id, sequence_no + """, + (node_ids,), + ) + result: dict[str, list[dict[str, Any]]] = {} + for row in rows: + result.setdefault(str(row["junction_id"]), []).append( + { + "demand": float(row["base_demand"]), + "pattern": row["pattern_id"], + "category": row["category"], + } + ) + return result + + +def sum_junction_base_demand(name: str, node_ids: list[str]) -> float: + """Sum selected junction demand from the authoritative model table.""" + if not node_ids: + return 0.0 + row = read( + name, + """ + SELECT COALESCE(SUM(base_demand), 0.0) AS total_base_demand + FROM network.demands + WHERE junction_id = ANY(%s) + """, + (node_ids,), + ) + return float(row["total_base_demand"]) diff --git a/app/native/wndb/gis/region_geometry.py b/app/native/wndb/gis/region_geometry.py index ac0e51c..2e86dbd 100644 --- a/app/native/wndb/gis/region_geometry.py +++ b/app/native/wndb/gis/region_geometry.py @@ -2,10 +2,8 @@ import platform import math from typing import Any import pyclipper -from ..model.elements import get_node_links, get_link_nodes, is_pipe -from ..model.pipes import get_pipe from ..core.database import read, try_read, read_all -from .coordinates import node_has_coord, get_node_coord +from .network_views import get_boundary_link_ids, get_topology_rows def from_postgis_polygon(polygon: str) -> list[tuple[float, float]]: @@ -42,21 +40,7 @@ def get_nodes_in_boundary(name: str, boundary: list[tuple[float, float]]) -> lis def _get_links_on_boundary(name: str, nodes: list[str]) -> list[str]: - links: list[str] = [] - - for node in nodes: - node_links = get_node_links(name, node) - for link in node_links: - if link in links: - continue - - link_nodes = get_link_nodes(name, link) - if link_nodes[0] in nodes and link_nodes[1] not in nodes: - links.append(link) - elif link_nodes[0] not in nodes and link_nodes[1] in nodes: - links.append(link) - - return links + return get_boundary_link_ids(name, nodes) def get_nodes_in_region(name: str, region_id: str) -> list[str]: @@ -128,37 +112,39 @@ def _angle_of_node_link(node: str, link: str, nodes, links) -> float: class Topology: def __init__(self, db: str, nodes: list[str]) -> None: - self._nodes: dict[str, Any] = {} - self._max_x_node = '' - self._node_list: list[str] = [] - for node in nodes: - if not node_has_coord(db, node): - continue - if get_node_links(db, node) == 0: - continue - self._nodes[node] = get_node_coord(db, node) | { 'links': [] } - self._node_list.append(node) - if self._max_x_node == '' or self._nodes[node]['x'] > self._nodes[self._max_x_node]['x']: - self._max_x_node = node + node_rows, link_rows = get_topology_rows(db, nodes) + + self._nodes: dict[str, Any] = { + str(row["id"]): { + "x": float(row["x"]), + "y": float(row["y"]), + "type": str(row["node_type"]), + "links": [], + } + for row in node_rows + } + self._node_list = list(self._nodes) + self._max_x_node = max( + self._nodes, + key=lambda node_id: self._nodes[node_id]["x"], + default="", + ) self._links: dict[str, Any] = {} - self._link_list: list[str] = [] - for node in self._nodes: - for link in get_node_links(db, node): - candidate = True - link_nodes = get_link_nodes(db, link) - for link_node in link_nodes: - if link_node not in self._nodes: - candidate = False - break - if candidate: - length = get_pipe(db, link)['length'] if is_pipe(db, link) else 0.0 - self._links[link] = { 'node1' : link_nodes[0], 'node2' : link_nodes[1], 'length' : length } - self._link_list.append(link) - if link not in self._nodes[link_nodes[0]]['links']: - self._nodes[link_nodes[0]]['links'].append(link) - if link not in self._nodes[link_nodes[1]]['links']: - self._nodes[link_nodes[1]]['links'].append(link) + for row in link_rows: + link_id = str(row["id"]) + node1 = str(row["start_node_id"]) + node2 = str(row["end_node_id"]) + if node1 not in self._nodes or node2 not in self._nodes: + continue + self._links[link_id] = { + "node1": node1, + "node2": node2, + "length": float(row["length"]), + } + self._nodes[node1]["links"].append(link_id) + self._nodes[node2]["links"].append(link_id) + self._link_list = list(self._links) def nodes(self): return self._nodes diff --git a/app/native/wndb/inp/exporter.py b/app/native/wndb/inp/exporter.py index 4638f88..34b708b 100644 --- a/app/native/wndb/inp/exporter.py +++ b/app/native/wndb/inp/exporter.py @@ -1,6 +1,6 @@ import os -from ..core.projects import close_project, have_project, is_project_open, open_project +from ..core.projects import have_project from ..core.database import ChangeSet from .sections import ( BACKDROP, @@ -56,7 +56,7 @@ from ..model.reactions import inp_out_reaction from ..model.mixing import inp_out_mixing from ..model.times import inp_out_time from ..model.reports import inp_out_report -from ..model.options_legacy import inp_out_option +from ..model.options_v2 import inp_out_option_v2 from ..model.options_v3 import inp_out_option_v3 from ..gis.coordinates import inp_out_coord from ..gis.vertices import inp_out_vertex @@ -72,11 +72,6 @@ def dump_inp(project: str, inp: str, version: str = '3'): if not have_project(project): return - project_open = is_project_open(project) - - if not project_open: - open_project(project) - dir = os.getcwd() path = os.path.join(dir, inp) @@ -173,7 +168,7 @@ def dump_inp(project: str, inp: str, version: str = '3'): if version == '3': file.write('\n'.join(inp_out_option_v3(project))) else: - file.write('\n'.join(inp_out_option(project))) + file.write('\n'.join(inp_out_option_v2(project))) elif name == COORDINATES: file.write('\n'.join(inp_out_coord(project))) @@ -194,10 +189,6 @@ def dump_inp(project: str, inp: str, version: str = '3'): file.close() - if not project_open: - close_project(project) - - def export_inp(project: str, version: str = '3') -> ChangeSet: if version != '3' and version != '2': version = '2' @@ -205,11 +196,6 @@ def export_inp(project: str, version: str = '3') -> ChangeSet: if not have_project(project): return ChangeSet() - project_open = is_project_open(project) - - if not project_open: - open_project(project) - inp = '' for name in section_name: @@ -294,7 +280,7 @@ def export_inp(project: str, version: str = '3') -> ChangeSet: if version == '3': inp += '\n'.join(inp_out_option_v3(project)) else: - inp += '\n'.join(inp_out_option(project)) + inp += '\n'.join(inp_out_option_v2(project)) elif name == COORDINATES: inp += '\n'.join(inp_out_coord(project)) @@ -313,7 +299,4 @@ def export_inp(project: str, version: str = '3') -> ChangeSet: inp += '\n' - if not project_open: - close_project(project) - return ChangeSet({'operation': 'export', 'inp': inp}) diff --git a/app/native/wndb/inp/importer.py b/app/native/wndb/inp/importer.py index 03a6d73..1bff391 100644 --- a/app/native/wndb/inp/importer.py +++ b/app/native/wndb/inp/importer.py @@ -1,18 +1,26 @@ import datetime +import logging import os +from tempfile import NamedTemporaryFile from psycopg import sql from ..core.projects import ( - close_project, - create_project, + copy_project, delete_project, have_project, - is_project_open, - open_project, + temporary_project_name, + temporary_template_database, ) +from app.infra.db.project_routing import get_project_template_database_name from ..core.connection import project_transaction -from ..core.database import ChangeSet, refresh_materialized_views, sql_literal, write +from ..core.model_replace import replace_project_model +from ..core.database import ( + ChangeSet, + refresh_materialized_views_after_commit, + sql_literal, + write, +) from .sections import ( BACKDROP, BOUND, @@ -68,7 +76,7 @@ from ..model.reactions import inp_in_reaction from ..model.mixing import inp_in_mixing from ..model.times import inp_in_time from ..model.reports import inp_in_report -from ..model.options_legacy import inp_in_option +from ..model.options_v2 import inp_in_option_v2 from ..model.options_v3 import inp_in_option_v3 from ..gis.coordinates import inp_in_coord from ..gis.vertices import inp_in_vertex @@ -82,10 +90,11 @@ from .exporter import export_inp _S = "S" _L = "L" +logger = logging.getLogger(__name__) def _inp_in_option(section: list[str], version: str = "3") -> str: - return inp_in_option_v3(section) if version == "3" else inp_in_option(section) + return inp_in_option_v3(section) if version == "3" else inp_in_option_v2(section) _handler = { @@ -389,60 +398,49 @@ def read_inp(project: str, inp: str, version: str = "3") -> bool: if version != "3" and version != "2": version = "2" - if is_project_open(project): - close_project(project) + if not have_project(project): + raise ValueError(f"Project database {project!r} does not exist") - if have_project(project): - delete_project(project) + staging_project = temporary_project_name(project, "model_import") + replacement_committed = False + try: + copy_project(get_project_template_database_name(project), staging_project) + with project_transaction(staging_project): + parse_file(staging_project, inp, version) + replace_project_model(project, staging_project) + replacement_committed = True + finally: + try: + if have_project(staging_project): + delete_project(staging_project) + except Exception: + logger.exception( + "Failed to remove model-import staging database %s", + staging_project, + ) - create_project(project) - open_project(project) + if replacement_committed: + refresh_materialized_views_after_commit(project) - with project_transaction(project): - parse_file(project, inp, version) - refresh_materialized_views(project) - - """try: - parse_file(project, inp, version) - except: - close_project(project) - delete_project(project) - return False""" - - close_project(project) return True # DingZQ, 2024-12-28, convert v3 to v2 def convert_inp_v3_to_v2(inp: str) -> ChangeSet: - project = "v3Tov2" - - if is_project_open(project): - close_project(project) - - if have_project(project): - delete_project(project) - - create_project(project) - open_project(project) - - filename = f"inp/{project}_temp.inp" - if os.path.exists(filename): - os.remove(filename) - - with open(filename, "w", encoding="utf-8") as f: - f.write(inp) - - parse_file(project, filename, "3") - - """try: - parse_file(project, inp, version) - except: - close_project(project) - delete_project(project) - return False""" - - return export_inp(project, "2") + temp_path: str | None = None + with temporary_template_database("conversion", "v3_to_v2") as project: + try: + with NamedTemporaryFile( + mode="w", suffix=".inp", encoding="utf-8", delete=False + ) as temp_file: + temp_file.write(inp) + temp_path = temp_file.name + with project_transaction(project): + parse_file(project, temp_path, "3") + return export_inp(project, "2") + finally: + if temp_path is not None: + os.remove(temp_path) def import_inp(project: str, cs: ChangeSet, version: str = "3") -> bool: @@ -452,17 +450,21 @@ def import_inp(project: str, cs: ChangeSet, version: str = "3") -> bool: if "inp" not in cs.operations[0]: return False - filename = f"inp/{project}_temp.inp" - if os.path.exists(filename): - os.remove(filename) - - _print_time(f'Start writing temp file "{filename}"...') - with open(filename, "w", encoding="utf-8") as f: - f.write(str(cs.operations[0]["inp"])) - _print_time(f'End writing temp file "{filename}"...') - - result = read_inp(project, filename, version) - - # os.remove(filename) - - return result + temp_path: str | None = None + try: + with NamedTemporaryFile( + mode="w", + suffix=".inp", + prefix="tjwater_import_", + encoding="utf-8", + delete=False, + ) as temp_file: + temp_file.write(str(cs.operations[0]["inp"])) + temp_path = temp_file.name + return read_inp(project, temp_path, version) + finally: + if temp_path is not None: + try: + os.remove(temp_path) + except FileNotFoundError: + pass diff --git a/app/native/wndb/model/demands.py b/app/native/wndb/model/demands.py index f0a1228..0331b4b 100644 --- a/app/native/wndb/model/demands.py +++ b/app/native/wndb/model/demands.py @@ -94,16 +94,3 @@ def delete_demand_by_junction(name: str, junction: str) -> ChangeSet: if row is None: return ChangeSet() return ChangeSet(g_update_prefix | {'type': 'demand', 'junction': junction, 'demands': []}) - - -def unset_demand_by_pattern(name: str, pattern: str) -> ChangeSet: - cs = ChangeSet() - - rows = read_all(name, "select distinct junction_id as junction from network.demands where pattern_id = %s", (pattern,)) - for row in rows: - ds = get_demand(name, row['junction']) - for d in ds['demands']: - d['pattern'] = None - cs.append(g_update_prefix | {'type': 'demand', 'junction': row['junction'], 'demands': ds['demands']}) - - return cs diff --git a/app/native/wndb/model/elements.py b/app/native/wndb/model/elements.py index fe8cb93..ceb7908 100644 --- a/app/native/wndb/model/elements.py +++ b/app/native/wndb/model/elements.py @@ -159,26 +159,6 @@ def get_nodes(name: str) -> list[str]: return _get_all(name, _NODE) -def get_nodes_id_and_type(name: str) -> dict[str, str]: - rows = read_all_typed(name, "SELECT id, node_type FROM network.nodes", ()) - return {row["id"]: row["node_type"] for row in rows} - - -def get_major_nodes(name: str, diameter: int) -> list[str]: - rows = read_all_typed( - name, - """ - SELECT DISTINCT endpoint - FROM network.links AS l - JOIN network.pipes AS p ON p.link_id = l.id - CROSS JOIN LATERAL (VALUES (l.start_node_id), (l.end_node_id)) AS e(endpoint) - WHERE p.diameter > %s - """, - (diameter,), - ) - return [row["endpoint"] for row in rows] - - def get_junctions(name: str) -> list[str]: return _get_nodes_by_type(name, JUNCTION) @@ -195,20 +175,6 @@ def get_links(name: str) -> list[str]: return _get_all(name, _LINK) -def get_links_id_and_type(name: str) -> dict[str, str]: - rows = read_all_typed(name, "SELECT id, link_type FROM network.links", ()) - return {row["id"]: row["link_type"] for row in rows} - - -def get_major_pipes(name: str, diameter: int) -> list[str]: - rows = read_all_typed( - name, - "SELECT link_id FROM network.pipes WHERE diameter > %s ORDER BY link_id", - (diameter,), - ) - return [row["link_id"] for row in rows] - - def get_pipes(name: str) -> list[str]: return _get_links_by_type(name, PIPE) @@ -247,10 +213,10 @@ def get_node_links(name: str, node_id: str) -> list[str]: def get_all_node_links(name: str) -> dict[str, list[str]]: - """Build the node adjacency map with one scan of the link table.""" + """Build the node adjacency map with one scan of the unified GIS view.""" rows = read_all_typed( name, - "SELECT id, start_node_id, end_node_id FROM network.links ORDER BY id", + "SELECT id, start_node_id, end_node_id FROM gis.network_links ORDER BY id", (), ) result: dict[str, list[str]] = {} diff --git a/app/native/wndb/model/junctions.py b/app/native/wndb/model/junctions.py index e7af019..cd7a241 100644 --- a/app/native/wndb/model/junctions.py +++ b/app/native/wndb/model/junctions.py @@ -6,6 +6,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -94,8 +95,12 @@ class Junction(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'x': self.x, 'y': self.y, 'elevation': self.elevation } -def _set_junction(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_junction(name, cs.operations[0]['id']) +def _set_junction( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_junction(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_junction_schema(name) @@ -113,11 +118,15 @@ def _set_junction(name: str, cs: ChangeSet) -> DatabaseCommand: def set_junction(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_junction(name, cs.operations[0]['id']) == {}: - return ChangeSet() - return execute_command(name, _set_junction(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_junction(name, operation['id']) + return None if current == {} else _set_junction(name, cs, current) + + return execute_locked_command(name, build_command) def _add_junction(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/native/wndb/model/options_legacy.py b/app/native/wndb/model/options_v2.py similarity index 90% rename from app/native/wndb/model/options_legacy.py rename to app/native/wndb/model/options_v2.py index 253c416..15b66a9 100644 --- a/app/native/wndb/model/options_legacy.py +++ b/app/native/wndb/model/options_v2.py @@ -1,10 +1,8 @@ -from psycopg import sql - from ..core.database import ChangeSet, g_update_prefix, read_all, sql_literal from .options import get_option_schema, generate_v3 -def _inp_in_option(section: list[str]) -> ChangeSet: +def _inp_in_option_v2(section: list[str]) -> ChangeSet: if len(section) <= 0: return ChangeSet() @@ -34,9 +32,9 @@ def _inp_in_option(section: list[str]) -> ChangeSet: return result -def inp_in_option(section: list[str]) -> str: +def inp_in_option_v2(section: list[str]) -> str: sql = '' - result = _inp_in_option(section) + result = _inp_in_option_v2(section) for op in result.operations: for key in op.keys(): if key == 'operation' or key == 'type': @@ -48,7 +46,7 @@ def inp_in_option(section: list[str]) -> str: return sql -def inp_out_option(name: str) -> list[str]: +def inp_out_option_v2(name: str) -> list[str]: lines = [] objs = read_all(name, "select key, value from network.simulation_settings where engine_version = 'legacy' order by key") @@ -71,7 +69,7 @@ def inp_out_option(name: str) -> list[str]: # why write this ? if key == 'PRESSURE': continue - # release version does not support new keys and has error message + # EPANET V2 does not support these newer keys. if key == 'HTOL' or key == 'QTOL' or key == 'RQTOL': continue # ignore some weird settings for DDA diff --git a/app/native/wndb/model/options_v3.py b/app/native/wndb/model/options_v3.py index ecf84b7..9e00b8a 100644 --- a/app/native/wndb/model/options_v3.py +++ b/app/native/wndb/model/options_v3.py @@ -1,5 +1,3 @@ -from psycopg import sql - from ..core.database import ChangeSet, g_update_prefix, read_all, sql_literal from .options import get_option_schema, get_option_v3_schema, generate_v2, generate_v3 diff --git a/app/native/wndb/model/patterns.py b/app/native/wndb/model/patterns.py index ffe8f70..d5f1237 100644 --- a/app/native/wndb/model/patterns.py +++ b/app/native/wndb/model/patterns.py @@ -93,7 +93,10 @@ def _delete_pattern(name: str, cs: ChangeSet) -> DatabaseCommand: id = cs.operations[0]['id'] f_id = sql_literal(id) - statement = f"delete from network.patterns where id = {f_id};" + statement = ( + f"update network.demands set pattern_id = null where pattern_id = {f_id};" + f"\ndelete from network.patterns where id = {f_id};" + ) change = g_delete_prefix | { 'type': 'pattern' } | { 'id': id } diff --git a/app/native/wndb/model/pipes.py b/app/native/wndb/model/pipes.py index ae45e30..91d3b0e 100644 --- a/app/native/wndb/model/pipes.py +++ b/app/native/wndb/model/pipes.py @@ -4,6 +4,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -144,9 +145,12 @@ class Pipe(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'node1': self.node1, 'node2': self.node2, 'length': self.length, 'diameter': self.diameter, 'roughness': self.roughness, 'minor_loss': self.minor_loss, 'status': self.status } -def _set_pipe(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_pipe(name, cs.operations[0]['id']) - +def _set_pipe( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_pipe(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_pipe_schema(name) for key, value in schema.items(): @@ -154,19 +158,59 @@ def _set_pipe(name: str, cs: ChangeSet) -> DatabaseCommand: raw_new[key] = new_dict[key] new = Pipe(raw_new) - statement = f"update network.links set start_node_id = {new.f_node1}, end_node_id = {new.f_node2} where id = {new.f_id};" - statement += f"\nupdate network.pipes set length = {new.f_length}, diameter = {new.f_diameter}, roughness = {new.f_roughness}, minor_loss = {new.f_minor_loss}, status = {new.f_status} where link_id = {new.f_id};" + link_columns = { + 'node1': ('start_node_id', new.f_node1), + 'node2': ('end_node_id', new.f_node2), + } + pipe_columns = { + 'length': ('length', new.f_length), + 'diameter': ('diameter', new.f_diameter), + 'roughness': ('roughness', new.f_roughness), + 'minor_loss': ('minor_loss', new.f_minor_loss), + 'status': ('status', new.f_status), + } + statements = [] + link_assignments = [ + f"{column} = {value}" + for field, (column, value) in link_columns.items() + if field in new_dict + ] + if link_assignments: + statements.append( + f"update network.links set {', '.join(link_assignments)} where id = {new.f_id};" + ) + pipe_assignments = [ + f"{column} = {value}" + for field, (column, value) in pipe_columns.items() + if field in new_dict + ] + if pipe_assignments: + statements.append( + f"update network.pipes set {', '.join(pipe_assignments)} where link_id = {new.f_id};" + ) + statement = "\n".join(statements) change = g_update_prefix | new.as_dict() return DatabaseCommand(statement, [change]) def set_pipe(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_pipe(name, cs.operations[0]['id']) == {}: + mutable_fields = { + 'node1', 'node2', 'length', 'diameter', 'roughness', 'minor_loss', 'status' + } + if not mutable_fields.intersection(operation): return ChangeSet() - return execute_command(name, _set_pipe(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_pipe(name, operation['id']) + if current == {}: + return None + return _set_pipe(name, cs, current) + + return execute_locked_command(name, build_command) def _add_pipe(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/native/wndb/model/pumps.py b/app/native/wndb/model/pumps.py index fb2657c..8dbbbc1 100644 --- a/app/native/wndb/model/pumps.py +++ b/app/native/wndb/model/pumps.py @@ -4,6 +4,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -87,8 +88,12 @@ class Pump(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'node1': self.node1, 'node2': self.node2, 'power': self.power, 'head': self.head, 'speed': self.speed, 'pattern': self.pattern } -def _set_pump(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_pump(name, cs.operations[0]['id']) +def _set_pump( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_pump(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_pump_schema(name) @@ -105,11 +110,15 @@ def _set_pump(name: str, cs: ChangeSet) -> DatabaseCommand: def set_pump(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_pump(name, cs.operations[0]['id']) == {}: - return ChangeSet() - return execute_command(name, _set_pump(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_pump(name, operation['id']) + return None if current == {} else _set_pump(name, cs, current) + + return execute_locked_command(name, build_command) def _add_pump(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/native/wndb/model/reservoirs.py b/app/native/wndb/model/reservoirs.py index 1e0bceb..326469c 100644 --- a/app/native/wndb/model/reservoirs.py +++ b/app/native/wndb/model/reservoirs.py @@ -4,6 +4,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -85,8 +86,12 @@ class Reservoir(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'x': self.x, 'y': self.y, 'head': self.head, 'pattern': self.pattern } -def _set_reservoir(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_reservoir(name, cs.operations[0]['id']) +def _set_reservoir( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_reservoir(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_reservoir_schema(name) @@ -104,11 +109,15 @@ def _set_reservoir(name: str, cs: ChangeSet) -> DatabaseCommand: def set_reservoir(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_reservoir(name, cs.operations[0]['id']) == {}: - return ChangeSet() - return execute_command(name, _set_reservoir(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_reservoir(name, operation['id']) + return None if current == {} else _set_reservoir(name, cs, current) + + return execute_locked_command(name, build_command) def _add_reservoir(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/native/wndb/model/tanks.py b/app/native/wndb/model/tanks.py index d610df7..b9173ba 100644 --- a/app/native/wndb/model/tanks.py +++ b/app/native/wndb/model/tanks.py @@ -4,6 +4,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -121,8 +122,12 @@ class Tank(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'x': self.x, 'y': self.y, 'elevation': self.elevation, 'init_level': self.init_level, 'min_level': self.min_level, 'max_level': self.max_level, 'diameter': self.diameter, 'min_vol': self.min_vol, 'vol_curve': self.vol_curve, 'overflow': self.overflow } -def _set_tank(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_tank(name, cs.operations[0]['id']) +def _set_tank( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_tank(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_tank_schema(name) @@ -140,11 +145,15 @@ def _set_tank(name: str, cs: ChangeSet) -> DatabaseCommand: def set_tank(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_tank(name, cs.operations[0]['id']) == {}: - return ChangeSet() - return execute_command(name, _set_tank(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_tank(name, operation['id']) + return None if current == {} else _set_tank(name, cs, current) + + return execute_locked_command(name, build_command) def _add_tank(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/native/wndb/model/valves.py b/app/native/wndb/model/valves.py index 687af97..3133166 100644 --- a/app/native/wndb/model/valves.py +++ b/app/native/wndb/model/valves.py @@ -4,6 +4,7 @@ from ..core.database import ( ChangeSet, DatabaseCommand, execute_command, + execute_locked_command, g_add_prefix, g_delete_prefix, g_update_prefix, @@ -94,8 +95,12 @@ class Valve(object): def as_dict(self) -> dict[str, Any]: return { 'type': self.type, 'id': self.id, 'node1': self.node1, 'node2': self.node2, 'diameter': self.diameter, 'v_type': self.v_type, 'setting': self.setting, 'minor_loss': self.minor_loss } -def _set_valve(name: str, cs: ChangeSet) -> DatabaseCommand: - raw_new = get_valve(name, cs.operations[0]['id']) +def _set_valve( + name: str, + cs: ChangeSet, + current: dict[str, Any] | None = None, +) -> DatabaseCommand: + raw_new = current if current is not None else get_valve(name, cs.operations[0]['id']) new_dict = cs.operations[0] schema = get_valve_schema(name) @@ -112,11 +117,15 @@ def _set_valve(name: str, cs: ChangeSet) -> DatabaseCommand: def set_valve(name: str, cs: ChangeSet) -> ChangeSet: - if 'id' not in cs.operations[0]: + operation = cs.operations[0] + if 'id' not in operation: return ChangeSet() - if get_valve(name, cs.operations[0]['id']) == {}: - return ChangeSet() - return execute_command(name, _set_valve(name, cs)) + + def build_command() -> DatabaseCommand | None: + current = get_valve(name, operation['id']) + return None if current == {} else _set_valve(name, cs, current) + + return execute_locked_command(name, build_command) def _add_valve(name: str, cs: ChangeSet) -> DatabaseCommand: diff --git a/app/services/__init__.py b/app/services/__init__.py index 1c6f317..a7827ab 100644 --- a/app/services/__init__.py +++ b/app/services/__init__.py @@ -1,5 +1,5 @@ """Service package. Keep package initialization lightweight. Import concrete service modules directly, -for example: `from app.services.tjnetwork import open_project`. +for example: `from app.services.tjnetwork import get_junction`. """ diff --git a/app/services/simulation.py b/app/services/simulation.py index 6ea495c..b92baaa 100644 --- a/app/services/simulation.py +++ b/app/services/simulation.py @@ -9,7 +9,6 @@ from app.services.tjnetwork import ( get_status, get_tank, get_time, - open_project, read_all, run_project, set_demand, @@ -34,7 +33,8 @@ import logging import app.services.globals as globals import app.services.project_info as project_info from app.services.time_api import parse_beijing_time, parse_clock_duration_seconds -from app.native.wndb.core.connection import project_connection +from app.native.wndb.core.connection import project_connection, project_transaction +from app.native.wndb.core.database import refresh_materialized_views_after_commit from app.infra.db.timescaledb.internal_queries import ( InternalQueries as TimescaleInternalQueries, ) @@ -48,6 +48,31 @@ logging.basicConfig( ) +def _primary_demand(demand_set: dict) -> dict: + """Return sequence-zero demand, creating it when a junction has none.""" + demands = demand_set.setdefault("demands", []) + if not demands: + demands.append({"demand": 0.0, "pattern": None, "category": None}) + return demands[0] + + +def _primary_demand_pattern(demand_set: dict) -> str: + """Return the first configured pattern in deterministic sequence order.""" + pattern = next( + ( + demand.get("pattern") + for demand in demand_set.get("demands", []) + if demand.get("pattern") + ), + None, + ) + if pattern is None: + raise ValueError( + f"Junction {demand_set.get('junction')!r} has no demand pattern" + ) + return str(pattern) + + def query_corresponding_element_id_and_query_id(name: str) -> None: """Load realtime device-to-element mappings from the new asset schema.""" target_maps = { @@ -221,281 +246,283 @@ def run_simulation( # elif simulation_type.upper() == 'EXTENDED': # 扩展模拟(复制数据库) # name_c = '_'.join([name, 'c']) # if have_project(name_c): - # if is_project_open(name_c): - # close_project(name_c) # delete_project(name_c) # copy_project(name, name_c) # 备份项目 # else: # raise Exception('Incorrect simulation type, choose in (realtime, extended)') name_c = name - # 打开数据库 - open_project(name_c) - dic_time = get_time(name_c) + with project_transaction(name_c): + dic_time = get_time(name_c) - print(dic_time) + print(dic_time) - # 获取水力模拟步长，如’0:15:00‘ - globals.hydraulic_timestep = dic_time["HYDRAULIC TIMESTEP"] - # 转换为分钟浮点数，兼容 EPANET 的 H:MM 和 H:MM:SS 写法 - globals.PATTERN_TIME_STEP = ( - parse_clock_duration_seconds( - globals.hydraulic_timestep, - field_name="HYDRAULIC TIMESTEP", + # 获取水力模拟步长，如’0:15:00‘ + globals.hydraulic_timestep = dic_time["HYDRAULIC TIMESTEP"] + # 转换为分钟浮点数，兼容 EPANET 的 H:MM 和 H:MM:SS 写法 + globals.PATTERN_TIME_STEP = ( + parse_clock_duration_seconds( + globals.hydraulic_timestep, + field_name="HYDRAULIC TIMESTEP", + ) + / 60 ) - / 60 - ) - # 对输入的时间参数进行处理 - pattern_start_time = convert_time_format(modify_pattern_start_time) - # 获取模拟开始时间是对应pattern的第几个数 - modify_index = get_pattern_index(pattern_start_time) - # 遍历水泵的pattern_id，并根据输入的pump_pattern修改pattern的值 - # for pump_pattern_id in pump_pattern_ids: - # # 检查pump_pattern中pump_pattern_id对应的第一个频率值是否为有效数字（非空、非NaN）。如果该值有效，则继续执行代码块。 - # if not np.isnan(modify_pump_pattern[pump_pattern_id][0]): - # # 取出数据库中的pattern - # pump_pattern = get_pattern(name_c, get_pump(name_c, pump_pattern_id)['pattern']) - # # 替换数据库中的pattern为modify_pump_pattern - # pump_pattern['factors'][modify_index: modify_index + len(modify_pump_pattern[pump_pattern_id])] \ - # = modify_pump_pattern[pump_pattern_id] - # cs = ChangeSet() - # cs.append(pump_pattern) - # set_pattern(name_c, cs) - # 修改模拟开始的时间 - str_pattern_start = get_pattern_index_str( - convert_time_format(modify_pattern_start_time) - ) - dic_time = get_time(name_c) - dic_time["PATTERN START"] = str_pattern_start - dic_time["DURATION"] = from_seconds_to_clock(modify_total_duration) - if simulation_type.upper() == "REALTIME": - dic_time["DURATION"] = 0 - cs = ChangeSet() - cs.operations.append(dic_time) - set_time(name_c, cs) - # 根据SCADA实时数据进行修改，如果没有对应的SCADA数据，如未来的时间点，则不改变pg数据库的数据 - if globals.reservoirs_id: - # reservoirs_id = {'ZBBDJSCP000002': '2497', 'R00003': '2571'} - # 1.获取reservoir的SCADA数据,形式如{'2497': '3.1231', '2571': '2.7387'} - reservoir_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( - device_ids=list(globals.reservoirs_id.values()), - query_time=modify_pattern_start_time, - db_name=name, + # 对输入的时间参数进行处理 + pattern_start_time = convert_time_format(modify_pattern_start_time) + # 获取模拟开始时间是对应pattern的第几个数 + modify_index = get_pattern_index(pattern_start_time) + # 遍历水泵的pattern_id，并根据输入的pump_pattern修改pattern的值 + # for pump_pattern_id in pump_pattern_ids: + # # 检查pump_pattern中pump_pattern_id对应的第一个频率值是否为有效数字（非空、非NaN）。如果该值有效，则继续执行代码块。 + # if not np.isnan(modify_pump_pattern[pump_pattern_id][0]): + # # 取出数据库中的pattern + # pump_pattern = get_pattern(name_c, get_pump(name_c, pump_pattern_id)['pattern']) + # # 替换数据库中的pattern为modify_pump_pattern + # pump_pattern['factors'][modify_index: modify_index + len(modify_pump_pattern[pump_pattern_id])] \ + # = modify_pump_pattern[pump_pattern_id] + # cs = ChangeSet() + # cs.append(pump_pattern) + # set_pattern(name_c, cs) + # 修改模拟开始的时间 + str_pattern_start = get_pattern_index_str( + convert_time_format(modify_pattern_start_time) ) - # 2.构建出新字典，形式如{'ZBBDJSCP000002': '3.1231', 'R00003': '2.7387'} - reservoir_dict = { - key: reservoir_SCADA_data_dict[value] - for key, value in globals.reservoirs_id.items() - } - # 3.修改reservoir液位模式 - for reservoir_name, value in reservoir_dict.items(): - if value and float(value) != 0: - # 先根据reservoir获取对应的pattern，再对pattern进行修改 - reservoir_pattern = get_pattern( - name_c, get_reservoir(name_c, reservoir_name)["pattern"] - ) - reservoir_pattern["factors"][modify_index] = ( - float(value) + globals.RESERVOIR_BASIC_HEIGHT - ) - cs = ChangeSet() - cs.append(reservoir_pattern) - set_pattern(name_c, cs) - if globals.tanks_id: - # 修改tank初始液位 - tank_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( - device_ids=list(globals.tanks_id.values()), - query_time=modify_pattern_start_time, - db_name=name, - ) - tank_dict = { - key: tank_SCADA_data_dict[value] for key, value in globals.tanks_id.items() - } - for tank_name, value in tank_dict.items(): - if value and float(value) != 0: - tank = get_tank(name_c, tank_name) - tank["init_level"] = float(value) - cs = ChangeSet() - cs.append(tank) - set_tank(name_c, cs) - if globals.fixed_pumps_id: - # 修改工频泵的pattern - fixed_pump_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( - device_ids=list(globals.fixed_pumps_id.values()), - query_time=modify_pattern_start_time, - db_name=name, - ) - # print(fixed_pump_SCADA_data_dict) - fixed_pump_dict = { - key: fixed_pump_SCADA_data_dict[value] - for key, value in globals.fixed_pumps_id.items() - } - # print(fixed_pump_dict) - for fixed_pump_name, value in fixed_pump_dict.items(): - if value: - pump_pattern = get_pattern( - name_c, get_pump(name_c, fixed_pump_name)["pattern"] - ) - # print(pump_pattern) - pump_pattern["factors"][modify_index] = float(value) - # print(pump_pattern['factors'][modify_index]) - cs = ChangeSet() - cs.append(pump_pattern) - set_pattern(name_c, cs) - if globals.variable_pumps_id: - # 修改变频泵的pattern - variable_pump_SCADA_data_dict = ( - TimescaleInternalQueries.query_scada_by_ids_time( - device_ids=list(globals.variable_pumps_id.values()), + dic_time = get_time(name_c) + dic_time["PATTERN START"] = str_pattern_start + dic_time["DURATION"] = from_seconds_to_clock(modify_total_duration) + if simulation_type.upper() == "REALTIME": + dic_time["DURATION"] = 0 + cs = ChangeSet() + cs.operations.append(dic_time) + set_time(name_c, cs) + # 根据SCADA实时数据进行修改，如果没有对应的SCADA数据，如未来的时间点，则不改变pg数据库的数据 + if globals.reservoirs_id: + # reservoirs_id = {'ZBBDJSCP000002': '2497', 'R00003': '2571'} + # 1.获取reservoir的SCADA数据,形式如{'2497': '3.1231', '2571': '2.7387'} + reservoir_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( + device_ids=list(globals.reservoirs_id.values()), query_time=modify_pattern_start_time, db_name=name, ) - ) - variable_pump_dict = { - key: variable_pump_SCADA_data_dict[value] - for key, value in globals.variable_pumps_id.items() - } - for variable_pump_name, value in variable_pump_dict.items(): - if value: - pump_pattern = get_pattern( - name_c, get_pump(name_c, variable_pump_name)["pattern"] - ) - pump_pattern["factors"][modify_index] = float(value) / 50 - cs = ChangeSet() - cs.append(pump_pattern) - set_pattern(name_c, cs) - if globals.demand_id: - # 基于实时数据，修改大用户节点的pattern - demand_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( - device_ids=list(globals.demand_id.values()), - query_time=modify_pattern_start_time, - db_name=name, - ) - demand_dict = { - key: demand_SCADA_data_dict[value] - for key, value in globals.demand_id.items() - } - for demand_name, value in demand_dict.items(): - if value: - demand_pattern = get_pattern( - name_c, get_demand(name_c, demand_name)["pattern"] - ) - if get_option(name_c)["UNITS"] == "LPS": - demand_pattern["factors"][modify_index] = ( - float(value) / 3.6 - ) # 默认SCADA数据获取的是流量单位是m3/h， 转换为 L/s - elif get_option(name_c)["UNITS"] == "CMH": - demand_pattern["factors"][modify_index] = float(value) - cs = ChangeSet() - cs.append(demand_pattern) - set_pattern(name_c, cs) - # 水质、压力实时数据使用方法待补充 - ############################# - # 显式请求参数覆盖实时设备数据，用于扩展模拟。 - # 修改清水池(reservoir)液位的pattern - if modify_reservoir_head_pattern: - for reservoir_name in modify_reservoir_head_pattern.keys(): - # 这句代码的作用是判断modify_reservoir_head_pattern[reservoir_name][0]是否不是NaN。 - # 如果modify_reservoir_head_pattern[reservoir_name][0]不是NaN，则条件成立，代码块会执行 - if not np.isnan(modify_reservoir_head_pattern[reservoir_name][0]): - # 给 list 中的所有元素加上 RESERVOIR_BASIC_HEIGHT - modified_values = [ - value + globals.RESERVOIR_BASIC_HEIGHT - for value in modify_reservoir_head_pattern[reservoir_name] - ] - reservoir_pattern = get_pattern( - name_c, get_reservoir(name_c, reservoir_name)["pattern"] - ) - reservoir_pattern["factors"][ - modify_index : modify_index + len(modified_values) - ] = modified_values - cs = ChangeSet() - cs.append(reservoir_pattern) - set_pattern(name_c, cs) - # 修改调节池(tank)初始液位 - if modify_tank_initial_level: - for tank_name in modify_tank_initial_level.keys(): - if (not np.isnan(modify_tank_initial_level[tank_name])) and ( - modify_tank_initial_level[tank_name] != 0 - ): - tank = get_tank(name_c, tank_name) - tank["init_level"] = modify_tank_initial_level[tank_name] - cs = ChangeSet() - cs.append(tank) - set_tank(name_c, cs) - # 修改节点（junction）基础水量（demand） - if modify_junction_base_demand: - for junction_name in modify_junction_base_demand.keys(): - if not np.isnan(modify_junction_base_demand[junction_name]): - junction = get_demand(name_c, junction_name) - junction["demand"] = modify_junction_base_demand[junction_name] - cs = ChangeSet() - cs.append(junction) - set_demand(name_c, cs) - # 修改节点（junction）的水量模式（pattern） - if modify_junction_damand_pattern: - for pattern_name in modify_junction_damand_pattern.keys(): - if not np.isnan(modify_junction_damand_pattern[pattern_name][0]): - junction_pattern = get_pattern(name_c, pattern_name) - junction_pattern["factors"][ - modify_index : modify_index - + len(modify_junction_damand_pattern[pattern_name]) - ] = modify_junction_damand_pattern[pattern_name] - cs = ChangeSet() - cs.append(junction_pattern) - set_pattern(name_c, cs) - # 修改工频水泵（fixed_pump）的pattern - if modify_fixed_pump_pattern: - for pump_name in modify_fixed_pump_pattern.keys(): - if not np.isnan(modify_fixed_pump_pattern[pump_name][0]): - pump_pattern = get_pattern( - name_c, get_pump(name_c, pattern_name)["pattern"] - ) - pump_pattern["factors"][ - modify_index : modify_index + len(modify_fixed_pump_pattern) - ] = modify_fixed_pump_pattern[pump_name] - cs = ChangeSet() - cs.append(pump_pattern) - set_pattern(name_c, cs) - # 修改变频水泵（variable_pump）的pattern - if modify_variable_pump_pattern: - for pump_name in modify_variable_pump_pattern.keys(): - if not np.isnan(modify_variable_pump_pattern[pump_name][0]): - # 给 list 中的所有元素除以 50Hz - modified_values = [ - value / 50 for value in modify_variable_pump_pattern[pump_name] - ] - pump_pattern = get_pattern( - name_c, get_pump(name_c, pattern_name)["pattern"] - ) - pump_pattern["factors"][ - modify_index : modify_index + len(modified_values) - ] = modified_values - cs = ChangeSet() - cs.append(pump_pattern) - set_pattern(name_c, cs) - # 显式阀门控制沿用 run_simulation_ex 的处理顺序和覆盖规则。 - if valve_control is not None: - _apply_valve_control(name_c, valve_control) - # 保留原开度参数逻辑，兼容现有方案调用。 - elif modify_valve_opening: - for valve_name in modify_valve_opening.keys(): - if not np.isnan(modify_valve_opening[valve_name]): - valve_status = get_status(name_c, valve_name) - if modify_valve_opening[valve_name] == 0: - valve_status["status"] = "CLOSED" - valve_status["setting"] = 0 - elif modify_valve_opening[valve_name] < 1: - valve_status["status"] = "OPEN" - valve_status["setting"] = 0.1036 * pow( - modify_valve_opening[valve_name], -3.105 + # 2.构建出新字典，形式如{'ZBBDJSCP000002': '3.1231', 'R00003': '2.7387'} + reservoir_dict = { + key: reservoir_SCADA_data_dict[value] + for key, value in globals.reservoirs_id.items() + } + # 3.修改reservoir液位模式 + for reservoir_name, value in reservoir_dict.items(): + if value and float(value) != 0: + # 先根据reservoir获取对应的pattern，再对pattern进行修改 + reservoir_pattern = get_pattern( + name_c, get_reservoir(name_c, reservoir_name)["pattern"] ) - elif modify_valve_opening[valve_name] == 1: - valve_status["status"] = "OPEN" - valve_status["setting"] = 0 - cs = ChangeSet() - cs.append(valve_status) - set_status(name_c, cs) - # 运行并返回结果 - result_data = json.loads(run_project(name_c)) + reservoir_pattern["factors"][modify_index] = ( + float(value) + globals.RESERVOIR_BASIC_HEIGHT + ) + cs = ChangeSet() + cs.append(reservoir_pattern) + set_pattern(name_c, cs) + if globals.tanks_id: + # 修改tank初始液位 + tank_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( + device_ids=list(globals.tanks_id.values()), + query_time=modify_pattern_start_time, + db_name=name, + ) + tank_dict = { + key: tank_SCADA_data_dict[value] for key, value in globals.tanks_id.items() + } + for tank_name, value in tank_dict.items(): + if value and float(value) != 0: + tank = get_tank(name_c, tank_name) + tank["init_level"] = float(value) + cs = ChangeSet() + cs.append(tank) + set_tank(name_c, cs) + if globals.fixed_pumps_id: + # 修改工频泵的pattern + fixed_pump_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( + device_ids=list(globals.fixed_pumps_id.values()), + query_time=modify_pattern_start_time, + db_name=name, + ) + # print(fixed_pump_SCADA_data_dict) + fixed_pump_dict = { + key: fixed_pump_SCADA_data_dict[value] + for key, value in globals.fixed_pumps_id.items() + } + # print(fixed_pump_dict) + for fixed_pump_name, value in fixed_pump_dict.items(): + if value: + pump_pattern = get_pattern( + name_c, get_pump(name_c, fixed_pump_name)["pattern"] + ) + # print(pump_pattern) + pump_pattern["factors"][modify_index] = float(value) + # print(pump_pattern['factors'][modify_index]) + cs = ChangeSet() + cs.append(pump_pattern) + set_pattern(name_c, cs) + if globals.variable_pumps_id: + # 修改变频泵的pattern + variable_pump_SCADA_data_dict = ( + TimescaleInternalQueries.query_scada_by_ids_time( + device_ids=list(globals.variable_pumps_id.values()), + query_time=modify_pattern_start_time, + db_name=name, + ) + ) + variable_pump_dict = { + key: variable_pump_SCADA_data_dict[value] + for key, value in globals.variable_pumps_id.items() + } + for variable_pump_name, value in variable_pump_dict.items(): + if value: + pump_pattern = get_pattern( + name_c, get_pump(name_c, variable_pump_name)["pattern"] + ) + pump_pattern["factors"][modify_index] = float(value) / 50 + cs = ChangeSet() + cs.append(pump_pattern) + set_pattern(name_c, cs) + if globals.demand_id: + # 基于实时数据，修改大用户节点的pattern + demand_SCADA_data_dict = TimescaleInternalQueries.query_scada_by_ids_time( + device_ids=list(globals.demand_id.values()), + query_time=modify_pattern_start_time, + db_name=name, + ) + demand_dict = { + key: demand_SCADA_data_dict[value] + for key, value in globals.demand_id.items() + } + for demand_name, value in demand_dict.items(): + if value is not None and not np.isnan(float(value)): + demand_set = get_demand(name_c, demand_name) + demand_pattern = get_pattern( + name_c, _primary_demand_pattern(demand_set) + ) + if get_option(name_c)["UNITS"] == "LPS": + demand_pattern["factors"][modify_index] = ( + float(value) / 3.6 + ) # 默认SCADA数据获取的是流量单位是m3/h， 转换为 L/s + elif get_option(name_c)["UNITS"] == "CMH": + demand_pattern["factors"][modify_index] = float(value) + cs = ChangeSet() + cs.append(demand_pattern) + set_pattern(name_c, cs) + # 水质、压力实时数据使用方法待补充 + ############################# + # 显式请求参数覆盖实时设备数据，用于扩展模拟。 + # 修改清水池(reservoir)液位的pattern + if modify_reservoir_head_pattern: + for reservoir_name in modify_reservoir_head_pattern.keys(): + # 这句代码的作用是判断modify_reservoir_head_pattern[reservoir_name][0]是否不是NaN。 + # 如果modify_reservoir_head_pattern[reservoir_name][0]不是NaN，则条件成立，代码块会执行 + if not np.isnan(modify_reservoir_head_pattern[reservoir_name][0]): + # 给 list 中的所有元素加上 RESERVOIR_BASIC_HEIGHT + modified_values = [ + value + globals.RESERVOIR_BASIC_HEIGHT + for value in modify_reservoir_head_pattern[reservoir_name] + ] + reservoir_pattern = get_pattern( + name_c, get_reservoir(name_c, reservoir_name)["pattern"] + ) + reservoir_pattern["factors"][ + modify_index : modify_index + len(modified_values) + ] = modified_values + cs = ChangeSet() + cs.append(reservoir_pattern) + set_pattern(name_c, cs) + # 修改调节池(tank)初始液位 + if modify_tank_initial_level: + for tank_name in modify_tank_initial_level.keys(): + if (not np.isnan(modify_tank_initial_level[tank_name])) and ( + modify_tank_initial_level[tank_name] != 0 + ): + tank = get_tank(name_c, tank_name) + tank["init_level"] = modify_tank_initial_level[tank_name] + cs = ChangeSet() + cs.append(tank) + set_tank(name_c, cs) + # 修改节点（junction）基础水量（demand） + if modify_junction_base_demand: + for junction_name in modify_junction_base_demand.keys(): + if not np.isnan(modify_junction_base_demand[junction_name]): + junction = get_demand(name_c, junction_name) + _primary_demand(junction)["demand"] = ( + modify_junction_base_demand[junction_name] + ) + cs = ChangeSet() + cs.append(junction) + set_demand(name_c, cs) + # 修改节点（junction）的水量模式（pattern） + if modify_junction_damand_pattern: + for pattern_name in modify_junction_damand_pattern.keys(): + if not np.isnan(modify_junction_damand_pattern[pattern_name][0]): + junction_pattern = get_pattern(name_c, pattern_name) + junction_pattern["factors"][ + modify_index : modify_index + + len(modify_junction_damand_pattern[pattern_name]) + ] = modify_junction_damand_pattern[pattern_name] + cs = ChangeSet() + cs.append(junction_pattern) + set_pattern(name_c, cs) + # 修改工频水泵（fixed_pump）的pattern + if modify_fixed_pump_pattern: + for pump_name in modify_fixed_pump_pattern.keys(): + if not np.isnan(modify_fixed_pump_pattern[pump_name][0]): + pump_pattern = get_pattern( + name_c, get_pump(name_c, pump_name)["pattern"] + ) + pump_pattern["factors"][ + modify_index + : modify_index + len(modify_fixed_pump_pattern[pump_name]) + ] = modify_fixed_pump_pattern[pump_name] + cs = ChangeSet() + cs.append(pump_pattern) + set_pattern(name_c, cs) + # 修改变频水泵（variable_pump）的pattern + if modify_variable_pump_pattern: + for pump_name in modify_variable_pump_pattern.keys(): + if not np.isnan(modify_variable_pump_pattern[pump_name][0]): + # 给 list 中的所有元素除以 50Hz + modified_values = [ + value / 50 for value in modify_variable_pump_pattern[pump_name] + ] + pump_pattern = get_pattern( + name_c, get_pump(name_c, pump_name)["pattern"] + ) + pump_pattern["factors"][ + modify_index : modify_index + len(modified_values) + ] = modified_values + cs = ChangeSet() + cs.append(pump_pattern) + set_pattern(name_c, cs) + # 显式阀门控制沿用 run_simulation_ex 的处理顺序和覆盖规则。 + if valve_control is not None: + _apply_valve_control(name_c, valve_control) + # 保留原开度参数逻辑，兼容现有方案调用。 + elif modify_valve_opening: + for valve_name in modify_valve_opening.keys(): + if not np.isnan(modify_valve_opening[valve_name]): + valve_status = get_status(name_c, valve_name) + if modify_valve_opening[valve_name] == 0: + valve_status["status"] = "CLOSED" + valve_status["setting"] = 0 + elif modify_valve_opening[valve_name] < 1: + valve_status["status"] = "OPEN" + valve_status["setting"] = 0.1036 * pow( + modify_valve_opening[valve_name], -3.105 + ) + elif modify_valve_opening[valve_name] == 1: + valve_status["status"] = "OPEN" + valve_status["setting"] = 0 + cs = ChangeSet() + cs.append(valve_status) + set_status(name_c, cs) + # 运行并返回结果 + result_data = json.loads(run_project(name_c)) + refresh_materialized_views_after_commit(name_c) time_cost_end = time.perf_counter() print( "{} -- Hydraulic simulation finished, cost time: {:.2f} s.".format( diff --git a/app/services/simulation_ops.py b/app/services/simulation_ops.py index 8621ac7..73f7d27 100644 --- a/app/services/simulation_ops.py +++ b/app/services/simulation_ops.py @@ -1,55 +1,60 @@ import json from datetime import datetime +from functools import wraps from math import pi import pytz from app.algorithms.simulation.runner import run_simulation_ex +from app.native.wndb.core.projects import temporary_project_database from app.services.tjnetwork import ( - close_project, - copy_project, - delete_project, get_pipe, get_tank, - have_project, - is_project_open, - open_project, ) +def _isolated_operation(purpose: str): + def decorator(func): + @wraps(func) + def wrapper(prj_name: str, *args, **kwargs): + with temporary_project_database(prj_name, purpose) as temporary: + kwargs["_temporary_project"] = temporary + return func(prj_name, *args, **kwargs) + + return wrapper + + return decorator + + ############################################################ # project management 07 ***暂时不使用，与业务需求无关*** ############################################################ +@_isolated_operation("project_management") def project_management( prj_name, start_datetime, pump_control, tank_initial_level_control=None, region_demand_control=None, + _temporary_project=None, ) -> str: print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"project_management_{prj_name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) + if _temporary_project is None: + raise RuntimeError("Project-management isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(prj_name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -64,9 +69,6 @@ def project_management( region_demand_control=region_demand_control, downloading_prohibition=True, ) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) return result @@ -75,31 +77,31 @@ def project_management( ############################################################ +@_isolated_operation("scheduling") def scheduling_simulation( - prj_name, start_time, pump_control, tank_id, water_plant_output_id, time_delta=300 + prj_name, + start_time, + pump_control, + tank_id, + water_plant_output_id, + time_delta=300, + _temporary_project=None, ) -> str: print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"scheduling_{prj_name}" - - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) + if _temporary_project is None: + raise RuntimeError("Scheduling isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(prj_name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -115,9 +117,6 @@ def scheduling_simulation( if not isinstance(output_data, dict): raise RuntimeError("run_simulation_ex did not return JSON output content") - if not is_project_open(new_name): - open_project(new_name) - tank = get_tank(new_name, tank_id) # 水塔信息 tank_floor_space = pi * pow(tank["diameter"] / 2, 2) # 水塔底面积(m^2) tank_init_level = tank["init_level"] # 水塔初始水位(m) @@ -158,38 +157,34 @@ def scheduling_simulation( "tank_level": tank_level, } - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - return json.dumps(simulation_results) +@_isolated_operation("daily_scheduling") def daily_scheduling_simulation( - prj_name, start_time, pump_control, reservoir_id, tank_id, water_plant_output_id + prj_name, + start_time, + pump_control, + reservoir_id, + tank_id, + water_plant_output_id, + _temporary_project=None, ) -> str: print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Analysis." ) - new_name = f"daily_scheduling_{prj_name}" - - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) + if _temporary_project is None: + raise RuntimeError("Daily-scheduling isolation was not prepared") + new_name = _temporary_project print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Copying Database." ) - copy_project(prj_name + "_template", new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Start Opening Database." ) - open_project(new_name) print( datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") + " -- Database Loading OK." @@ -209,9 +204,6 @@ def daily_scheduling_simulation( if not isinstance(output_data, dict): raise RuntimeError("run_simulation_ex did not return JSON output content") - if not is_project_open(new_name): - open_project(new_name) - node_results = output_data.get("node_results") or [] # [{'node': str, 'result': [{'pressure': float, 'head': float}]}] water_plant_output_pressure = [] reservoir_level = [] @@ -235,8 +227,4 @@ def daily_scheduling_simulation( "tank_level": tank_level, } - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - return json.dumps(simulation_results) diff --git a/app/services/tjnetwork.py b/app/services/tjnetwork.py index 272b68c..06d7fc6 100644 --- a/app/services/tjnetwork.py +++ b/app/services/tjnetwork.py @@ -32,14 +32,11 @@ from app.native.wndb.commands.api import ( ) from app.native.wndb.core.database import ChangeSet, read_all from app.native.wndb.core.projects import ( - close_project, copy_project, create_project, delete_project, have_project, - is_project_open, list_project, - open_project, ) from app.native.wndb.gis.backdrop import ( get_backdrop, @@ -54,6 +51,12 @@ from app.native.wndb.gis.labels import ( get_label_schema, set_label, ) +from app.native.wndb.gis.network_views import ( + get_major_node_coords, + get_major_pipe_nodes, + get_network_link_nodes, + get_network_node_coords, +) from app.native.wndb.gis.region_geometry import get_nodes_in_region from app.native.wndb.gis.regions import ( add_region, @@ -89,16 +92,11 @@ from app.native.wndb.model.elements import ( get_curves, get_element_type, get_element_type_value, - get_link_nodes, get_link_type, get_links, - get_links_id_and_type, - get_major_nodes, - get_major_pipes, get_node_links, get_node_type, get_nodes, - get_nodes_id_and_type, get_patterns, get_regions, is_curve, @@ -280,39 +278,6 @@ def get_element_properties(name: str, element_id: str) -> dict[str, Any]: return get_scada_info(name, element_id) -def get_network_node_coords(name: str) -> dict[str, dict[str, Any]]: - nodes = get_nodes_id_and_type(name) - return { - node_id: {**get_node_coord(name, node_id), "type": node_type} - for node_id, node_type in nodes.items() - } - - -def get_major_node_coords(name: str, diameter: int) -> dict[str, dict[str, Any]]: - node_types = get_nodes_id_and_type(name) - return { - node_id: {**get_node_coord(name, node_id), "type": node_types[node_id]} - for node_id in get_major_nodes(name, diameter) - } - - -def get_network_link_nodes(name: str) -> list[str]: - links = get_links_id_and_type(name) - return [ - f"{link_id}:{link_type}:{nodes[0]}:{nodes[1]}" - for link_id, link_type in links.items() - if (nodes := get_link_nodes(name, link_id)) - ] - - -def get_major_pipe_nodes(name: str, diameter: int) -> list[str]: - return [ - f"{link_id}:pipe:{nodes[0]}:{nodes[1]}" - for link_id in get_major_pipes(name, diameter) - if (nodes := get_link_nodes(name, link_id)) - ] - - def get_network_in_extent( name: str, x1: float, y1: float, x2: float, y2: float ) -> dict[str, Any]: diff --git a/contracts/manifest.json b/contracts/manifest.json index 920c9a4..55a34aa 100644 --- a/contracts/manifest.json +++ b/contracts/manifest.json @@ -3,7 +3,7 @@ "contracts": { "server": { "file": "server-v1.openapi.json", - "sha256": "404a196c0177faed2aa5b46ee86430a034dfe990e0a77a43428a727748a882b6" + "sha256": "34f67bf3b6f1da263d0271e5a1f3cb599c128c4b422e44d2d6d540d99f7855f4" } } } diff --git a/contracts/server-v1.openapi.json b/contracts/server-v1.openapi.json index a53537d..a584c0c 100644 --- a/contracts/server-v1.openapi.json +++ b/contracts/server-v1.openapi.json @@ -2149,6 +2149,176 @@ "title": "PumpFailureState", "type": "object" }, + "RealtimeLinkBatchItem": { + "properties": { + "flow": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Flow" + }, + "friction": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Friction" + }, + "headloss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Headloss" + }, + "id": { + "title": "Id", + "type": "string" + }, + "quality": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Quality" + }, + "reaction": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Reaction" + }, + "setting": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Setting" + }, + "status": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Status" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "velocity": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Velocity" + } + }, + "required": [ + "time", + "id" + ], + "title": "RealtimeLinkBatchItem", + "type": "object" + }, + "RealtimeNodeBatchItem": { + "properties": { + "actual_demand": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Actual Demand" + }, + "id": { + "title": "Id", + "type": "string" + }, + "pressure": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Pressure" + }, + "quality": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Quality" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "total_head": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Total Head" + } + }, + "required": [ + "time", + "id" + ], + "title": "RealtimeNodeBatchItem", + "type": "object" + }, "RunSimulationManuallyByDateRest": { "properties": { "duration": { @@ -6204,6 +6374,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -6950,6 +7131,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -7278,6 +7470,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -7605,6 +7808,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -7932,6 +8146,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -8144,6 +8369,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -8246,6 +8482,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -8918,6 +9165,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -9016,6 +9274,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -11088,6 +11357,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -11642,6 +11922,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "204": { "description": "Successful Response" @@ -11731,6 +12022,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -11948,6 +12250,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -12945,6 +13258,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "204": { "description": "Successful Response" @@ -13034,6 +13358,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -13130,6 +13465,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -13687,6 +14033,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -14003,6 +14360,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -18095,6 +18463,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -18422,6 +18801,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -18520,6 +18910,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -20652,6 +21053,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -21435,345 +21847,6 @@ ] } }, - "/api/v1/project-codes": { - "get": { - "description": "获取服务器上所有可用的供水管网项目名称列表。", - "operationId": "get_project_codes", - "parameters": [ - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "default": 100, - "maximum": 1000, - "minimum": 1, - "title": "Limit", - "type": "integer" - } - }, - { - "in": "query", - "name": "offset", - "required": false, - "schema": { - "default": 0, - "minimum": 0, - "title": "Offset", - "type": "integer" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_str_" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "获取项目列表", - "tags": [ - "Project" - ] - } - }, - "/api/v1/project-conversions": { - "post": { - "description": "将 EPANET 3.0 格式的 INP 内容转换为 2.x 格式。", - "operationId": "post_project_conversions", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "转换 INP V3 为 V2", - "tags": [ - "Project" - ] - } - }, - "/api/v1/project-copies": { - "post": { - "description": "将现有项目复制为新项目。", - "operationId": "post_project_copies", - "parameters": [ - { - "description": "管网名称（或数据库名称）", - "in": "query", - "name": "source", - "required": true, - "schema": { - "description": "管网名称（或数据库名称）", - "title": "Source", - "type": "string" - } - }, - { - "description": "管网名称（或数据库名称）", - "in": "query", - "name": "target", - "required": true, - "schema": { - "description": "管网名称（或数据库名称）", - "title": "Target", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "复制项目", - "tags": [ - "Project" - ] - } - }, "/api/v1/project-managements": { "post": { "description": "高级版本的项目管理，通过JSON请求体提供详细的控制参数，包括泵控制策略、水箱初始水位和区域需水量控制。", @@ -22082,95 +22155,6 @@ } }, "/api/v1/projects": { - "delete": { - "description": "永久删除指定的供水管网项目。此操作不可恢复。", - "operationId": "delete_projects", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "删除项目", - "tags": [ - "Project" - ] - }, "get": { "description": "获取当前用户有权限的所有项目列表", "operationId": "get_projects", @@ -22280,194 +22264,9 @@ "tags": [ "Metadata" ] - }, - "post": { - "description": "创建一个新的供水管网项目。如果项目已存在，可能会覆盖或报错（取决于底层实现）。", - "operationId": "post_projects", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "创建新项目", - "tags": [ - "Project" - ] } }, "/api/v1/projects/current": { - "delete": { - "description": "将指定项目从内存中卸载，释放资源。", - "operationId": "delete_projects_current", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "关闭项目", - "tags": [ - "Project" - ] - }, "get": { "description": "从数据库获取项目的详细信息，包括地图范围等。", "operationId": "get_projects_current", @@ -22563,102 +22362,6 @@ "tags": [ "Project" ] - }, - "post": { - "description": "将指定项目加载到内存中，并初始化数据库连接池。", - "operationId": "post_projects_current", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "打开项目", - "tags": [ - "Project" - ] } }, "/api/v1/projects/current/database-health": { @@ -22868,716 +22571,6 @@ ] } }, - "/api/v1/projects/current/exports/inp": { - "post": { - "description": "将项目当前状态保存为 INP 文件到服务器文件系统。", - "operationId": "post_projects_current_exports_inp", - "parameters": [ - { - "description": "目标文件名", - "in": "query", - "name": "inp", - "required": true, - "schema": { - "description": "目标文件名", - "title": "Inp", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Projects Current Exports Inp", - "type": "boolean" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "导出项目到 INP 文件", - "tags": [ - "Project" - ] - } - }, - "/api/v1/projects/current/files/inp": { - "get": { - "description": "从服务器数据目录下载指定的 INP 文件。", - "operationId": "get_projects_current_files_inp", - "parameters": [ - { - "description": "文件名", - "in": "query", - "name": "name", - "required": true, - "schema": { - "description": "文件名", - "title": "Name", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "下载 INP 文件", - "tags": [ - "Project" - ] - } - }, - "/api/v1/projects/current/imports": { - "post": { - "description": "从服务器文件系统中读取指定的 INP 文件并加载到项目中。", - "operationId": "post_projects_current_imports", - "parameters": [ - { - "description": "INP 文件名 (不包含路径)", - "in": "query", - "name": "inp", - "required": true, - "schema": { - "description": "INP 文件名 (不包含路径)", - "title": "Inp", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "title": "Response Post Projects Current Imports", - "type": "boolean" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "读取 INP 文件到项目", - "tags": [ - "Project" - ] - } - }, - "/api/v1/projects/current/lock": { - "delete": { - "description": "释放对项目的锁定。", - "operationId": "delete_projects_current_lock", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "解锁项目", - "tags": [ - "Project" - ] - }, - "get": { - "description": "检查指定项目是否处于锁定状态。", - "operationId": "get_projects_current_lock", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "检查项目是否被锁定", - "tags": [ - "Project" - ] - }, - "post": { - "description": "锁定指定项目以防止并发修改。", - "operationId": "post_projects_current_lock", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "锁定项目", - "tags": [ - "Project" - ] - } - }, - "/api/v1/projects/current/lock/ownership": { - "get": { - "description": "检查指定项目是否被当前访问地址 (IP) 锁定。", - "operationId": "get_projects_current_lock_ownership", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "检查项目是否被当前用户锁定", - "tags": [ - "Project" - ] - } - }, "/api/v1/projects/current/metadata": { "get": { "description": "获取当前项目的元数据和配置信息", @@ -23676,202 +22669,6 @@ ] } }, - "/api/v1/projects/current/status": { - "get": { - "description": "检查指定项目是否已被加载到内存中。", - "operationId": "get_projects_current_status", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "检查项目是否已打开", - "tags": [ - "Project" - ] - } - }, - "/api/v1/projects/existence": { - "get": { - "description": "检查指定名称的项目是否存在。", - "operationId": "get_projects_existence", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "检查项目是否存在", - "tags": [ - "Project" - ] - } - }, "/api/v1/pump-failure-events": { "post": { "description": "记录和管理泵的故障状态，包括故障发生时间和受影响的泵列表。系统将记录故障日志并更新泵状态。", @@ -25058,6 +23855,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -25264,6 +24072,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -25362,6 +24181,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -25558,6 +24388,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "204": { "description": "Successful Response" @@ -25764,6 +24605,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -25859,6 +24711,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -27506,6 +26369,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -28177,6 +27051,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -29809,6 +28694,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -29905,6 +28801,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -30234,6 +29141,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -30553,6 +29471,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -30772,6 +29701,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -33517,6 +32457,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -34306,6 +33257,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -35189,9 +34151,9 @@ "content": { "application/json": { "schema": { - "description": "管道数据列表，每项包含管道ID、时间戳等信息", + "description": "同一时间点的管道快照数据", "items": { - "type": "object" + "$ref": "#/components/schemas/RealtimeLinkBatchItem" }, "title": "Data", "type": "array" @@ -35680,9 +34642,9 @@ "content": { "application/json": { "schema": { - "description": "节点数据列表，每项包含节点ID、时间戳等信息", + "description": "同一时间点的节点快照数据", "items": { - "type": "object" + "$ref": "#/components/schemas/RealtimeNodeBatchItem" }, "title": "Data", "type": "array" @@ -37606,6 +36568,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -39431,6 +38404,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -39999,6 +38983,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "204": { "description": "Successful Response" @@ -40088,6 +39083,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "201": { "content": { @@ -40415,6 +39421,17 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Payload", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { diff --git a/resources/db_v2/DATABASE_ARCHITECTURE.md b/resources/db_v2/DATABASE_ARCHITECTURE.md index 5bb3e35..4185c96 100644 --- a/resources/db_v2/DATABASE_ARCHITECTURE.md +++ b/resources/db_v2/DATABASE_ARCHITECTURE.md @@ -1,29 +1,31 @@ # TJWater 数据库改造说明与当前结构 -> 本文记录 2026-08-25 的数据库实际状态。结构、约束、行数、TimescaleDB chunk 和策略均直接读取数据库，不以仓库中的 SQL 脚本为依据。文中不包含主机、端口、账号、密码或 DSN。 +> 本文记录截至 2026-08-27 的数据库实际状态。结构、约束、行数、TimescaleDB chunk 和策略均直接读取数据库，不以仓库中的 SQL 脚本为依据。文中不包含主机、端口、账号、密码或 DSN。 ## 改造范围与当前状态 -本次改造保留原 `tjwater` 业务库和时序库，新建 `tjwater_next` 作为隔离验证环境。元数据库仍为 `system_hub`，新项目通过 `biz_data` 和 `iot_data` 两条路由分别关联新业务库与新时序库。项目完成迁移和联调后已切换为 `active`，原数据库没有被覆盖，仍可用于对照和回退。 +本次改造保留原 `tjwater` 业务库和时序库，新建的隔离库完成验证后已正式重命名为 `tjwater_v2`。元数据库仍为 `system_hub`，逻辑项目 `tjwater_next` 通过 `biz_data` 和 `iot_data` 两条路由分别关联 `tjwater_v2` 业务库与时序库。项目已切换为 `active`，原数据库没有被覆盖，仍可用于对照和回退。 + +命名约定：正式物理库名为 `tjwater_v2`，WNDB 版本模板固定为 `tjwater_v2_template`。元数据库中的逻辑项目代码和 GeoServer 工作空间仍为 `tjwater_next`；这些是路由与图层限定名，不是物理库名。模板已创建并清空项目模型、SCADA、分析运行和物化视图数据，仅保留数据库结构、PostGIS 对象和必要配置键种子，普通连接已关闭。 已经完成的数据库修改包括： -- 新建并迁移 `tjwater_next` 业务库，将旧 `public` 中混合存放的管网、GIS、SCADA 配置和分析数据按领域拆分。 -- 新建并迁移 `tjwater_next` 时序库，将 SCADA、实时计算和分析计算结果分开存放。 +- 新建并迁移 `tjwater_v2` 业务库，将旧 `public` 中混合存放的管网、GIS、SCADA 配置和分析数据按领域拆分。 +- 新建并迁移 `tjwater_v2` 时序库，将 SCADA、实时计算和分析计算结果分开存放。 - `realtime` 采用冷热数据策略，72 小时后的 chunk 自动转为有序列存。 - `analysis` 按 `stored_at` 分区，入库满 24 小时的 chunk 自动转为有序列存。 - GIS 查询层改用物化视图，当前 7 张物化视图均已填充。 -- GeoServer 已建立 `tjwater_next` 工作空间和同名数据存储，从业务库 `gis` schema 发布 7 个图层。GeoWebCache 的服务端与客户端缓存有效期均为 300 秒。 +- GeoServer 已建立 `tjwater_next` 工作空间和同名数据存储，数据存储连接 `tjwater_v2` 业务库的 `gis` schema 并发布 7 个图层。GeoWebCache 的服务端与客户端缓存有效期均为 300 秒。 - 旧库中的 `operation`、`current_operation`、`batch_operation`、`operation_table`、`restore_operation` 和 `snapshot_operation` 没有进入新业务库。 - `system_hub.public` 补充了项目数据库外键、数据库路由约束、连接池约束、必要的非空约束，以及 5 张表和 44 个字段的中文数据库注释。 - 用户角色和项目角色仍是可扩展字符串，没有增加枚举检查约束。 - `audit_logs.user_id` 和 `audit_logs.project_id` 仍为逻辑关联，没有增加外键。 - 业务库 48 个表或物化视图、192 个字段，以及时序库 7 张表、47 个字段均已写入中文数据库注释。 -- 后端已对接新 schema。WNDB、PostgreSQL 管理连接和同步 TimescaleDB 访问均使用有界连接池，闲置项目按最近使用顺序回收，实时覆盖写入使用单一事务。 +- 后端已对接新 schema。WNDB、PostgreSQL 管理连接和同步 TimescaleDB 访问均使用有界连接池，闲置项目按最近使用顺序回收；动态异步池采用代际切换，配置更新不会中断旧借用或阻塞新请求。 - 后端批量元素查询读取 GIS 物化视图，模型增删改和 INP 导入提交后执行并发刷新；批量事务只刷新一次。 - `pattern_values`、`pattern_flow_samples`、`curve_points`、`demands` 和 `link_vertices` 的顺序号按所属父对象编号，主键已改为父对象 ID 与 `sequence_no` 的复合键。 -`tjwater_next` 当前为 `active`。排水项目 `lingang` 已迁入 `system_hub.public`，供水和排水后端共用同一套项目、成员、数据库路由和审计表。 +逻辑项目 `tjwater_next` 当前为 `active`。排水项目 `lingang` 已迁入 `system_hub.public`，供水和排水后端共用同一套项目、成员、数据库路由和审计表。 ## 数据库总体关系 @@ -38,9 +40,9 @@ flowchart LR OT["tjwater 时序库
scada / realtime / scheme"] end - subgraph NEXT["隔离验证数据库"] - NB["tjwater_next 业务库
network / gis / asset / analysis"] - NT["tjwater_next 时序库
scada / realtime / analysis"] + subgraph V2["v2 正式数据库"] + NB["tjwater_v2 业务库
network / gis / asset / analysis"] + NT["tjwater_v2 时序库
scada / realtime / analysis"] end GS["GeoServer
tjwater_next 工作空间"] @@ -48,8 +50,8 @@ flowchart LR MP -->|"tjwater 的 biz_data"| OB MP -->|"tjwater 的 iot_data"| OT - MP -->|"tjwater_next 的 biz_data"| NB - MP -->|"tjwater_next 的 iot_data"| NT + MP -->|"项目 tjwater_next 的 biz_data"| NB + MP -->|"项目 tjwater_next 的 iot_data"| NT MP -->|"lingang 的两条数据库路由"| DRAIN["排水项目数据库"] NB -->|"gis 物化视图"| GS -->|"WFS / WMTS"| WEB ``` @@ -146,7 +148,7 @@ erDiagram ## 相对原数据库的结构变化 -以下对比以当前仍保留的原业务库 `tjwater`、原时序库 `tjwater` 与新库 `tjwater_next` 的实际对象为准。对象名称的对应关系表示业务实体或数据职责的迁移方向，不表示所有字段均一对一复制。 +以下对比以当前仍保留的原业务库 `tjwater`、原时序库 `tjwater` 与新库 `tjwater_v2` 的实际对象为准。对象名称的对应关系表示业务实体或数据职责的迁移方向，不表示所有字段均一对一复制。 ### 归并和调整 @@ -167,7 +169,7 @@ erDiagram - 云端操作记录相关的 `operation`、`current_operation`、`batch_operation`、`operation_table`、`restore_operation`、`snapshot_operation` 未进入新业务库。该设计不再作为业务数据模型的一部分。 - 临时处理表 `temp_link_1`、`temp_link_2`、`temp_node`、`temp_region`、`temp_vd_topology` 未迁入。它们属于历史处理过程的中间对象，不应成为长期库结构。 - 原库中的 `_node`、`_link`、`_pattern`、`_curve`、`_region` 等内部或过渡表不再单独存在。新库以明确的领域表和外键关系表达同一类数据。 -- 原库的 `tiger`、`topology` 扩展 schema 未在新库安装，`tjwater_next` 仅保留 PostGIS 及其 `public` 系统对象。 +- 原库的 `tiger`、`topology` 扩展 schema 未在新库安装，`tjwater_v2` 仅保留 PostGIS 及其 `public` 系统对象。 ### 尚未完整承接的范围 @@ -183,7 +185,7 @@ erDiagram - `realtime` 两张 hypertable 已启用 72 小时后的列存压缩策略，`analysis` 两张 hypertable 按入库时间执行 24 小时冷热转换。`scada` 保持独立行存。 - 元数据库的 `public.project_databases` 增加项目外键、数据库角色和类型约束，以及连接池上下限约束，用于保证每个项目的业务库和时序库路由有效。 -## 新业务库 tjwater_next +## 新业务库 tjwater_v2 新业务库使用 PostgreSQL 和 PostGIS。业务对象分布在 4 个 schema 中，`public` 只保留 PostGIS 提供的系统对象。 @@ -316,7 +318,7 @@ flowchart LR ### GeoServer 与前端图层 -`system_hub.public.projects` 中的 `tjwater_next` 项目已配置 `gs_workspace=tjwater_next`，当前状态为 `active`。GeoServer 的 `tjwater_next` 数据存储连接同名业务库并限定到 `gis` schema，图层名称直接采用物化视图名称。7 个图层使用相同的项目管网发布边界，空图层和视口内没有要素的瓦片会返回空 MVT，不会产生越界错误。 +`system_hub.public.projects` 中的 `tjwater_next` 项目已配置 `gs_workspace=tjwater_next`，当前状态为 `active`。GeoServer 的 `tjwater_next` 数据存储连接 `tjwater_v2` 业务库并限定到 `gis` schema，图层名称直接采用物化视图名称。7 个图层使用相同的项目管网发布边界，空图层和视口内没有要素的瓦片会返回空 MVT，不会产生越界错误。 | 前端数据源 | GeoServer 图层 | 几何 | 当前要素数 | | --- | --- | --- | ---: | @@ -384,7 +386,7 @@ erDiagram 这些表均为行存，没有启用压缩策略。方案结果使用 `scheme_name` 关联业务库中的方案记录。 -## 新时序库 tjwater_next +## 新时序库 tjwater_v2 新时序库仍运行 TimescaleDB 2.21.3。业务时序数据分为 `scada`、`realtime` 和 `analysis`，迁移过程状态单独放在 `migration`。 @@ -404,7 +406,7 @@ erDiagram ### realtime -`realtime.node_results` 和 `realtime.link_results` 保存当前实时计算窗口。主键保证同一时刻、同一元素只能有一条记录。相同时间窗口由后端先删除、再通过 `COPY` 批量插入；节点和连接两次替换位于同一个最外层事务，其中任一步失败都会整体回滚。 +`realtime.node_results` 和 `realtime.link_results` 保存当前实时计算窗口。主键保证同一时刻、同一元素只能有一条记录。每个批次强制只包含一个归一化时间戳；相同时间点由后端在一个最外层事务中先同时删除节点和连接旧快照，再通过 `COPY` 写入非空侧。某一侧为空也会删除其旧数据，符合“同时间点整体覆盖”语义。 两张表当前各有 19 个 chunk，均已转为列存，因为现有数据都早于 72 小时热窗口。数据库每小时执行一次策略检查，将 72 小时以前的 chunk 转为有序列存。节点结果按 `node_id, time DESC` 排序，连接结果按 `link_id, time DESC` 排序。新写入的数据使用 1 天 chunk，并在 72 小时内保持行存。 @@ -422,9 +424,9 @@ erDiagram ## 后端连接与事务 -元数据库通过 SQLAlchemy 异步连接池访问；项目请求按 `system_hub.public.project_databases` 路由到业务库和时序库。异步业务查询和异步时序查询由项目级动态池管理，原生 WNDB 同步访问使用按数据库缓存的 `psycopg_pool.ConnectionPool`，数据库创建、复制和删除使用独立的 PostgreSQL 管理池，同步 TimescaleDB 访问也使用按数据库缓存的连接池。应用目录中已没有直接调用 `psycopg.connect` 的业务代码。 +元数据库通过 SQLAlchemy 异步连接池访问；项目请求按 `system_hub.public.project_databases` 路由到业务库和时序库。异步业务查询和异步时序查询由项目级动态池管理，池条目记录借用数；配置更新建立新一代池，旧池只在已有借用归还后关闭。原生 WNDB 同步访问使用按数据库缓存且同样带借用计数的 `psycopg_pool.ConnectionPool`，数据库创建、复制和删除使用独立的 PostgreSQL 管理池及数据库级 advisory lock，同步 TimescaleDB 访问也使用按数据库缓存的连接池。应用目录中已没有直接调用 `psycopg.connect` 的业务代码。 -WNDB 批量修改和 INP 数据导入在同一条池连接和同一事务中执行，提交后再刷新 GIS 物化视图。实时节点和连接结果也在一个事务中执行先删后写，同一结果时间使用事务级锁避免并发覆盖竞态；分析结果按 `run_id` 加事务级锁，防止同一运行被并发写入两次。 +WNDB 批量修改和模拟参数准备在同一条池连接和同一事务中执行，提交后只刷新一次 GIS 物化视图。普通写入和 INP 整体替换使用同一项目级事务锁；INP 先在唯一暂存库校验，再从一致性快照事务替换当前库的 `network/gis` 模型表。临时分析库由固定模板建结构后复制当前项目模型、SCADA 映射并刷新视图。实时节点和连接结果在一个事务中执行整体先删后写，同一结果时间使用事务级锁；分析结果按 `run_id` 加事务级锁。Timescale 复合查询按节点/管段批量读取，SCADA 清洗使用单条集合更新，不再逐点往返。 自动化真实数据库测试分别执行 64 次业务库和 64 次时序库并发借用，查询结果一致，连接均能归还池中。嵌套 WNDB 写入和分析运行生命周期测试会在外层强制回滚，数据库没有残留记录。`DatabaseCommand` 的 pattern 新增、修改、删除也在同一池化事务中完成，并验证了五张明细表的复合主键、级联解除需求模式关联、结果变更和整体回滚。实时覆盖测试确认第二批数据替换第一批数据，外层回滚后测试记录为 0。 diff --git a/resources/db_v2/WNDB_STRUCTURE.md b/resources/db_v2/WNDB_STRUCTURE.md index 80a8771..b011516 100644 --- a/resources/db_v2/WNDB_STRUCTURE.md +++ b/resources/db_v2/WNDB_STRUCTURE.md @@ -6,7 +6,9 @@ 当前结构适合继续维护。WNDB 已按连接基础设施、管网模型、GIS、INP 和命令执行分组，原来的编号文件名、根目录聚合门面和星号导入已经移除。WDA、SCADA 资产查询和测压点选址也已离开底层模型目录。 -本次调整了代码文件、导入关系、命令分派和 WNDB 内部命令对象，没有改变 HTTP 接口。历史撤销日志已从数据库中移除，内部接口不再保留无效的兼容字段。真实库回归时发现五张明细表错误地把局部顺序号设成全局主键，已在 `tjwater_next` 中改为父对象 ID 与 `sequence_no` 的复合主键。 +本次调整了代码文件、导入关系、命令分派、项目生命周期接口和 WNDB 内部命令对象。无状态服务不再发布“打开、关闭、是否打开项目”三个旧 HTTP 操作，数据库连接在请求中按需从池借用。历史撤销日志已从数据库中移除，内部接口不再保留无效的兼容字段。真实库回归时发现五张明细表错误地把局部顺序号设成全局主键，已在 `tjwater_v2` 中改为父对象 ID 与 `sequence_no` 的复合主键。 + +`tjwater_v2` 是 v2 业务库和时序库的正式物理库名。元数据库中的逻辑项目代码仍为 `tjwater_next`，由项目路由指向 `tjwater_v2`；两者不必同名。版本模板固定为 `tjwater_v2_template`，不按项目代码动态派生。目前该模板已从实际 v2 结构创建、清空项目数据、刷新空物化视图并封存，压缩后约 19 MB。 ## 当前目录 @@ -16,6 +18,7 @@ app/native/wndb/ ├── core/ │ ├── connection.py │ ├── database.py +│ ├── model_replace.py │ └── projects.py ├── model/ │ ├── elements.py @@ -28,6 +31,8 @@ app/native/wndb/ │ ├── patterns.py │ ├── curves.py │ ├── options.py +│ ├── options_v2.py +│ ├── options_v3.py │ └── 其他 EPANET 模型模块 ├── gis/ │ ├── coordinates.py @@ -35,6 +40,7 @@ app/native/wndb/ │ ├── labels.py │ ├── backdrop.py │ ├── regions.py +│ ├── network_views.py │ └── region_geometry.py ├── inp/ │ ├── sections.py @@ -46,7 +52,7 @@ app/native/wndb/ └── executor.py ``` -目录内共有 47 个 Python 文件，约 7,100 行。`app/native/wndb/__init__.py` 只保留包说明，不再统一导出所有函数。调用方需要从具体职责模块导入，依赖来源可以直接从文件头确认。 +目录内共有 49 个 Python 文件。`app/native/wndb/__init__.py` 只保留包说明，不再统一导出所有函数。调用方需要从具体职责模块导入，依赖来源可以直接从文件头确认。 ## 各目录的职责 @@ -56,27 +62,44 @@ app/native/wndb/ `database.py` 提供 `ChangeSet`、`DatabaseCommand`、参数化查询和物化视图刷新。`DatabaseCommand` 只保存待执行的 SQL 和执行成功后返回给调用方的变更列表，不再生成或保存撤销 SQL。模型直接修改时按需刷新视图；批量命令在外层事务提交后只刷新一次。物化视图保留模型坐标 `x`、`y`，同时将供 GeoServer 使用的 `geom` 转换为 `EPSG:3857`，WNDB 查询不会把发布坐标误当成模型坐标。 -`projects.py` 只负责项目数据库的创建、复制、打开、关闭和删除，不再混入模型查询。 +`projects.py` 只负责项目数据库的创建、复制、删除和异常安全的临时库上下文，不再保存“项目已打开”状态，也不混入模型查询。`postgres`、模板库、旧 WNDB 模板库 `project` 和元数据库均属于保护对象；模板复制源只能精确匹配 `WNDB_TEMPLATE_DB_NAME`，不能重新引入每项目 `_template`。批量清理不再扫描并删除服务器上的未知数据库，调用方必须显式提供每一个目标库名。数据库级 advisory lock 与 `datallowconn` 共同串行化多 worker 下的复制和删除。普通项目复制若复制源仍有其他会话会直接失败，不再主动终止正常请求。 + +`model_replace.py` 在源库可重复读快照中读取 `network`、`gis` 基表，并按外键拓扑顺序复制到目标业务库。替换在单一事务内完成，不再删除并重建整个业务库；普通模型修改和整体替换共用同一项目级事务锁。INP 替换时，`analysis.results` 保留历史记录，只有新模型中不存在的元素引用会置空，`asset.scada_devices` 保留仍能匹配新节点或管段的设备；临时分析库则从当前项目复制模型和有效 SCADA 映射。 ### model：管网模型和仿真配置 `model` 按业务实体命名，不再使用 `s2_junctions.py` 这类 INP 章节编号。节点、连接、模式、曲线、需求、规则和仿真设置都能从文件名直接定位。 +`options_v2.py` 和 `options_v3.py` 分别负责 EPANET V2、V3 的 `[OPTIONS]` 章节导入导出；数据库中的 `engine_version = 'legacy'` 仍表示 V2 配置，仅作为现有存储标识保留。 + 每个实体模块保留三类紧密相关的函数：读取实体、生成并执行实体变更、转换该实体对应的一行或一段 INP 内容。完整文件的读取顺序、事务和项目生命周期由 `inp` 目录负责。因此，实体级编解码仍靠近实体定义，跨章节编排已经集中。 `elements.py` 保存节点、连接、模式、曲线和区域的通用类型判断及拓扑查询。它不再承担业务算法。 ### gis：空间数据和区域几何 -`coordinates.py`、`vertices.py`、`labels.py` 和 `backdrop.py` 对应原始 GIS 数据。`regions.py` 负责区域持久化，`region_geometry.py` 负责边界、凸包、膨胀和区域内元素查询。 +`coordinates.py`、`vertices.py`、`labels.py` 和 `backdrop.py` 对应原始 GIS 数据。`regions.py` 负责区域持久化，`region_geometry.py` 负责边界、凸包、膨胀和区域内元素查询。`network_views.py` 是 GIS 查询视图的只读适配层，负责统一节点、链路、拓扑和需求投影的批量读取。坐标读取不会补写默认几何；缺少坐标时仅向调用方返回 `(0, 0)`，真实坐标的初始化仍由新增、导入或修复命令负责。 管网实体修改会调用坐标 SQL 辅助函数，区域几何也会读取管网拓扑。这里存在明确的模型与 GIS 协作，但没有模块导入环。现阶段继续拆出抽象接口只会增加层级，没有实际收益。 +### GIS 统一查询视图 + +`tjwater_v2` 以现有 GIS 物化视图为基础增加了两个不保存重复数据的普通视图： + +| 视图 | 来源 | 用途 | +| --- | --- | --- | +| `gis.network_nodes` | `gis.junctions`、`gis.reservoirs`、`gis.tanks` | 统一返回 `id`、模型坐标 `x/y` 和 `node_type` | +| `gis.network_links` | `gis.pipes`、`gis.pumps`、`gis.valves` | 统一返回 `id`、起止节点和 `link_type` | + +普通视图始终读取底层物化视图当前内容，不需要单独刷新。管网修改提交后仍由 `gis.refresh_all_materialized_views` 刷新节点、管道、泵、阀门等物化视图，两个统一视图会随之得到最新结果。视图和全部字段均已在数据库中写入说明。 + +旧服务先读取节点或链路 ID，再逐个查询坐标、类型和端点，会对完整管网产生数万次往返。当前节点坐标、主干节点、全部链路、主干管道、区域边界链路、区域拓扑和用水量汇总均改为视图批量查询。没有调用方的 `get_nodes_in_extent`、`get_links_in_extent`，以及仅服务于旧聚合过程的四个模型辅助函数已删除；单元素详情接口仍保留原始模型查询。 + ### inp：文件级导入导出 `sections.py` 只保存 INP 章节名称和输出顺序。旧文件中混放的 `s1_title`、`s2_junction` 等命令类型常量已经移除。 -`importer.py` 负责文件分段、导入顺序、项目事务、版本转换和导入后的物化视图刷新。`exporter.py` 负责按 EPANET 版本组织各章节并写出文件或 `ChangeSet`。 +`importer.py` 负责文件分段、导入顺序、项目事务、版本转换和导入后的物化视图刷新。INP 更新先从 `tjwater_v2_template` 创建唯一暂存库并完成解析，再在当前业务库中事务替换模型表。模型提交后即使暂存库清理失败也仍会刷新物化视图；清理失败会记录日志，不再遮蔽主操作。ChangeSet 导入使用每请求唯一临时文件并在 `finally` 删除，避免同项目并发导入互相覆盖。`exporter.py` 负责按 EPANET 版本组织各章节并写出文件或 `ChangeSet`。 ### commands：批量修改和级联关系 @@ -100,7 +123,9 @@ flowchart LR EXECUTE --> COMMIT --> REFRESH ``` -实体模块根据请求生成 `DatabaseCommand`，其中 `sql` 是要执行的语句，`changes` 是成功后的变更结果。批量命令先在同一个项目事务中展开级联关系，再依次执行 SQL；任何一步失败都会回滚整个事务。事务提交后统一刷新物化视图，避免一次批量修改触发多次刷新。直接调用单个实体修改时，如果当前没有外层项目事务，则由 `execute_command` 完成提交并按影响范围刷新视图。 +实体模块根据请求生成 `DatabaseCommand`，其中 `sql` 是要执行的语句，`changes` 是成功后的变更结果。批量命令先在同一个项目事务中展开级联关系，再依次执行 SQL；任何写入步骤失败都会回滚整个事务。事务提交后统一刷新物化视图，避免一次批量修改触发多次刷新。直接调用单个实体修改时，如果当前没有外层项目事务，则由 `execute_command` 完成提交并按影响范围刷新视图。 + +物化视图采用并发刷新，因此刷新位于模型事务提交之后。若刷新失败，模型修改已经持久化，代码会抛出 `MaterializedViewRefreshAfterCommitError`。HTTP 层返回专用 Problem Details、`503` 和 `X-TJWater-Changes-Committed: true`，明确提示不能盲目重放原始写入。 旧实现中的 `DbChangeSet` 同时保存执行和撤销两套 SQL、两套变更结果，但数据库已经不再提供 operation 或 snapshot 撤销日志，这些字段没有消费者。当前代码已经删除 `undo_sql`、`undo_cs` 及各实体模块中的撤销 SQL 构造，也删除了只为撤销结果读取旧记录的查询和辅助方法。局部修改仍会读取一次当前记录，用于补齐请求中未提供的字段；这类读取属于更新语义，不是撤销机制。 @@ -117,6 +142,12 @@ flowchart LR `tjnetwork.py` 从约 995 行缩减到约 320 行。它不再通过 WNDB 根包获得全部函数，只显式导入当前服务使用的能力。过时的 `scripts/test_tjnetwork.py` 依赖已移除的 operation、snapshot、DMA 和旧 SCADA API，已经一并删除。 +### HTTP 执行边界 + +WNDB 当前使用同步 `psycopg` 连接池。`network/`、`components/` 和同步 EPANET 仿真接口统一声明为同步处理函数；公开 REST 路由的异步适配器把这些函数送入线程池，并把项目路由上下文传入工作线程，不会在事件循环线程上阻塞数据库或求解器。异步业务库和时序库访问使用带借用计数和代际切换的项目池：活跃旧池不会被 LRU 淘汰或强制关闭，配置变化后新请求立即使用新池，旧池在已有借用归还后关闭。元数据库保持独立 SQLAlchemy 异步池。 + +临时分析库先由固定 `tjwater_v2_template` 提供结构，再从当前项目的一致性快照复制 `network/gis` 模型与 SCADA 映射并刷新物化视图。V3→V2 格式转换不需要项目模型，单独使用空模板临时库。旧 `online_Analysis.py`、restore 和 open/close 项目脚本已经删除，不再保留每项目模板与 operation 恢复入口。 + ## 依赖方向 ```mermaid @@ -162,7 +193,8 @@ WNDB 根包不再作为依赖汇聚点。上层若只需要管道查询，应直 ## 验证结果 -- 本地 conda 环境全量测试：239 项通过，10 项按条件跳过。 -- Docker 镜像构建成功，镜像内全量测试结果一致。 -- `tjwater_next` 真实数据库测试：8 项通过，覆盖业务库和时序库并发借用、嵌套事务回滚、分析运行生命周期、恶意标识符转义、五张明细表的复合主键，以及 WNDB pattern 增删改、级联解除需求关联和整体回滚。 +- 本地 conda 环境单元、鉴权和 API 测试：286 项通过，2 项按条件跳过。 +- 一次性实库从 `tjwater_v2_template` 创建后，通过 INP 暂存解析和事务替换得到 11 个节点、13 条连接、11 条坐标及 9 条 junction 物化视图记录，验证后已完整删除。 +- `tjwater_v2` 统一视图覆盖 87,907 个节点和 91,054 条链路，与六个来源物化视图的合计数量一致。实测完整节点读取约 0.17 秒、完整链路读取约 0.10 秒、完整拓扑两次批量查询约 1.12 秒；耗时仅作为当前环境基线，不作为固定性能承诺。 +- `tjwater_v2` 真实数据库测试：11 项通过，覆盖业务库和时序库并发借用、失效连接自动重建、临时库模型/SCADA/视图完整克隆与清理、嵌套事务回滚、分析运行生命周期、恶意标识符转义、明细表复合主键、统一 GIS 查询视图，以及 WNDB pattern 增删改、级联解除需求关联和整体回滚。 - Python 编译、未使用导入扫描、撤销字段残留扫描和 `git diff --check` 均通过。 diff --git a/scripts/build_pyd.py b/scripts/build_pyd.py index a9ec4ec..9a38c6f 100644 --- a/scripts/build_pyd.py +++ b/scripts/build_pyd.py @@ -3,15 +3,12 @@ from Cython.Build import cythonize setup(ext_modules=cythonize([ "tjnetwork.py", - "online_Analysis.py", "sensitivity.py", - "run_simlation.py", "run_simulation.py", "get_hist_data.py", "get_realValue.py", "get_data.py", "get_current_total_Q.py", - "get_current_status.py", "simulation.py", "time_api.py", "api/*.py", diff --git a/scripts/clean_projects.py b/scripts/clean_projects.py index 0cb677c..4ef9770 100644 --- a/scripts/clean_projects.py +++ b/scripts/clean_projects.py @@ -1,5 +1,26 @@ -from app.services.tjnetwork import clean_project, delete_project +import argparse +from collections.abc import Sequence +import sys +from pathlib import Path -if __name__ == '__main__': - clean_project() - delete_project('project') +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.native.wndb.core.projects import clean_project + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Delete explicitly named temporary project databases.", + ) + parser.add_argument( + "--yes", + action="store_true", + required=True, + help="confirm permanent deletion of every named database", + ) + parser.add_argument("projects", nargs="+", help="project database names to delete") + return parser.parse_args(argv) + + +if __name__ == "__main__": + clean_project(parse_args().projects) diff --git a/scripts/create_template.py b/scripts/create_template.py deleted file mode 100644 index 8bd7db4..0000000 --- a/scripts/create_template.py +++ /dev/null @@ -1,136 +0,0 @@ -import psycopg as pg - -sql_create = [ - "script/sql/create/0.base.sql", - "script/sql/create/1.title.sql", - "script/sql/create/2.junctions.sql", - "script/sql/create/3.reservoirs.sql", - "script/sql/create/4.tanks.sql", - "script/sql/create/5.pipes.sql", - "script/sql/create/6.pumps.sql", - "script/sql/create/7.valves.sql", - "script/sql/create/8.tags.sql", - "script/sql/create/9.demands.sql", - "script/sql/create/10.status.sql", - "script/sql/create/11.patterns.sql", - "script/sql/create/12.curves.sql", - "script/sql/create/13.controls.sql", - "script/sql/create/14.rules.sql", - "script/sql/create/15.energy.sql", - "script/sql/create/16.emitters.sql", - "script/sql/create/17.quality.sql", - "script/sql/create/18.sources.sql", - "script/sql/create/19.reactions.sql", - "script/sql/create/20.mixing.sql", - "script/sql/create/21.times.sql", - "script/sql/create/22.report.sql", - "script/sql/create/23.options.sql", - "script/sql/create/24.coordinates.sql", - "script/sql/create/25.vertices.sql", - "script/sql/create/26.labels.sql", - "script/sql/create/27.backdrop.sql", - "script/sql/create/28.end.sql", - "script/sql/create/29.scada_device.sql", - "script/sql/create/30.scada_device_data.sql", - "script/sql/create/31.scada_element.sql", - "script/sql/create/32.region.sql", - "script/sql/create/33.dma.sql", - "script/sql/create/34.sa.sql", - "script/sql/create/35.vd.sql", - "script/sql/create/36.wda.sql", - "script/sql/create/37.history_patterns_flows.sql", - "script/sql/create/38.scada_info.sql", - "script/sql/create/40.scheme_list.sql", - "script/sql/create/41.pipe_risk_probability.sql", - "script/sql/create/42.sensor_placement.sql", - "script/sql/create/43.burst_locate_result.sql", - "script/sql/create/44.leakage_identify_result.sql", - "script/sql/create/extension_data.sql", - "script/sql/create/operation.sql" -] - -sql_drop = [ - "script/sql/drop/operation.sql", - "script/sql/drop/extension_data.sql", - "script/sql/drop/43.burst_locate_result.sql", - "script/sql/drop/42.sensor_placement.sql", - "script/sql/drop/44.leakage_identify_result.sql", - "script/sql/drop/41.pipe_risk_probability.sql", - "script/sql/drop/40.scheme_list.sql", - "script/sql/drop/38.scada_info.sql", - "script/sql/drop/37.history_patterns_flows.sql", - "script/sql/drop/36.wda.sql", - "script/sql/drop/35.vd.sql", - "script/sql/drop/34.sa.sql", - "script/sql/drop/33.dma.sql", - "script/sql/drop/32.region.sql", - "script/sql/drop/31.scada_element.sql", - "script/sql/drop/30.scada_device_data.sql", - "script/sql/drop/29.scada_device.sql", - "script/sql/drop/28.end.sql", - "script/sql/drop/27.backdrop.sql", - "script/sql/drop/26.labels.sql", - "script/sql/drop/25.vertices.sql", - "script/sql/drop/24.coordinates.sql", - "script/sql/drop/23.options.sql", - "script/sql/drop/22.report.sql", - "script/sql/drop/21.times.sql", - "script/sql/drop/20.mixing.sql", - "script/sql/drop/19.reactions.sql", - "script/sql/drop/18.sources.sql", - "script/sql/drop/17.quality.sql", - "script/sql/drop/16.emitters.sql", - "script/sql/drop/15.energy.sql", - "script/sql/drop/14.rules.sql", - "script/sql/drop/13.controls.sql", - "script/sql/drop/12.curves.sql", - "script/sql/drop/11.patterns.sql", - "script/sql/drop/10.status.sql", - "script/sql/drop/9.demands.sql", - "script/sql/drop/8.tags.sql", - "script/sql/drop/7.valves.sql", - "script/sql/drop/6.pumps.sql", - "script/sql/drop/5.pipes.sql", - "script/sql/drop/4.tanks.sql", - "script/sql/drop/3.reservoirs.sql", - "script/sql/drop/2.junctions.sql", - "script/sql/drop/1.title.sql", - "script/sql/drop/0.base.sql" -] - -def create_template(): - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("create database project") - with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: - with conn.cursor() as cur: - cur.execute('create extension postgis cascade') - cur.execute('create extension pgrouting cascade') - for sql in sql_create: - with open(sql, "r", encoding="utf-8") as f: - cur.execute(f.read()) - print(f'executed {sql}') - conn.commit() - -def have_template(): - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("select * from pg_database where datname = 'project'") - return cur.rowcount > 0 - -def delete_template(): - with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: - with conn.cursor() as cur: - for sql in sql_drop: - with open(sql, "r", encoding="utf-8") as f: - cur.execute(f.read()) - print(f'executed {sql}') - conn.commit() - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("drop database project") - -if __name__ == "__main__": - if (have_template()): - delete_template() - create_template() diff --git a/scripts/demo.py b/scripts/demo.py index c4e3014..b5397e1 100644 --- a/scripts/demo.py +++ b/scripts/demo.py @@ -1,7 +1,5 @@ from app.services.tjnetwork import list_project, read_inp read_inp("beibeizone","beibeizone.inp") -#open_project('beibeizone') #generate_service_area("beibeizone",0.00001) print(list_project()) - diff --git a/scripts/dev.py b/scripts/dev.py deleted file mode 100644 index bda8f47..0000000 --- a/scripts/dev.py +++ /dev/null @@ -1,21 +0,0 @@ -from app.services.tjnetwork import calculate_service_area, open_project, read_inp - -p = 'dev' - -read_inp(p, f'./inp/net3.inp', '3') -open_project(p) - -sass = calculate_service_area(p) -assert len(sass) == 25 - -assert sass[0]['River'] == ['River', '60', '61', '123', '601'] -assert sass[0]['3'] == ['121', '120', '119', '117', '257', '151', '157', '115', '259', '261', '149', '159', '111', '113', '263', '147', '161', '197', '193', '105', '145', '163', '195', '191', '267', '107', '141', '164', '265', '187', '189', '143', '166', '169', '204', '15', '167', '171', '269', '173', '271', '199', '201', '203', '3', '20', '127', '125', '129', '153', '131', '139'] -assert sass[0]['1'] == ['185', '184', '205', '273', '1', '40', '179', '177', '183', '181', '35'] -assert sass[0]['2'] == ['207', '275', '2', '50', '255', '247', '253', '251', '241', '249', '239', '243', '237', '211', '229', '209', '213', '231', '208', '215', '206', '217', '219', '225'] - -print(sass[1]) -assert sass[0]['River'] == ['River', '60', '61', '123', '601'] -assert sass[0]['3'] == ['121', '120', '119', '117', '257', '151', '157', '115', '259', '261', '149', '159', '111', '113', '263', '147', '161', '197', '193', '145', '163', '195', '191', '141', '164', '265', '187', '143', '166', '169', '267', '204', '15', '167', '171', '269', '173', '199', '201', '203', '3', '20', '127', '125', '129', '153', '131', '139'] -assert sass[0]['Lake'] == ['105', '107', 'Lake', '10', '101', '103', '109'] -assert sass[0]['1'] == ['189', '185', '271', '184', '205', '273', '1', '40', '179', '177', '183', '181', '35'] -assert sass[0]['2'] == ['207', '275', '2', '50', '255', '247', '253', '251', '241', '249', '239', '243', '237', '211', '229', '209', '213', '231', '208', '215', '206', '217', '219', '225'] diff --git a/scripts/get_current_status.py b/scripts/get_current_status.py deleted file mode 100644 index e64c8a8..0000000 --- a/scripts/get_current_status.py +++ /dev/null @@ -1,66 +0,0 @@ -from app.services.tjnetwork import api, get_all_service_area_ids, open_project -from get_realValue import * -from get_hist_data import * -import datetime -from api.s36_wda_cal import * - - -ids=['2498','3854','3853','2510','2514','4780','4854'] -cur_data=None - - -def get_latest_cal_time()->datetime: - current_time=datetime.datetime.now() - return current_time - - -def get_current_data(str_datetime: str=None)->bool: - global cur_data - if str_datetime==None: - cur_data=get_realValue(ids) - else: - cur_date=get_hist_data(ids,str_datetime) - if cur_data ==None: - return False - return True - -def get_current_total_Q(str_dt:str='')->float: - q_ids=['2498','3854','3853'] - q_dn900=cur_data[q_ids[0]] - q_dn500=cur_data[q_ids[1]] - q_dn1000=cur_data[q_ids[2]] - total_q=q_dn1000+q_dn500+q_dn900 - return total_q - -def get_h_pressure()->float: - head_id='2510' - h_pressure=cur_data[head_id] - return h_pressure - -def get_l_pressure()->float: - head_id='2514' - l_pressure=cur_data[head_id] - return l_pressure - -def get_h_tank_leve()->float: - h_tank_id='4780' - h_tank_level=cur_data[h_tank_id] - return h_tank_level - -def get_l_tank_leve()->float: - l_tank_id='4854' - l_tank_level=cur_data[l_tank_id] - return l_tank_level - - -# test interface -if __name__ == '__main__': - # if get_current_data()==True: - # tQ=get_current_total_Q() - # print(f"the current tQ is {tQ}\n") - # data=get_hist_data(ids,conver_beingtime_to_ucttime('2024-04-10 15:05:00'),conver_beingtime_to_ucttime('2024-04-10 15:10:00')) - open_project("beibeizone") - regions=get_all_service_area_ids("beibeizone") - for region in regions: - t_basedmds=api.s36_wda_cal.get_total_base_demand("beibeizone",region) - print(f"{region}:{t_basedmds}") \ No newline at end of file diff --git a/scripts/online_Analysis.py b/scripts/online_Analysis.py deleted file mode 100644 index 82e9dda..0000000 --- a/scripts/online_Analysis.py +++ /dev/null @@ -1,1548 +0,0 @@ -import os -from app.services.tjnetwork import ( - ChangeSet, - OPTION_DEMAND_MODEL_PDA, - OPTION_QUALITY_CHEMICAL, - SOURCE_TYPE_CONCEN, - add_pattern, - add_source, - close_project, - copy_project, - delete_project, - dump_inp, - get_demand, - get_emitter, - get_node_links, - get_option, - get_pattern, - get_pipe, - get_source, - get_tank, - get_time, - have_project, - is_junction, - is_project_open, - open_project, - read_inp, - set_demand, - set_emitter, - set_option, - set_source, - set_time, -) -from app.algorithms.simulation.runner import run_simulation_ex, from_clock_to_seconds_2 -from math import sqrt, pi -from app.infra.epanet.epanet import Output -import json -from datetime import datetime -import time -import pytz -import psycopg -from psycopg import sql -import pandas as pd -import csv -import chardet -import app.services.simulation as simulation -import geopandas as gpd -from sqlalchemy import create_engine -import ast -import app.services.project_info as project_info -import app.algorithms.sensor.kmeans as kmeans_sensor -import app.algorithms.cleaning.flow as flow_data_clean -import app.algorithms.cleaning.pressure as pressure_data_clean -import app.algorithms.sensor.sensitivity as sensitivity -from app.core.config import get_pgconn_string - - -############################################################ -# burst analysis 01 -############################################################ -def convert_to_local_unit(proj: str, emitters: float) -> float: - open_project(proj) - proj_opt = get_option(proj) - str_unit = proj_opt.get("UNITS") - - if str_unit == "CMH": - return emitters * 3.6 - elif str_unit == "LPS": - return emitters - elif str_unit == "CMS": - return emitters / 1000.0 - elif str_unit == "MGD": - return emitters * 0.0438126 - - # Unknown unit: log and return original value - print(str_unit) - return emitters - - -def burst_analysis( - name: str, - modify_pattern_start_time: str, - burst_ID: list | str = None, - burst_size: list | float | int = None, - modify_total_duration: int = 900, - modify_fixed_pump_pattern: dict[str, list] = None, - modify_variable_pump_pattern: dict[str, list] = None, - modify_valve_opening: dict[str, float] = None, - scheme_name: str = None, - username: str | None = None, -) -> None: - """ - 爆管模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param burst_ID: 爆管管道的ID，选取的是管道，单独传入一个爆管管道，可以是str或list，传入多个爆管管道是用list - :param burst_size: 爆管管道破裂的孔口面积，和burst_ID列表各位置的ID对应，以cm*cm计算 - :param modify_total_duration: 模拟总历时，秒 - :param modify_fixed_pump_pattern: dict中包含多个水泵模式，str为工频水泵的id，list为修改后的pattern - :param modify_variable_pump_pattern: dict中包含多个水泵模式，str为变频水泵的id，list为修改后的pattern - :param modify_valve_opening: dict中包含多个阀门开启度，str为阀门的id，float为修改后的阀门开启度 - :param scheme_name: 方案名称 - :return: - """ - if not username: - raise ValueError("username is required when storing burst analysis scheme") - - scheme_detail: dict = { - "burst_ID": burst_ID, - "burst_size": burst_size, - "modify_total_duration": modify_total_duration, - "modify_fixed_pump_pattern": modify_fixed_pump_pattern, - "modify_variable_pump_pattern": modify_variable_pump_pattern, - "modify_valve_opening": modify_valve_opening, - } - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"burst_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - simulation.run_simulation( - name=new_name, - simulation_type="manually_temporary", - modify_pattern_start_time=modify_pattern_start_time, - ) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - ##step 1 set the emitter coefficient of end node of busrt pipe - if isinstance(burst_ID, list): - if (burst_size is not None) and (type(burst_size) is not list): - return json.dumps("Type mismatch.") - elif isinstance(burst_ID, str): - burst_ID = [burst_ID] - if burst_size is not None: - if isinstance(burst_size, float) or isinstance(burst_size, int): - burst_size = [burst_size] - else: - return json.dumps("Type mismatch.") - else: - return json.dumps("Type mismatch.") - if burst_size is None: - burst_size = [-1] * len(burst_ID) - elif len(burst_size) < len(burst_ID): - burst_size += [-1] * (len(burst_ID) - len(burst_size)) - elif len(burst_size) > len(burst_ID): - # burst_size = burst_size[:len(burst_ID)] - return json.dumps("Length mismatch.") - for burst_ID_, burst_size_ in zip(burst_ID, burst_size): - pipe = get_pipe(new_name, burst_ID_) - str_start_node = pipe["node1"] - str_end_node = pipe["node2"] - d_pipe = pipe["diameter"] / 1000.0 - if burst_size_ <= 0: - burst_size_ = 3.14 * d_pipe * d_pipe / 4 / 8 - else: - burst_size_ = burst_size_ / 10000 - emitter_coeff = ( - 0.65 * burst_size_ * sqrt(19.6) * 1000 - ) # 1/8开口面积作为coeff，单位 L/S - emitter_coeff = convert_to_local_unit(new_name, emitter_coeff) - emitter_node = "" - if is_junction(new_name, str_end_node): - emitter_node = str_end_node - elif is_junction(new_name, str_start_node): - emitter_node = str_start_node - old_emitter = get_emitter(new_name, emitter_node) - if old_emitter != None: - old_emitter["coefficient"] = emitter_coeff # 爆管的emitter coefficient设置 - else: - old_emitter = {"junction": emitter_node, "coefficient": emitter_coeff} - new_emitter = ChangeSet() - new_emitter.append(old_emitter) - set_emitter(new_name, new_emitter) - # step 2. run simulation - # 涉及关阀计算，可能导致关阀后仍有流量，改为压力驱动PDA - options = get_option(new_name) - options["DEMAND MODEL"] = OPTION_DEMAND_MODEL_PDA - options["REQUIRED PRESSURE"] = "10.0000" - cs_options = ChangeSet() - cs_options.append(options) - set_option(new_name, cs_options) - # valve_control = None - # if modify_valve_opening is not None: - # valve_control = {} - # for valve in modify_valve_opening: - # valve_control[valve] = {'status': 'CLOSED'} - # result = run_simulation_ex(new_name,'realtime', modify_pattern_start_time, - # end_datetime=modify_pattern_start_time, - # modify_total_duration=modify_total_duration, - # modify_pump_pattern=modify_pump_pattern, - # valve_control=valve_control, - # downloading_prohibition=True) - simulation.run_simulation( - name=new_name, - simulation_type="extended", - modify_pattern_start_time=modify_pattern_start_time, - modify_total_duration=modify_total_duration, - modify_fixed_pump_pattern=modify_fixed_pump_pattern, - modify_variable_pump_pattern=modify_variable_pump_pattern, - modify_valve_opening=modify_valve_opening, - scheme_type="burst_Analysis", - scheme_name=scheme_name, - ) - # step 3. restore the base model status - # execute_undo(name) #有疑惑 - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # return result - store_scheme_info( - name=name, - scheme_name=scheme_name, - scheme_type="burst_Analysis", - username=username, - scheme_start_time=modify_pattern_start_time, - scheme_detail=scheme_detail, - ) - - -############################################################ -# valve closing analysis 02 -############################################################ -def valve_close_analysis( - name: str, - modify_pattern_start_time: str, - modify_total_duration: int = 900, - modify_valve_opening: dict[str, float] = None, - scheme_name: str = None, -) -> None: - """ - 关阀模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :param modify_valve_opening: dict中包含多个阀门开启度，str为阀门的id，float为修改后的阀门开启度 - :param scheme_name: 方案名称 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"valve_close_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - # step 1. change the valves status to 'closed' - # for valve in valves: - # if not is_valve(new_name,valve): - # result='ID:{}is not a valve type'.format(valve) - # return result - # cs=ChangeSet() - # status=get_status(new_name,valve) - # status['status']='CLOSED' - # cs.append(status) - # set_status(new_name,cs) - # step 2. run simulation - # 涉及关阀计算，可能导致关阀后仍有流量，改为压力驱动PDA - options = get_option(new_name) - options["DEMAND MODEL"] = OPTION_DEMAND_MODEL_PDA - options["REQUIRED PRESSURE"] = "20.0000" - cs_options = ChangeSet() - cs_options.append(options) - set_option(new_name, cs_options) - # result = run_simulation_ex(new_name,'realtime', modify_pattern_start_time, modify_pattern_start_time, modify_total_duration, - # downloading_prohibition=True) - simulation.run_simulation( - name=new_name, - simulation_type="extended", - modify_pattern_start_time=modify_pattern_start_time, - modify_total_duration=modify_total_duration, - modify_valve_opening=modify_valve_opening, - scheme_type="valve_close_Analysis", - scheme_name=scheme_name, - ) - # step 3. restore the base model - # for valve in valves: - # execute_undo(name) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # return result - - -############################################################ -# flushing analysis 03 -# Pipe_Flushing_Analysis(prj_name,date_time, Valve_id_list, Drainage_Node_Id, Flushing_flow[opt], Flushing_duration[opt])->out_file:string -############################################################ -def flushing_analysis( - name: str, - modify_pattern_start_time: str, - modify_total_duration: int = 900, - modify_valve_opening: dict[str, float] = None, - drainage_node_ID: str = None, - flushing_flow: float = 0, - scheme_name: str = None, -) -> None: - """ - 管道冲洗模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :param modify_valve_opening: dict中包含多个阀门开启度，str为阀门的id，float为修改后的阀门开启度 - :param drainage_node_ID: 冲洗排放口所在节点ID - :param flushing_flow: 冲洗水量，传入参数单位为m3/h - :param scheme_name: 方案名称 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"flushing_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - if not is_junction(new_name, drainage_node_ID): - return "Wrong Drainage node type" - # step 1. change the valves status to 'closed' - # for valve, valve_k in zip(valves, valves_k): - # cs=ChangeSet() - # status=get_status(new_name,valve) - # # status['status']='CLOSED' - # if valve_k == 0: - # status['status'] = 'CLOSED' - # elif valve_k < 1: - # status['status'] = 'OPEN' - # status['setting'] = 0.1036 * pow(valve_k, -3.105) - # cs.append(status) - # set_status(new_name,cs) - units = get_option(new_name) - # step 2. set the emitter coefficient of drainage node or add flush flow to the drainage node - emitter_demand = get_demand(new_name, drainage_node_ID) - cs = ChangeSet() - if flushing_flow > 0: - for r in emitter_demand["demands"]: - if units == "LPS": - r["demand"] += flushing_flow / 3.6 - elif units == "CMH": - r["demand"] += flushing_flow - cs.append(emitter_demand) - set_demand(new_name, cs) - else: - pipes = get_node_links(new_name, drainage_node_ID) - flush_diameter = 50 - for pipe in pipes: - d = get_pipe(new_name, pipe)["diameter"] - if flush_diameter < d: - flush_diameter = d - flush_diameter /= 1000 - emitter_coeff = ( - 0.65 * 3.14 * (flush_diameter * flush_diameter / 4) * sqrt(19.6) * 1000 - ) # 全开口面积作为coeff - - old_emitter = get_emitter(new_name, drainage_node_ID) - if old_emitter != None: - old_emitter["coefficient"] = emitter_coeff # 爆管的emitter coefficient设置 - else: - old_emitter = {"junction": drainage_node_ID, "coefficient": emitter_coeff} - new_emitter = ChangeSet() - new_emitter.append(old_emitter) - set_emitter(new_name, new_emitter) - # step 3. run simulation - # 涉及关阀计算，可能导致关阀后仍有流量，改为压力驱动PDA - options = get_option(new_name) - options["DEMAND MODEL"] = OPTION_DEMAND_MODEL_PDA - options["REQUIRED PRESSURE"] = "20.0000" - cs_options = ChangeSet() - cs_options.append(options) - set_option(new_name, cs_options) - # result = run_simulation_ex(new_name,'realtime', modify_pattern_start_time, modify_pattern_start_time, modify_total_duration, - # downloading_prohibition=True) - simulation.run_simulation( - name=new_name, - simulation_type="extended", - modify_pattern_start_time=modify_pattern_start_time, - modify_total_duration=modify_total_duration, - modify_valve_opening=modify_valve_opening, - scheme_type="flushing_Analysis", - scheme_name=scheme_name, - ) - # step 4. restore the base model - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # return result - - -############################################################ -# Contaminant simulation 04 -# -############################################################ -def contaminant_simulation( - name: str, - modify_pattern_start_time: str, # 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - modify_total_duration: int = 900, # 模拟总历时，秒 - source: str = None,# 污染源节点ID - concentration: float = None, # 污染源浓度，单位mg/L - source_pattern: str = None, # 污染源时间变化模式名称 - scheme_name: str = None, -) -> None: - """ - 污染模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :param source: 污染源所在的节点ID - :param concentration: 污染源位置处的浓度，单位mg/L，即默认的污染模拟setting为concentration（应改为 Set point booster） - :param source_pattern: 污染源的时间变化模式，若不传入则默认以恒定浓度持续模拟，时间长度等于duration; - 若传入，则格式为{1.0,0.5,1.1}等系数列表pattern_step模拟等于模型的hydraulic time step - :param scheme_name: 方案名称 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"contaminant_Sim_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - dic_time = get_time(new_name) - dic_time["QUALITY TIMESTEP"] = "0:05:00" - cs = ChangeSet() - cs.operations.append(dic_time) - set_time(new_name, cs) # set QUALITY TIMESTEP - time_option = get_time(new_name) - hydraulic_step = time_option["HYDRAULIC TIMESTEP"] - secs = from_clock_to_seconds_2(hydraulic_step) - operation_step = 0 - # step 1. set duration - if modify_total_duration == None: - modify_total_duration = secs - # step 2. set pattern - if source_pattern != None: - pt = get_pattern(new_name, source_pattern) - if pt == None: - str_response = str("cant find source_pattern") - return str_response - else: - cs_pattern = ChangeSet() - pt = {} - factors = [] - tmp_duration = modify_total_duration - while tmp_duration > 0: - factors.append(1.0) - tmp_duration = tmp_duration - secs - pt["id"] = "contam_pt" - pt["factors"] = factors - cs_pattern.append(pt) - add_pattern(new_name, cs_pattern) - operation_step += 1 - # step 3. set source/initial quality - # source quality - cs_source = ChangeSet() - source_schema = { - "node": source, - "s_type": SOURCE_TYPE_CONCEN, - "strength": concentration, - "pattern": pt["id"], - } - cs_source.append(source_schema) - source_node = get_source(new_name, source) - if len(source_node) == 0: - add_source(new_name, cs_source) - else: - set_source(new_name, cs_source) - dict_demand = get_demand(new_name, source) - for demands in dict_demand["demands"]: - dict_demand["demands"][dict_demand["demands"].index(demands)]["demand"] = -1 - dict_demand["demands"][dict_demand["demands"].index(demands)]["pattern"] = None - cs = ChangeSet() - cs.append(dict_demand) - set_demand(new_name, cs) # set inflow node - # # initial quality - # dict_quality = get_quality(new_name, source) - # dict_quality['quality'] = concentration - # cs = ChangeSet() - # cs.append(dict_quality) - # set_quality(new_name, cs) - operation_step += 1 - # step 4 set option of quality to chemical - opt = get_option(new_name) - opt["QUALITY"] = OPTION_QUALITY_CHEMICAL - cs_option = ChangeSet() - cs_option.append(opt) - set_option(new_name, cs_option) - operation_step += 1 - # step 5. run simulation - # result = run_simulation_ex(new_name,'realtime', modify_pattern_start_time, modify_pattern_start_time, modify_total_duration, - # downloading_prohibition=True) - simulation.run_simulation( - name=new_name, - simulation_type="extended", - modify_pattern_start_time=modify_pattern_start_time, - modify_total_duration=modify_total_duration, - scheme_type="contaminant_Analysis", - scheme_name=scheme_name, - ) - - # for i in range(1,operation_step): - # execute_undo(name) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # return result - - -############################################################ -# age analysis 05 ***水龄模拟目前还没和实时模拟打通，不确定是否需要，先不要使用*** -############################################################ - - -def age_analysis( - name: str, modify_pattern_start_time: str, modify_total_duration: int = 900 -) -> None: - """ - 水龄模拟 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"age_Anal_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - # step 1. run simulation - - result = run_simulation_ex( - new_name, - "realtime", - modify_pattern_start_time, - duration=modify_total_duration, - downloading_prohibition=True, - ) - # step 2. restore the base model status - # execute_undo(name) #有疑惑 - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - output = Output("./temp/{}.db.out".format(new_name)) - # element_name = output.element_name() - # node_name = element_name['nodes'] - # link_name = element_name['links'] - nodes_age = [] - node_result = output.node_results() - for node in node_result: - nodes_age.append(node["result"][-1]["quality"]) - links_age = [] - link_result = output.link_results() - for link in link_result: - links_age.append(link["result"][-1]["quality"]) - age_result = {"nodes": nodes_age, "links": links_age} - # age_result = {'nodes': nodes_age, 'links': links_age, 'nodeIDs': node_name, 'linkIDs': link_name} - return json.dumps(age_result) - - -############################################################ -# pressure regulation 06 -############################################################ - - -def pressure_regulation( - name: str, - modify_pattern_start_time: str, - modify_total_duration: int = 900, - modify_tank_initial_level: dict[str, float] = None, - modify_fixed_pump_pattern: dict[str, list] = None, - modify_variable_pump_pattern: dict[str, list] = None, - scheme_name: str = None, -) -> None: - """ - 区域调压模拟，用来模拟未来15分钟内，开关水泵对区域压力的影响 - :param name: 模型名称，数据库中对应的名字 - :param modify_pattern_start_time: 模拟开始时间,格式为'2024-11-25T09:00:00+08:00' - :param modify_total_duration: 模拟总历时，秒 - :param modify_tank_initial_level: dict中包含多个水塔，str为水塔的id，float为修改后的initial_level - :param modify_fixed_pump_pattern: dict中包含多个水泵模式，str为工频水泵的id，list为修改后的pattern - :param modify_variable_pump_pattern: dict中包含多个水泵模式，str为变频水泵的id，list为修改后的pattern - :param scheme_name: 模拟方案名称 - :return: - """ - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"pressure_regulation_{name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(name): - # close_project(name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - # 全部关泵后，压力计算不合理，改为压力驱动PDA - options = get_option(new_name) - options["DEMAND MODEL"] = OPTION_DEMAND_MODEL_PDA - options["REQUIRED PRESSURE"] = "15.0000" - cs_options = ChangeSet() - cs_options.append(options) - set_option(new_name, cs_options) - # result = run_simulation_ex(name=new_name, - # simulation_type='realtime', - # start_datetime=start_datetime, - # duration=900, - # pump_control=pump_control, - # tank_initial_level_control=tank_initial_level_control, - # downloading_prohibition=True) - simulation.run_simulation( - name=new_name, - simulation_type="extended", - modify_pattern_start_time=modify_pattern_start_time, - modify_total_duration=modify_total_duration, - modify_tank_initial_level=modify_tank_initial_level, - modify_fixed_pump_pattern=modify_fixed_pump_pattern, - modify_variable_pump_pattern=modify_variable_pump_pattern, - scheme_type="pressure_regulation", - scheme_name=scheme_name, - ) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # return result - - -############################################################ -# project management 07 ***暂时不使用，与业务需求无关*** -############################################################ - - -def project_management( - prj_name, - start_datetime, - pump_control, - tank_initial_level_control=None, - region_demand_control=None, -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"project_management_{prj_name}" - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(prj_name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(prj_name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - result = run_simulation_ex( - name=new_name, - simulation_type="realtime", - start_datetime=start_datetime, - duration=86400, - pump_control=pump_control, - tank_initial_level_control=tank_initial_level_control, - region_demand_control=region_demand_control, - downloading_prohibition=True, - ) - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - return result - - -############################################################ -# scheduling analysis 08 ***暂时不使用，与业务需求无关*** -############################################################ - - -def scheduling_simulation( - prj_name, start_time, pump_control, tank_id, water_plant_output_id, time_delta=300 -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"scheduling_{prj_name}" - - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(prj_name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(prj_name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - - run_simulation_ex( - new_name, "realtime", start_time, duration=0, pump_control=pump_control - ) - - if not is_project_open(new_name): - open_project(new_name) - - tank = get_tank(new_name, tank_id) # 水塔信息 - tank_floor_space = pi * pow(tank["diameter"] / 2, 2) # 水塔底面积(m^2) - tank_init_level = tank["init_level"] # 水塔初始水位(m) - tank_pipes_id = tank["links"] # pipes list - - tank_pipe_flow_direction = ( - {} - ) # 管道流向修正系数, 水塔为下游节点时为1, 水塔为上游节点时为-1 - for pipe_id in tank_pipes_id: - if get_pipe(new_name, pipe_id)["node2"] == tank_id: # 水塔为下游节点 - tank_pipe_flow_direction[pipe_id] = 1 - else: - tank_pipe_flow_direction[pipe_id] = -1 - - output = Output("./temp/{}.db.out".format(new_name)) - - node_results = ( - output.node_results() - ) # [{'node': str, 'result': [{'pressure': float}]}] - water_plant_output_pressure = 0 - for node_result in node_results: - if node_result["node"] == water_plant_output_id: # 水厂出水压力(m) - water_plant_output_pressure = node_result["result"][-1]["pressure"] - water_plant_output_pressure /= 100 # 预计水厂出水压力(Mpa) - - pipe_results = output.link_results() # [{'link': str, 'result': [{'flow': float}]}] - tank_inflow = 0 - for pipe_result in pipe_results: - for pipe_id in tank_pipes_id: # 遍历与水塔相连的管道 - if pipe_result["link"] == pipe_id: # 水塔入流流量(L/s) - tank_inflow += ( - pipe_result["result"][-1]["flow"] - * tank_pipe_flow_direction[pipe_id] - ) - tank_inflow /= 1000 # 水塔入流流量(m^3/s) - tank_level_delta = tank_inflow * time_delta / tank_floor_space # 水塔水位改变值(m) - tank_level = tank_init_level + tank_level_delta # 预计水塔水位(m) - - simulation_results = { - "water_plant_output_pressure": water_plant_output_pressure, - "tank_init_level": tank_init_level, - "tank_level": tank_level, - } - - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - - return json.dumps(simulation_results) - - -def daily_scheduling_simulation( - prj_name, start_time, pump_control, reservoir_id, tank_id, water_plant_output_id -) -> str: - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Analysis." - ) - new_name = f"daily_scheduling_{prj_name}" - - if have_project(new_name): - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - # if is_project_open(prj_name): - # close_project(prj_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Copying Database." - ) - # CopyProjectEx()(prj_name, new_name, - # ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table']) - copy_project(prj_name + "_template", new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Start Opening Database." - ) - open_project(new_name) - print( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - + " -- Database Loading OK." - ) - - run_simulation_ex( - new_name, "realtime", start_time, duration=86400, pump_control=pump_control - ) - - if not is_project_open(new_name): - open_project(new_name) - - output = Output("./temp/{}.db.out".format(new_name)) - - node_results = ( - output.node_results() - ) # [{'node': str, 'result': [{'pressure': float, 'head': float}]}] - water_plant_output_pressure = [] - reservoir_level = [] - tank_level = [] - for node_result in node_results: - if node_result["node"] == water_plant_output_id: - for result in node_result["result"]: - water_plant_output_pressure.append( - result["pressure"] / 100 - ) # 水厂出水压力(Mpa) - elif node_result["node"] == reservoir_id: - for result in node_result["result"]: - reservoir_level.append(result["head"] - 250.35) # 清水池液位(m) - elif node_result["node"] == tank_id: - for result in node_result["result"]: - tank_level.append(result["pressure"]) # 调节池液位(m) - - simulation_results = { - "water_plant_output_pressure": water_plant_output_pressure, - "reservoir_level": reservoir_level, - "tank_level": tank_level, - } - - if is_project_open(new_name): - close_project(new_name) - delete_project(new_name) - - return json.dumps(simulation_results) - - -############################################################ -# network_update 10 -############################################################ - - -def network_update(file_path: str) -> None: - """ - 更新pg数据库中的inp文件 - :param file_path: inp文件 - :return: - """ - read_inp("szh", file_path) - - csv_path = "./history_pattern_flow.csv" - - # # 检查文件是否存在 - # if os.path.exists(csv_path): - # print(f"history_patterns_flows文件存在，开始处理...") - # - # # 读取 CSV 文件 - # df = pd.read_csv(csv_path) - # - # # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） - # with psycopg.connect("dbname=bb host=127.0.0.1") as conn: - # with conn.cursor() as cur: - # for index, row in df.iterrows(): - # # 直接将数据插入，不进行唯一性检查 - # insert_sql = sql.SQL(""" - # INSERT INTO history_patterns_flows (id, factor, flow) - # VALUES (%s, %s, %s); - # """) - # # 将数据插入数据库 - # cur.execute(insert_sql, (row['id'], row['factor'], row['flow'])) - # conn.commit() - # print("数据成功导入到 'history_patterns_flows' 表格。") - # else: - # print(f"history_patterns_flows文件不存在。") - # 检查文件是否存在 - if os.path.exists(csv_path): - print(f"history_patterns_flows文件存在，开始处理...") - - # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） - with psycopg.connect(f"dbname={project_info.name} host=127.0.0.1") as conn: - with conn.cursor() as cur: - with open(csv_path, newline="", encoding="utf-8-sig") as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - # 直接将数据插入，不进行唯一性检查 - insert_sql = sql.SQL( - """ - INSERT INTO history_patterns_flows (id, factor, flow) - VALUES (%s, %s, %s); - """ - ) - # 将数据插入数据库 - cur.execute(insert_sql, (row["id"], row["factor"], row["flow"])) - conn.commit() - print("数据成功导入到 'history_patterns_flows' 表格。") - else: - print(f"history_patterns_flows文件不存在。") - - -def submit_scada_info(name: str, coord_id: str) -> None: - """ - 将scada信息表导入pg数据库 - :param name: 项目名称（数据库名称） - :param coord_id: 坐标系的id，如4326，根据原始坐标信息输入 - :return: - """ - scada_info_path = "./scada_info.csv" - # 检查文件是否存在 - if os.path.exists(scada_info_path): - print(f"scada_info文件存在，开始处理...") - - # 自动检测文件编码 - with open(scada_info_path, "rb") as file: - raw_data = file.read() - detected = chardet.detect(raw_data) - file_encoding = detected["encoding"] - print(f"检测到的文件编码：{file_encoding}") - try: - # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) - - # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 检查 scada_info 表是否为空 - cur.execute("SELECT COUNT(*) FROM scada_info;") - count = cur.fetchone()[0] - - if count > 0: - print("scada_info表中已有数据，正在清空记录...") - cur.execute("DELETE FROM scada_info;") - print("表记录已清空。") - - with open( - scada_info_path, newline="", encoding=file_encoding - ) as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - # 将CSV单元格值为空的字段转换为 None - cleaned_row = { - key: (value if value.strip() else None) - for key, value in row.items() - } - - # 处理 associated_source_outflow_id 列动态变化 - associated_columns = [ - f"associated_source_outflow_id{i}" for i in range(1, 21) - ] - associated_values = [ - ( - cleaned_row.get(col).strip() - if cleaned_row.get(col) - and cleaned_row.get(col).strip() - else None - ) - for col in associated_columns - ] - - # 将 X_coor 和 Y_coor 转换为 geometry 类型 - x_coor = ( - float(cleaned_row["X_coor"]) - if cleaned_row["X_coor"] - else None - ) - y_coor = ( - float(cleaned_row["Y_coor"]) - if cleaned_row["Y_coor"] - else None - ) - coord = ( - f"SRID={coord_id};POINT({x_coor} {y_coor})" - if x_coor and y_coor - else None - ) - - # 准备插入 SQL 语句 - insert_sql = sql.SQL( - """ - INSERT INTO scada_info ( - id, type, associated_element_id, associated_pattern, - associated_pipe_flow_id, {associated_columns}, - API_query_id, transmission_mode, transmission_frequency, - reliability, X_coor, Y_coor, coord - ) - VALUES ( - %s, %s, %s, %s, %s, {associated_placeholders}, - %s, %s, %s, %s, %s, %s, %s - ); - """ - ).format( - associated_columns=sql.SQL(", ").join( - sql.Identifier(col) for col in associated_columns - ), - associated_placeholders=sql.SQL(", ").join( - sql.Placeholder() for _ in associated_columns - ), - ) - # 将数据插入数据库 - cur.execute( - insert_sql, - ( - cleaned_row["id"], - cleaned_row["type"], - cleaned_row["associated_element_id"], - cleaned_row.get("associated_pattern"), - cleaned_row.get("associated_pipe_flow_id"), - *associated_values, - cleaned_row.get("API_query_id"), - cleaned_row["transmission_mode"], - cleaned_row["transmission_frequency"], - cleaned_row["reliability"], - x_coor, - y_coor, - coord, - ), - ) - conn.commit() - print("数据成功导入到 'scada_info' 表格。") - except Exception as e: - print(f"导入时出错：{e}") - else: - print(f"scada_info文件不存在。") - - -# 2025/03/23 -def scheme_name_exists(name: str, scheme_name: str) -> bool: - """ - 判断传入的 scheme_name 是否已存在于 scheme_list 表中,用于输入框判断 - :param name: 数据库名称 - :param scheme_name: 需要判断的方案名称 - :return: 如果存在返回 True，否则返回 False - """ - try: - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT COUNT(*) FROM scheme_list WHERE scheme_name = %s", - (scheme_name,), - ) - result = cur.fetchone() - if result is not None and result[0] > 0: - return True - else: - return False - except Exception as e: - print(f"查询 scheme_name 时出错：{e}") - return False - - -# 2025/03/23 -def store_scheme_info( - name: str, - scheme_name: str, - scheme_type: str, - username: str, - scheme_start_time: str, - scheme_detail: dict, -): - """ - 将一条方案记录插入 scheme_list 表中 - :param name: 数据库名称 - :param scheme_name: 方案名称 - :param scheme_type: 方案类型 - :param username: 用户名（需在 users 表中已存在） - :param scheme_start_time: 方案起始时间（字符串） - :param scheme_detail: 方案详情（字典，会转换为 JSON） - :return: - """ - try: - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - sql = """ - INSERT INTO scheme_list (scheme_name, scheme_type, username, scheme_start_time, scheme_detail) - VALUES (%s, %s, %s, %s, %s) - """ - # 将字典转换为 JSON 字符串 - scheme_detail_json = json.dumps(scheme_detail) - cur.execute( - sql, - ( - scheme_name, - scheme_type, - username, - scheme_start_time, - scheme_detail_json, - ), - ) - conn.commit() - print("方案信息存储成功！") - except Exception as e: - print(f"存储方案信息时出错：{e}") - - -# 2025/03/23 -def delete_scheme_info(name: str, scheme_name: str) -> None: - """ - 从 scheme_list 表中删除指定的方案 - :param name: 数据库名称 - :param scheme_name: 要删除的方案名称 - """ - try: - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 使用参数化查询删除方案记录 - cur.execute( - "DELETE FROM scheme_list WHERE scheme_name = %s", (scheme_name,) - ) - conn.commit() - print(f"方案 {scheme_name} 删除成功！") - except Exception as e: - print(f"删除方案时出错：{e}") - - -# 2025/03/23 -def query_scheme_list(name: str) -> list: - """ - 查询pg数据库中的scheme_list，按照 create_time 降序排列，离现在时间最近的记录排在最前面 - :param name: 项目名称（数据库名称） - :return: 返回查询结果的所有行 - """ - try: - # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) - # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 按 create_time 降序排列 - cur.execute("SELECT * FROM scheme_list ORDER BY create_time DESC") - rows = cur.fetchall() - return rows - - except Exception as e: - print(f"查询错误：{e}") - - -# 2025/03/23 -def upload_shp_to_pg(name: str, table_name: str, role: str, shp_file_path: str): - """ - 将 Shapefile 文件上传到 PostgreSQL 数据库 - :param name: 项目名称（数据库名称） - :param table_name: 创建表的名字 - :param role: 数据库角色名，位于c盘user中查看 - :param shp_file_path: shp文件的路径 - :return: - """ - try: - # 动态连接到指定的数据库 - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - # 读取 Shapefile 文件 - gdf = gpd.read_file(shp_file_path) - - # 检查投影坐标系（CRS），并确保是 EPSG:4326 - if gdf.crs.to_string() != "EPSG:4490": - gdf = gdf.to_crs(epsg=4490) - - # 使用 GeoDataFrame 的 .to_postgis 方法将数据写入 PostgreSQL - # 需要在数据库中提前安装 PostGIS 扩展 - engine = create_engine(f"postgresql+psycopg2://{role}:@127.0.0.1/{name}") - gdf.to_postgis( - table_name, engine, if_exists="replace", index=True, index_label="id" - ) - - print( - f"Shapefile 文件成功上传到 PostgreSQL 数据库 '{name}' 的表 '{table_name}'." - ) - - except Exception as e: - print(f"上传 Shapefile 到 PostgreSQL 时出错：{e}") - - -def submit_risk_probability_result(name: str, result_file_path: str) -> None: - """ - 将管网风险评估结果导入pg数据库 - :param name: 项目名称（数据库名称） - :param result_file_path: 结果文件路径 - :return: - """ - # 自动检测文件编码 - # with open({result_file_path}, 'rb') as file: - # raw_data = file.read() - # detected = chardet.detect(raw_data) - # file_encoding = detected['encoding'] - # print(f"检测到的文件编码：{file_encoding}") - - try: - # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) - - # 连接到 PostgreSQL 数据库 - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 检查 scada_info 表是否为空 - cur.execute("SELECT COUNT(*) FROM pipe_risk_probability;") - count = cur.fetchone()[0] - - if count > 0: - print("pipe_risk_probability表中已有数据，正在清空记录...") - cur.execute("DELETE FROM pipe_risk_probability;") - print("表记录已清空。") - - # 读取Excel并转换x/y列为列表 - df = pd.read_excel(result_file_path, sheet_name="Sheet1") - df["x"] = df["x"].apply(ast.literal_eval) - df["y"] = df["y"].apply(ast.literal_eval) - - # 批量插入数据 - for index, row in df.iterrows(): - insert_query = """ - INSERT INTO pipe_risk_probability - (pipeID, pipeage, risk_probability_now, x, y) - VALUES (%s, %s, %s, %s, %s) - """ - cur.execute( - insert_query, - ( - row["pipeID"], - row["pipeage"], - row["risk_probability_now"], - row["x"], # 直接传递列表 - row["y"], # 同上 - ), - ) - - conn.commit() - print("风险评估结果导入成功") - - except Exception as e: - print(f"导入时出错：{e}") - - -def pressure_sensor_placement_sensitivity( - name: str, scheme_name: str, sensor_number: int, min_diameter: int, username: str -) -> None: - """ - 基于改进灵敏度法进行压力监测点优化布置 - :param name: 数据库名称 - :param scheme_name: 监测优化布置方案名称 - :param sensor_number: 传感器数目 - :param min_diameter: 最小管径 - :param username: 用户名 - :return: - """ - sensor_location = sensitivity.get_ID( - name=name, sensor_num=sensor_number, min_diameter=min_diameter - ) - try: - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - sql = """ - INSERT INTO sensor_placement (scheme_name, sensor_number, min_diameter, username, sensor_location) - VALUES (%s, %s, %s, %s, %s) - """ - - cur.execute( - sql, - ( - scheme_name, - sensor_number, - min_diameter, - username, - sensor_location, - ), - ) - conn.commit() - print("方案信息存储成功！") - except Exception as e: - print(f"存储方案信息时出错：{e}") - - -# 2025/08/21 -# 基于kmeans聚类法进行压力监测点优化布置 -def pressure_sensor_placement_kmeans( - name: str, scheme_name: str, sensor_number: int, min_diameter: int, username: str -) -> None: - """ - 基于聚类法进行压力监测点优化布置 - :param name: 数据库名称（注意，此处数据库名称也是inp文件名称，inp文件与pg库名要一样） - :param scheme_name: 监测优化布置方案名称 - :param sensor_number: 传感器数目 - :param min_diameter: 最小管径 - :param username: 用户名 - :return: - """ - # dump_inp - inp_name = f"./db_inp/{name}.db.inp" - dump_inp(name, inp_name, "2") - sensor_location = kmeans_sensor.kmeans_sensor_placement( - name=name, sensor_num=sensor_number, min_diameter=min_diameter - ) - try: - conn_string = get_pgconn_string(db_name=name) - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - sql = """ - INSERT INTO sensor_placement (scheme_name, sensor_number, min_diameter, username, sensor_location) - VALUES (%s, %s, %s, %s, %s) - """ - - cur.execute( - sql, - ( - scheme_name, - sensor_number, - min_diameter, - username, - sensor_location, - ), - ) - conn.commit() - print("方案信息存储成功！") - except Exception as e: - print(f"存储方案信息时出错：{e}") - - -############################################################ -# 流量监测数据清洗 ***卡尔曼滤波法*** -############################################################ -# 2025/08/21 hxyan - - -def flow_data_clean(input_csv_file: str) -> str: - """ - 读取 input_csv_path 中的每列时间序列，使用一维 Kalman 滤波平滑并用预测值替换基于 3σ 检测出的异常点。 - 保存输出为：_cleaned.xlsx（与输入同目录），并返回输出文件的绝对路径。如有同名文件存在，则覆盖。 - :param: input_csv_file: 输入的 CSV 文件明或路径 - :return: 输出文件的绝对路径 - """ - - # 提供的 input_csv_path 绝对路径，以下为 默认脚本目录下同名 CSV 文件，构建绝对路径，可根据情况修改 - script_dir = os.path.dirname(os.path.abspath(__file__)) - input_csv_path = os.path.join(script_dir, input_csv_file) - - # 检查文件是否存在 - if not os.path.exists(input_csv_path): - raise FileNotFoundError(f"指定的文件不存在: {input_csv_path}") - # 调用 Fdataclean.clean_flow_data_kf 函数进行数据清洗 - out_xlsx_path = flow_data_clean.clean_flow_data_kf(input_csv_path) - print("清洗后的数据已保存到:", out_xlsx_path) - - -############################################################ -# 压力监测数据清洗 ***kmean++法*** -############################################################ -# 2025/08/21 hxyan - - -def pressure_data_clean(input_csv_file: str) -> str: - """ - 读取 input_csv_path 中的每列时间序列，使用Kmean++清洗数据。 - 保存输出为：_cleaned.xlsx（与输入同目录），并返回输出文件的绝对路径。如有同名文件存在，则覆盖。 - 原始数据在 sheet 'raw_pressure_data'，处理后数据在 sheet 'cleaned_pressusre_data'。 - :param input_csv_path: 输入的 CSV 文件路径 - :return: 输出文件的绝对路径 - """ - - # 提供的 input_csv_path 绝对路径，以下为 默认脚本目录下同名 CSV 文件，构建绝对路径，可根据情况修改 - script_dir = os.path.dirname(os.path.abspath(__file__)) - input_csv_path = os.path.join(script_dir, input_csv_file) - - # 检查文件是否存在 - if not os.path.exists(input_csv_path): - raise FileNotFoundError(f"指定的文件不存在: {input_csv_path}") - # 调用 Fdataclean.clean_flow_data_kf 函数进行数据清洗 - out_xlsx_path = pressure_data_clean.clean_pressure_data_km(input_csv_path) - print("清洗后的数据已保存到:", out_xlsx_path) - - -if __name__ == "__main__": - # contaminant_simulation('bb_model','2024-06-24T00:00:00Z','ZBBDTZDP009034',30,1800) - # flushing_analysis('bb_model','2024-04-01T08:00:00Z',{'GSD230719205857733F8F5214FF','GSD230719205857C0AF65B6A170'},'GSD2307192058570DEDF28E4F73',0,duration=900) - # flushing_analysis('bb_model', '2024-08-26T08:00:00Z', ['GSD2307192058572E5C0E14D83E'], [0.5], 'ZBBDTZDP009410', 0, - # duration=1800) - # valve_close_analysis('bb_model','2024-04-01T08:00:00Z',['GSD2307192058576122D929EE99(L)'],duration=1800) - - # burst_analysis('bb','2024-04-01T08:00:00Z','ZBBGXSZW000001',burst_size=200,duration=1800) - # run_simulation('beibeizone','2024-04-01T08:00:00Z') - # str_dump=dump_output('h:\\OneDrive\\tjwaterserver\\temp\\beibeizone.db_no_burst.out') - # with open("out_dump.txt", "w") as f: - # f.write(str_dump) - # str_dump=dump_output('h:\\OneDrive\\tjwaterserver\\temp\\beibeizone.db_busrtID(ZBBGXSZW000001).out') - # with open("burst_out_dump.txt", "w") as f: - # f.write(str_dump) - - # # 更新inp文件，并插入history_patterns_flows - # network_update('fx0217-mass injection.inp') - - # # 更新scada_info文件 - # submit_scada_info(project_info.name, '4490') - - # 示例：scheme_name_exists - # if scheme_name_exists(name='bb', scheme_name='burst_scheme'): - # print(f"方案名已存在，请更改！") - # else: - # print(f"方案名不存在，可以使用。") - - # 示例1：burst_analysis - # burst_analysis(name='bb', modify_pattern_start_time='2025-04-17T00:00:00+08:00', - # burst_ID='GSD230112144241FA18292A84CB', burst_size=400, modify_total_duration=1800, scheme_name='GSD230112144241FA18292A84CB_400') - - # # 示例：query_scheme_list - # result = query_scheme_list(name=project_info.name) - # print(result) - - # # 示例：delete_scheme_info - # delete_scheme_info(name=project_info.name, scheme_name='burst_scheme') - - # # 示例：upload_shp_to_pg - # # 这里的role是 电脑的用户名，服务器上是 Administrator - # upload_shp_to_pg(name=project_info.name, table_name='GIS_pipe', role='Administrator', shp_file_path='市政管线.shp') - - # # 示例：submit_risk_probability_result - # submit_risk_probability_result(name=project_info.name, result_file_path='./北碚市政管线风险评价结果.xlsx') - - # # 示例：pressure_sensor_placement_sensitivity - # pressure_sensor_placement_sensitivity(name=project_info.name, scheme_name='20250517', sensor_number=10, min_diameter=300, username='admin') - - # 示例：pressure_sensor_placement_kmeans - # pressure_sensor_placement_kmeans(name=project_info.name, scheme_name='sensor_1103', sensor_number=35, min_diameter=300, username='admin') - - # 测试：convert emitters coefficients - convert_to_local_unit("szh", 100) diff --git a/scripts/open_szh.py b/scripts/open_szh.py deleted file mode 100644 index b2c20ca..0000000 --- a/scripts/open_szh.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys -from app.services.tjnetwork import open_project - -def main(): - open_project('szh') - -if __name__ == '__main__': - main() diff --git a/scripts/restore_project.py b/scripts/restore_project.py deleted file mode 100644 index 8655496..0000000 --- a/scripts/restore_project.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys -from app.services.tjnetwork import close_project, open_project, restore - -def main(): - if len(sys.argv) != 2: - print("restore_project name") - return - - p = sys.argv[1] - open_project(p) - restore(p) - close_project(p) - -if __name__ == '__main__': - main() diff --git a/scripts/restore_projects.py b/scripts/restore_projects.py deleted file mode 100644 index 8ad5b6a..0000000 --- a/scripts/restore_projects.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.services.tjnetwork import close_project, list_project, open_project, restore - -def main(): - for p in list_project(): - print(f'restore {p}...') - open_project(p) - restore(p) - close_project(p) - -if __name__ == '__main__': - main() diff --git a/scripts/run_simlation.py b/scripts/run_simlation.py deleted file mode 100644 index d97bf66..0000000 --- a/scripts/run_simlation.py +++ /dev/null @@ -1,8 +0,0 @@ -from app.services.tjnetwork import open_project -from get_current_status import * - -def run_simulation(cur_datetime:str=None)->str: - - open_project('beibei_skeleton') - - return \ No newline at end of file diff --git a/tests/api/test_meta_endpoints.py b/tests/api/test_meta_endpoints.py index e8ddd3a..33d65bf 100644 --- a/tests/api/test_meta_endpoints.py +++ b/tests/api/test_meta_endpoints.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from uuid import uuid4 from fastapi.testclient import TestClient -from sqlalchemy.exc import SQLAlchemyError +import psycopg from tests.conftest import build_test_app, install_stub, load_module_from_path @@ -15,7 +15,7 @@ def _load_meta_module(monkeypatch): { "ProjectContext": object, "get_project_context": lambda: None, - "get_project_pg_session": lambda: None, + "get_project_pg_connection": lambda: None, "get_project_timescale_connection": lambda: None, "get_metadata_repository": lambda: None, }, @@ -68,16 +68,18 @@ def test_meta_project_returns_map_extent(monkeypatch): def test_meta_db_health_returns_503_for_postgres_errors(monkeypatch): module = _load_meta_module(monkeypatch) - class BrokenSession: - async def execute(self, _query): - raise SQLAlchemyError("pg unavailable") + class BrokenConnection: + def cursor(self): + raise psycopg.OperationalError("pg unavailable") class DummyTimescaleConnection: def cursor(self): raise AssertionError("timescale should not be queried after postgres failure") app = build_test_app(module.router, "/api/v1") - app.dependency_overrides[module.get_project_pg_session] = lambda: BrokenSession() + app.dependency_overrides[module.get_project_pg_connection] = ( + lambda: BrokenConnection() + ) app.dependency_overrides[module.get_project_timescale_connection] = lambda: DummyTimescaleConnection() client = TestClient(app) diff --git a/tests/api/test_model_import_endpoints.py b/tests/api/test_model_import_endpoints.py index 65c73a9..ae42670 100644 --- a/tests/api/test_model_import_endpoints.py +++ b/tests/api/test_model_import_endpoints.py @@ -1,7 +1,9 @@ +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock from uuid import uuid4 +import pytest from fastapi import HTTPException from fastapi.testclient import TestClient @@ -12,6 +14,7 @@ from app.auth.metadata_dependencies import ( ) from app.infra.db.metadb.repositories.metadata_repository import ProjectDbRouting from app.infra.db.project_routing import get_project_pgconn_string +from app.native.wndb.core.database import MaterializedViewRefreshAfterCommitError from tests.conftest import build_test_app @@ -144,3 +147,47 @@ def test_model_update_uses_project_business_routing(monkeypatch): "dsn": "postgresql://user:password@biz.example/routed_business", } repo.get_project_db_routing.assert_awaited_once_with(project_id, "biz_data") + + +def test_gb18030_upload_is_normalized_to_utf8() -> None: + content = "[TITLE]\n天津供水\n[JUNCTIONS]\n".encode("gb18030") + + class FakeUpload: + filename = "model.inp" + + async def read(self, _limit: int) -> bytes: + return content + + normalized, filename = asyncio.run(model_import._read_upload(FakeUpload())) + + assert filename == "model.inp" + assert normalized.decode("utf-8") == "[TITLE]\n天津供水\n[JUNCTIONS]\n" + + +def test_model_update_runs_blocking_import_in_threadpool(monkeypatch) -> None: + calls: list[tuple[object, tuple[object, ...]]] = [] + + async def fake_threadpool(function, *args): + calls.append((function, args)) + + monkeypatch.setattr(model_import, "run_in_threadpool", fake_threadpool) + + asyncio.run(model_import._update_from_inp(b"[TITLE]\n", "demo")) + + assert calls == [(model_import._update_from_inp_sync, (b"[TITLE]\n", "demo"))] + + +def test_committed_refresh_failure_is_not_wrapped_as_retryable_500( + monkeypatch, +) -> None: + error = MaterializedViewRefreshAfterCommitError("demo") + + async def fail_update(_content: bytes, _project_code: str) -> None: + raise error + + monkeypatch.setattr(model_import, "_update_from_inp", fail_update) + + with pytest.raises(MaterializedViewRefreshAfterCommitError) as exc_info: + asyncio.run(model_import._apply_model_update(b"[TITLE]\n", "demo")) + + assert exc_info.value is error diff --git a/tests/api/test_openapi_contract.py b/tests/api/test_openapi_contract.py index 848b291..eb8519e 100644 --- a/tests/api/test_openapi_contract.py +++ b/tests/api/test_openapi_contract.py @@ -1,6 +1,5 @@ from __future__ import annotations -import inspect from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 @@ -12,6 +11,7 @@ from fastapi.testclient import TestClient from app.api.v1.endpoints import simulation as simulation_endpoint from app.api.pagination import PaginatedList +from app.api.problem_details import install_problem_details_handlers from app.api.v1.rest_router import api_router, build_rest_router from app.api.v1.router import api_router as source_api_router from app.auth.project_dependencies import ( @@ -207,19 +207,51 @@ def test_valve_isolation_route_uses_the_isolation_handler() -> None: assert route.name == "valve_isolation_endpoint" -def test_open_project_route_requires_business_and_timescale_routing() -> None: - route = next( - route +def test_legacy_project_pool_lifecycle_routes_are_not_published() -> None: + operations = { + (method, route.path) for route in api_router.routes if isinstance(route, APIRoute) - and route.path == "/projects/current" - and route.methods == {"POST"} - ) - routing_parameter = inspect.signature(route.endpoint).parameters[ - "_rest_project_routing" - ] + for method in route.methods or set() + } - assert routing_parameter.default.dependency is get_project_simulation_routing + assert ("POST", "/projects/current") not in operations + assert ("DELETE", "/projects/current") not in operations + assert ("GET", "/projects/current/status") not in operations + + +def test_legacy_server_filesystem_inp_routes_are_not_published() -> None: + operations = { + (method, route.path) + for route in source_api_router.routes + if isinstance(route, APIRoute) + for method in route.methods or set() + } + + assert operations.isdisjoint( + { + ("POST", "/projects/current/imports"), + ("POST", "/projects/current/exports/inp"), + ("GET", "/projects/current/files/inp"), + } + ) + + +def test_committed_write_refresh_failure_has_explicit_http_contract() -> None: + from app.native.wndb.core.database import MaterializedViewRefreshAfterCommitError + + app = FastAPI() + install_problem_details_handlers(app) + + @app.get("/probe") + def probe(): + raise MaterializedViewRefreshAfterCommitError("project_a") + + response = TestClient(app, raise_server_exceptions=False).get("/probe") + + assert response.status_code == 503 + assert response.headers["X-TJWater-Changes-Committed"] == "true" + assert response.json()["code"] == "materialized_view_refresh_failed_after_commit" def test_valve_isolation_runtime_accepts_frontend_query(monkeypatch) -> None: diff --git a/tests/api/test_project_endpoints.py b/tests/api/test_project_endpoints.py index b4d1834..5ba8b5d 100644 --- a/tests/api/test_project_endpoints.py +++ b/tests/api/test_project_endpoints.py @@ -18,6 +18,9 @@ class DummyChangeSet: def _load_project_module(monkeypatch): + class DummyProjectContext: + project_code = "demo" + install_stub(monkeypatch, "app.services", package=True) install_stub( monkeypatch, @@ -33,14 +36,9 @@ def _load_project_module(monkeypatch): "have_project": lambda network: network == "demo", "create_project": lambda network: None, "delete_project": lambda network: None, - "is_project_open": lambda network: False, - "open_project": lambda network: None, - "close_project": lambda network: None, "copy_project": lambda source, target: None, "import_inp": lambda network, cs: {"ok": True}, "export_inp": lambda network, version: DummyChangeSet({"kind": "export"}), - "read_inp": lambda network, inp: True, - "dump_inp": lambda network, inp: True, "get_all_vertices": lambda network: [], "get_all_scada_info": lambda network: [], "get_all_district_metering_areas": lambda network: [], @@ -52,7 +50,12 @@ def _load_project_module(monkeypatch): install_stub( monkeypatch, "app.auth.project_dependencies", - {"get_metadata_repository": lambda: None}, + { + "ProjectContext": DummyProjectContext, + "get_metadata_repository": lambda: None, + "get_project_context": lambda: DummyProjectContext(), + "use_project_business_routing": lambda: None, + }, ) return load_module_from_path( "tests_project_endpoints_module", @@ -98,40 +101,21 @@ def test_project_info_returns_project_workspace(monkeypatch): assert "geoserver" not in payload -def test_open_project_uses_unified_wndb_connection_path(monkeypatch): +def test_legacy_open_project_endpoint_is_removed(monkeypatch): module = _load_project_module(monkeypatch) - called = [] - - monkeypatch.setattr(module, "open_project", lambda network: called.append(network)) - client = TestClient(build_test_app(module.router, "/api/v1")) response = client.post("/api/v1/projects/current", params={"network": "demo"}) - assert response.status_code == 200 - assert response.json() == "demo" - assert called == ["demo"] + assert response.status_code == 405 -def test_project_lock_lifecycle(monkeypatch): +def test_legacy_physical_project_routes_are_removed(monkeypatch): module = _load_project_module(monkeypatch) - module.lockedPrjs.clear() client = TestClient(build_test_app(module.router, "/api/v1")) - first_lock = client.post("/api/v1/projects/current/lock", params={"network": "demo"}) - second_lock = client.post("/api/v1/projects/current/lock", params={"network": "demo"}) - locked_by_me = client.get( - "/api/v1/projects/current/lock/ownership", - params={"network": "demo"}, - ) - unlock = client.delete( - "/api/v1/projects/current/lock", - params={"network": "demo"}, - ) - locked = client.get("/api/v1/projects/current/lock", params={"network": "demo"}) - - assert first_lock.json() == 0 - assert second_lock.json() == 1 - assert locked_by_me.json() is True - assert unlock.json() is True - assert locked.json() is False + assert client.get("/api/v1/project-codes").status_code == 404 + assert client.get("/api/v1/projects/existence").status_code == 404 + assert client.post("/api/v1/project-copies").status_code == 404 + assert client.get("/api/v1/projects/current/lock").status_code == 404 + assert client.post("/api/v1/project-conversions").status_code == 404 diff --git a/tests/api/test_wndb_endpoint_threading.py b/tests/api/test_wndb_endpoint_threading.py new file mode 100644 index 0000000..7cfa11c --- /dev/null +++ b/tests/api/test_wndb_endpoint_threading.py @@ -0,0 +1,68 @@ +from threading import get_ident +from uuid import uuid4 + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.v1.endpoints.components import curves +from app.api.v1.rest_router import api_router +from app.auth.project_dependencies import ( + ProjectContext, + get_project_business_routing, + get_project_context, +) +from app.infra.db.project_routing import ActiveProjectRouting, get_active_project_routing + + +def test_sync_wndb_endpoint_parses_body_and_runs_in_worker_thread(monkeypatch) -> None: + call: dict[str, object] = {} + project_id = uuid4() + user_id = uuid4() + context = ProjectContext( + project_id=project_id, + project_code="project_a", + user_id=user_id, + project_role="member", + ) + routing = ActiveProjectRouting( + project_code="project_a", + business_dsn="postgresql://user:password@db.example/project_a", + ) + + async def override_context() -> ProjectContext: + call["event_loop_thread"] = get_ident() + return context + + async def override_routing() -> ActiveProjectRouting: + return routing + + def fake_add_curve(network, changes): + call["worker_thread"] = get_ident() + call["network"] = network + call["operations"] = changes.operations + call["routing"] = get_active_project_routing() + return {"ok": True} + + monkeypatch.setattr(curves, "add_curve", fake_add_curve) + app = FastAPI() + app.include_router(api_router) + app.dependency_overrides[get_project_context] = override_context + app.dependency_overrides[get_project_business_routing] = override_routing + + with TestClient(app) as client: + response = client.post( + "/curves", + params={"curve": "C-1"}, + json={"points": [[0, 10], [1, 20]]}, + ) + + assert response.status_code == 201 + assert response.json() == {"ok": True} + assert call == { + "event_loop_thread": call["event_loop_thread"], + "worker_thread": call["worker_thread"], + "network": "project_a", + "operations": [{"id": "C-1", "points": [[0, 10], [1, 20]]}], + "routing": routing, + } + assert call["worker_thread"] != call["event_loop_thread"] diff --git a/tests/integration/test_database_pooling_live.py b/tests/integration/test_database_pooling_live.py index e5f3818..8a1534b 100644 --- a/tests/integration/test_database_pooling_live.py +++ b/tests/integration/test_database_pooling_live.py @@ -1,12 +1,17 @@ +import asyncio import os from concurrent.futures import ThreadPoolExecutor from uuid import uuid4 import pytest +from psycopg import connect +from app.core.config import get_pgconn_string +from app.infra.db.dynamic_manager import ProjectConnectionManager from app.infra.db.timescaledb.sync_pool import timescale_connection from app.native.wndb.commands.api import delete_pattern_cascade from app.native.wndb.core.connection import project_connection, project_transaction +from app.native.wndb.core.projects import have_project, temporary_project_database from app.native.wndb.core.database import ChangeSet, g_delete_prefix, write from app.native.wndb.model import demands, junctions, patterns from app.services.scheme_management import create_analysis_run, update_analysis_run @@ -17,7 +22,7 @@ pytestmark = pytest.mark.skipif( reason="set RUN_DB_INTEGRATION=1 to test configured PostgreSQL databases", ) -PROJECT = os.getenv("DB_INTEGRATION_PROJECT", "tjwater_next") +PROJECT = os.getenv("DB_INTEGRATION_PROJECT", "tjwater_v2") def _read_business_database(_: int) -> str: @@ -46,6 +51,60 @@ def test_timeseries_pool_handles_concurrent_borrows() -> None: assert names == [PROJECT] * 64 +def test_temporary_project_clone_copies_model_scada_and_views() -> None: + def counts(project: str) -> dict: + with project_connection(project) as conn, conn.cursor() as cur: + cur.execute( + """ + select + (select count(*) from network.nodes) as nodes, + (select count(*) from network.links) as links, + (select count(*) from asset.scada_devices) as scada, + (select count(*) from gis.junctions) as mv_junctions, + (select count(*) from gis.pipes) as mv_pipes + """ + ) + return dict(cur.fetchone()) + + source_counts = counts(PROJECT) + temporary = None + with temporary_project_database(PROJECT, "clone_validation") as temporary: + assert counts(temporary) == source_counts + + assert temporary is not None + assert have_project(temporary) is False + + +def test_dynamic_pool_replaces_terminated_idle_connection() -> None: + async def exercise_pool() -> None: + manager = ProjectConnectionManager() + dsn = get_pgconn_string(db_name=PROJECT) + try: + project_id = uuid4() + async with manager.pg_connection( + project_id, "biz_data", dsn, 1, 1 + ) as conn: + async with conn.cursor() as cur: + await cur.execute("select pg_backend_pid()") + backend_pid = int((await cur.fetchone())["pg_backend_pid"]) + + with connect(dsn, autocommit=True) as admin_conn: + with admin_conn.cursor() as cur: + cur.execute("select pg_terminate_backend(%s)", (backend_pid,)) + assert cur.fetchone()[0] is True + + async with manager.pg_connection( + project_id, "biz_data", dsn, 1, 1 + ) as conn: + async with conn.cursor() as cur: + await cur.execute("select current_database()") + assert (await cur.fetchone())["current_database"] == PROJECT + finally: + await manager.close_all() + + asyncio.run(exercise_pool()) + + def test_nested_wndb_writes_roll_back_as_one_transaction() -> None: with pytest.raises(RuntimeError, match="force rollback"): with project_transaction(PROJECT) as conn: @@ -186,10 +245,55 @@ def test_wndb_ordered_detail_tables_use_parent_scoped_primary_keys() -> None: assert actual == expected +def test_gis_unified_views_cover_all_materialized_network_layers() -> None: + with project_connection(PROJECT) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT + (SELECT COUNT(*) FROM gis.network_nodes) AS nodes, + (SELECT COUNT(*) FROM gis.junctions) + + (SELECT COUNT(*) FROM gis.reservoirs) + + (SELECT COUNT(*) FROM gis.tanks) AS source_nodes, + (SELECT COUNT(*) FROM gis.network_links) AS links, + (SELECT COUNT(*) FROM gis.pipes) + + (SELECT COUNT(*) FROM gis.pumps) + + (SELECT COUNT(*) FROM gis.valves) AS source_links + """ + ) + counts = cur.fetchone() + cur.execute( + """ + SELECT obj_description('gis.network_nodes'::regclass) AS node_comment, + obj_description('gis.network_links'::regclass) AS link_comment + """ + ) + comments = cur.fetchone() + cur.execute( + """ + SELECT COUNT(*) AS undocumented_columns + FROM pg_attribute + WHERE attrelid = ANY( + ARRAY['gis.network_nodes'::regclass, 'gis.network_links'::regclass] + ) + AND attnum > 0 + AND NOT attisdropped + AND col_description(attrelid, attnum) IS NULL + """ + ) + undocumented_columns = cur.fetchone()["undocumented_columns"] + + assert counts["nodes"] == counts["source_nodes"] + assert counts["links"] == counts["source_links"] + assert comments["node_comment"] + assert comments["link_comment"] + assert undocumented_columns == 0 + + def test_wndb_pattern_cascade_unsets_dependent_demand_atomically() -> None: suffix = uuid4() junction_id = f"integration-junction-{suffix}" pattern_id = f"integration-cascade-{suffix}" + retained_pattern_id = f"integration-retained-{suffix}" with pytest.raises(RuntimeError, match="force rollback"): with project_transaction(PROJECT): @@ -203,6 +307,10 @@ def test_wndb_pattern_cascade_unsets_dependent_demand_atomically() -> None: PROJECT, ChangeSet({"id": pattern_id, "factors": [1.0]}), ) + patterns.add_pattern( + PROJECT, + ChangeSet({"id": retained_pattern_id, "factors": [1.0]}), + ) demands.set_demand( PROJECT, ChangeSet( @@ -213,7 +321,12 @@ def test_wndb_pattern_cascade_unsets_dependent_demand_atomically() -> None: "demand": 1.0, "pattern": pattern_id, "category": "integration", - } + }, + { + "demand": 2.0, + "pattern": retained_pattern_id, + "category": "retained", + }, ], } ), @@ -226,7 +339,12 @@ def test_wndb_pattern_cascade_unsets_dependent_demand_atomically() -> None: assert patterns.get_pattern(PROJECT, pattern_id) == {} assert demands.get_demand(PROJECT, junction_id)["demands"] == [ - {"demand": 1.0, "pattern": None, "category": "integration"} + {"demand": 1.0, "pattern": None, "category": "integration"}, + { + "demand": 2.0, + "pattern": retained_pattern_id, + "category": "retained", + }, ] assert result.operations[-1] == { "operation": "delete", diff --git a/tests/unit/test_age_analysis.py b/tests/unit/test_age_analysis.py index 240b112..6bbcd8a 100644 --- a/tests/unit/test_age_analysis.py +++ b/tests/unit/test_age_analysis.py @@ -1,4 +1,5 @@ import json +from contextlib import contextmanager from tests.conftest import install_stub, load_module_from_path @@ -29,7 +30,6 @@ def _load_scenarios_module(monkeypatch): "SOURCE_TYPE_SETPOINT": "SOURCE_TYPE_SETPOINT", "add_pattern": lambda *args, **kwargs: None, "add_source": lambda *args, **kwargs: None, - "close_project": lambda *args, **kwargs: None, "copy_project": lambda *args, **kwargs: None, "delete_project": lambda *args, **kwargs: None, "get_demand": lambda *args, **kwargs: None, @@ -42,8 +42,6 @@ def _load_scenarios_module(monkeypatch): "get_time": lambda *args, **kwargs: None, "have_project": lambda *args, **kwargs: False, "is_junction": lambda *args, **kwargs: False, - "is_project_open": lambda *args, **kwargs: False, - "open_project": lambda *args, **kwargs: None, "set_demand": lambda *args, **kwargs: None, "set_emitter": lambda *args, **kwargs: None, "set_option": lambda *args, **kwargs: None, @@ -61,12 +59,11 @@ def test_age_analysis_passes_duration_by_keyword(monkeypatch): module = _load_scenarios_module(monkeypatch) captured = {} - monkeypatch.setattr(module, "copy_project", lambda *args, **kwargs: None) - monkeypatch.setattr(module, "open_project", lambda *args, **kwargs: None) - monkeypatch.setattr(module, "close_project", lambda *args, **kwargs: None) - monkeypatch.setattr(module, "delete_project", lambda *args, **kwargs: None) - monkeypatch.setattr(module, "have_project", lambda *args, **kwargs: False) - monkeypatch.setattr(module, "is_project_open", lambda *args, **kwargs: False) + @contextmanager + def fake_temporary_project(project, purpose): + yield f"{purpose}_{project}_run" + + monkeypatch.setattr(module, "temporary_project_database", fake_temporary_project) def fake_run_simulation_ex(*args, **kwargs): captured["args"] = args @@ -78,7 +75,7 @@ def test_age_analysis_passes_duration_by_keyword(monkeypatch): module.age_analysis("demo", "2026-06-03T07:00:00+08:00", 300) assert captured["args"] == ( - "age_Anal_demo", + "age_analysis_demo_run", "realtime", "2026-06-03T07:00:00+08:00", ) @@ -86,3 +83,29 @@ def test_age_analysis_passes_duration_by_keyword(monkeypatch): "duration": 300, "downloading_prohibition": True, } + + +def test_isolated_analysis_cleans_database_after_early_return(monkeypatch): + module = _load_scenarios_module(monkeypatch) + lifecycle: list[tuple[str, str]] = [] + + @contextmanager + def fake_temporary_project(project, purpose): + lifecycle.append(("create", purpose)) + try: + yield "isolated_run" + finally: + lifecycle.append(("delete", purpose)) + + monkeypatch.setattr( + module, "temporary_project_database", fake_temporary_project + ) + + @module._isolated_analysis("probe") + def return_early(name, *, _temporary_project=None): + assert name == "demo" + assert _temporary_project == "isolated_run" + return "done" + + assert return_early("demo") == "done" + assert lifecycle == [("create", "probe"), ("delete", "probe")] diff --git a/tests/unit/test_analysis_simulation.py b/tests/unit/test_analysis_simulation.py index d9a16ab..dd81a34 100644 --- a/tests/unit/test_analysis_simulation.py +++ b/tests/unit/test_analysis_simulation.py @@ -1,7 +1,11 @@ import inspect import json +from contextlib import contextmanager +from unittest.mock import Mock from uuid import uuid4 +import pytest + def test_run_simulation_exposes_explicit_valve_control(): from app.services import simulation @@ -9,6 +13,34 @@ def test_run_simulation_exposes_explicit_valve_control(): assert "valve_control" in inspect.signature(simulation.run_simulation).parameters +def test_extended_runner_cleans_temporary_database_after_failure(monkeypatch): + from app.algorithms.simulation import runner + + lifecycle: list[tuple[str, str]] = [] + + @contextmanager + def temporary_project(project: str, purpose: str): + lifecycle.append(("create", project)) + try: + yield "isolated_project" + finally: + lifecycle.append(("delete", project)) + + monkeypatch.setattr(runner, "temporary_project_database", temporary_project) + + @runner._clean_extended_simulation + def fail(name, simulation_type, *, _temporary_project=None): + assert name == "demo" + assert simulation_type == "extended" + assert _temporary_project == "isolated_project" + raise RuntimeError("simulation failed") + + with pytest.raises(RuntimeError, match="simulation failed"): + fail("demo", "extended") + + assert lifecycle == [("create", "demo"), ("delete", "demo")] + + def test_apply_valve_control_matches_runner_semantics(monkeypatch): from app.services import simulation @@ -46,12 +78,58 @@ def test_apply_valve_control_matches_runner_semantics(monkeypatch): assert updates["V-k"]["setting"] == 0.1036 * pow(0.5, -3.105) +def test_primary_demand_update_preserves_additional_categories(): + from app.services import simulation + + demand_set = { + "junction": "J1", + "demands": [ + {"demand": 1.0, "pattern": "P1", "category": "domestic"}, + {"demand": 2.0, "pattern": "P2", "category": "industrial"}, + ], + } + + simulation._primary_demand(demand_set)["demand"] = 3.0 + + assert demand_set["demands"] == [ + {"demand": 3.0, "pattern": "P1", "category": "domestic"}, + {"demand": 2.0, "pattern": "P2", "category": "industrial"}, + ] + assert simulation._primary_demand_pattern(demand_set) == "P1" + + +def test_primary_demand_is_created_for_empty_junction(): + from app.services import simulation + + demand_set = {"junction": "J1", "demands": []} + + primary = simulation._primary_demand(demand_set) + + assert primary == {"demand": 0.0, "pattern": None, "category": None} + with pytest.raises(ValueError, match="has no demand pattern"): + simulation._primary_demand_pattern(demand_set) + + def test_extended_simulation_stores_results_by_run_id(monkeypatch): from app.services import simulation run_id = uuid4() storage_calls: list[tuple] = [] - monkeypatch.setattr(simulation, "open_project", lambda name: None) + transaction_calls: list[tuple[str, str]] = [] + + @contextmanager + def project_transaction(name): + transaction_calls.append(("begin", name)) + try: + yield object() + finally: + transaction_calls.append(("end", name)) + + refresh_mock = Mock() + monkeypatch.setattr(simulation, "project_transaction", project_transaction) + monkeypatch.setattr( + simulation, "refresh_materialized_views_after_commit", refresh_mock + ) monkeypatch.setattr( simulation, "get_time", @@ -104,6 +182,8 @@ def test_extended_simulation_stores_results_by_run_id(monkeypatch): assert kwargs["db_name"] == "demo" assert returned_run_id == run_id assert lifecycle_calls[-1][1]["status"] == "completed" + assert transaction_calls == [("begin", "demo"), ("end", "demo")] + refresh_mock.assert_called_once_with("demo") def test_extended_simulation_marks_run_failed_when_result_storage_fails(monkeypatch): @@ -111,7 +191,15 @@ def test_extended_simulation_marks_run_failed_when_result_storage_fails(monkeypa run_id = uuid4() lifecycle_calls: list[tuple] = [] - monkeypatch.setattr(simulation, "open_project", lambda name: None) + + @contextmanager + def project_transaction(_name): + yield object() + + monkeypatch.setattr(simulation, "project_transaction", project_transaction) + monkeypatch.setattr( + simulation, "refresh_materialized_views_after_commit", lambda _name: None + ) monkeypatch.setattr( simulation, "get_time", diff --git a/tests/unit/test_clean_projects_cli.py b/tests/unit/test_clean_projects_cli.py new file mode 100644 index 0000000..964eae3 --- /dev/null +++ b/tests/unit/test_clean_projects_cli.py @@ -0,0 +1,17 @@ +import pytest + +from scripts.clean_projects import parse_args + + +def test_clean_projects_requires_explicit_confirmation() -> None: + with pytest.raises(SystemExit) as exc_info: + parse_args(["temporary_project"]) + + assert exc_info.value.code == 2 + + +def test_clean_projects_accepts_exact_targets_after_confirmation() -> None: + args = parse_args(["--yes", "temp_a", "temp_b"]) + + assert args.yes is True + assert args.projects == ["temp_a", "temp_b"] diff --git a/tests/unit/test_dynamic_manager.py b/tests/unit/test_dynamic_manager.py index e8dd81d..e609495 100644 --- a/tests/unit/test_dynamic_manager.py +++ b/tests/unit/test_dynamic_manager.py @@ -1,12 +1,102 @@ -from app.infra.db.dynamic_manager import ProjectConnectionManager +import asyncio +from contextlib import asynccontextmanager +from uuid import uuid4 + +import pytest + +from app.infra.db import dynamic_manager -def test_normalize_pg_url_preserves_password(): - manager = ProjectConnectionManager() +class FakeAsyncPool: + created: list["FakeAsyncPool"] = [] - url = manager._normalize_pg_url( - "postgresql://tjwater:secret@192.168.1.114:5433/tjwater" - ) + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + self.closed = False + self.created.append(self) - assert url == "postgresql+psycopg://tjwater:secret@192.168.1.114:5433/tjwater" - assert "***" not in url + async def open(self) -> None: + return None + + async def close(self) -> None: + self.closed = True + + @asynccontextmanager + async def connection(self): + yield object() + + +def test_active_project_pool_is_not_evicted(monkeypatch) -> None: + async def exercise() -> None: + manager = dynamic_manager.ProjectConnectionManager() + first_id = uuid4() + second_id = uuid4() + + async with manager.pg_connection(first_id, "biz_data", "dsn-1", 1, 2): + first_pool = manager._pg_raw_cache[ + dynamic_manager.CacheKey(first_id, "biz_data") + ].pool + async with manager.pg_connection( + second_id, "biz_data", "dsn-2", 1, 2 + ): + assert first_pool.closed is False + assert len(manager._pg_raw_cache) == 2 + + assert first_pool.closed is False + assert list(manager._pg_raw_cache) == [ + dynamic_manager.CacheKey(first_id, "biz_data") + ] + + await manager.close_all() + + FakeAsyncPool.created = [] + monkeypatch.setattr(dynamic_manager, "AsyncConnectionPool", FakeAsyncPool) + monkeypatch.setattr(dynamic_manager.settings, "PROJECT_PG_CACHE_SIZE", 1) + asyncio.run(exercise()) + + +def test_active_pool_uses_generation_replacement(monkeypatch) -> None: + async def exercise() -> None: + manager = dynamic_manager.ProjectConnectionManager() + project_id = uuid4() + + async with manager.pg_connection( + project_id, "biz_data", "old-dsn", 1, 2 + ): + old_pool = FakeAsyncPool.created[0] + async with manager.pg_connection( + project_id, "biz_data", "new-dsn", 1, 2 + ): + assert old_pool.closed is False + assert len(manager._retired_pg) == 1 + + assert old_pool.closed is True + assert manager._retired_pg == [] + + await manager.close_all() + + FakeAsyncPool.created = [] + monkeypatch.setattr(dynamic_manager, "AsyncConnectionPool", FakeAsyncPool) + asyncio.run(exercise()) + + +def test_close_project_does_not_interrupt_active_borrow(monkeypatch) -> None: + async def exercise() -> None: + manager = dynamic_manager.ProjectConnectionManager() + project_id = uuid4() + key = dynamic_manager.CacheKey(project_id, "iot_data") + + async with manager.timescale_connection( + project_id, "iot_data", "ts-dsn", 1, 2 + ): + pool = manager._ts_cache[key].pool + assert await manager.close_project(project_id) is False + assert pool.closed is False + + assert await manager.close_project(project_id) is True + assert pool.closed is True + assert key not in manager._ts_cache + + FakeAsyncPool.created = [] + monkeypatch.setattr(dynamic_manager, "AsyncConnectionPool", FakeAsyncPool) + asyncio.run(exercise()) diff --git a/tests/unit/test_pool_connection_checks.py b/tests/unit/test_pool_connection_checks.py new file mode 100644 index 0000000..caf4833 --- /dev/null +++ b/tests/unit/test_pool_connection_checks.py @@ -0,0 +1,104 @@ +import asyncio +from collections import OrderedDict +from contextlib import asynccontextmanager +from uuid import uuid4 + +from app.infra.db import dynamic_manager +from app.infra.db.timescaledb import sync_pool +from app.native.wndb.core import connection + + +class RecordingAsyncPool: + created: list[dict] = [] + + @staticmethod + async def check_connection(_conn) -> None: + return None + + def __init__(self, **kwargs) -> None: + self.created.append(kwargs) + self.closed = False + + async def open(self) -> None: + return None + + async def close(self) -> None: + self.closed = True + + @asynccontextmanager + async def connection(self): + yield object() + + +class RecordingPool: + created: list[dict] = [] + + @staticmethod + def check_connection(_conn) -> None: + return None + + def __init__(self, **kwargs) -> None: + self.created.append(kwargs) + self.closed = False + + def close(self) -> None: + self.closed = True + + +def test_dynamic_project_pools_check_connections_before_borrow(monkeypatch) -> None: + async def create_pools() -> None: + manager = dynamic_manager.ProjectConnectionManager() + async with manager.pg_connection( + uuid4(), "biz_data", "postgresql://user:password@db.example/biz", 1, 5 + ): + pass + async with manager.timescale_connection( + uuid4(), "iot_data", "postgresql://user:password@db.example/ts", 1, 5 + ): + pass + + RecordingAsyncPool.created = [] + monkeypatch.setattr(dynamic_manager, "AsyncConnectionPool", RecordingAsyncPool) + asyncio.run(create_pools()) + + assert len(RecordingAsyncPool.created) == 2 + assert all( + options["check"] is dynamic_manager._check_async_connection + for options in RecordingAsyncPool.created + ) + + +def test_synchronous_project_pools_check_connections_before_borrow(monkeypatch) -> None: + RecordingPool.created = [] + monkeypatch.setattr(connection, "ConnectionPool", RecordingPool) + monkeypatch.setattr(connection, "_pools", OrderedDict()) + monkeypatch.setattr(connection, "_pool_conninfo", {}) + monkeypatch.setattr(connection, "_pool_borrows", {}) + monkeypatch.setattr(connection, "_admin_pools", OrderedDict()) + monkeypatch.setattr(connection, "_admin_pool_borrows", {}) + monkeypatch.setattr( + connection, + "get_project_pgconn_string", + lambda db_name: f"postgresql://user:password@db.example/{db_name}", + ) + + connection.get_project_pool("tjwater_next") + connection.get_admin_pool() + + monkeypatch.setattr(sync_pool, "ConnectionPool", RecordingPool) + monkeypatch.setattr(sync_pool, "_pools", OrderedDict()) + monkeypatch.setattr(sync_pool, "_pool_conninfo", {}) + monkeypatch.setattr(sync_pool, "_pool_borrows", {}) + monkeypatch.setattr( + sync_pool, + "get_project_timescale_pgconn_string", + lambda db_name: f"postgresql://user:password@db.example/{db_name}", + ) + sync_pool.get_timescale_pool("tjwater_next") + + assert len(RecordingPool.created) == 3 + assert [options["check"] for options in RecordingPool.created] == [ + connection._check_connection, + connection._check_connection, + sync_pool._check_connection, + ] diff --git a/tests/unit/test_project_routing.py b/tests/unit/test_project_routing.py index 4e9636d..11edf0a 100644 --- a/tests/unit/test_project_routing.py +++ b/tests/unit/test_project_routing.py @@ -5,7 +5,9 @@ from app.infra.db.project_routing import ( ActiveProjectRouting, activate_project_routing, get_active_project_routing, + get_project_database_name, get_project_pgconn_string, + get_project_template_database_name, get_project_timescale_pgconn_string, ) @@ -35,9 +37,16 @@ def test_project_database_uses_exact_routing_dsn_for_project_code() -> None: ) -def test_business_template_keeps_server_and_timescale_ignores_legacy_db_name() -> None: +def test_business_template_keeps_server_and_timescale_ignores_legacy_db_name( + monkeypatch, +) -> None: + monkeypatch.setattr( + "app.infra.db.project_routing.settings.WNDB_TEMPLATE_DB_NAME", + "tjwater_v2_template", + ) with activate_project_routing(_routing()): - business = conninfo_to_dict(get_project_pgconn_string("project_a_template")) + template_name = get_project_template_database_name("project_a") + business = conninfo_to_dict(get_project_pgconn_string(template_name)) timescale = conninfo_to_dict( get_project_timescale_pgconn_string("temporary_scheme") ) @@ -45,7 +54,7 @@ def test_business_template_keeps_server_and_timescale_ignores_legacy_db_name() - assert business == { "user": "biz_user", "password": "biz_password", - "dbname": "project_a_template", + "dbname": "tjwater_v2_template", "host": "biz.example", "port": "5432", "sslmode": "require", @@ -60,6 +69,28 @@ def test_business_template_keeps_server_and_timescale_ignores_legacy_db_name() - } +def test_project_code_resolves_to_physical_business_database(monkeypatch) -> None: + monkeypatch.setattr( + "app.infra.db.project_routing.settings.WNDB_TEMPLATE_DB_NAME", + "tjwater_v2_template", + ) + with activate_project_routing(_routing()): + assert get_project_database_name("project_a") == "biz_database" + assert get_project_database_name("temporary_run") == "temporary_run" + assert get_project_template_database_name("project_a") == "tjwater_v2_template" + + +def test_template_falls_back_to_config_outside_project_routing(monkeypatch) -> None: + monkeypatch.setattr( + "app.infra.db.project_routing.settings.WNDB_TEMPLATE_DB_NAME", + "tjwater_v2_template", + ) + + assert get_project_template_database_name("ignored-project-code") == ( + "tjwater_v2_template" + ) + + def test_project_routing_is_nested_and_request_local() -> None: first = _routing("project_a") second = _routing("project_b") diff --git a/tests/unit/test_project_scada_metadata.py b/tests/unit/test_project_scada_metadata.py index e1eecf5..4c06041 100644 --- a/tests/unit/test_project_scada_metadata.py +++ b/tests/unit/test_project_scada_metadata.py @@ -33,12 +33,19 @@ def _patch_project_scadas(monkeypatch): def test_realtime_scada_simulation_uses_current_project_metadata(monkeypatch): _patch_project_scadas(monkeypatch) - query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}]) + query_mock = AsyncMock( + return_value={"J1": [{"time": START_TIME, "value": 26.5}]} + ) monkeypatch.setattr( composite_queries.RealtimeRepository, - "get_node_field_by_time_range", + "get_node_fields_by_ids_time_range", query_mock, ) + monkeypatch.setattr( + composite_queries.RealtimeRepository, + "get_link_fields_by_ids_time_range", + AsyncMock(return_value={}), + ) result = asyncio.run( composite_queries.CompositeQueries.get_scada_associated_realtime_simulation_data( @@ -55,17 +62,22 @@ def test_realtime_scada_simulation_uses_current_project_metadata(monkeypatch): assert query_mock.await_args.args[1:] == ( START_TIME, END_TIME, - "J1", + ["J1"], "pressure", ) def test_analysis_scada_simulation_uses_current_project_metadata(monkeypatch): _patch_project_scadas(monkeypatch) - query_mock = AsyncMock(return_value=[{"time": START_TIME, "value": 26.5}]) + async def query_series(_conn, _run_id, element_type, element_ids, *_args): + if element_type == "node": + return {"J1": [{"time": START_TIME, "value": 26.5}]} + return {} + + query_mock = AsyncMock(side_effect=query_series) monkeypatch.setattr( composite_queries.AnalysisResultsRepository, - "get_node_series", + "get_series_by_ids", query_mock, ) @@ -81,7 +93,11 @@ def test_analysis_scada_simulation_uses_current_project_metadata(monkeypatch): ) assert result[PROJECT_SCADA["device_id"]][0]["scada_id"] == PROJECT_SCADA["device_id"] - assert query_mock.await_args.args[2:] == ("J1", START_TIME, END_TIME, "pressure") + assert query_mock.await_count == 2 + node_call = next( + call for call in query_mock.await_args_list if call.args[2] == "node" + ) + assert node_call.args[3:] == (["J1"], START_TIME, END_TIME, "pressure") def test_element_scada_query_uses_current_project_metadata(monkeypatch): diff --git a/tests/unit/test_realtime_repository.py b/tests/unit/test_realtime_repository.py index f692f94..cebacc1 100644 --- a/tests/unit/test_realtime_repository.py +++ b/tests/unit/test_realtime_repository.py @@ -1,5 +1,5 @@ import asyncio -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager from datetime import datetime, timezone from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository @@ -71,12 +71,23 @@ def test_get_nodes_by_time_range_normalizes_inputs_to_utc(): class _SyncTransactionConnection: def __init__(self): self.transactions = 0 + self.calls: list[tuple[str, tuple]] = [] @contextmanager def transaction(self): self.transactions += 1 yield + @contextmanager + def cursor(self): + connection = self + + class Cursor: + def execute(self, query, params): + connection.calls.append((query, params)) + + yield Cursor() + def test_realtime_node_and_link_replacement_share_outer_transaction(monkeypatch): conn = _SyncTransactionConnection() @@ -101,3 +112,65 @@ def test_realtime_node_and_link_replacement_share_outer_transaction(monkeypatch) assert conn.transactions == 1 assert calls == ["nodes", "links"] + assert [query for query, _params in conn.calls] == [ + "SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))", + "DELETE FROM realtime.node_results WHERE time = %s", + "DELETE FROM realtime.link_results WHERE time = %s", + ] + + +def test_realtime_batch_rejects_multiple_timestamps_before_writing(): + data = [ + {"time": "2026-06-01T00:00:00Z", "id": "N1"}, + {"time": "2026-06-01T00:15:00Z", "id": "N2"}, + ] + + try: + RealtimeRepository.insert_nodes_batch_sync(object(), data) + except ValueError as exc: + assert str(exc) == "Realtime batch must contain exactly one timestamp" + else: + raise AssertionError("multiple realtime timestamps were accepted") + + +class _AsyncEmptySnapshotConnection: + def __init__(self): + self.calls: list[tuple[str, tuple]] = [] + + @asynccontextmanager + async def transaction(self): + yield + + def cursor(self): + connection = self + + class Cursor: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def execute(self, query, params): + connection.calls.append((query, params)) + + return Cursor() + + +def test_empty_realtime_side_is_deleted_as_part_of_snapshot_replacement(): + conn = _AsyncEmptySnapshotConnection() + + asyncio.run( + RealtimeRepository.store_realtime_simulation_result( + conn, + node_result_list=[], + link_result_list=[], + result_start_time="2026-06-01T00:00:00Z", + ) + ) + + assert [query for query, _params in conn.calls] == [ + "SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))", + "DELETE FROM realtime.node_results WHERE time = %s", + "DELETE FROM realtime.link_results WHERE time = %s", + ] diff --git a/tests/unit/test_scada_cleaning.py b/tests/unit/test_scada_cleaning.py index 10268fa..a830113 100644 --- a/tests/unit/test_scada_cleaning.py +++ b/tests/unit/test_scada_cleaning.py @@ -32,9 +32,10 @@ def test_clean_scada_uses_current_project_metadata(monkeypatch): ), ) update_mock = AsyncMock() + update_mock.return_value = 1 monkeypatch.setattr( composite_queries.ScadaRepository, - "update_scada_field", + "update_scada_field_batch", update_mock, ) monkeypatch.setattr( @@ -111,7 +112,7 @@ def test_clean_scada_rejects_zero_database_updates(monkeypatch): update_mock = AsyncMock() monkeypatch.setattr( composite_queries.ScadaRepository, - "update_scada_field", + "update_scada_field_batch", update_mock, ) monkeypatch.setattr( @@ -157,7 +158,7 @@ def test_clean_scada_propagates_write_failures(monkeypatch): ) monkeypatch.setattr( composite_queries.ScadaRepository, - "update_scada_field", + "update_scada_field_batch", AsyncMock(side_effect=RuntimeError("database write failed")), ) monkeypatch.setattr( diff --git a/tests/unit/test_scada_repository.py b/tests/unit/test_scada_repository.py index 36f9077..34694b3 100644 --- a/tests/unit/test_scada_repository.py +++ b/tests/unit/test_scada_repository.py @@ -86,3 +86,31 @@ def test_update_scada_field_skips_insert_when_update_succeeds(): assert len(conn.cursor_instance.calls) == 1 assert "UPDATE scada.measurements SET" in conn.cursor_instance.calls[0][0] + + +def test_update_scada_field_batch_uses_one_set_based_statement(): + ScadaRepository = _load_scada_repository() + conn = _FakeConnection(initial_rowcount=2) + first_time = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc) + second_time = datetime(2026, 1, 1, 0, 5, tzinfo=timezone.utc) + + updated = asyncio.run( + ScadaRepository.update_scada_field_batch( + conn, + [ + (first_time, "170490", 26.5), + (second_time, "170491", 27.0), + ], + "cleaned_value", + ) + ) + + assert updated == 2 + assert len(conn.cursor_instance.calls) == 1 + query, params = conn.cursor_instance.calls[0] + assert "unnest" in query.lower() + assert params == ( + [first_time, second_time], + ["170490", "170491"], + [26.5, 27.0], + ) diff --git a/tests/unit/test_wndb_batch_transactions.py b/tests/unit/test_wndb_batch_transactions.py index b192276..0aa66fd 100644 --- a/tests/unit/test_wndb_batch_transactions.py +++ b/tests/unit/test_wndb_batch_transactions.py @@ -3,7 +3,9 @@ from contextlib import contextmanager import pytest from app.native.wndb.commands import executor +from app.native.wndb.core import database from app.native.wndb.core.database import ChangeSet +from app.native.wndb.model import junctions, pipes, pumps, reservoirs, tanks, valves def test_batch_commits_before_materialized_view_refresh(monkeypatch) -> None: @@ -15,7 +17,7 @@ def test_batch_commits_before_materialized_view_refresh(monkeypatch) -> None: yield object() events.append("transaction-exit") - monkeypatch.setattr(executor, "project_transaction", fake_transaction) + monkeypatch.setattr(executor, "model_mutation_transaction", fake_transaction) monkeypatch.setattr( executor, "expand_command", @@ -28,7 +30,7 @@ def test_batch_commits_before_materialized_view_refresh(monkeypatch) -> None: ) monkeypatch.setattr( executor, - "refresh_materialized_views", + "refresh_materialized_views_after_commit", lambda _name: events.append("refresh"), ) @@ -56,7 +58,7 @@ def test_failed_batch_does_not_refresh_materialized_views(monkeypatch) -> None: finally: events.append("transaction-exit") - monkeypatch.setattr(executor, "project_transaction", fake_transaction) + monkeypatch.setattr(executor, "model_mutation_transaction", fake_transaction) monkeypatch.setattr( executor, "expand_command", @@ -70,7 +72,7 @@ def test_failed_batch_does_not_refresh_materialized_views(monkeypatch) -> None: monkeypatch.setattr(executor, "_execute_update_command", fail_write) monkeypatch.setattr( executor, - "refresh_materialized_views", + "refresh_materialized_views_after_commit", lambda _name: events.append("refresh"), ) @@ -81,3 +83,209 @@ def test_failed_batch_does_not_refresh_materialized_views(monkeypatch) -> None: ) assert events == ["transaction-enter", "write", "transaction-exit"] + + +def test_batch_option_update_does_not_refresh_materialized_views(monkeypatch) -> None: + events: list[str] = [] + + @contextmanager + def fake_transaction(_name: str): + events.append("transaction-enter") + yield object() + events.append("transaction-exit") + + monkeypatch.setattr(executor, "model_mutation_transaction", fake_transaction) + monkeypatch.setattr(executor, "expand_command", lambda _name, cs: cs) + monkeypatch.setattr( + executor, + "_execute_update_command", + lambda _name, _change_set: events.append("write") or ChangeSet(), + ) + monkeypatch.setattr( + executor, + "refresh_materialized_views_after_commit", + lambda _name: events.append("refresh"), + ) + + executor.execute_batch_commands( + "project_a", + ChangeSet({"operation": "update", "type": "option", "id": "duration"}), + ) + + assert events == ["transaction-enter", "write", "transaction-exit"] + + +def test_model_mutation_lock_is_acquired_once_per_transaction(monkeypatch) -> None: + state = {"held": False} + statements: list[tuple[str, tuple[str]]] = [] + + class FakeCursor: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def execute(self, statement: str, params: tuple[str]): + statements.append((statement, params)) + + class FakeConnection: + def cursor(self): + return FakeCursor() + + monkeypatch.setattr( + database, + "is_model_mutation_lock_active", + lambda _name: state["held"], + raising=False, + ) + monkeypatch.setattr( + database, + "mark_model_mutation_lock_active", + lambda _name: state.__setitem__("held", True), + raising=False, + ) + monkeypatch.setattr(database, "get_project_database_name", lambda name: name) + + conn = FakeConnection() + database.acquire_model_mutation_lock(conn, "project_a") + database.acquire_model_mutation_lock(conn, "project_a") + + assert len(statements) == 1 + + +def test_locked_command_builds_after_lock_and_refreshes_after_commit( + monkeypatch, +) -> None: + events: list[str] = [] + + @contextmanager + def fake_model_transaction(_name: str): + events.append("lock") + yield object() + events.append("commit") + + def build_command() -> database.DatabaseCommand: + events.append("read-and-build") + return database.DatabaseCommand( + "UPDATE network.junctions SET elevation = 1", + [{"operation": "update", "type": "junction", "id": "J1"}], + ) + + monkeypatch.setattr(database, "model_mutation_transaction", fake_model_transaction) + monkeypatch.setattr( + database, + "is_project_transaction_active", + lambda _name: False, + ) + monkeypatch.setattr( + database, + "execute_command", + lambda _name, command: events.append("write") + or ChangeSet.from_list(command.changes), + ) + monkeypatch.setattr( + database, + "refresh_materialized_views_after_commit", + lambda _name: events.append("refresh"), + ) + + result = database.execute_locked_command("project_a", build_command) + + assert events == ["lock", "read-and-build", "write", "commit", "refresh"] + assert result.operations[0]["id"] == "J1" + + +def test_pipe_patch_reads_under_lock_and_updates_only_supplied_columns( + monkeypatch, +) -> None: + events: list[str] = [] + current = { + "id": "P1", + "node1": "J1", + "node2": "J2", + "length": 10.0, + "diameter": 100.0, + "roughness": 120.0, + "minor_loss": 0.0, + "status": "OPEN", + } + + def fake_locked_command(_name: str, builder): + events.append("lock") + command = builder() + assert command is not None + captured.append(command) + events.append("refresh") + return ChangeSet.from_list(command.changes) + + def fake_get_pipe(_name: str, _id: str): + events.append("read") + return current.copy() + + captured: list[database.DatabaseCommand] = [] + + monkeypatch.setattr( + pipes, + "execute_locked_command", + fake_locked_command, + ) + monkeypatch.setattr(pipes, "get_pipe", fake_get_pipe) + + result = pipes.set_pipe( + "project_a", + ChangeSet({"operation": "update", "type": "pipe", "id": "P1", "length": 20}), + ) + + assert events == ["lock", "read", "refresh"] + assert len(captured) == 1 + assert "length = 20.0" in captured[0].sql + assert "diameter" not in captured[0].sql + assert "update network.links" not in captured[0].sql.lower() + assert result.operations[0]["length"] == 20.0 + + +@pytest.mark.parametrize( + ("module", "setter_name", "getter_name", "builder_name"), + [ + (junctions, "set_junction", "get_junction", "_set_junction"), + (reservoirs, "set_reservoir", "get_reservoir", "_set_reservoir"), + (tanks, "set_tank", "get_tank", "_set_tank"), + (pumps, "set_pump", "get_pump", "_set_pump"), + (valves, "set_valve", "get_valve", "_set_valve"), + ], +) +def test_element_patch_reads_after_shared_model_lock( + monkeypatch, + module, + setter_name: str, + getter_name: str, + builder_name: str, +) -> None: + events: list[str] = [] + current = {"id": "E1"} + + def fake_getter(_name: str, _id: str): + events.append("read") + return current + + def fake_builder(_name: str, _changes: ChangeSet, supplied_current): + events.append("build") + assert supplied_current is current + return database.DatabaseCommand("UPDATE network.nodes SET id = id", []) + + def fake_locked_command(_name: str, builder): + events.append("lock") + assert builder() is not None + return ChangeSet() + + monkeypatch.setattr(module, getter_name, fake_getter) + monkeypatch.setattr(module, builder_name, fake_builder) + monkeypatch.setattr(module, "execute_locked_command", fake_locked_command) + + getattr(module, setter_name)( + "project_a", + ChangeSet({"operation": "update", "type": "element", "id": "E1"}), + ) + + assert events == ["lock", "read", "build"] diff --git a/tests/unit/test_wndb_endpoint_boundaries.py b/tests/unit/test_wndb_endpoint_boundaries.py new file mode 100644 index 0000000..56f6839 --- /dev/null +++ b/tests/unit/test_wndb_endpoint_boundaries.py @@ -0,0 +1,54 @@ +import ast +from pathlib import Path + + +def test_wndb_editor_routes_are_synchronous() -> None: + """Sync psycopg-backed routes must let FastAPI schedule them in its thread pool.""" + endpoints = Path(__file__).resolve().parents[2] / "app" / "api" / "v1" / "endpoints" + route_roots = [endpoints / "network", endpoints / "components"] + async_routes: list[str] = [] + + for root in route_roots: + for path in root.glob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if not isinstance(node, ast.AsyncFunctionDef): + continue + is_route = any( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and isinstance(decorator.func.value, ast.Name) + and decorator.func.value.id == "router" + for decorator in node.decorator_list + ) + if is_route: + async_routes.append(f"{path.name}:{node.lineno}:{node.name}") + + assert async_routes == [] + + +def test_simulation_routes_are_synchronous() -> None: + """EPANET and synchronous WNDB work must run in FastAPI's thread pool.""" + path = ( + Path(__file__).resolve().parents[2] + / "app" + / "api" + / "v1" + / "endpoints" + / "simulation.py" + ) + tree = ast.parse(path.read_text(encoding="utf-8")) + async_routes = [ + node.name + for node in tree.body + if isinstance(node, ast.AsyncFunctionDef) + and any( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and isinstance(decorator.func.value, ast.Name) + and decorator.func.value.id == "router" + for decorator in node.decorator_list + ) + ] + + assert async_routes == [] diff --git a/tests/unit/test_wndb_importer.py b/tests/unit/test_wndb_importer.py new file mode 100644 index 0000000..606c803 --- /dev/null +++ b/tests/unit/test_wndb_importer.py @@ -0,0 +1,66 @@ +from contextlib import contextmanager +from pathlib import Path + +from app.native.wndb.core.database import ChangeSet +from app.native.wndb.inp import importer + + +def test_import_inp_uses_unique_file_and_always_removes_it(monkeypatch) -> None: + paths: list[str] = [] + + def fake_read_inp(_project: str, path: str, _version: str) -> bool: + assert Path(path).read_text(encoding="utf-8") == "[TITLE]\nmodel" + paths.append(path) + return True + + monkeypatch.setattr(importer, "read_inp", fake_read_inp) + change_set = ChangeSet({"inp": "[TITLE]\nmodel"}) + + assert importer.import_inp("project_a", change_set) is True + assert importer.import_inp("project_a", change_set) is True + + assert paths[0] != paths[1] + assert all(not Path(path).exists() for path in paths) + + +def test_read_inp_refreshes_after_committed_replace_when_cleanup_fails( + monkeypatch, +) -> None: + events: list[str] = [] + + @contextmanager + def fake_transaction(_project: str): + yield + + monkeypatch.setattr(importer, "have_project", lambda _project: True) + monkeypatch.setattr( + importer, "temporary_project_name", lambda _project, _purpose: "staging" + ) + monkeypatch.setattr( + importer, + "copy_project", + lambda _source, _target: events.append("copy"), + ) + monkeypatch.setattr(importer, "project_transaction", fake_transaction) + monkeypatch.setattr( + importer, "parse_file", lambda *_args: events.append("parse") + ) + monkeypatch.setattr( + importer, + "replace_project_model", + lambda *_args: events.append("replace"), + ) + + def fail_cleanup(_project: str) -> None: + events.append("cleanup-failed") + raise RuntimeError("drop failed") + + monkeypatch.setattr(importer, "delete_project", fail_cleanup) + monkeypatch.setattr( + importer, + "refresh_materialized_views_after_commit", + lambda _project: events.append("refresh"), + ) + + assert importer.read_inp("project_a", "model.inp") is True + assert events == ["copy", "parse", "replace", "cleanup-failed", "refresh"] diff --git a/tests/unit/test_wndb_materialized_refresh.py b/tests/unit/test_wndb_materialized_refresh.py index 8943b42..cbe683d 100644 --- a/tests/unit/test_wndb_materialized_refresh.py +++ b/tests/unit/test_wndb_materialized_refresh.py @@ -1,3 +1,5 @@ +import pytest + from app.native.wndb.core import database @@ -29,7 +31,7 @@ def test_direct_model_write_refreshes_materialized_views(monkeypatch) -> None: ) monkeypatch.setattr( database, - "refresh_materialized_views", + "refresh_materialized_views_after_commit", lambda _name: events.append("refresh"), ) @@ -61,7 +63,7 @@ def test_batch_model_write_defers_materialized_view_refresh(monkeypatch) -> None ) monkeypatch.setattr( database, - "refresh_materialized_views", + "refresh_materialized_views_after_commit", lambda _name: events.append("refresh"), ) @@ -73,6 +75,23 @@ def test_batch_model_write_defers_materialized_view_refresh(monkeypatch) -> None assert events == ["write"] +def test_refresh_after_commit_reports_that_changes_are_durable(monkeypatch) -> None: + def fail_refresh(_name: str) -> None: + raise RuntimeError("refresh failed") + + monkeypatch.setattr(database, "refresh_materialized_views", fail_refresh) + + with pytest.raises( + database.MaterializedViewRefreshAfterCommitError, + match="changes were committed", + ) as exc_info: + database.refresh_materialized_views_after_commit("project_a") + + assert exc_info.value.project == "project_a" + assert exc_info.value.changes_committed is True + assert isinstance(exc_info.value.__cause__, RuntimeError) + + def test_non_gis_model_write_does_not_refresh_materialized_views(monkeypatch) -> None: events: list[str] = [] monkeypatch.setattr( diff --git a/tests/unit/test_wndb_network_views.py b/tests/unit/test_wndb_network_views.py new file mode 100644 index 0000000..86c62af --- /dev/null +++ b/tests/unit/test_wndb_network_views.py @@ -0,0 +1,163 @@ +from app.native.wndb.gis import network_views, region_geometry +from app.native.wndb.model import elements + + +def test_network_node_coords_use_one_unified_view_query(monkeypatch) -> None: + calls = [] + + def fake_read_all(name, statement, params=None): + calls.append((name, statement, params)) + return [ + {"id": "J-1", "x": 10.5, "y": 20.5, "node_type": "junction"}, + {"id": "R-1", "x": 30.0, "y": 40.0, "node_type": "reservoir"}, + ] + + monkeypatch.setattr(network_views, "read_all", fake_read_all) + + assert network_views.get_network_node_coords("project_a") == { + "J-1": {"x": 10.5, "y": 20.5, "type": "junction"}, + "R-1": {"x": 30.0, "y": 40.0, "type": "reservoir"}, + } + assert len(calls) == 1 + assert "FROM gis.network_nodes" in calls[0][1] + + +def test_network_link_nodes_use_one_unified_view_query(monkeypatch) -> None: + calls = [] + + def fake_read_all(name, statement, params=None): + calls.append((name, statement, params)) + return [ + { + "id": "P-1", + "link_type": "pipe", + "start_node_id": "J-1", + "end_node_id": "J-2", + }, + { + "id": "PU-1", + "link_type": "pump", + "start_node_id": "R-1", + "end_node_id": "J-1", + }, + ] + + monkeypatch.setattr(network_views, "read_all", fake_read_all) + + assert network_views.get_network_link_nodes("project_a") == [ + "P-1:pipe:J-1:J-2", + "PU-1:pump:R-1:J-1", + ] + assert len(calls) == 1 + assert "FROM gis.network_links" in calls[0][1] + + +def test_topology_rows_are_loaded_in_two_batch_queries(monkeypatch) -> None: + calls = [] + responses = [ + [{"id": "J-1", "x": 1.0, "y": 2.0, "node_type": "junction"}], + [ + { + "id": "P-1", + "start_node_id": "J-1", + "end_node_id": "J-2", + "length": 12.5, + } + ], + ] + + def fake_read_all(name, statement, params=None): + calls.append((name, statement, params)) + return responses[len(calls) - 1] + + monkeypatch.setattr(network_views, "read_all", fake_read_all) + + nodes, links = network_views.get_topology_rows("project_a", ["J-1", "J-2"]) + + assert nodes == responses[0] + assert links == responses[1] + assert len(calls) == 2 + assert "FROM network.nodes" in calls[0][1] + assert "FROM network.links" in calls[1][1] + assert "LEFT JOIN network.pipes" in calls[1][1] + + +def test_topology_builds_adjacency_from_batch_rows(monkeypatch) -> None: + monkeypatch.setattr( + region_geometry, + "get_topology_rows", + lambda _name, _node_ids: ( + [ + {"id": "J-1", "x": 1.0, "y": 2.0, "node_type": "junction"}, + {"id": "R-1", "x": 3.0, "y": 4.0, "node_type": "reservoir"}, + ], + [ + { + "id": "P-1", + "start_node_id": "J-1", + "end_node_id": "R-1", + "length": 20.0, + } + ], + ), + ) + + topology = region_geometry.Topology("project_a", ["J-1", "R-1"]) + + assert topology.max_x_node() == "R-1" + assert topology.nodes()["J-1"] == { + "x": 1.0, + "y": 2.0, + "type": "junction", + "links": ["P-1"], + } + assert topology.links()["P-1"] == { + "node1": "J-1", + "node2": "R-1", + "length": 20.0, + } + + +def test_junction_demands_are_mapped_from_authoritative_table(monkeypatch) -> None: + monkeypatch.setattr( + network_views, + "read_all", + lambda _name, _statement, _params: [ + { + "junction_id": "J-1", + "sequence_no": 0, + "base_demand": 3.5, + "pattern_id": "PAT-1", + "category": None, + } + ], + ) + + assert network_views.get_junction_demands("project_a", ["J-1"]) == { + "J-1": [ + {"demand": 3.5, "pattern": "PAT-1", "category": None} + ] + } + + +def test_all_node_links_scan_the_unified_view_once(monkeypatch) -> None: + calls = [] + + def fake_read_all_typed(name, statement, params): + calls.append((name, statement, params)) + return [ + { + "id": "P-1", + "start_node_id": "J-1", + "end_node_id": "J-2", + } + ] + + monkeypatch.setattr(elements, "read_all_typed", fake_read_all_typed) + + assert elements.get_all_node_links("project_a") == { + "J-1": ["P-1"], + "J-2": ["P-1"], + } + assert len(calls) == 1 + assert "FROM gis.network_links" in calls[0][1] diff --git a/tests/unit/test_wndb_projects.py b/tests/unit/test_wndb_projects.py new file mode 100644 index 0000000..468015b --- /dev/null +++ b/tests/unit/test_wndb_projects.py @@ -0,0 +1,279 @@ +from contextlib import contextmanager + +import pytest + +from app.infra.db.project_routing import ActiveProjectRouting, activate_project_routing +from app.native.wndb.core import projects + + +class _FakeCursor: + def __init__(self, *, rows=None, current_database: str = "postgres") -> None: + self.rows = rows or [] + self.current_database = current_database + self.calls: list[tuple[object, object]] = [] + self._last_statement = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def __iter__(self): + return iter(self.rows) + + def execute(self, statement, params=None): + self.calls.append((statement, params)) + self._last_statement = statement + return self + + def fetchone(self): + if isinstance(self._last_statement, str) and self._last_statement.startswith( + "select datallowconn" + ): + return {"datallowconn": False} + return {"current_database": self.current_database} + + +class _FakeConnection: + def __init__(self, cursor: _FakeCursor) -> None: + self._cursor = cursor + + def cursor(self, **_kwargs): + return self._cursor + + +def _admin_connection(cursor: _FakeCursor): + @contextmanager + def connection(): + yield _FakeConnection(cursor) + + return connection + + +@pytest.mark.parametrize( + "name", + [ + "postgres", + "project", + "system_hub", + "SYSTEM_HUB", + "tjwater_v2_template", + "another_template", + ], +) +def test_delete_project_rejects_protected_database_before_side_effects( + monkeypatch, name +) -> None: + closed: list[str] = [] + monkeypatch.setattr(projects, "close_project_pool", closed.append) + monkeypatch.setattr( + projects, + "admin_connection", + lambda: pytest.fail("protected database opened an administration connection"), + ) + + with pytest.raises(ValueError, match="protected"): + projects.delete_project(name) + + assert closed == [] + + +def test_copy_project_rejects_metadata_source(monkeypatch) -> None: + monkeypatch.setattr( + projects, + "admin_connection", + lambda: pytest.fail("protected database opened an administration connection"), + ) + + with pytest.raises(ValueError, match="protected"): + projects.copy_project("system_hub", "copy") + + +def test_copy_project_rejects_unconfigured_template_source(monkeypatch) -> None: + monkeypatch.setattr( + projects, + "admin_connection", + lambda: pytest.fail("unconfigured template opened an administration connection"), + ) + + with pytest.raises(ValueError, match="protected"): + projects.copy_project("tjwater_next_template", "copy") + + +def test_create_project_allows_the_protected_template_as_source(monkeypatch) -> None: + cursor = _FakeCursor() + closed: list[str] = [] + monkeypatch.setattr(projects, "admin_connection", _admin_connection(cursor)) + monkeypatch.setattr(projects, "close_project_pool", closed.append) + + projects.create_project("project_a") + + assert closed == ["tjwater_v2_template", "project_a"] + assert any(call[1] == ("tjwater_v2_template",) for call in cursor.calls) + assert any( + isinstance(call[0], str) + and call[0].startswith("select pg_terminate_backend") + and call[1] == ("tjwater_v2_template",) + for call in cursor.calls + ) + assert any("create database" in str(call[0]).lower() for call in cursor.calls) + + +def test_list_project_excludes_metadata_database(monkeypatch) -> None: + cursor = _FakeCursor(rows=[{"datname": "project_a"}]) + monkeypatch.setattr(projects, "admin_connection", _admin_connection(cursor)) + + assert projects.list_project() == ["project_a"] + excluded = cursor.calls[0][1][0] + assert "system_hub" in excluded + assert "project" in excluded + assert "tjwater_v2_template" in excluded + + +def test_delete_project_uses_routed_physical_database_name(monkeypatch) -> None: + cursor = _FakeCursor() + closed: list[str] = [] + monkeypatch.setattr(projects, "admin_connection", _admin_connection(cursor)) + monkeypatch.setattr(projects, "close_project_pool", closed.append) + routing = ActiveProjectRouting( + project_code="logical-project", + business_dsn="postgresql://user:password@db.example/tjwater_v2", + ) + + with activate_project_routing(routing): + projects.delete_project("logical-project") + + assert closed == ["logical-project"] + assert any(call[1] == ("tjwater_v2",) for call in cursor.calls) + + +def test_temporary_project_names_are_unique_and_postgres_safe() -> None: + first = projects.temporary_project_name( + "TJWater V2 / Production With A Very Long Project Name", + "Burst Analysis", + ) + second = projects.temporary_project_name( + "TJWater V2 / Production With A Very Long Project Name", + "Burst Analysis", + ) + + assert first != second + assert len(first.encode("utf-8")) <= 63 + assert first.startswith("tjw_tmp_burst_analysis_tjwat") + + +def test_temporary_database_capacity_rejects_creation_at_limit( + monkeypatch, +) -> None: + cursor = _FakeCursor() + monkeypatch.setattr(projects.settings, "WNDB_TEMP_DB_MAX_COUNT", 2) + cursor.fetchone = lambda: {"count": 2} + + with pytest.raises(RuntimeError, match="limit reached"): + with projects._temporary_database_capacity(cursor, "tjw_tmp_analysis_123"): + pytest.fail("capacity guard yielded after reaching the limit") + + assert any("pg_advisory_unlock" in str(statement) for statement, _ in cursor.calls) + + +def test_temporary_project_database_cleans_up_after_failure(monkeypatch) -> None: + calls: list[tuple[str, ...]] = [] + monkeypatch.setattr( + projects, + "temporary_project_name", + lambda project, purpose: "isolated_run", + ) + monkeypatch.setattr( + projects, + "copy_project", + lambda source, target: calls.append(("copy", source, target)), + ) + monkeypatch.setattr(projects, "have_project", lambda name: True) + monkeypatch.setattr( + projects, + "delete_project", + lambda name: calls.append(("delete", name)), + ) + monkeypatch.setattr( + "app.native.wndb.core.model_replace.replace_project_model", + lambda target, source, *, copy_source_scada: calls.append( + ("clone", target, source, str(copy_source_scada)) + ), + ) + monkeypatch.setattr( + "app.native.wndb.core.database.refresh_materialized_views_after_commit", + lambda name: calls.append(("refresh", name)), + ) + + with pytest.raises(RuntimeError, match="analysis failed"): + with projects.temporary_project_database("project_a", "age") as name: + assert name == "isolated_run" + raise RuntimeError("analysis failed") + + assert calls == [ + ("copy", "tjwater_v2_template", "isolated_run"), + ("clone", "isolated_run", "project_a", "True"), + ("refresh", "isolated_run"), + ("delete", "isolated_run"), + ] + + +def test_temporary_template_database_does_not_clone_a_project(monkeypatch) -> None: + calls: list[tuple[str, ...]] = [] + monkeypatch.setattr( + projects, + "temporary_project_name", + lambda project, purpose: "empty_conversion", + ) + monkeypatch.setattr( + projects, + "copy_project", + lambda source, target: calls.append(("copy", source, target)), + ) + monkeypatch.setattr(projects, "have_project", lambda name: True) + monkeypatch.setattr( + projects, + "delete_project", + lambda name: calls.append(("delete", name)), + ) + + with projects.temporary_template_database("conversion", "v3_to_v2") as name: + assert name == "empty_conversion" + + assert calls == [ + ("copy", "tjwater_v2_template", "empty_conversion"), + ("delete", "empty_conversion"), + ] + + +def test_clean_project_deletes_only_explicit_unique_targets(monkeypatch) -> None: + cursor = _FakeCursor() + closed: list[str] = [] + monkeypatch.setattr(projects, "admin_connection", _admin_connection(cursor)) + monkeypatch.setattr(projects, "close_project_pool", closed.append) + + projects.clean_project(["temp_a", "temp_b", "temp_a"]) + + assert closed == ["temp_a", "temp_b"] + termination_targets = [ + params[0] + for statement, params in cursor.calls + if isinstance(statement, str) and statement.startswith("select pg_terminate_backend") + ] + assert termination_targets == ["temp_a", "temp_b"] + + +def test_clean_project_validates_all_targets_before_deleting(monkeypatch) -> None: + closed: list[str] = [] + monkeypatch.setattr(projects, "close_project_pool", closed.append) + monkeypatch.setattr( + projects, + "admin_connection", + lambda: pytest.fail("invalid targets opened an administration connection"), + ) + + with pytest.raises(ValueError, match="protected"): + projects.clean_project(["temp_a", "system_hub"]) + + assert closed == [] diff --git a/tests/unit/test_wndb_query_safety.py b/tests/unit/test_wndb_query_safety.py index 0dc9d7a..34c7a3a 100644 --- a/tests/unit/test_wndb_query_safety.py +++ b/tests/unit/test_wndb_query_safety.py @@ -3,6 +3,7 @@ from pathlib import Path from app.infra.db.postgresql import scada_assets from app.native.wndb.core.database import ChangeSet, sql_literal +from app.native.wndb.gis import coordinates from app.native.wndb.model import controls, junctions, patterns @@ -53,6 +54,26 @@ def test_get_all_scada_info_reads_materialized_view(monkeypatch) -> None: assert "FROM gis.scada_devices" in statements[0] +def test_get_missing_node_coord_does_not_write_default_geometry(monkeypatch) -> None: + calls: list[tuple[str, str, tuple[str, ...]]] = [] + + def fake_try_read(name, statement, params): + calls.append((name, statement, params)) + return None + + monkeypatch.setattr(coordinates, "try_read", fake_try_read) + + assert coordinates.get_node_coord("project_a", "J-1") == {"x": 0.0, "y": 0.0} + assert calls == [ + ( + "project_a", + "select st_astext(geom) as coord_geom " + "from gis.node_geometries where node_id = %s", + ("J-1",), + ) + ] + + def test_sql_literal_keeps_attacker_text_inside_one_postgres_literal() -> None: malicious = "x'); DROP SCHEMA network CASCADE; --"