OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(db)!: adopt project-routed pooled databases

Browse Source 
Reorganize WNDB by responsibility and remove legacy scheme endpoints.\n\nRoute analysis and time-series access through project pools, preserve transactional realtime replacement, and refresh GIS materialized views after writes.\n\nAdd database architecture documentation, live pooling coverage, API contract updates, and executable container verification.\n\nBREAKING CHANGE: legacy scheme APIs and flat app.native.wndb module imports are removed.
This commit is contained in:
jiang
2026-08-25 18:35:05 +08:00
parent fdbcc5c033
commit fa188af0b1
181 changed files with 8446 additions and 33546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/burst_detection.py
+21 -25
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from collections import Counter
from datetime import datetime, timedelta
from typing import Any
from uuid import UUID
import numpy as np
import pandas as pd
@@ -49,8 +50,7 @@ def run_burst_detection(
sensor_nodes: list[str] | None = None,
scheme_name: str | None = None,
data_source: str = "monitoring",
simulation_scheme_name: str | None = None,
simulation_scheme_type: str | None = None,
simulation_run_id: UUID | None = None,
) -> dict[str, Any]:
"""
运行爆管侦测服务入口。
@@ -153,18 +153,17 @@ def run_burst_detection(
else _get_pressure_sensor_nodes(network)
)
if data_source == "simulation":
if not simulation_scheme_name:
raise ValueError("模拟方案模式必须提供 simulation_scheme_name。")
if not simulation_run_id:
raise ValueError("模拟数据模式必须提供 simulation_run_id。")
observed_df = _build_observed_pressure_from_simulation(
network=network,
sensor_nodes=scada_sensor_nodes,
scada_start=scada_start,
scada_end=scada_end,
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=simulation_scheme_type,
simulation_run_id=simulation_run_id,
)
observed_input = observed_df
observed_source = "simulation_scheme_timerange"
observed_source = "analysis_run_timerange"
else:
observed_df = _build_observed_pressure_from_scada(
network=network,
@@ -224,9 +223,8 @@ def run_burst_detection(
}
if data_source == "simulation":
payload["data_source"] = "simulation"
payload["simulation_scheme"] = {
"name": simulation_scheme_name,
"type": simulation_scheme_type,
payload["simulation_run"] = {
"run_id": str(simulation_run_id),
}
else:
payload["data_source"] = "monitoring"
@@ -299,11 +297,10 @@ def _build_observed_pressure_from_simulation(
sensor_nodes: list[str],
scada_start: datetime | str | None,
scada_end: datetime | str | None,
simulation_scheme_name: str | None,
simulation_scheme_type: str | None = None,
simulation_run_id: UUID,
) -> pd.DataFrame:
if scada_start is None or scada_end is None:
raise ValueError("使用模拟方案查询时必须同时提供 scada_start 与 scada_end。")
raise ValueError("使用分析模拟数据时必须同时提供 scada_start 与 scada_end。")
start_dt = _to_datetime(scada_start)
end_dt = _to_datetime(scada_end)
@@ -316,12 +313,9 @@ def _build_observed_pressure_from_simulation(
# Check for missing nodes in simulation result if needed, but InternalQueries handles some of it.
# We assume sensor_nodes are valid pressure nodes.
scheme_type = simulation_scheme_type or "burst_analysis"
simulation_data = InternalQueries.query_scheme_simulation_by_ids_timerange(
simulation_data = InternalQueries.query_analysis_simulation_by_ids_timerange(
db_name=network,
scheme_type=scheme_type,
scheme_name=simulation_scheme_name,
run_id=simulation_run_id,
element_ids=sensor_nodes,
start_time=start_dt.isoformat(),
end_time=end_dt.isoformat(),
@@ -646,8 +640,8 @@ def _resolve_sampling_interval_minutes(
inferred_intervals = [
parsed
for item in get_all_scada_info(network)
if str(item.get("type", "")).lower() == "pressure"
and str(item.get("associated_element_id", "")) in selected_nodes
if str(item.get("device_type", "")).lower() == "pressure"
and str(item.get("node_id", "")) in selected_nodes
and (
parsed := _parse_sampling_interval_minutes(
item.get("transmission_frequency")
@@ -685,9 +679,9 @@ def _parse_sampling_interval_minutes(value: Any) -> int | None:
def _get_pressure_sensor_mapping(network: str) -> dict[str, str]:
node_query_id: dict[str, str] = {}
for item in get_all_scada_info(network):
if str(item.get("type", "")).lower() != "pressure":
if str(item.get("device_type", "")).lower() != "pressure":
continue
node_id = item.get("associated_element_id")
node_id = item.get("node_id")
query_id = item.get("api_query_id")
if node_id and query_id is not None:
node_query_id[str(node_id)] = str(query_id)
@@ -697,14 +691,16 @@ def _get_pressure_sensor_mapping(network: str) -> dict[str, str]:
def _get_pressure_sensor_nodes(network: str) -> list[str]:
sensor_nodes: list[str] = []
for item in get_all_scada_info(network):
if str(item.get("type", "")).lower() != "pressure":
if str(item.get("device_type", "")).lower() != "pressure":
continue
node_id = item.get("associated_element_id")
node_id = item.get("node_id")
if isinstance(node_id, str) and node_id:
sensor_nodes.append(node_id)
sensor_nodes = list(dict.fromkeys(sensor_nodes))
if not sensor_nodes:
raise ValueError("未找到压力传感器对应节点（scada_info.type=pressure）。")
raise ValueError(
"未找到压力传感器对应节点（asset.scada_devices.device_type=pressure）。"
)
return sensor_nodes
app/services/burst_location.py
+44 -67
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import os
from datetime import datetime, timedelta
from typing import Any
from uuid import UUID
import pandas as pd
@@ -11,7 +12,7 @@ from app.infra.db.timescaledb.internal_queries import InternalQueries
from app.services.scheme_management import (
query_burst_location_scheme_detail,
query_burst_location_schemes,
query_scheme_list,
get_analysis_run,
scheme_name_exists,
store_scheme_info,
)
@@ -21,7 +22,6 @@ from app.services.time_api import extract_date, parse_utc_time, utc_now
SeriesInput = pd.Series | dict[str, Any] | list[dict[str, Any]]
FLOW_SCADA_TYPES = {"pipe_flow", "flow", "demand"}
SIMULATION_DATA_SOURCES = {"monitoring", "simulation"}
DEFAULT_SIMULATION_SCHEME_TYPE = "burst_analysis"
def _normalize_series(data: SeriesInput, field_name: str) -> pd.Series:
@@ -65,16 +65,12 @@ def run_burst_location_by_network(
scada_normal_end: datetime | str | None = None,
use_scada_flow: bool = False,
scheme_name: str | None = None,
simulation_scheme_name: str | None = None,
simulation_scheme_type: str | None = None,
simulation_run_id: UUID | None = None,
) -> dict[str, Any]:
if not network:
raise ValueError("network is required.")
normalized_data_source = _normalize_data_source(
data_source, simulation_scheme_name=simulation_scheme_name
)
resolved_simulation_scheme_type = (
simulation_scheme_type or DEFAULT_SIMULATION_SCHEME_TYPE
data_source, simulation_run_id=simulation_run_id
)
selected_pressure_ids = (
@@ -123,8 +119,8 @@ def run_burst_location_by_network(
else None
)
if normalized_data_source == "simulation":
if not simulation_scheme_name:
raise ValueError("模拟方案模式必须提供 simulation_scheme_name。")
if not simulation_run_id:
raise ValueError("模拟数据模式必须提供 simulation_run_id。")
normal_start_dt = burst_start_dt
normal_end_dt = burst_end_dt
(
@@ -137,9 +133,8 @@ def run_burst_location_by_network(
end_dt=burst_end_dt,
data_type="pressure",
series_name="burst_pressure",
simulation_source="scheme",
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=resolved_simulation_scheme_type,
simulation_source="analysis",
simulation_run_id=simulation_run_id,
)
(
normal_pressure_series,
@@ -152,10 +147,9 @@ def run_burst_location_by_network(
data_type="pressure",
series_name="normal_pressure",
simulation_source="realtime",
simulation_scheme_name=None,
simulation_scheme_type=resolved_simulation_scheme_type,
simulation_run_id=None,
)
observed_source = "simulation_scheme_burst_realtime_normal_timerange"
observed_source = "analysis_run_burst_realtime_normal_timerange"
else:
if normal_pressure_from_payload is None and (
normal_start_dt is None or normal_end_dt is None
@@ -230,8 +224,8 @@ def run_burst_location_by_network(
else None
)
if normalized_data_source == "simulation":
if not simulation_scheme_name:
raise ValueError("模拟方案模式必须提供 simulation_scheme_name。")
if not simulation_run_id:
raise ValueError("模拟数据模式必须提供 simulation_run_id。")
burst_flow_series, burst_flow_samples = (
_build_observed_series_from_simulation(
network=network,
@@ -240,9 +234,8 @@ def run_burst_location_by_network(
end_dt=burst_end_dt,
data_type="flow",
series_name="burst_flow",
simulation_source="scheme",
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=resolved_simulation_scheme_type,
simulation_source="analysis",
simulation_run_id=simulation_run_id,
)
)
normal_flow_series, normal_flow_samples = (
@@ -254,8 +247,7 @@ def run_burst_location_by_network(
data_type="flow",
series_name="normal_flow",
simulation_source="realtime",
simulation_scheme_name=None,
simulation_scheme_type=resolved_simulation_scheme_type,
simulation_run_id=None,
)
)
else:
@@ -354,14 +346,12 @@ def run_burst_location_by_network(
}
)
if normalized_data_source == "simulation":
simulation_burst_ids = _get_simulation_scheme_burst_ids(
simulation_burst_ids = _get_simulation_run_burst_ids(
network=network,
scheme_name=simulation_scheme_name,
scheme_type=resolved_simulation_scheme_type,
run_id=simulation_run_id,
)
payload["simulation_scheme"] = {
"name": simulation_scheme_name,
"type": resolved_simulation_scheme_type,
payload["simulation_run"] = {
"run_id": str(simulation_run_id),
"burst_ids": simulation_burst_ids,
}
if scheme_name:
@@ -471,20 +461,15 @@ def _validate_time_window(
return start_dt, end_dt
def _get_simulation_scheme_burst_ids(
*, network: str, scheme_name: str | None, scheme_type: str
def _get_simulation_run_burst_ids(
*, network: str, run_id: UUID | None
) -> list[str]:
if not scheme_name:
if run_id is None:
return []
rows = query_scheme_list(network, scheme_type=scheme_type) or []
for row in rows:
if len(row) < 7:
continue
if row[1] != scheme_name or row[2] != scheme_type:
continue
detail = row[6] if isinstance(row[6], dict) else {}
return _normalize_burst_ids(detail.get("burst_ID"))
return []
run = get_analysis_run(network, run_id)
if not run:
raise ValueError(f"未找到模拟运行: {run_id}")
return _normalize_burst_ids(run["parameters"].get("burst_ID"))
def _normalize_burst_ids(value: Any) -> list[str]:
@@ -566,8 +551,7 @@ def _build_observed_series_from_simulation(
data_type: str,
series_name: str,
simulation_source: str,
simulation_scheme_name: str | None,
simulation_scheme_type: str,
simulation_run_id: UUID | None,
) -> tuple[pd.Series, int]:
sensor_ids = _dedupe_ids(sensor_ids)
sensor_metadata = _build_sensor_metadata(network=network, data_type=data_type)
@@ -586,8 +570,7 @@ def _build_observed_series_from_simulation(
end_dt=end_dt,
data_type=data_type,
simulation_source=simulation_source,
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=simulation_scheme_type,
simulation_run_id=simulation_run_id,
)
simulation_data = _normalize_timeseries_by_id(simulation_data)
values: dict[str, float] = {}
@@ -625,10 +608,9 @@ def _query_simulation_data_by_sensor_ids(
end_dt: datetime,
data_type: str,
simulation_source: str,
simulation_scheme_name: str | None,
simulation_scheme_type: str,
simulation_run_id: UUID | None,
) -> dict[str, list[dict[str, Any]]]:
if simulation_source not in {"scheme", "realtime"}:
if simulation_source not in {"analysis", "realtime"}:
raise ValueError(f"Unsupported simulation_source: {simulation_source}")
sensor_ids = _dedupe_ids(sensor_ids)
@@ -645,8 +627,7 @@ def _query_simulation_data_by_sensor_ids(
start_dt=start_dt,
end_dt=end_dt,
simulation_source=simulation_source,
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=simulation_scheme_type,
simulation_run_id=simulation_run_id,
)
)
return result
@@ -679,8 +660,7 @@ def _query_simulation_data_by_sensor_ids(
start_dt=start_dt,
end_dt=end_dt,
simulation_source=simulation_source,
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=simulation_scheme_type,
simulation_run_id=simulation_run_id,
)
)
if demand_ids:
@@ -693,8 +673,7 @@ def _query_simulation_data_by_sensor_ids(
start_dt=start_dt,
end_dt=end_dt,
simulation_source=simulation_source,
simulation_scheme_name=simulation_scheme_name,
simulation_scheme_type=simulation_scheme_type,
simulation_run_id=simulation_run_id,
)
)
return result
@@ -709,19 +688,17 @@ def _query_simulation_values(
start_dt: datetime,
end_dt: datetime,
simulation_source: str,
simulation_scheme_name: str | None,
simulation_scheme_type: str,
simulation_run_id: UUID | None,
) -> dict[str, list[dict[str, Any]]]:
element_ids = _dedupe_ids(element_ids)
if not element_ids:
return {}
if simulation_source == "scheme":
if not simulation_scheme_name:
raise ValueError("读取方案模拟数据时必须提供 simulation_scheme_name。")
return InternalQueries.query_scheme_simulation_by_ids_timerange(
if simulation_source == "analysis":
if not simulation_run_id:
raise ValueError("读取分析模拟数据时必须提供 simulation_run_id。")
return InternalQueries.query_analysis_simulation_by_ids_timerange(
db_name=network,
scheme_type=simulation_scheme_type,
scheme_name=simulation_scheme_name,
run_id=simulation_run_id,
element_ids=element_ids,
start_time=start_dt.isoformat(),
end_time=end_dt.isoformat(),
@@ -743,7 +720,7 @@ def _query_simulation_values(
def _build_sensor_metadata(network: str, data_type: str) -> dict[str, dict[str, str]]:
metadata: dict[str, dict[str, str]] = {}
for item in get_all_scada_info(network):
scada_type = str(item.get("type", "")).lower()
scada_type = str(item.get("device_type", "")).lower()
if data_type == "pressure":
if scada_type != "pressure":
continue
@@ -752,7 +729,7 @@ def _build_sensor_metadata(network: str, data_type: str) -> dict[str, dict[str,
continue
else:
raise ValueError(f"Unsupported data_type: {data_type}")
element_id = _normalize_identifier(item.get("associated_element_id"))
element_id = _normalize_identifier(item.get("node_id") or item.get("link_id"))
query_id = _normalize_identifier(item.get("api_query_id"))
if element_id and query_id:
metadata[element_id] = {"query_id": query_id, "scada_type": scada_type}
@@ -765,11 +742,11 @@ def _build_scada_mapping(network: str, data_type: str) -> dict[str, str]:
def _normalize_data_source(
data_source: str | None, simulation_scheme_name: str | None = None
data_source: str | None, simulation_run_id: UUID | None = None
) -> str:
normalized = str(data_source or "").strip().lower()
if not normalized:
return "simulation" if simulation_scheme_name else "monitoring"
return "simulation" if simulation_run_id else "monitoring"
if normalized not in SIMULATION_DATA_SOURCES:
allowed_sources = ", ".join(sorted(SIMULATION_DATA_SOURCES))
raise ValueError(
@@ -783,7 +760,7 @@ def _get_sensor_nodes(network: str, data_type: str) -> list[str]:
sensor_ids = sorted(mapping.keys())
if not sensor_ids:
type_name = "压力" if data_type == "pressure" else "流量"
raise ValueError(f"未找到{type_name}传感器对应节点（scada_info.type）。")
raise ValueError(f"未找到{type_name}传感器对应节点（asset.scada_devices.device_type）。")
return sensor_ids
app/services/globals.py
+13 -53
View File
@@ -1,54 +1,14 @@
# simulation.py中的全局变量
# reservoir basic height
RESERVOIR_BASIC_HEIGHT = float(250.35)
PATTERN_TIME_STEP = None # 浮点数
# 实时数据类：element_id和api_query_id对应
reservoirs_id = {}
tanks_id = {}
fixed_pumps_id ={}
variable_pumps_id = {}
pressure_id = {}
demand_id = {}
quality_id = {}
# 实时数据类：pattern_id和api_query_id对应
source_outflow_pattern_id = {}
realtime_pipe_flow_pattern_id = {}
pipe_flow_region_patterns = {} # 根据realtime的pipe_flow，对non_realtime的demand进行分区
# 分区查询
source_outflow_region = {} # 以绑定的管段作为value
source_outflow_region_id = {} # 以api_query_id作为value
source_outflow_region_patterns = {} # 以associated_pattern作为value
# 非实时数据的pattern
non_realtime_region_patterns = {} # 基于source_outflow_region进行区分
realtime_region_pipe_flow_and_demand_id = {} # 基于source_outflow_region搜索该分区中的实时pipe_flow和demand的api_query_id，后续用region的流量 - 实时流量计的流量
realtime_region_pipe_flow_and_demand_patterns = {} # 基于source_outflow_region搜索该分区中的实时pipe_flow和demand的associated_pattern，后续用region的流量 - 实时流量计的流量
# ---------------------------------------------------------
# 全局变量，用于存储不同类型的realtime api_query_id
reservoir_liquid_level_realtime_ids = []
tank_liquid_level_realtime_ids = []
fixed_pump_realtime_ids = []
variable_pump_realtime_ids = []
source_outflow_realtime_ids = []
pipe_flow_realtime_ids = []
pressure_realtime_ids = []
demand_realtime_ids = []
quality_realtime_ids = []
# transmission_frequency的最大值
transmission_frequency = None
hydraulic_timestep = None # 时间字符串
reservoir_liquid_level_non_realtime_ids = []
tank_liquid_level_non_realtime_ids = []
fixed_pump_non_realtime_ids = []
variable_pump_non_realtime_ids = []
source_outflow_non_realtime_ids = []
pipe_flow_non_realtime_ids = []
pressure_non_realtime_ids = []
demand_non_realtime_ids = []
quality_non_realtime_ids = []
"""Mutable state used by the legacy synchronous simulation runner."""
# api_query_id和associated_element_id对应，不包含液位和泵
scheme_source_outflow_ids = {}
scheme_pipe_flow_ids = {}
scheme_pressure_ids = {}
scheme_demand_ids = {}
scheme_quality_ids = {}
RESERVOIR_BASIC_HEIGHT = 250.35
PATTERN_TIME_STEP: float | None = None
hydraulic_timestep: str | None = None
# Element ID -> SCADA api_query_id, loaded per project before simulation.
reservoirs_id: dict[str, str] = {}
tanks_id: dict[str, str] = {}
fixed_pumps_id: dict[str, str] = {}
variable_pumps_id: dict[str, str] = {}
pressure_id: dict[str, str] = {}
demand_id: dict[str, str] = {}
quality_id: dict[str, str] = {}
app/services/leakage_identifier.py
+7 -7
View File
@@ -194,18 +194,18 @@ def get_leakage_identify_scheme_detail(
def _get_pressure_sensor_nodes(network: str) -> list[str]:
scada_info = get_all_scada_info(network)
scada_devices = get_all_scada_info(network)
sensor_nodes: list[str] = []
for item in scada_info:
scada_type = str(item.get("type", "")).lower()
for item in scada_devices:
scada_type = str(item.get("device_type", "")).lower()
if scada_type != "pressure":
continue
node_id = item.get("associated_element_id")
node_id = item.get("node_id")
if isinstance(node_id, str) and node_id:
sensor_nodes.append(node_id)
sensor_nodes = list(dict.fromkeys(sensor_nodes))
if not sensor_nodes:
raise ValueError("未找到压力传感器对应节点（scada_info.type=pressure）。")
raise ValueError("未找到关联节点的压力 SCADA 设备。")
return sensor_nodes
@@ -462,9 +462,9 @@ def _build_observed_pressure_from_scada(
node_query_id: dict[str, str] = {}
for item in get_all_scada_info(network):
if str(item.get("type", "")).lower() != "pressure":
if str(item.get("device_type", "")).lower() != "pressure":
continue
node_id = item.get("associated_element_id")
node_id = item.get("node_id")
query_id = item.get("api_query_id")
if (
isinstance(node_id, str)
app/services/network_import.py
+1 -191
View File
@@ -1,196 +1,6 @@
import csv
import os
import chardet
import psycopg
from psycopg import sql
from app.infra.db.project_routing import get_project_pgconn_string
from app.services.tjnetwork import read_inp
############################################################
# network_update 10
############################################################
def network_update(file_path: str, project_code: str) -> None:
"""
更新pg数据库中的inp文件
:param file_path: inp文件
:param project_code: 元数据项目代码
:return:
"""
"""Replace one project's hydraulic model from an EPANET INP file."""
read_inp(project_code, file_path)
csv_path = "./history_pattern_flow.csv"
# # 检查文件是否存在
# if os.path.exists(csv_path):
# print(f"history_patterns_flows文件存在，开始处理...")
#
# # 读取 CSV 文件
# df = pd.read_csv(csv_path)
#
# # 连接到 PostgreSQL 数据库（这里是数据库 "bb"）
# with psycopg.connect("dbname=bb host=127.0.0.1") as conn:
# with conn.cursor() as cur:
# for index, row in df.iterrows():
# # 直接将数据插入，不进行唯一性检查
# insert_sql = sql.SQL("""
# INSERT INTO history_patterns_flows (id, factor, flow)
# VALUES (%s, %s, %s);
# """)
# # 将数据插入数据库
# cur.execute(insert_sql, (row['id'], row['factor'], row['flow']))
# conn.commit()
# print("数据成功导入到 'history_patterns_flows' 表格。")
# else:
# print(f"history_patterns_flows文件不存在。")
# 检查文件是否存在
if os.path.exists(csv_path):
print(f"history_patterns_flows文件存在，开始处理...")
with psycopg.connect(get_project_pgconn_string(project_code)) as conn:
with conn.cursor() as cur:
with open(csv_path, newline="", encoding="utf-8-sig") as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
# 直接将数据插入，不进行唯一性检查
insert_sql = sql.SQL(
"""
INSERT INTO history_patterns_flows (id, factor, flow)
VALUES (%s, %s, %s);
"""
)
# 将数据插入数据库
cur.execute(insert_sql, (row["id"], row["factor"], row["flow"]))
conn.commit()
print("数据成功导入到 'history_patterns_flows' 表格。")
else:
print(f"history_patterns_flows文件不存在。")
def submit_scada_info(name: str, coord_id: str) -> None:
"""
将scada信息表导入pg数据库
:param name: 项目名称（数据库名称）
:param coord_id: 坐标系的id，如4326，根据原始坐标信息输入
:return:
"""
scada_info_path = "./scada_info.csv"
# 检查文件是否存在
if os.path.exists(scada_info_path):
print(f"scada_info文件存在，开始处理...")
# 自动检测文件编码
with open(scada_info_path, "rb") as file:
raw_data = file.read()
detected = chardet.detect(raw_data)
file_encoding = detected["encoding"]
print(f"检测到的文件编码：{file_encoding}")
try:
# 动态替换数据库名称
conn_string = get_project_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库（这里是数据库 "bb"）
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 检查 scada_info 表是否为空
cur.execute("SELECT COUNT(*) FROM scada_info;")
count = cur.fetchone()[0]
if count > 0:
print("scada_info表中已有数据，正在清空记录...")
cur.execute("DELETE FROM scada_info;")
print("表记录已清空。")
with open(
scada_info_path, newline="", encoding=file_encoding
) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
# 将CSV单元格值为空的字段转换为 None
cleaned_row = {
key: (value if value.strip() else None)
for key, value in row.items()
}
# 处理 associated_source_outflow_id 列动态变化
associated_columns = [
f"associated_source_outflow_id{i}" for i in range(1, 21)
]
associated_values = [
(
cleaned_row.get(col).strip()
if cleaned_row.get(col)
and cleaned_row.get(col).strip()
else None
)
for col in associated_columns
]
# 将 X_coor 和 Y_coor 转换为 geometry 类型
x_coor = (
float(cleaned_row["X_coor"])
if cleaned_row["X_coor"]
else None
)
y_coor = (
float(cleaned_row["Y_coor"])
if cleaned_row["Y_coor"]
else None
)
coord = (
f"SRID={coord_id};POINT({x_coor} {y_coor})"
if x_coor and y_coor
else None
)
# 准备插入 SQL 语句
insert_sql = sql.SQL(
"""
INSERT INTO scada_info (
id, type, associated_element_id, associated_pattern,
associated_pipe_flow_id, {associated_columns},
API_query_id, transmission_mode, transmission_frequency,
reliability, X_coor, Y_coor, coord
)
VALUES (
%s, %s, %s, %s, %s, {associated_placeholders},
%s, %s, %s, %s, %s, %s, %s
);
"""
).format(
associated_columns=sql.SQL(", ").join(
sql.Identifier(col) for col in associated_columns
),
associated_placeholders=sql.SQL(", ").join(
sql.Placeholder() for _ in associated_columns
),
)
# 将数据插入数据库
cur.execute(
insert_sql,
(
cleaned_row["id"],
cleaned_row["type"],
cleaned_row["associated_element_id"],
cleaned_row.get("associated_pattern"),
cleaned_row.get("associated_pipe_flow_id"),
*associated_values,
cleaned_row.get("API_query_id"),
cleaned_row["transmission_mode"],
cleaned_row["transmission_frequency"],
cleaned_row["reliability"],
x_coor,
y_coor,
coord,
),
)
conn.commit()
print("数据成功导入到 'scada_info' 表格。")
except Exception as e:
print(f"导入时出错：{e}")
else:
print(f"scada_info文件不存在。")
app/services/scheme_management.py
+210 -574
View File
@@ -1,43 +1,23 @@
import ast
import json
from datetime import date, datetime
from typing import Any
from uuid import UUID, uuid4
import geopandas as gpd
import pandas as pd
import psycopg
from sqlalchemy import create_engine
from psycopg.types.json import Jsonb
from app.infra.db.project_routing import get_project_pgconn_string
from app.native.wndb.core.connection import project_connection
from app.services.time_api import parse_utc_time
# 2025/03/23
def scheme_name_exists(name: str, scheme_name: str) -> bool:
"""
判断传入的 scheme_name 是否已存在于 scheme_list 表中,用于输入框判断
:param name: 数据库名称
:param scheme_name: 需要判断的方案名称
:return: 如果存在返回 True，否则返回 False
"""
try:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT COUNT(*) FROM scheme_list WHERE scheme_name = %s",
(scheme_name,),
)
result = cur.fetchone()
if result is not None and result[0] > 0:
return True
else:
return False
except Exception as e:
print(f"查询 scheme_name 时出错：{e}")
return False
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"select exists(select 1 from analysis.runs where name = %s)",
(scheme_name,),
)
row = cur.fetchone()
return bool(row and row[0])
# 2025/03/23
def store_scheme_info(
name: str,
scheme_name: str,
@@ -45,202 +25,165 @@ def store_scheme_info(
username: str,
scheme_start_time: datetime | str,
scheme_detail: dict,
):
"""
将一条方案记录插入 scheme_list 表中
:param name: 数据库名称
:param scheme_name: 方案名称
:param scheme_type: 方案类型
:param username: MetaDB 中的用户名快照
:param scheme_start_time: 带时区的方案起始时间；写入前统一转换为 UTC
:param scheme_detail: 方案详情（字典，会转换为 JSON）
:return:
"""
try:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
sql = """
INSERT INTO scheme_list (scheme_name, scheme_type, username, scheme_start_time, scheme_detail)
VALUES (%s, %s, %s, %s, %s)
"""
# 将字典转换为 JSON 字符串
scheme_detail_json = json.dumps(scheme_detail)
normalized_scheme_start_time = parse_utc_time(
scheme_start_time, field_name="scheme_start_time"
)
cur.execute(
sql,
(
scheme_name,
scheme_type,
username,
normalized_scheme_start_time,
scheme_detail_json,
),
)
conn.commit()
print("方案信息存储成功！")
except Exception as e:
print(f"存储方案信息时出错：{e}")
) -> UUID:
"""Create one completed, immutable analysis run."""
return create_analysis_run(
name=name,
scheme_name=scheme_name,
scheme_type=scheme_type,
username=username,
scheme_start_time=scheme_start_time,
scheme_detail=scheme_detail,
status="completed",
)
# 2025/03/23
def delete_scheme_info(name: str, scheme_name: str) -> None:
"""
从 scheme_list 表中删除指定的方案
:param name: 数据库名称
:param scheme_name: 要删除的方案名称
"""
try:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 使用参数化查询删除方案记录
cur.execute(
"DELETE FROM scheme_list WHERE scheme_name = %s", (scheme_name,)
)
conn.commit()
print(f"方案 {scheme_name} 删除成功！")
except Exception as e:
print(f"删除方案时出错：{e}")
def create_analysis_run(
name: str,
scheme_name: str,
scheme_type: str,
username: str,
scheme_start_time: datetime | str,
scheme_detail: dict,
*,
status: str = "running",
) -> UUID:
"""Create a distinct execution record; names are labels, not identities."""
started_at = parse_utc_time(scheme_start_time, field_name="scheme_start_time")
run_id = uuid4()
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"""
insert into analysis.runs
(run_id, name, run_type, created_by, created_at, started_at, status, parameters)
values (%s, %s, %s, %s, now(), %s, %s, %s)
""",
(
run_id,
scheme_name,
scheme_type,
username,
started_at,
status,
Jsonb(scheme_detail),
),
)
return run_id
def update_analysis_run(
name: str,
run_id: UUID,
*,
status: str,
username: str,
scheme_detail: dict,
) -> None:
"""Update lifecycle state and metadata for one execution identity."""
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"""
update analysis.runs
set created_by = %s, status = %s, parameters = %s
where run_id = %s
""",
(username, status, Jsonb(scheme_detail), run_id),
)
if cur.rowcount != 1:
raise LookupError(f"analysis run {run_id} does not exist")
def _run_row(row: dict[str, Any]) -> dict[str, Any]:
parameters = row.get("parameters") if isinstance(row.get("parameters"), dict) else {}
return {
"run_id": row["run_id"],
"name": row["name"],
"run_type": row["run_type"],
"created_by": row["created_by"],
"created_at": row["created_at"],
"started_at": row["started_at"],
"status": row["status"],
"parameters": parameters,
}
def _list_runs(
name: str,
run_type: str | None = None,
query_date: date | None = None,
) -> list[dict[str, Any]]:
clauses: list[str] = []
params: list[Any] = []
if run_type:
clauses.append("run_type = %s")
params.append(run_type)
if query_date is not None:
clauses.append("created_at::date = %s")
params.append(query_date)
where = f"where {' and '.join(clauses)}" if clauses else ""
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
f"select run_id, name, run_type, created_by, created_at, started_at, status, parameters from analysis.runs {where} order by created_at desc",
params,
)
return [_run_row(row) for row in cur.fetchall()]
# 2025/03/23
def query_scheme_list(
name: str,
scheme_type: str | None = None,
query_date: date | None = None,
) -> list:
"""
查询pg数据库中的scheme_list，按照 create_time 降序排列，离现在时间最近的记录排在最前面
:param name: 项目名称（数据库名称）
:param scheme_type: 方案类型；为空时返回全部类型
:param query_date: 查询日期；为空时不按日期过滤
:return: 返回查询结果的所有行
"""
try:
# 动态替换数据库名称
conn_string = get_project_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库（这里是数据库 "bb"）
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
if scheme_type and query_date is not None:
cur.execute(
"""
SELECT *
FROM scheme_list
WHERE scheme_type = %s AND DATE(create_time) = %s
ORDER BY create_time DESC
""",
(scheme_type, query_date),
)
elif scheme_type:
cur.execute(
"""
SELECT *
FROM scheme_list
WHERE scheme_type = %s
ORDER BY create_time DESC
""",
(scheme_type,),
)
elif query_date is not None:
cur.execute(
"""
SELECT *
FROM scheme_list
WHERE DATE(create_time) = %s
ORDER BY create_time DESC
""",
(query_date,),
)
else:
cur.execute("SELECT * FROM scheme_list ORDER BY create_time DESC")
rows = cur.fetchall()
return rows
except Exception as e:
print(f"查询错误：{e}")
) -> list[dict[str, Any]]:
return _list_runs(name, scheme_type, query_date)
def _filter_scheme_detail_scope(
result: dict,
def _get_run_by_name(
name: str,
scheme_type: str | None = None,
) -> dict:
if not result:
return {}
if scheme_type and result.get("scheme_type") != scheme_type:
return {}
network = result.get("network")
if network not in (None, name):
return {}
return result
run_name: str,
run_type: str | None = None,
) -> dict[str, Any]:
params: list[Any] = [run_name]
type_clause = ""
if run_type:
type_clause = "and run_type = %s"
params.append(run_type)
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
f"""
select run_id, name, run_type, created_by, created_at, started_at,
status, parameters
from analysis.runs
where name = %s {type_clause}
order by created_at desc
limit 1
""",
params,
)
row = cur.fetchone()
return _run_row(row) if row else {}
def get_analysis_run(name: str, run_id: UUID) -> dict[str, Any]:
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"""
select run_id, name, run_type, created_by, created_at, started_at,
status, parameters
from analysis.runs
where run_id = %s
""",
(run_id,),
)
row = cur.fetchone()
return _run_row(row) if row else {}
def query_scheme_detail(
name: str,
scheme_name: str,
scheme_type: str | None = None,
) -> dict:
if scheme_type == "dma_leak_identification":
return _filter_scheme_detail_scope(
query_leakage_identify_scheme_detail(name, scheme_name),
name,
scheme_type,
)
if scheme_type == "burst_detection":
return _filter_scheme_detail_scope(
query_burst_detection_scheme_detail(name, scheme_name),
name,
scheme_type,
)
if scheme_type == "burst_location":
return _filter_scheme_detail_scope(
query_burst_location_scheme_detail(name, scheme_name),
name,
scheme_type,
)
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
if scheme_type:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_name = %s AND scheme_type = %s
LIMIT 1
""",
(scheme_name, scheme_type),
)
else:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_name = %s
LIMIT 1
""",
(scheme_name,),
)
row = cur.fetchone()
if row is None:
return {}
detail = row[6] if isinstance(row[6], dict) else {}
return _filter_scheme_detail_scope({
"scheme_id": row[0],
"scheme_name": row[1],
"scheme_type": row[2],
"username": row[3],
"create_time": row[4],
"scheme_start_time": row[5],
"scheme_detail": detail,
"network": detail.get("network"),
"result_payload": detail.get("result_payload", {}),
}, name, scheme_type)
) -> dict[str, Any]:
return _get_run_by_name(name, scheme_name, scheme_type)
def store_leakage_identify_result(
@@ -255,42 +198,51 @@ def store_leakage_identify_result(
run_status: str = "completed",
error_message: str | None = None,
) -> None:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO public.leakage_identify_result
(
scheme_name, network, run_status, error_message,
sensor_nodes, result_rows, node_area_map, areas, drawing_payload
)
VALUES (%s, %s, %s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s::jsonb, %s::jsonb)
ON CONFLICT (scheme_name)
DO UPDATE SET
network = EXCLUDED.network,
run_status = EXCLUDED.run_status,
error_message = EXCLUDED.error_message,
sensor_nodes = EXCLUDED.sensor_nodes,
result_rows = EXCLUDED.result_rows,
node_area_map = EXCLUDED.node_area_map,
areas = EXCLUDED.areas,
drawing_payload = EXCLUDED.drawing_payload,
created_at = NOW();
""",
(
scheme_name,
network,
run_status,
error_message,
json.dumps(sensor_nodes),
json.dumps(result_rows),
json.dumps(node_area_map),
json.dumps(areas),
json.dumps(drawing_payload or {}),
),
)
conn.commit()
run = _get_run_by_name(name, scheme_name, "dma_leak_identification")
if not run:
raise LookupError(f"analysis run {scheme_name!r} does not exist")
payload = {
"network": network,
"run_status": run_status,
"error_message": error_message,
"sensor_nodes": sensor_nodes,
"rows": result_rows,
"node_area_map": node_area_map,
"areas": areas,
"drawing_payload": drawing_payload or {},
}
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"insert into analysis.results (run_id, result_type, payload) values (%s, 'leakage_identification', %s)",
(run["run_id"], Jsonb(payload)),
)
def _list_typed_runs(
name: str,
network: str,
run_type: str,
query_date: date | None,
) -> list[dict[str, Any]]:
rows = _list_runs(name, run_type, query_date)
return [
row
for row in rows
if not network or row["parameters"].get("network") in (None, network)
]
def _typed_run_detail(name: str, run_name: str, run_type: str) -> dict[str, Any]:
run = _get_run_by_name(name, run_name, run_type)
if not run:
return {}
with project_connection(name) as conn, conn.cursor() as cur:
cur.execute(
"select result_type, payload, created_at from analysis.results where run_id = %s order by created_at, result_id",
(run["run_id"],),
)
results = [dict(row) for row in cur.fetchall()]
return run | {"results": results}
def query_leakage_identify_schemes(
@@ -298,100 +250,12 @@ def query_leakage_identify_schemes(
network: str,
scheme_type: str = "dma_leak_identification",
query_date: date | None = None,
) -> list[dict]:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
if query_date is None:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s
ORDER BY create_time DESC
""",
(scheme_type,),
)
else:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s AND DATE(create_time) = %s
ORDER BY create_time DESC
""",
(scheme_type, query_date),
)
rows = cur.fetchall()
result = []
for row in rows:
detail = row[6] if isinstance(row[6], dict) else {}
if network and detail.get("network") not in (None, network):
continue
result.append(
{
"scheme_id": row[0],
"scheme_name": row[1],
"scheme_type": row[2],
"username": row[3],
"create_time": row[4],
"scheme_start_time": row[5],
"scheme_detail": detail,
}
)
return result
) -> list[dict[str, Any]]:
return _list_typed_runs(name, network, scheme_type, query_date)
def query_leakage_identify_scheme_detail(name: str, scheme_name: str) -> dict:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_name = %s
LIMIT 1
""",
(scheme_name,),
)
base_row = cur.fetchone()
if base_row is None:
return {}
cur.execute(
"""
SELECT network, created_at, run_status, error_message, sensor_nodes, result_rows, node_area_map, areas, drawing_payload
FROM public.leakage_identify_result
WHERE scheme_name = %s
LIMIT 1
""",
(scheme_name,),
)
result_row = cur.fetchone()
if result_row is None:
return {}
return {
"scheme_id": base_row[0],
"scheme_name": base_row[1],
"scheme_type": base_row[2],
"username": base_row[3],
"create_time": base_row[4],
"scheme_start_time": base_row[5],
"scheme_detail": base_row[6] if isinstance(base_row[6], dict) else {},
"network": result_row[0],
"result_created_at": result_row[1],
"run_status": result_row[2],
"error_message": result_row[3],
"sensor_nodes": result_row[4] if isinstance(result_row[4], list) else [],
"rows": result_row[5] if isinstance(result_row[5], list) else [],
"node_area_map": result_row[6] if isinstance(result_row[6], dict) else {},
"areas": result_row[7] if isinstance(result_row[7], list) else [],
"drawing_payload": (
result_row[8]
if isinstance(result_row[8], dict)
else {"type": "FeatureCollection", "features": []}
),
}
def query_leakage_identify_scheme_detail(name: str, scheme_name: str) -> dict[str, Any]:
return _typed_run_detail(name, scheme_name, "dma_leak_identification")
def query_burst_location_schemes(
@@ -399,78 +263,12 @@ def query_burst_location_schemes(
network: str,
scheme_type: str = "burst_location",
query_date: date | None = None,
) -> list[dict]:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
if query_date is None:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s
ORDER BY create_time DESC
""",
(scheme_type,),
)
else:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s AND DATE(create_time) = %s
ORDER BY create_time DESC
""",
(scheme_type, query_date),
)
rows = cur.fetchall()
result = []
for row in rows:
detail = row[6] if isinstance(row[6], dict) else {}
if network and detail.get("network") not in (None, network):
continue
result.append(
{
"scheme_id": row[0],
"scheme_name": row[1],
"scheme_type": row[2],
"username": row[3],
"create_time": row[4],
"scheme_start_time": row[5],
"scheme_detail": detail,
}
)
return result
) -> list[dict[str, Any]]:
return _list_typed_runs(name, network, scheme_type, query_date)
def query_burst_location_scheme_detail(name: str, scheme_name: str) -> dict:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_name = %s
LIMIT 1
""",
(scheme_name,),
)
base_row = cur.fetchone()
if base_row is None:
return {}
detail = base_row[6] if isinstance(base_row[6], dict) else {}
return {
"scheme_id": base_row[0],
"scheme_name": base_row[1],
"scheme_type": base_row[2],
"username": base_row[3],
"create_time": base_row[4],
"scheme_start_time": base_row[5],
"scheme_detail": detail,
"network": detail.get("network"),
"result_payload": detail.get("result_payload", {}),
}
def query_burst_location_scheme_detail(name: str, scheme_name: str) -> dict[str, Any]:
return _typed_run_detail(name, scheme_name, "burst_location")
def query_burst_detection_schemes(
@@ -478,171 +276,9 @@ def query_burst_detection_schemes(
network: str,
scheme_type: str = "burst_detection",
query_date: date | None = None,
) -> list[dict]:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
if query_date is None:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s
ORDER BY create_time DESC
""",
(scheme_type,),
)
else:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_type = %s AND DATE(create_time) = %s
ORDER BY create_time DESC
""",
(scheme_type, query_date),
)
rows = cur.fetchall()
result = []
for row in rows:
detail = row[6] if isinstance(row[6], dict) else {}
if network and detail.get("network") not in (None, network):
continue
result.append(
{
"scheme_id": row[0],
"scheme_name": row[1],
"scheme_type": row[2],
"username": row[3],
"create_time": row[4],
"scheme_start_time": row[5],
"scheme_detail": detail,
}
)
return result
) -> list[dict[str, Any]]:
return _list_typed_runs(name, network, scheme_type, query_date)
def query_burst_detection_scheme_detail(name: str, scheme_name: str) -> dict:
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
FROM public.scheme_list
WHERE scheme_name = %s
LIMIT 1
""",
(scheme_name,),
)
base_row = cur.fetchone()
if base_row is None:
return {}
detail = base_row[6] if isinstance(base_row[6], dict) else {}
return {
"scheme_id": base_row[0],
"scheme_name": base_row[1],
"scheme_type": base_row[2],
"username": base_row[3],
"create_time": base_row[4],
"scheme_start_time": base_row[5],
"scheme_detail": detail,
"network": detail.get("network"),
"result_payload": detail.get("result_payload", {}),
}
# 2025/03/23
def upload_shp_to_pg(name: str, table_name: str, role: str, shp_file_path: str):
"""
将 Shapefile 文件上传到 PostgreSQL 数据库
:param name: 项目名称（数据库名称）
:param table_name: 创建表的名字
:param role: 数据库角色名，位于c盘user中查看
:param shp_file_path: shp文件的路径
:return:
"""
try:
# 动态连接到指定的数据库
conn_string = get_project_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
# 读取 Shapefile 文件
gdf = gpd.read_file(shp_file_path)
# 检查投影坐标系（CRS），并确保是 EPSG:4326
if gdf.crs.to_string() != "EPSG:4490":
gdf = gdf.to_crs(epsg=4490)
# 使用 GeoDataFrame 的 .to_postgis 方法将数据写入 PostgreSQL
# 需要在数据库中提前安装 PostGIS 扩展
engine = create_engine(f"postgresql+psycopg2://{role}:@127.0.0.1/{name}")
gdf.to_postgis(
table_name, engine, if_exists="replace", index=True, index_label="id"
)
print(
f"Shapefile 文件成功上传到 PostgreSQL 数据库 '{name}' 的表 '{table_name}'."
)
except Exception as e:
print(f"上传 Shapefile 到 PostgreSQL 时出错：{e}")
def submit_risk_probability_result(name: str, result_file_path: str) -> None:
"""
将管网风险评估结果导入pg数据库
:param name: 项目名称（数据库名称）
:param result_file_path: 结果文件路径
:return:
"""
# 自动检测文件编码
# with open({result_file_path}, 'rb') as file:
# raw_data = file.read()
# detected = chardet.detect(raw_data)
# file_encoding = detected['encoding']
# print(f"检测到的文件编码：{file_encoding}")
try:
# 动态替换数据库名称
conn_string = get_project_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 检查 scada_info 表是否为空
cur.execute("SELECT COUNT(*) FROM pipe_risk_probability;")
count = cur.fetchone()[0]
if count > 0:
print("pipe_risk_probability表中已有数据，正在清空记录...")
cur.execute("DELETE FROM pipe_risk_probability;")
print("表记录已清空。")
# 读取Excel并转换x/y列为列表
df = pd.read_excel(result_file_path, sheet_name="Sheet1")
df["x"] = df["x"].apply(ast.literal_eval)
df["y"] = df["y"].apply(ast.literal_eval)
# 批量插入数据
for index, row in df.iterrows():
insert_query = """
INSERT INTO pipe_risk_probability
(pipeID, pipeage, risk_probability_now, x, y)
VALUES (%s, %s, %s, %s, %s)
"""
cur.execute(
insert_query,
(
row["pipeID"],
row["pipeage"],
row["risk_probability_now"],
row["x"], # 直接传递列表
row["y"], # 同上
),
)
conn.commit()
print("风险评估结果导入成功")
except Exception as e:
print(f"导入时出错：{e}")
def query_burst_detection_scheme_detail(name: str, scheme_name: str) -> dict[str, Any]:
return _typed_run_detail(name, scheme_name, "burst_detection")
app/services/sensor_placement.py
+43 -30
View File
@@ -1,6 +1,7 @@
from datetime import datetime
from io import BytesIO
from typing import Any
from uuid import UUID
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
@@ -8,7 +9,7 @@ from openpyxl.worksheet.worksheet import Worksheet
from openpyxl.utils import get_column_letter
from pyproj import Transformer
from app.native import wndb
from app.infra.db.postgresql import sensor_placement as sensor_placement_repository
class SensorPlacementNotFoundError(LookupError):
@@ -61,7 +62,9 @@ def _sensor_points(
network: str,
sensor_location: list[str],
) -> list[dict[str, Any]]:
nodes = wndb.get_sensor_placement_nodes(network, sensor_location)
nodes = sensor_placement_repository.get_sensor_placement_nodes(
network, sensor_location
)
by_id = {str(node["node_id"]): node for node in nodes}
missing = [node_id for node_id in sensor_location if node_id not in by_id]
if missing:
@@ -113,49 +116,59 @@ def validate_sensor_placement_nodes(
_sensor_points(network, _normalize_locations(sensor_location))
def get_sensor_placement_scheme(network: str, scheme_id: int) -> dict[str, Any]:
scheme = wndb.get_sensor_placement(network, scheme_id)
if scheme is None:
raise SensorPlacementNotFoundError("监测点方案不存在")
def get_sensor_placement_run(network: str, run_id: UUID) -> dict[str, Any]:
run = sensor_placement_repository.get_sensor_placement(network, run_id)
if run is None:
raise SensorPlacementNotFoundError("监测点优化运行不存在")
locations = [str(item) for item in (scheme.get("sensor_location") or [])]
locations = [str(item) for item in (run.get("sensor_locations") or [])]
return {
**scheme,
"sensor_number": len(locations),
"sensor_location": locations,
**run,
"sensor_count": len(locations),
"sensor_locations": locations,
"sensor_points": _sensor_points(network, locations),
}
def update_sensor_placement_scheme(
def list_sensor_placement_runs(network: str) -> list[dict[str, Any]]:
return [
{
**run,
"sensor_points": _sensor_points(network, run["sensor_locations"]),
}
for run in sensor_placement_repository.get_all_sensor_placements(network)
]
def update_sensor_placement_run(
network: str,
scheme_id: int,
run_id: UUID,
*,
expected_sensor_location: list[str],
sensor_location: list[str],
expected_sensor_locations: list[str],
sensor_locations: list[str],
) -> dict[str, Any]:
expected = _normalize_locations(expected_sensor_location)
next_locations = _normalize_locations(sensor_location)
expected = _normalize_locations(expected_sensor_locations)
next_locations = _normalize_locations(sensor_locations)
_sensor_points(network, next_locations)
updated = wndb.update_sensor_placement(
updated = sensor_placement_repository.update_sensor_placement(
network,
scheme_id,
expected_sensor_location=expected,
sensor_location=next_locations,
run_id,
expected_sensor_locations=expected,
sensor_locations=next_locations,
)
if updated is None:
if wndb.get_sensor_placement(network, scheme_id) is None:
raise SensorPlacementNotFoundError("监测点方案不存在")
raise SensorPlacementConflictError("方案已被其他用户修改，请重新加载")
return get_sensor_placement_scheme(network, scheme_id)
if sensor_placement_repository.get_sensor_placement(network, run_id) is None:
raise SensorPlacementNotFoundError("监测点优化运行不存在")
raise SensorPlacementConflictError("运行结果已被其他用户修改，请重新加载")
return get_sensor_placement_run(network, run_id)
def can_edit_sensor_placement(user: Any, scheme: dict[str, Any]) -> bool:
def can_edit_sensor_placement(user: Any, run: dict[str, Any]) -> bool:
return bool(
getattr(user, "is_superuser", False)
or getattr(user, "role", None) == "admin"
or getattr(user, "username", None) == scheme.get("username")
or getattr(user, "username", None) == run.get("created_by")
)
@@ -174,16 +187,16 @@ def _populate_info_sheet(
location_count: int,
is_draft: bool,
) -> None:
created_at = scheme["create_time"]
created_at = scheme["created_at"]
if isinstance(created_at, datetime):
created_at = created_at.isoformat(timespec="minutes")
rows = [
("项目", network),
("方案名称", scheme["scheme_name"]),
("运行名称", scheme["name"]),
("监测点数量", location_count),
("最小管径", scheme["min_diameter"]),
("创建人", scheme["username"]),
("创建人", scheme["created_by"]),
("创建时间", created_at),
("导出时间", datetime.now().astimezone().isoformat(timespec="minutes")),
("文档状态", "未保存草稿" if is_draft else "当前方案"),
@@ -245,7 +258,7 @@ def build_sensor_placement_workbook(
) -> BytesIO:
locations = _normalize_locations(sensor_location)
points = _sensor_points(network, locations)
is_draft = locations != list(scheme["sensor_location"])
is_draft = locations != list(scheme["sensor_locations"])
workbook = Workbook()
info_sheet = workbook.active
app/services/simulation.py
+81 -883
View File
File diff suppressed because it is too large Load Diff
app/services/simulation_ops.py
-6
View File
@@ -44,8 +44,6 @@ def project_management(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
@@ -96,8 +94,6 @@ def scheduling_simulation(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
@@ -188,8 +184,6 @@ def daily_scheduling_simulation(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
app/services/tjnetwork.py
+281 -1320
View File
File diff suppressed because it is too large Load Diff