拆分online_Analysis.py文件

This commit is contained in:
2026-01-26 17:22:06 +08:00
parent f3665798b7
commit 7c9667822f
10 changed files with 1597 additions and 1573 deletions

View File

@@ -0,0 +1,32 @@
from app.services.network_import import network_update, submit_scada_info
from app.services.scheme_management import (
create_user,
delete_user,
scheme_name_exists,
store_scheme_info,
delete_scheme_info,
query_scheme_list,
upload_shp_to_pg,
submit_risk_probability_result,
)
from app.services.simulation_ops import (
project_management,
scheduling_simulation,
daily_scheduling_simulation,
)
__all__ = [
"network_update",
"submit_scada_info",
"create_user",
"delete_user",
"scheme_name_exists",
"store_scheme_info",
"delete_scheme_info",
"query_scheme_list",
"upload_shp_to_pg",
"submit_risk_probability_result",
"project_management",
"scheduling_simulation",
"daily_scheduling_simulation",
]

View File

@@ -0,0 +1,197 @@
import csv
import os
import chardet
import psycopg
from psycopg import sql
import app.services.project_info as project_info
from app.native.api.postgresql_info import get_pgconn_string
from app.services.tjnetwork import read_inp
############################################################
# network_update 10
############################################################
def network_update(file_path: str) -> None:
"""
更新pg数据库中的inp文件
:param file_path: inp文件
:return:
"""
read_inp("szh", file_path)
csv_path = "./history_pattern_flow.csv"
# # 检查文件是否存在
# if os.path.exists(csv_path):
# print(f"history_patterns_flows文件存在开始处理...")
#
# # 读取 CSV 文件
# df = pd.read_csv(csv_path)
#
# # 连接到 PostgreSQL 数据库(这里是数据库 "bb"
# with psycopg.connect("dbname=bb host=127.0.0.1") as conn:
# with conn.cursor() as cur:
# for index, row in df.iterrows():
# # 直接将数据插入,不进行唯一性检查
# insert_sql = sql.SQL("""
# INSERT INTO history_patterns_flows (id, factor, flow)
# VALUES (%s, %s, %s);
# """)
# # 将数据插入数据库
# cur.execute(insert_sql, (row['id'], row['factor'], row['flow']))
# conn.commit()
# print("数据成功导入到 'history_patterns_flows' 表格。")
# else:
# print(f"history_patterns_flows文件不存在。")
# 检查文件是否存在
if os.path.exists(csv_path):
print(f"history_patterns_flows文件存在开始处理...")
# 连接到 PostgreSQL 数据库(这里是数据库 "bb"
with psycopg.connect(f"dbname={project_info.name} host=127.0.0.1") as conn:
with conn.cursor() as cur:
with open(csv_path, newline="", encoding="utf-8-sig") as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
# 直接将数据插入,不进行唯一性检查
insert_sql = sql.SQL(
"""
INSERT INTO history_patterns_flows (id, factor, flow)
VALUES (%s, %s, %s);
"""
)
# 将数据插入数据库
cur.execute(insert_sql, (row["id"], row["factor"], row["flow"]))
conn.commit()
print("数据成功导入到 'history_patterns_flows' 表格。")
else:
print(f"history_patterns_flows文件不存在。")
def submit_scada_info(name: str, coord_id: str) -> None:
"""
将scada信息表导入pg数据库
:param name: 项目名称(数据库名称)
:param coord_id: 坐标系的id如4326根据原始坐标信息输入
:return:
"""
scada_info_path = "./scada_info.csv"
# 检查文件是否存在
if os.path.exists(scada_info_path):
print(f"scada_info文件存在开始处理...")
# 自动检测文件编码
with open(scada_info_path, "rb") as file:
raw_data = file.read()
detected = chardet.detect(raw_data)
file_encoding = detected["encoding"]
print(f"检测到的文件编码:{file_encoding}")
try:
# 动态替换数据库名称
conn_string = get_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库(这里是数据库 "bb"
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 检查 scada_info 表是否为空
cur.execute("SELECT COUNT(*) FROM scada_info;")
count = cur.fetchone()[0]
if count > 0:
print("scada_info表中已有数据正在清空记录...")
cur.execute("DELETE FROM scada_info;")
print("表记录已清空。")
with open(
scada_info_path, newline="", encoding=file_encoding
) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
# 将CSV单元格值为空的字段转换为 None
cleaned_row = {
key: (value if value.strip() else None)
for key, value in row.items()
}
# 处理 associated_source_outflow_id 列动态变化
associated_columns = [
f"associated_source_outflow_id{i}" for i in range(1, 21)
]
associated_values = [
(
cleaned_row.get(col).strip()
if cleaned_row.get(col)
and cleaned_row.get(col).strip()
else None
)
for col in associated_columns
]
# 将 X_coor 和 Y_coor 转换为 geometry 类型
x_coor = (
float(cleaned_row["X_coor"])
if cleaned_row["X_coor"]
else None
)
y_coor = (
float(cleaned_row["Y_coor"])
if cleaned_row["Y_coor"]
else None
)
coord = (
f"SRID={coord_id};POINT({x_coor} {y_coor})"
if x_coor and y_coor
else None
)
# 准备插入 SQL 语句
insert_sql = sql.SQL(
"""
INSERT INTO scada_info (
id, type, associated_element_id, associated_pattern,
associated_pipe_flow_id, {associated_columns},
API_query_id, transmission_mode, transmission_frequency,
reliability, X_coor, Y_coor, coord
)
VALUES (
%s, %s, %s, %s, %s, {associated_placeholders},
%s, %s, %s, %s, %s, %s, %s
);
"""
).format(
associated_columns=sql.SQL(", ").join(
sql.Identifier(col) for col in associated_columns
),
associated_placeholders=sql.SQL(", ").join(
sql.Placeholder() for _ in associated_columns
),
)
# 将数据插入数据库
cur.execute(
insert_sql,
(
cleaned_row["id"],
cleaned_row["type"],
cleaned_row["associated_element_id"],
cleaned_row.get("associated_pattern"),
cleaned_row.get("associated_pipe_flow_id"),
*associated_values,
cleaned_row.get("API_query_id"),
cleaned_row["transmission_mode"],
cleaned_row["transmission_frequency"],
cleaned_row["reliability"],
x_coor,
y_coor,
coord,
),
)
conn.commit()
print("数据成功导入到 'scada_info' 表格。")
except Exception as e:
print(f"导入时出错:{e}")
else:
print(f"scada_info文件不存在。")

View File

@@ -0,0 +1,266 @@
import ast
import json
import geopandas as gpd
import pandas as pd
import psycopg
from sqlalchemy import create_engine
from app.native.api.postgresql_info import get_pgconn_string
# 2025/03/23
def create_user(name: str, username: str, password: str):
"""
创建用户
:param name: 数据库名称
:param username: 用户名
:param password: 密码
:return:
"""
try:
# 动态替换数据库名称
conn_string = get_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库(这里是数据库 "bb"
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO users (username, password) VALUES (%s, %s)",
(username, password),
)
# 提交事务
conn.commit()
print("新用户创建成功!")
except Exception as e:
print(f"创建用户出错:{e}")
# 2025/03/23
def delete_user(name: str, username: str):
"""
删除用户
:param name: 数据库名称
:param username: 用户名
:return:
"""
try:
# 动态替换数据库名称
conn_string = get_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库(这里是数据库 "bb"
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute("DELETE FROM users WHERE username = %s", (username,))
conn.commit()
print(f"用户 {username} 删除成功!")
except Exception as e:
print(f"删除用户出错:{e}")
# 2025/03/23
def scheme_name_exists(name: str, scheme_name: str) -> bool:
"""
判断传入的 scheme_name 是否已存在于 scheme_list 表中,用于输入框判断
:param name: 数据库名称
:param scheme_name: 需要判断的方案名称
:return: 如果存在返回 True否则返回 False
"""
try:
conn_string = get_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT COUNT(*) FROM scheme_list WHERE scheme_name = %s",
(scheme_name,),
)
result = cur.fetchone()
if result is not None and result[0] > 0:
return True
else:
return False
except Exception as e:
print(f"查询 scheme_name 时出错:{e}")
return False
# 2025/03/23
def store_scheme_info(
name: str,
scheme_name: str,
scheme_type: str,
username: str,
scheme_start_time: str,
scheme_detail: dict,
):
"""
将一条方案记录插入 scheme_list 表中
:param name: 数据库名称
:param scheme_name: 方案名称
:param scheme_type: 方案类型
:param username: 用户名(需在 users 表中已存在)
:param scheme_start_time: 方案起始时间(字符串)
:param scheme_detail: 方案详情(字典,会转换为 JSON
:return:
"""
try:
conn_string = get_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
sql = """
INSERT INTO scheme_list (scheme_name, scheme_type, username, scheme_start_time, scheme_detail)
VALUES (%s, %s, %s, %s, %s)
"""
# 将字典转换为 JSON 字符串
scheme_detail_json = json.dumps(scheme_detail)
cur.execute(
sql,
(
scheme_name,
scheme_type,
username,
scheme_start_time,
scheme_detail_json,
),
)
conn.commit()
print("方案信息存储成功!")
except Exception as e:
print(f"存储方案信息时出错:{e}")
# 2025/03/23
def delete_scheme_info(name: str, scheme_name: str) -> None:
"""
从 scheme_list 表中删除指定的方案
:param name: 数据库名称
:param scheme_name: 要删除的方案名称
"""
try:
conn_string = get_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 使用参数化查询删除方案记录
cur.execute(
"DELETE FROM scheme_list WHERE scheme_name = %s", (scheme_name,)
)
conn.commit()
print(f"方案 {scheme_name} 删除成功!")
except Exception as e:
print(f"删除方案时出错:{e}")
# 2025/03/23
def query_scheme_list(name: str) -> list:
"""
查询pg数据库中的scheme_list按照 create_time 降序排列,离现在时间最近的记录排在最前面
:param name: 项目名称(数据库名称)
:return: 返回查询结果的所有行
"""
try:
# 动态替换数据库名称
conn_string = get_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库(这里是数据库 "bb"
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 按 create_time 降序排列
cur.execute("SELECT * FROM scheme_list ORDER BY create_time DESC")
rows = cur.fetchall()
return rows
except Exception as e:
print(f"查询错误:{e}")
# 2025/03/23
def upload_shp_to_pg(name: str, table_name: str, role: str, shp_file_path: str):
"""
将 Shapefile 文件上传到 PostgreSQL 数据库
:param name: 项目名称(数据库名称)
:param table_name: 创建表的名字
:param role: 数据库角色名位于c盘user中查看
:param shp_file_path: shp文件的路径
:return:
"""
try:
# 动态连接到指定的数据库
conn_string = get_pgconn_string(db_name=name)
with psycopg.connect(conn_string) as conn:
# 读取 Shapefile 文件
gdf = gpd.read_file(shp_file_path)
# 检查投影坐标系CRS并确保是 EPSG:4326
if gdf.crs.to_string() != "EPSG:4490":
gdf = gdf.to_crs(epsg=4490)
# 使用 GeoDataFrame 的 .to_postgis 方法将数据写入 PostgreSQL
# 需要在数据库中提前安装 PostGIS 扩展
engine = create_engine(f"postgresql+psycopg2://{role}:@127.0.0.1/{name}")
gdf.to_postgis(
table_name, engine, if_exists="replace", index=True, index_label="id"
)
print(
f"Shapefile 文件成功上传到 PostgreSQL 数据库 '{name}' 的表 '{table_name}'."
)
except Exception as e:
print(f"上传 Shapefile 到 PostgreSQL 时出错:{e}")
def submit_risk_probability_result(name: str, result_file_path: str) -> None:
"""
将管网风险评估结果导入pg数据库
:param name: 项目名称(数据库名称)
:param result_file_path: 结果文件路径
:return:
"""
# 自动检测文件编码
# with open({result_file_path}, 'rb') as file:
# raw_data = file.read()
# detected = chardet.detect(raw_data)
# file_encoding = detected['encoding']
# print(f"检测到的文件编码:{file_encoding}")
try:
# 动态替换数据库名称
conn_string = get_pgconn_string(db_name=name)
# 连接到 PostgreSQL 数据库
with psycopg.connect(conn_string) as conn:
with conn.cursor() as cur:
# 检查 scada_info 表是否为空
cur.execute("SELECT COUNT(*) FROM pipe_risk_probability;")
count = cur.fetchone()[0]
if count > 0:
print("pipe_risk_probability表中已有数据正在清空记录...")
cur.execute("DELETE FROM pipe_risk_probability;")
print("表记录已清空。")
# 读取Excel并转换x/y列为列表
df = pd.read_excel(result_file_path, sheet_name="Sheet1")
df["x"] = df["x"].apply(ast.literal_eval)
df["y"] = df["y"].apply(ast.literal_eval)
# 批量插入数据
for index, row in df.iterrows():
insert_query = """
INSERT INTO pipe_risk_probability
(pipeID, pipeage, risk_probability_now, x, y)
VALUES (%s, %s, %s, %s, %s)
"""
cur.execute(
insert_query,
(
row["pipeID"],
row["pipeage"],
row["risk_probability_now"],
row["x"], # 直接传递列表
row["y"], # 同上
),
)
conn.commit()
print("风险评估结果导入成功")
except Exception as e:
print(f"导入时出错:{e}")

View File

@@ -0,0 +1,233 @@
import json
from datetime import datetime
from math import pi
import pytz
from app.algorithms.api_ex.run_simulation import run_simulation_ex
from app.native.api.project import copy_project
from app.services.epanet.epanet import Output
from app.services.tjnetwork import *
############################################################
# project management 07 ***暂时不使用,与业务需求无关***
############################################################
def project_management(
prj_name,
start_datetime,
pump_control,
tank_initial_level_control=None,
region_demand_control=None,
) -> str:
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Analysis."
)
new_name = f"project_management_{prj_name}"
if have_project(new_name):
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
# if is_project_open(prj_name):
# close_project(prj_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Opening Database."
)
open_project(new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Database Loading OK."
)
result = run_simulation_ex(
name=new_name,
simulation_type="realtime",
start_datetime=start_datetime,
duration=86400,
pump_control=pump_control,
tank_initial_level_control=tank_initial_level_control,
region_demand_control=region_demand_control,
downloading_prohibition=True,
)
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
return result
############################################################
# scheduling analysis 08 ***暂时不使用,与业务需求无关***
############################################################
def scheduling_simulation(
prj_name, start_time, pump_control, tank_id, water_plant_output_id, time_delta=300
) -> str:
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Analysis."
)
new_name = f"scheduling_{prj_name}"
if have_project(new_name):
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
# if is_project_open(prj_name):
# close_project(prj_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Opening Database."
)
open_project(new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Database Loading OK."
)
run_simulation_ex(
new_name, "realtime", start_time, duration=0, pump_control=pump_control
)
if not is_project_open(new_name):
open_project(new_name)
tank = get_tank(new_name, tank_id) # 水塔信息
tank_floor_space = pi * pow(tank["diameter"] / 2, 2) # 水塔底面积(m^2)
tank_init_level = tank["init_level"] # 水塔初始水位(m)
tank_pipes_id = tank["links"] # pipes list
tank_pipe_flow_direction = (
{}
) # 管道流向修正系数, 水塔为下游节点时为1, 水塔为上游节点时为-1
for pipe_id in tank_pipes_id:
if get_pipe(new_name, pipe_id)["node2"] == tank_id: # 水塔为下游节点
tank_pipe_flow_direction[pipe_id] = 1
else:
tank_pipe_flow_direction[pipe_id] = -1
output = Output("./temp/{}.db.out".format(new_name))
node_results = (
output.node_results()
) # [{'node': str, 'result': [{'pressure': float}]}]
water_plant_output_pressure = 0
for node_result in node_results:
if node_result["node"] == water_plant_output_id: # 水厂出水压力(m)
water_plant_output_pressure = node_result["result"][-1]["pressure"]
water_plant_output_pressure /= 100 # 预计水厂出水压力(Mpa)
pipe_results = output.link_results() # [{'link': str, 'result': [{'flow': float}]}]
tank_inflow = 0
for pipe_result in pipe_results:
for pipe_id in tank_pipes_id: # 遍历与水塔相连的管道
if pipe_result["link"] == pipe_id: # 水塔入流流量(L/s)
tank_inflow += (
pipe_result["result"][-1]["flow"]
* tank_pipe_flow_direction[pipe_id]
)
tank_inflow /= 1000 # 水塔入流流量(m^3/s)
tank_level_delta = tank_inflow * time_delta / tank_floor_space # 水塔水位改变值(m)
tank_level = tank_init_level + tank_level_delta # 预计水塔水位(m)
simulation_results = {
"water_plant_output_pressure": water_plant_output_pressure,
"tank_init_level": tank_init_level,
"tank_level": tank_level,
}
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
return json.dumps(simulation_results)
def daily_scheduling_simulation(
prj_name, start_time, pump_control, reservoir_id, tank_id, water_plant_output_id
) -> str:
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Analysis."
)
new_name = f"daily_scheduling_{prj_name}"
if have_project(new_name):
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
# if is_project_open(prj_name):
# close_project(prj_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Copying Database."
)
# CopyProjectEx()(prj_name, new_name,
# ['operation', 'current_operation', 'restore_operation', 'batch_operation', 'operation_table'])
copy_project(prj_name + "_template", new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Start Opening Database."
)
open_project(new_name)
print(
datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")
+ " -- Database Loading OK."
)
run_simulation_ex(
new_name, "realtime", start_time, duration=86400, pump_control=pump_control
)
if not is_project_open(new_name):
open_project(new_name)
output = Output("./temp/{}.db.out".format(new_name))
node_results = (
output.node_results()
) # [{'node': str, 'result': [{'pressure': float, 'head': float}]}]
water_plant_output_pressure = []
reservoir_level = []
tank_level = []
for node_result in node_results:
if node_result["node"] == water_plant_output_id:
for result in node_result["result"]:
water_plant_output_pressure.append(
result["pressure"] / 100
) # 水厂出水压力(Mpa)
elif node_result["node"] == reservoir_id:
for result in node_result["result"]:
reservoir_level.append(result["head"] - 250.35) # 清水池液位(m)
elif node_result["node"] == tank_id:
for result in node_result["result"]:
tank_level.append(result["pressure"]) # 调节池液位(m)
simulation_results = {
"water_plant_output_pressure": water_plant_output_pressure,
"reservoir_level": reservoir_level,
"tank_level": tank_level,
}
if is_project_open(new_name):
close_project(new_name)
delete_project(new_name)
return json.dumps(simulation_results)