refactor(backend)!: separate algorithm and data layers

Reorganize algorithm packages by business responsibility, move orchestration into services, and keep database access behind pooled repositories.

Harden analysis API validation, remove unsafe legacy simulation endpoints, and add regression and architecture boundary coverage.

BREAKING CHANGE: legacy algorithm module paths and obsolete simulation endpoints are removed.
This commit is contained in:
2026-09-04 17:30:55 +08:00
parent 9b095c7439
commit 5966d039de
91 changed files with 1418 additions and 4020 deletions
+6 -1
View File
@@ -4,6 +4,7 @@ from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Body
from pydantic import BaseModel, Field
from starlette.concurrency import run_in_threadpool
from app.auth.keycloak_dependencies import get_current_keycloak_username
from app.services.burst_detection import (
@@ -73,6 +74,10 @@ async def detect_burst(
HTTPException: 当处理过程中发生错误时
"""
try:
return run_burst_detection(**data.model_dump(), username=username)
return await run_in_threadpool(
run_burst_detection,
**data.model_dump(),
username=username,
)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc))
+6 -1
View File
@@ -6,6 +6,7 @@ from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Body
from pydantic import BaseModel, Field
from starlette.concurrency import run_in_threadpool
from app.auth.keycloak_dependencies import get_current_keycloak_username
from app.services.burst_location import (
@@ -63,6 +64,10 @@ async def locate_burst(
HTTPException: 当数据类型或值不正确时
"""
try:
return run_burst_location_by_network(**data.model_dump(), username=username)
return await run_in_threadpool(
run_burst_location_by_network,
**data.model_dump(),
username=username,
)
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc))
+33 -14
View File
@@ -3,34 +3,49 @@ from typing import Any
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Body
from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field
from starlette.concurrency import run_in_threadpool
from app.auth.keycloak_dependencies import get_current_keycloak_username
from app.services.leakage_identifier import (
from app.services.dma_leakage_estimation import (
run_leakage_identification,
)
router = APIRouter()
DEFAULT_N_WORKERS = max(1, min((os.cpu_count() or 1) - 1, 4))
MAX_POPULATION_SIZE = 1_000
MAX_GENERATIONS = 1_000
MAX_DURATION_HOURS = 168
class LeakageIdentifyRequest(BaseModel):
"""漏损识别请求模型"""
model_config = ConfigDict(extra="forbid")
network: str = Field(..., description="管网名称(或数据库名称)")
observed_pressure_data: str | dict[str, list[Any]] | list[dict[str, Any]] | None = Field(
None, description="观测的压力数据"
observed_pressure_data: dict[str, list[Any]] | list[dict[str, Any]] | None = (
Field(None, description="观测的压力数据;文件路径不属于公共 API 输入")
)
start_time: float = Field(0, description="起始时间(小时)")
duration: float = Field(24, description="持续时间(小时)")
timestep: float = Field(5, description="时间步长(分钟")
q_sum: float = Field(0.2, description="总流量(m3/s")
start_time: float = Field(0, ge=0, description="起始时间(小时)")
duration: float = Field(
24, gt=0, le=MAX_DURATION_HOURS, description="持续时间(小时"
)
timestep: float = Field(5, gt=0, le=1440, description="时间步长(分钟)")
q_sum: float = Field(0.2, ge=0, description="总流量(m3/s")
q_sum_unit: str = Field("m3/s", description="流量单位")
output_dir: str = Field("db_inp", description="输出目录")
pop_size: int = Field(50, description="种群大小")
max_gen: int = Field(100, description="最大代数")
n_workers: int = Field(DEFAULT_N_WORKERS, description="工作线程")
pop_size: int = Field(
50, ge=2, le=MAX_POPULATION_SIZE, description="种群大小"
)
max_gen: int = Field(100, ge=1, le=MAX_GENERATIONS, description="最大代")
n_workers: int = Field(
DEFAULT_N_WORKERS,
ge=1,
le=DEFAULT_N_WORKERS,
description="工作进程数",
)
output_flow_unit: str = Field("m3/s", description="输出流量单位")
dma_count: int | None = Field(None, description="DMA区域数量")
dma_count: int | None = Field(None, ge=1, description="DMA区域数量")
scada_start: datetime | None = Field(None, description="SCADA数据起始时间")
scada_end: datetime | None = Field(None, description="SCADA数据结束时间")
sensor_nodes: list[str] | None = Field(None, description="传感器节点列表")
@@ -63,6 +78,10 @@ async def identify_leakage(
HTTPException: 当处理过程中发生错误时
"""
try:
return run_leakage_identification(**data.model_dump(), username=username)
return await run_in_threadpool(
run_leakage_identification,
**data.model_dump(),
username=username,
)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc))
+8 -10
View File
@@ -7,10 +7,6 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
from fastapi.responses import StreamingResponse
from starlette.concurrency import run_in_threadpool
from app.algorithms.sensor import (
pressure_sensor_placement_kmeans,
pressure_sensor_placement_sensitivity,
)
from app.auth.metadata_dependencies import get_current_metadata_user
from app.auth.project_dependencies import (
ProjectContext,
@@ -34,6 +30,8 @@ from app.services.sensor_placement import (
get_sensor_placement_candidate,
get_sensor_placement_run,
list_sensor_placement_runs,
optimize_sensor_placement_by_kmeans,
optimize_sensor_placement_by_sensitivity,
update_sensor_placement_run,
)
@@ -136,18 +134,18 @@ async def optimize_sensor_placement_scheme(
network = _project_network(payload.network, project_context)
_require_project_write(project_context)
optimizer = (
pressure_sensor_placement_sensitivity
optimize_sensor_placement_by_sensitivity
if payload.method == "sensitivity"
else pressure_sensor_placement_kmeans
else optimize_sensor_placement_by_kmeans
)
try:
created = await run_in_threadpool(
optimizer,
name=network,
scheme_name=payload.run_name,
sensor_number=payload.sensor_count,
project_code=network,
run_name=payload.run_name,
sensor_count=payload.sensor_count,
min_diameter=payload.min_diameter,
username=current_user.username,
created_by=current_user.username,
)
run = get_sensor_placement_run(network, created["run_id"])
return {**run, "can_edit": True}
+14 -263
View File
@@ -1,33 +1,22 @@
from typing import Any, List, Literal, Optional
from datetime import datetime, timedelta
import json
import threading
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from fastapi.responses import PlainTextResponse
from app.auth.keycloak_dependencies import get_current_keycloak_username
import app.services.simulation as simulation
from app.services.tjnetwork import (
run_project,
run_project_return_dict,
run_inp,
dump_output,
)
from app.algorithms.simulation.scenarios import (
from app.services.simulation_scenarios import (
burst_analysis,
valve_close_analysis,
flushing_analysis,
contaminant_simulation,
age_analysis,
# scheduling_analysis,
pressure_regulation,
)
from app.services.simulation_ops import (
project_management,
scheduling_simulation,
daily_scheduling_simulation,
)
from app.services.valve_isolation import analyze_valve_isolation
from app.services.time_api import (
from app.domain.time import (
parse_aware_time,
parse_clock_duration_seconds,
parse_utc_time,
@@ -49,57 +38,13 @@ class RunSimulationManuallyByDate(BaseModel):
return value
class BurstAnalysis(BaseModel):
name: str = Field(..., description="管网名称(或数据库名称)")
modify_pattern_start_time: str = Field(..., description="模式修改开始时间 (ISO 8601)")
burst_ID: List[str] | str | None = Field(None, description="爆管节点/管段ID列表")
burst_size: List[float] | float | int | None = Field(None, description="爆管流量大小")
modify_total_duration: int = Field(900, description="模拟总时长 (秒)")
modify_fixed_pump_pattern: Optional[dict[str, list]] = Field(None, description="定速泵模式修改")
modify_variable_pump_pattern: Optional[dict[str, list]] = Field(None, description="变速泵模式修改")
modify_valve_opening: Optional[dict[str, float]] = Field(None, description="阀门开度修改")
scheme_name: Optional[str] = Field(None, description="方案名称")
class SchedulingAnalysis(BaseModel):
network: str = Field(..., description="管网名称(或数据库名称)")
start_time: str = Field(..., description="开始时间")
pump_control: dict = Field(..., description="泵控制策略")
tank_id: str = Field(..., description="水箱ID")
water_plant_output_id: str = Field(..., description="水厂出水ID")
time_delta: Optional[int] = Field(300, description="时间步长 (秒)")
class PressureRegulation(BaseModel):
network: str = Field(..., description="管网名称(或数据库名称)")
start_time: str = Field(..., description="开始时间")
pump_control: dict = Field(..., description="泵控制策略")
tank_init_level: Optional[dict] = Field(None, description="水箱初始水位")
duration: Optional[int] = Field(900, description="持续时间 (秒)")
scheme_name: Optional[str] = Field(None, description="方案名称")
class ProjectManagement(BaseModel):
network: str = Field(..., description="管网名称(或数据库名称)")
start_time: str = Field(..., description="开始时间")
pump_control: dict = Field(..., description="泵控制策略")
tank_init_level: Optional[dict] = Field(None, description="水箱初始水位")
region_demand: Optional[dict] = Field(None, description="区域需水量控制")
class DailySchedulingAnalysis(BaseModel):
network: str = Field(..., description="管网名称(或数据库名称)")
start_time: str = Field(..., description="开始时间")
pump_control: dict = Field(..., description="泵控制策略")
reservoir_id: str = Field(..., description="水库ID")
tank_id: str = Field(..., description="水箱ID")
water_plant_output_id: str = Field(..., description="水厂出水ID")
time_delta: Optional[int] = Field(300, description="时间步长 (秒)")
class PumpFailureState(BaseModel):
time: str = Field(..., description="故障发生时间")
pump_status: dict = Field(..., description="泵状态字典")
scheme_name: str = Field(..., min_length=1, description="方案名称")
def run_simulation_manually_by_date(
@@ -161,41 +106,15 @@ def run_project_return_dict_endpoint(network: str = Query(..., description="管
return run_project_return_dict(network)
# put in inp folder, name without extension
@router.post("/inp-runs", summary="运行INP文件", description="运行指定INP文件格式的管网模型进行水力模拟。INP文件应该放在inp文件夹中,参数为文件名不含扩展名。")
def run_inp_endpoint(network: str = Query(..., description="inp文件名(不含扩展名)")) -> str:
"""
运行INP文件
- **network**: inp文件名(不含扩展名)
从inp文件夹中读取指定的INP文件并运行模拟。
"""
return run_inp(network)
# path is absolute path
@router.get("/outputs", summary="导出模拟输出", description="导出指定路径的模拟输出文件内容。参数应为绝对路径。")
def dump_output_endpoint(output: str = Query(..., description="模拟输出文件的绝对路径")) -> str:
"""
导出模拟输出
- **output**: 模拟输出文件的绝对路径
读取并返回指定路径的模拟输出内容。
"""
return dump_output(output)
# Analysis Endpoints
@router.post("/burst-analyses", summary="爆管分析(高级)", description="高级版本的爆管分析,支持在指定时间点修改泵控制模式和阀门开度,以分析这些改变对爆管影响的作用。支持固定泵和变速泵的独立控制。")
def fastapi_burst_analysis(
network: str = Query(..., description="管网名称(或数据库名称)"),
modify_pattern_start_time: str = Query(..., description="模式修改开始时间(ISO 8601格式)"),
burst_ID: list[str] = Query(..., description="爆管节点/管段ID列表"),
burst_size: list[float] = Query(..., description="对应各爆管点的爆管流量大小列表(L/s)"),
modify_total_duration: int = Query(..., description="模拟总时长(秒)"),
scheme_name: str = Query(..., description="分析方案名称"),
burst_ID: list[str] = Query(..., min_length=1, description="爆管节点/管段ID列表"),
burst_size: list[float] = Query(..., min_length=1, description="对应各爆管点的爆管流量大小列表(L/s)"),
modify_total_duration: int = Query(..., gt=0, description="模拟总时长(秒)"),
scheme_name: str = Query(..., min_length=1, description="分析方案名称"),
username: str = Depends(get_current_keycloak_username),
) -> str:
"""
@@ -210,6 +129,11 @@ def fastapi_burst_analysis(
支持在指定时间修改泵控制模式和阀门开度。
"""
if len(burst_ID) != len(burst_size):
raise HTTPException(
status_code=422,
detail="burst_id 与 burst_size 的数量必须一致",
)
burst_analysis(
name=network,
modify_pattern_start_time=modify_pattern_start_time,
@@ -442,48 +366,6 @@ def fastapi_contaminant_simulation(
return result or "success"
@router.post("/water-age-analyses", response_class=PlainTextResponse, summary="水龄分析(高级)", description="高级版本的水龄分析,在指定时间点进行分析,支持自定义模拟持续时间。返回纯文本格式的分析结果。")
def fastapi_age_analysis(
network: str = Query(..., description="管网名称(或数据库名称)"),
start_time: str = Query(..., description="分析开始时间(ISO 8601格式)"),
duration: int = Query(..., description="模拟持续时间(秒)"),
) -> str:
"""
水龄分析(高级版本)
- **network**: 管网名称(或数据库名称)
- **start_time**: 分析开始时间
- **duration**: 模拟持续时间(秒)
分析指定时间段内管网中各节点的水体停留时间。
"""
result = age_analysis(network, start_time, duration)
return result or "success"
# @router.get("/schedulinganalysis/")
# async def scheduling_analysis_endpoint(network: str):
# return scheduling_analysis(network)
@router.post("/pressure-regulation-calculations", summary="压力调节(基础)", description="对管网的压力进行调节分析,通过控制泵的运行来维持目标节点的目标压力。此为基础版本。")
def pressure_regulation_endpoint(
network: str = Query(..., description="管网名称(或数据库名称)"),
target_node: str = Query(..., description="目标节点ID"),
target_pressure: float = Query(..., description="目标压力值(kPa"),
):
"""
压力调节(基础版本)
- **network**: 管网名称(或数据库名称)
- **target_node**: 目标节点ID
- **target_pressure**: 目标压力值(kPa
通过泵控制维持目标节点的压力。
"""
return pressure_regulation(network, target_node, target_pressure)
@router.post("/pressure-regulation-analyses", summary="压力调节(高级)", description="高级版本的压力调节分析,通过JSON请求体提供详细的控制参数,包括固定泵和变速泵的独立控制、水箱初始水位等。")
def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description="压力调节控制参数")) -> str:
"""
@@ -495,7 +377,7 @@ def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description
- **pump_control**: 泵控制策略字典
- **tank_init_level**: 水箱初始水位字典(可选)
- **duration**: 模拟持续时间(秒,可选,默认900)
- **scheme_name**: 控制方案名称(可选)
- **scheme_name**: 控制方案名称
支持固定泵和变速泵的独立控制。
"""
@@ -525,137 +407,6 @@ def fastapi_pressure_regulation(data: PressureRegulation = Body(..., description
return "success"
@router.post("/project-managements", summary="项目管理(高级)", description="高级版本的项目管理,通过JSON请求体提供详细的控制参数,包括泵控制策略、水箱初始水位和区域需水量控制。")
def fastapi_project_management(data: ProjectManagement = Body(..., description="项目管理控制参数")) -> str:
"""
项目管理(高级版本)
请求体参数:
- **network**: 管网名称(或数据库名称)
- **start_time**: 管理开始时间
- **pump_control**: 泵控制策略字典
- **tank_init_level**: 水箱初始水位字典(可选)
- **region_demand**: 区域需水量控制字典(可选)
支持多维度的项目管理。
"""
item = data.model_dump()
return project_management(
prj_name=item["network"],
start_datetime=item["start_time"],
pump_control=item["pump_control"],
tank_initial_level_control=item["tank_init_level"],
region_demand_control=item["region_demand"],
)
# @router.get("/dailyschedulinganalysis/")
# async def daily_scheduling_analysis_endpoint(network: str):
# return daily_scheduling_analysis(network)
@router.post("/scheduling-analyses", summary="排程分析", description="对管网的供水排程进行分析,优化泵的运行时间和出水流量,平衡水厂出水、水箱进出水,满足用户需求。")
def fastapi_scheduling_analysis(data: SchedulingAnalysis = Body(..., description="排程分析参数")) -> str:
"""
排程分析
请求体参数:
- **network**: 管网名称(或数据库名称)
- **start_time**: 分析开始时间
- **pump_control**: 泵控制策略字典
- **tank_id**: 水箱ID
- **water_plant_output_id**: 水厂出水ID
- **time_delta**: 时间步长(秒,可选,默认300)
用于优化供水排程。
"""
item = data.model_dump()
return scheduling_simulation(
item["network"],
item["start_time"],
item["pump_control"],
item["tank_id"],
item["water_plant_output_id"],
item["time_delta"],
)
@router.post("/daily-scheduling-analyses", summary="日排程分析", description="对管网的每日供水排程进行分析,优化水库、水厂、水箱和用户需求的协调,制定合理的每日排程方案。")
def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis = Body(..., description="日排程分析参数")) -> str:
"""
日排程分析
请求体参数:
- **network**: 管网名称(或数据库名称)
- **start_time**: 分析开始时间
- **pump_control**: 泵控制策略字典
- **reservoir_id**: 水库ID
- **tank_id**: 水箱ID
- **water_plant_output_id**: 水厂出水ID
- **time_delta**: 时间步长(秒,可选,默认300)
用于制定每日供水排程方案。
"""
item = data.model_dump()
return daily_scheduling_simulation(
item["network"],
item["start_time"],
item["pump_control"],
item["reservoir_id"],
item["tank_id"],
item["water_plant_output_id"],
)
# @router.get("/pumpfailure/")
# async def pump_failure_endpoint(network: str, pump_id: str, time: str):
# return pump_failure(network, pump_id, time)
@router.post("/pump-failure-events", summary="泵故障管理", description="记录和管理泵的故障状态,包括故障发生时间和受影响的泵列表。系统将记录故障日志并更新泵状态。")
def fastapi_pump_failure(data: PumpFailureState = Body(..., description="泵故障状态信息")) -> str:
"""
泵故障管理
请求体参数:
- **time**: 故障发生时间
- **pump_status**: 泵状态字典,包含第一阶段和第二阶段泵的故障状态
系统将验证泵信息的有效性并更新故障状态文件。
"""
item = data.model_dump()
with open("./pump_failure_message.txt", "a", encoding="utf-8-sig") as f1:
f1.write("[{}] {}\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), item))
with open("./pump_failure_status.txt", "r", encoding="utf-8-sig") as f2:
lines = f2.readlines()
first_stage_pump_status_dict = json.loads(json.dumps(eval(lines[0])))
second_stage_pump_status_dict = json.loads(json.dumps(eval(lines[-1])))
pump_status_dict = {
"first": first_stage_pump_status_dict,
"second": second_stage_pump_status_dict,
}
status_info = item.copy()
for pump_type in status_info["pump_status"].keys():
if pump_type in pump_status_dict.keys():
if all(
pump_id in pump_status_dict[pump_type].keys()
for pump_id in status_info["pump_status"][pump_type].keys()
):
for pump_id in status_info["pump_status"][pump_type].keys():
pump_status_dict[pump_type][pump_id] = int(
status_info["pump_status"][pump_type][pump_id]
)
else:
return json.dumps("ERROR: Wrong Pump ID")
else:
return json.dumps("ERROR: Wrong Pump Type")
with open("./pump_failure_status.txt", "w", encoding="utf-8-sig") as f2_:
f2_.write(
"{}\n{}".format(pump_status_dict["first"], pump_status_dict["second"])
)
return json.dumps("SUCCESS")
@router.post("/simulation-runs", summary="手动运行日期指定模拟", description="根据指定的开始时间和持续时间,手动运行水力模拟。开始时间必须是显式带时区的 ISO 8601 / RFC3339 时间。")
def fastapi_run_simulation_manually_by_date(
data: RunSimulationManuallyByDate = Body(..., description="模拟运行参数"),
+8 -8
View File
@@ -3,7 +3,7 @@ from datetime import datetime
from psycopg import AsyncConnection
from uuid import UUID
from app.infra.db.timescaledb.composite_queries import CompositeQueries
from app.services.timeseries_analysis import TimeseriesAnalysisService
from .dependencies import get_timescale_connection, get_postgres_connection
router = APIRouter()
@@ -46,7 +46,7 @@ async def get_scada_associated_simulation_data(
)
if run_id is not None:
result = await CompositeQueries.get_scada_associated_analysis_simulation_data(
result = await TimeseriesAnalysisService.get_scada_associated_analysis_simulation_data(
timescale_conn,
postgres_conn,
device_ids_list,
@@ -56,7 +56,7 @@ async def get_scada_associated_simulation_data(
)
else:
result = (
await CompositeQueries.get_scada_associated_realtime_simulation_data(
await TimeseriesAnalysisService.get_scada_associated_realtime_simulation_data(
timescale_conn,
postgres_conn,
device_ids_list,
@@ -116,7 +116,7 @@ async def get_feature_simulation_data(
raise HTTPException(status_code=400, detail="feature_infos cannot be empty")
if run_id is not None:
result = await CompositeQueries.get_analysis_simulation_data(
result = await TimeseriesAnalysisService.get_analysis_simulation_data(
timescale_conn,
feature_infos_list,
start_time,
@@ -124,7 +124,7 @@ async def get_feature_simulation_data(
run_id,
)
else:
result = await CompositeQueries.get_realtime_simulation_data(
result = await TimeseriesAnalysisService.get_realtime_simulation_data(
timescale_conn,
feature_infos_list,
start_time,
@@ -168,7 +168,7 @@ async def get_element_associated_scada_data(
HTTPException: 当查询参数无效时返回400错误,未找到关联数据返回404错误
"""
try:
result = await CompositeQueries.get_element_associated_scada_data(
result = await TimeseriesAnalysisService.get_element_associated_scada_data(
timescale_conn, postgres_conn, element_id, start_time, end_time, use_cleaned
)
if result is None:
@@ -216,7 +216,7 @@ async def clean_scada_data(
if device_ids
else []
)
return await CompositeQueries.clean_scada_data(
return await TimeseriesAnalysisService.clean_scada_data(
timescale_conn, postgres_conn, device_ids_list, start_time, end_time
)
except ValueError as e:
@@ -246,7 +246,7 @@ async def predict_pipeline_health(
HTTPException: 当模型文件不存在返回404错误,其他错误返回400或500错误
"""
try:
return await CompositeQueries.predict_pipeline_health(
return await TimeseriesAnalysisService.predict_pipeline_health(
timescale_conn, postgres_conn, query_time
)
except ValueError as e: