OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(db)!: adopt project-routed pooled databases

Browse Source 
Reorganize WNDB by responsibility and remove legacy scheme endpoints.\n\nRoute analysis and time-series access through project pools, preserve transactional realtime replacement, and refresh GIS materialized views after writes.\n\nAdd database architecture documentation, live pooling coverage, API contract updates, and executable container verification.\n\nBREAKING CHANGE: legacy scheme APIs and flat app.native.wndb module imports are removed.
This commit is contained in:
jiang
2026-08-25 18:35:05 +08:00
parent fdbcc5c033
commit fa188af0b1
181 changed files with 8446 additions and 33546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/timescaledb/internal_queries.py
+41 -80
View File
@@ -2,12 +2,12 @@ from typing import List
from fastapi.logger import logger
from datetime import datetime, timedelta
import psycopg
from psycopg import sql
from psycopg.rows import dict_row
import time
from app.infra.db.project_routing import get_project_timescale_pgconn_string
from app.infra.db.timescaledb.repositories.scheme import SchemeRepository
from app.infra.db.timescaledb.sync_pool import timescale_connection
from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.time_api import parse_utc_time
@@ -25,12 +25,7 @@ class InternalStorage:
"""存储实时模拟结果"""
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
with timescale_connection(db_name) as conn:
RealtimeRepository.store_realtime_simulation_result_sync(
conn, node_result_list, link_result_list, result_start_time
)
@@ -43,9 +38,8 @@ class InternalStorage:
raise # 达到最大重试次数后抛出异常
@staticmethod
def store_scheme_simulation(
scheme_type: str,
scheme_name: str,
def store_analysis_simulation(
run_id,
node_result_list: List[dict],
link_result_list: List[dict],
result_start_time: str,
@@ -54,24 +48,16 @@ class InternalStorage:
db_name: str = None,
max_retries: int = 3,
):
"""存储方案模拟结果"""
"""Store immutable simulation results for one analysis run."""
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
SchemeRepository.store_scheme_simulation_result_sync(
conn,
scheme_type,
scheme_name,
node_result_list,
link_result_list,
result_start_time,
num_periods,
result_timestep_seconds,
with timescale_connection(db_name) as conn:
node_rows, link_rows = AnalysisResultsRepository.prepare_simulation_rows(
node_result_list, link_result_list, result_start_time,
num_periods, result_timestep_seconds or 3600,
)
AnalysisResultsRepository.store_results_sync(
conn, run_id, node_rows, link_rows
)
break # 成功
except Exception as e:
@@ -98,12 +84,7 @@ class InternalQueries:
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
with timescale_connection(db_name) as conn:
rows = ScadaRepository.get_scada_by_ids_time_range_sync(
conn, device_ids, start_time, end_time
)
@@ -139,12 +120,7 @@ class InternalQueries:
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
with timescale_connection(db_name) as conn:
rows = ScadaRepository.get_scada_by_ids_time_range_sync(
conn, device_ids, start_dt, end_dt
)
@@ -184,12 +160,7 @@ class InternalQueries:
)
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
with timescale_connection(db_name) as conn:
return ScadaRepository.get_latest_scada_time_sync(
conn,
device_ids,
@@ -224,20 +195,19 @@ class InternalQueries:
)
@staticmethod
def query_scheme_simulation_by_ids_timerange(
def query_analysis_simulation_by_ids_timerange(
element_ids: List[str],
start_time: str | datetime,
end_time: str | datetime,
element_type: str,
field: str,
scheme_type: str,
scheme_name: str,
run_id,
db_name: str = None,
max_retries: int = 3,
) -> dict[str, list[dict]]:
"""查询方案模拟结果，返回 {id: [{time, value}, ...]}。"""
"""Query one analysis run, returning {id: [{time, value}, ...]}."""
return InternalQueries._query_simulation_by_ids_timerange(
schema_name="scheme",
schema_name="analysis",
element_ids=element_ids,
start_time=start_time,
end_time=end_time,
@@ -245,8 +215,7 @@ class InternalQueries:
field=field,
db_name=db_name,
max_retries=max_retries,
scheme_type=scheme_type,
scheme_name=scheme_name,
run_id=run_id,
)
@staticmethod
@@ -260,8 +229,7 @@ class InternalQueries:
field: str,
db_name: str = None,
max_retries: int = 3,
scheme_type: str | None = None,
scheme_name: str | None = None,
run_id=None,
) -> dict[str, list[dict]]:
normalized_element_ids = list(
dict.fromkeys(
@@ -275,51 +243,44 @@ class InternalQueries:
start_dt = parse_utc_time(start_time, field_name="start_time")
end_dt = parse_utc_time(end_time, field_name="end_time")
table_name, valid_fields = InternalQueries._resolve_simulation_table(element_type)
table_name, id_column, valid_fields = InternalQueries._resolve_simulation_table(element_type)
if field not in valid_fields:
raise ValueError(f"Invalid field for {element_type}: {field}")
if schema_name not in {"realtime", "scheme"}:
if schema_name not in {"realtime", "analysis"}:
raise ValueError(f"Unsupported schema_name: {schema_name}")
if schema_name == "scheme" and (not scheme_type or not scheme_name):
raise ValueError("scheme 查询必须提供 scheme_type 和 scheme_name。")
if schema_name == "analysis" and run_id is None:
raise ValueError("analysis query requires run_id")
for attempt in range(max_retries):
try:
conn_string = (
get_project_timescale_pgconn_string(db_name=db_name)
if db_name
else get_project_timescale_pgconn_string()
)
with psycopg.Connection.connect(conn_string) as conn:
with timescale_connection(db_name) as conn:
with conn.cursor(row_factory=dict_row) as cur:
if schema_name == "scheme":
if schema_name == "analysis":
query = sql.SQL(
"SELECT btrim(id::text) AS id, time, {} FROM {}.{} "
"WHERE scheme_type = %s AND scheme_name = %s "
"AND time >= %s AND time <= %s AND btrim(id::text) = ANY(%s)"
"SELECT btrim({}::text) AS id, time, {} FROM {}.{} "
"WHERE run_id = %s AND time >= %s AND time <= %s "
"AND btrim({}::text) = ANY(%s)"
).format(
sql.Identifier(id_column),
sql.Identifier(field),
sql.Identifier(schema_name),
sql.Identifier(table_name),
sql.Identifier(id_column),
)
cur.execute(
query,
(
scheme_type,
scheme_name,
start_dt,
end_dt,
normalized_element_ids,
),
(run_id, start_dt, end_dt, normalized_element_ids),
)
else:
query = sql.SQL(
"SELECT btrim(id::text) AS id, time, {} FROM {}.{} "
"WHERE time >= %s AND time <= %s AND btrim(id::text) = ANY(%s)"
"SELECT btrim({}::text) AS id, time, {} FROM {}.{} "
"WHERE time >= %s AND time <= %s AND btrim({}::text) = ANY(%s)"
).format(
sql.Identifier(id_column),
sql.Identifier(field),
sql.Identifier(schema_name),
sql.Identifier(table_name),
sql.Identifier(id_column),
)
cur.execute(query, (start_dt, end_dt, normalized_element_ids))
rows = cur.fetchall()
@@ -342,12 +303,12 @@ class InternalQueries:
raise
@staticmethod
def _resolve_simulation_table(element_type: str) -> tuple[str, set[str]]:
def _resolve_simulation_table(element_type: str) -> tuple[str, str, set[str]]:
normalized_type = element_type.lower()
if normalized_type == "node":
return "node_simulation", {"actual_demand", "total_head", "pressure", "quality"}
return "node_results", "node_id", {"actual_demand", "total_head", "pressure", "quality"}
if normalized_type == "link":
return "link_simulation", {
return "link_results", "link_id", {
"flow",
"friction",
"headloss",