refactor(db)!: adopt project-routed pooled databases
Reorganize WNDB by responsibility and remove legacy scheme endpoints.\n\nRoute analysis and time-series access through project pools, preserve transactional realtime replacement, and refresh GIS materialized views after writes.\n\nAdd database architecture documentation, live pooling coverage, API contract updates, and executable container verification.\n\nBREAKING CHANGE: legacy scheme APIs and flat app.native.wndb module imports are removed.
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg import AsyncConnection
|
||||
|
||||
|
||||
class AnalysisRepository:
|
||||
@staticmethod
|
||||
async def list_runs(conn: AsyncConnection) -> list[dict]:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT run_id, name, run_type, created_by, created_at,
|
||||
started_at, status, parameters
|
||||
FROM analysis.runs
|
||||
ORDER BY created_at DESC, run_id
|
||||
"""
|
||||
)
|
||||
return await cur.fetchall()
|
||||
|
||||
@staticmethod
|
||||
async def get_run(conn: AsyncConnection, run_id: UUID) -> dict | None:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT run_id, name, run_type, created_by, created_at,
|
||||
started_at, status, parameters
|
||||
FROM analysis.runs
|
||||
WHERE run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
@staticmethod
|
||||
async def list_results(
|
||||
conn: AsyncConnection, run_id: UUID, result_type: str | None = None
|
||||
) -> list[dict]:
|
||||
query = """
|
||||
SELECT result_id, run_id, result_type, node_id, link_id,
|
||||
payload, created_at
|
||||
FROM analysis.results
|
||||
WHERE run_id = %s
|
||||
"""
|
||||
params: tuple[UUID] | tuple[UUID, str] = (run_id,)
|
||||
if result_type is not None:
|
||||
query += " AND result_type = %s"
|
||||
params = (run_id, result_type)
|
||||
query += " ORDER BY created_at, result_id"
|
||||
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(query, params)
|
||||
return await cur.fetchall()
|
||||
@@ -1,123 +0,0 @@
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator, Dict, Optional
|
||||
import psycopg_pool
|
||||
from psycopg.rows import dict_row
|
||||
from app.infra.db.project_routing import get_project_pgconn_string
|
||||
|
||||
# Configure logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, db_name=None):
|
||||
self.pool = None
|
||||
self.db_name = db_name
|
||||
self.conninfo = None
|
||||
|
||||
def init_pool(self, db_name=None):
|
||||
"""Initialize the connection pool."""
|
||||
# Use provided db_name, or the one from constructor, or default from config
|
||||
target_db_name = db_name or self.db_name
|
||||
|
||||
# Get connection string, handling default case where target_db_name might be None
|
||||
if target_db_name:
|
||||
conn_string = get_project_pgconn_string(db_name=target_db_name)
|
||||
else:
|
||||
conn_string = get_project_pgconn_string()
|
||||
self.conninfo = conn_string
|
||||
|
||||
try:
|
||||
self.pool = psycopg_pool.AsyncConnectionPool(
|
||||
conninfo=conn_string,
|
||||
min_size=5,
|
||||
max_size=20,
|
||||
open=False, # Don't open immediately, wait for startup
|
||||
kwargs={"row_factory": dict_row}, # Return rows as dictionaries
|
||||
)
|
||||
logger.info(f"PostgreSQL connection pool initialized for database: {target_db_name or 'default'}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize postgresql connection pool: {e}")
|
||||
raise
|
||||
|
||||
async def open(self):
|
||||
if self.pool:
|
||||
await self.pool.open()
|
||||
|
||||
async def close(self):
|
||||
"""Close the connection pool."""
|
||||
if self.pool:
|
||||
await self.pool.close()
|
||||
logger.info("PostgreSQL connection pool closed.")
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_connection(self) -> AsyncGenerator:
|
||||
"""Get a connection from the pool."""
|
||||
if not self.pool:
|
||||
raise Exception("Database pool is not initialized.")
|
||||
|
||||
async with self.pool.connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
# 默认数据库实例
|
||||
db = Database()
|
||||
|
||||
# 缓存不同数据库的实例 - 避免重复创建连接池
|
||||
_database_instances: Dict[str, Database] = {}
|
||||
|
||||
|
||||
def create_database_instance(db_name):
|
||||
"""Create a new Database instance for a specific database."""
|
||||
return Database(db_name=db_name)
|
||||
|
||||
|
||||
async def get_database_instance(db_name: Optional[str] = None) -> Database:
|
||||
"""Get or create a database instance for the specified database name."""
|
||||
if not db_name:
|
||||
return db # 返回默认数据库实例
|
||||
|
||||
expected_conninfo = get_project_pgconn_string(db_name=db_name)
|
||||
existing = _database_instances.get(db_name)
|
||||
if existing is not None and existing.conninfo != expected_conninfo:
|
||||
await existing.close()
|
||||
del _database_instances[db_name]
|
||||
|
||||
if db_name not in _database_instances:
|
||||
# 创建新的数据库实例
|
||||
instance = create_database_instance(db_name)
|
||||
instance.init_pool()
|
||||
await instance.open()
|
||||
_database_instances[db_name] = instance
|
||||
logger.info(f"Created new database instance for: {db_name}")
|
||||
|
||||
return _database_instances[db_name]
|
||||
|
||||
|
||||
async def get_db_connection():
|
||||
"""Dependency for FastAPI to get a database connection."""
|
||||
async with db.get_connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
async def get_database_connection(db_name: Optional[str] = None):
|
||||
"""
|
||||
FastAPI dependency to get database connection with optional database name.
|
||||
使用方法: conn: AsyncConnection = Depends(lambda: get_database_connection("your_db_name"))
|
||||
或在路由函数中: conn: AsyncConnection = Depends(get_database_connection)
|
||||
"""
|
||||
instance = await get_database_instance(db_name)
|
||||
async with instance.get_connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
async def cleanup_database_instances():
|
||||
"""Clean up all database instances (call this on application shutdown)."""
|
||||
for db_name, instance in _database_instances.items():
|
||||
await instance.close()
|
||||
logger.info(f"Closed database instance for: {db_name}")
|
||||
_database_instances.clear()
|
||||
|
||||
# 关闭默认数据库
|
||||
await db.close()
|
||||
logger.info("All database instances cleaned up.")
|
||||
@@ -11,6 +11,10 @@ def _optional_float(value: Any) -> float | None:
|
||||
return float(value) if value is not None else None
|
||||
|
||||
|
||||
def _optional_int(value: Any) -> int | None:
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
class ScadaInfoRepository:
|
||||
"""Read SCADA metadata from the current project's business database."""
|
||||
|
||||
@@ -19,33 +23,34 @@ class ScadaInfoRepository:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT id,
|
||||
type,
|
||||
associated_element_id,
|
||||
SELECT id AS device_id,
|
||||
device_type,
|
||||
node_id,
|
||||
link_id,
|
||||
api_query_id,
|
||||
transmission_mode,
|
||||
transmission_frequency,
|
||||
reliability,
|
||||
x_coor,
|
||||
y_coor
|
||||
FROM public.scada_info
|
||||
x,
|
||||
y
|
||||
FROM gis.scada_devices
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
records = await cur.fetchall()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": str(record["id"]).strip(),
|
||||
"type": str(record["type"]).strip().lower(),
|
||||
"associated_element_id": _optional_text(
|
||||
record["associated_element_id"]
|
||||
),
|
||||
"api_query_id": record["api_query_id"],
|
||||
"device_id": str(record["device_id"]).strip(),
|
||||
"device_type": str(record["device_type"]).strip().lower(),
|
||||
"node_id": _optional_text(record["node_id"]),
|
||||
"link_id": _optional_text(record["link_id"]),
|
||||
"api_query_id": _optional_text(record["api_query_id"]),
|
||||
"transmission_mode": record["transmission_mode"],
|
||||
"transmission_frequency": record["transmission_frequency"],
|
||||
"reliability": _optional_float(record["reliability"]),
|
||||
"x": _optional_float(record["x_coor"]),
|
||||
"y": _optional_float(record["y_coor"]),
|
||||
"reliability": _optional_int(record["reliability"]),
|
||||
"x": _optional_float(record["x"]),
|
||||
"y": _optional_float(record["y"]),
|
||||
}
|
||||
for record in records
|
||||
]
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
from typing import Any
|
||||
|
||||
from app.native.wndb.core.database import read_all, try_read
|
||||
|
||||
|
||||
def get_scada_info_schema(name: str) -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
"device_id": {"type": "str", "optional": False, "readonly": True},
|
||||
"device_type": {"type": "str", "optional": False, "readonly": True},
|
||||
"node_id": {"type": "str", "optional": True, "readonly": True},
|
||||
"link_id": {"type": "str", "optional": True, "readonly": True},
|
||||
"api_query_id": {"type": "str", "optional": True, "readonly": True},
|
||||
"transmission_mode": {"type": "str", "optional": False, "readonly": True},
|
||||
"transmission_frequency": {"type": "str", "optional": False, "readonly": True},
|
||||
"reliability": {"type": "int", "optional": False, "readonly": True},
|
||||
"x": {"type": "float", "optional": True, "readonly": True},
|
||||
"y": {"type": "float", "optional": True, "readonly": True},
|
||||
}
|
||||
|
||||
|
||||
_SELECT = """
|
||||
SELECT device_id, device_type, node_id, link_id, api_query_id,
|
||||
transmission_mode, transmission_frequency, reliability,
|
||||
x, y
|
||||
FROM asset.scada_devices
|
||||
"""
|
||||
|
||||
_SELECT_MATERIALIZED = """
|
||||
SELECT id AS device_id, device_type, node_id, link_id, api_query_id,
|
||||
transmission_mode, transmission_frequency, reliability,
|
||||
x, y
|
||||
FROM gis.scada_devices
|
||||
"""
|
||||
|
||||
|
||||
def _device(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"device_id": str(row["device_id"]),
|
||||
"device_type": str(row["device_type"]),
|
||||
"node_id": str(row["node_id"]) if row["node_id"] is not None else None,
|
||||
"link_id": str(row["link_id"]) if row["link_id"] is not None else None,
|
||||
"api_query_id": (
|
||||
str(row["api_query_id"]) if row["api_query_id"] is not None else None
|
||||
),
|
||||
"transmission_mode": str(row["transmission_mode"]),
|
||||
"transmission_frequency": str(row["transmission_frequency"]),
|
||||
"reliability": int(row["reliability"]),
|
||||
"x": float(row["x"]) if row["x"] is not None else None,
|
||||
"y": float(row["y"]) if row["y"] is not None else None,
|
||||
}
|
||||
|
||||
|
||||
def get_scada_info(name: str, device_id: str) -> dict[str, Any]:
|
||||
row = try_read(
|
||||
name,
|
||||
_SELECT + " WHERE device_id = %s",
|
||||
(device_id,),
|
||||
)
|
||||
return _device(row) if row else {}
|
||||
|
||||
|
||||
def get_all_scada_info(name: str) -> list[dict[str, Any]]:
|
||||
return [
|
||||
_device(row)
|
||||
for row in read_all(name, _SELECT_MATERIALIZED + " ORDER BY device_id")
|
||||
]
|
||||
@@ -1,104 +0,0 @@
|
||||
from typing import List, Optional, Any
|
||||
from psycopg import AsyncConnection
|
||||
|
||||
|
||||
class SchemeRepository:
|
||||
|
||||
@staticmethod
|
||||
async def get_schemes(conn: AsyncConnection) -> List[dict]:
|
||||
"""
|
||||
查询pg数据库中, scheme_list 的所有记录
|
||||
:param conn: 异步数据库连接
|
||||
:return: 包含所有记录的列表, 每条记录为一个字典
|
||||
"""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT scheme_id, scheme_name, scheme_type, username, create_time, scheme_start_time, scheme_detail
|
||||
FROM public.scheme_list
|
||||
"""
|
||||
)
|
||||
records = await cur.fetchall()
|
||||
|
||||
scheme_list = []
|
||||
for record in records:
|
||||
scheme_list.append(
|
||||
{
|
||||
"scheme_id": record["scheme_id"],
|
||||
"scheme_name": record["scheme_name"],
|
||||
"scheme_type": record["scheme_type"],
|
||||
"username": record["username"],
|
||||
"create_time": record["create_time"],
|
||||
"scheme_start_time": record["scheme_start_time"],
|
||||
"scheme_detail": record["scheme_detail"],
|
||||
}
|
||||
)
|
||||
|
||||
return scheme_list
|
||||
|
||||
@staticmethod
|
||||
async def get_burst_locate_results(conn: AsyncConnection) -> List[dict]:
|
||||
"""
|
||||
查询pg数据库中, burst_locate_result 的所有记录
|
||||
:param conn: 异步数据库连接
|
||||
:return: 包含所有记录的列表, 每条记录为一个字典
|
||||
"""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT id, type, burst_incident, leakage, detect_time, locate_result
|
||||
FROM public.burst_locate_result
|
||||
"""
|
||||
)
|
||||
records = await cur.fetchall()
|
||||
|
||||
results = []
|
||||
for record in records:
|
||||
results.append(
|
||||
{
|
||||
"id": record["id"],
|
||||
"type": record["type"],
|
||||
"burst_incident": record["burst_incident"],
|
||||
"leakage": record["leakage"],
|
||||
"detect_time": record["detect_time"],
|
||||
"locate_result": record["locate_result"],
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
async def get_burst_locate_result_by_incident(
|
||||
conn: AsyncConnection, burst_incident: str
|
||||
) -> List[dict]:
|
||||
"""
|
||||
根据 burst_incident 查询爆管定位结果
|
||||
:param conn: 异步数据库连接
|
||||
:param burst_incident: 爆管事件标识
|
||||
:return: 包含匹配记录的列表
|
||||
"""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""
|
||||
SELECT id, type, burst_incident, leakage, detect_time, locate_result
|
||||
FROM public.burst_locate_result
|
||||
WHERE burst_incident = %s
|
||||
""",
|
||||
(burst_incident,),
|
||||
)
|
||||
records = await cur.fetchall()
|
||||
|
||||
results = []
|
||||
for record in records:
|
||||
results.append(
|
||||
{
|
||||
"id": record["id"],
|
||||
"type": record["type"],
|
||||
"burst_incident": record["burst_incident"],
|
||||
"leakage": record["leakage"],
|
||||
"detect_time": record["detect_time"],
|
||||
"locate_result": record["locate_result"],
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,189 @@
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app.native.wndb.core.connection import project_connection
|
||||
|
||||
|
||||
RUN_TYPE = "sensor_placement"
|
||||
RESULT_TYPE = "sensor_placement"
|
||||
|
||||
|
||||
def _placement_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = row.get("payload") if isinstance(row.get("payload"), dict) else {}
|
||||
locations = [str(item) for item in payload.get("sensor_locations", [])]
|
||||
return {
|
||||
"run_id": row["run_id"],
|
||||
"name": row["name"],
|
||||
"sensor_count": len(locations),
|
||||
"min_diameter": int(payload.get("minimum_diameter", 0)),
|
||||
"created_by": row["created_by"],
|
||||
"created_at": row["created_at"],
|
||||
"status": row["status"],
|
||||
"sensor_locations": locations,
|
||||
}
|
||||
|
||||
|
||||
def get_all_sensor_placements(name: str) -> list[dict[str, Any]]:
|
||||
with project_connection(name) as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT r.run_id, r.name, r.created_by, r.created_at, r.status,
|
||||
result.payload
|
||||
FROM analysis.runs AS r
|
||||
JOIN LATERAL (
|
||||
SELECT payload
|
||||
FROM analysis.results
|
||||
WHERE run_id = r.run_id AND result_type = %s
|
||||
ORDER BY created_at DESC, result_id DESC
|
||||
LIMIT 1
|
||||
) AS result ON true
|
||||
WHERE r.run_type = %s
|
||||
ORDER BY r.created_at DESC, r.run_id
|
||||
""",
|
||||
(RESULT_TYPE, RUN_TYPE),
|
||||
)
|
||||
return [_placement_row(row) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def create_sensor_placement(
|
||||
name: str,
|
||||
*,
|
||||
run_name: str,
|
||||
min_diameter: int,
|
||||
created_by: str,
|
||||
sensor_locations: list[str],
|
||||
) -> dict[str, Any]:
|
||||
run_id = uuid4()
|
||||
payload = {
|
||||
"sensor_number": len(sensor_locations),
|
||||
"minimum_diameter": min_diameter,
|
||||
"sensor_locations": sensor_locations,
|
||||
}
|
||||
with project_connection(name) as conn, conn.transaction():
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO analysis.runs
|
||||
(run_id, name, run_type, created_by, started_at, status, parameters)
|
||||
VALUES (%s, %s, %s, %s, now(), 'completed', '{}'::jsonb)
|
||||
RETURNING run_id, name, created_by, created_at, status
|
||||
""",
|
||||
(run_id, run_name, RUN_TYPE, created_by),
|
||||
)
|
||||
created = cur.fetchone()
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO analysis.results (run_id, result_type, payload)
|
||||
VALUES (%s, %s, %s)
|
||||
""",
|
||||
(run_id, RESULT_TYPE, Jsonb(payload)),
|
||||
)
|
||||
if created is None:
|
||||
raise RuntimeError("监测点优化运行写入失败")
|
||||
return _placement_row(dict(created) | {"payload": payload})
|
||||
|
||||
|
||||
def get_sensor_placement(name: str, run_id: UUID) -> dict[str, Any] | None:
|
||||
with project_connection(name) as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT r.run_id, r.name, r.created_by, r.created_at, r.status,
|
||||
result.payload
|
||||
FROM analysis.runs AS r
|
||||
JOIN LATERAL (
|
||||
SELECT payload
|
||||
FROM analysis.results
|
||||
WHERE run_id = r.run_id AND result_type = %s
|
||||
ORDER BY created_at DESC, result_id DESC
|
||||
LIMIT 1
|
||||
) AS result ON true
|
||||
WHERE r.run_id = %s AND r.run_type = %s
|
||||
""",
|
||||
(RESULT_TYPE, run_id, RUN_TYPE),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return _placement_row(row) if row else None
|
||||
|
||||
|
||||
def get_sensor_placement_nodes(
|
||||
name: str,
|
||||
node_ids: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
if not node_ids:
|
||||
return []
|
||||
with project_connection(name) as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
WITH incident_pipe_diameters AS (
|
||||
SELECT node_id, MAX(diameter) AS max_pipe_diameter
|
||||
FROM (
|
||||
SELECT l.start_node_id AS node_id, p.diameter
|
||||
FROM network.pipes AS p
|
||||
JOIN network.links AS l ON l.id = p.link_id
|
||||
WHERE l.start_node_id = ANY(%s)
|
||||
UNION ALL
|
||||
SELECT l.end_node_id AS node_id, p.diameter
|
||||
FROM network.pipes AS p
|
||||
JOIN network.links AS l ON l.id = p.link_id
|
||||
WHERE l.end_node_id = ANY(%s)
|
||||
) AS incident_pipes
|
||||
GROUP BY node_id
|
||||
)
|
||||
SELECT j.node_id,
|
||||
ipd.max_pipe_diameter,
|
||||
j.elevation,
|
||||
ST_X(g.geom) AS project_x,
|
||||
ST_Y(g.geom) AS project_y,
|
||||
ST_X(ST_Transform(g.geom, 3857)) AS map_x,
|
||||
ST_Y(ST_Transform(g.geom, 3857)) AS map_y
|
||||
FROM network.junctions AS j
|
||||
JOIN gis.node_geometries AS g ON g.node_id = j.node_id
|
||||
LEFT JOIN incident_pipe_diameters AS ipd ON ipd.node_id = j.node_id
|
||||
WHERE j.node_id = ANY(%s)
|
||||
ORDER BY j.node_id
|
||||
""",
|
||||
(node_ids, node_ids, node_ids),
|
||||
)
|
||||
return list(cur.fetchall())
|
||||
|
||||
|
||||
def update_sensor_placement(
|
||||
name: str,
|
||||
run_id: UUID,
|
||||
*,
|
||||
expected_sensor_locations: list[str],
|
||||
sensor_locations: list[str],
|
||||
) -> dict[str, Any] | None:
|
||||
with project_connection(name) as conn, conn.transaction():
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT result_id, payload
|
||||
FROM analysis.results
|
||||
WHERE run_id = %s AND result_type = %s
|
||||
ORDER BY created_at DESC, result_id DESC
|
||||
LIMIT 1
|
||||
FOR UPDATE
|
||||
""",
|
||||
(run_id, RESULT_TYPE),
|
||||
)
|
||||
result = cur.fetchone()
|
||||
if result is None:
|
||||
return None
|
||||
payload = result["payload"] if isinstance(result["payload"], dict) else {}
|
||||
current = [str(item) for item in payload.get("sensor_locations", [])]
|
||||
if current != expected_sensor_locations:
|
||||
return None
|
||||
payload = {
|
||||
**payload,
|
||||
"sensor_number": len(sensor_locations),
|
||||
"sensor_locations": sensor_locations,
|
||||
}
|
||||
cur.execute(
|
||||
"UPDATE analysis.results SET payload = %s WHERE result_id = %s",
|
||||
(Jsonb(payload), result["result_id"]),
|
||||
)
|
||||
return get_sensor_placement(name, run_id)
|
||||
Reference in New Issue
Block a user