OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(db)!: clean up business SQL access

Browse Source 
- make realtime replacement and analysis result writes transactional\n- consolidate SCADA repositories and remove process-global project state\n- validate SCADA batches and use indexed GIS-backed business queries\n\nBREAKING CHANGE: remove the public analysis result writer and the pipeline-health network_name query parameter.
This commit is contained in:
jiang
2026-08-28 11:37:36 +08:00
parent b74799a39d
commit 9b095c7439
34 changed files with 859 additions and 921 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/timescaledb/composite_queries.py
+26 -14
View File
@@ -14,7 +14,6 @@ from app.infra.db.postgresql.scada import ScadaInfoRepository
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.native.wndb.model.pipes import get_pipes_by_property
class CompositeQueries:
@@ -454,20 +453,25 @@ class CompositeQueries:
if not cleaned_rows:
raise ValueError("SCADA 数据清洗未产生任何数据库更新")
updated_rows = await ScadaRepository.update_scada_field_batch(
timescale_conn,
cleaned_rows,
"cleaned_value",
)
if updated_rows == 0:
raise ValueError("SCADA 清洗结果未匹配任何已有监测数据")
expected_rows = len({(row[0], row[1]) for row in cleaned_rows})
async with timescale_conn.transaction():
updated_rows = await ScadaRepository.update_scada_field_batch(
timescale_conn,
cleaned_rows,
"cleaned_value",
)
if updated_rows != expected_rows:
raise ValueError(
"SCADA 清洗目标在写入期间发生变化，"
f"预期更新 {expected_rows} 行，实际更新 {updated_rows} 行"
)
return "success"
@staticmethod
async def predict_pipeline_health(
timescale_conn: AsyncConnection,
network_name: str,
postgres_conn: AsyncConnection,
query_time: datetime,
) -> List[Dict[str, Any]]:
"""
@@ -478,7 +482,6 @@ class CompositeQueries:
Args:
timescale_conn: TimescaleDB 异步连接
db_name: 管网数据库名称
query_time: 查询时间
property_conditions: 可选的管道筛选条件，如 {"diameter": 300}
@@ -505,12 +508,21 @@ class CompositeQueries:
# 3. 只查询有流速数据的管道的基本信息
valid_link_ids = list(velocity_data.keys())
# 批量查询这些管道的详细信息
fields = ["id", "diameter", "node1", "node2"]
all_links = get_pipes_by_property(network_name, fields=fields)
# GIS 物化视图是低频更新管网的查询面；只读取本次有结果的管道。
async with postgres_conn.cursor() as cur:
await cur.execute(
"""
SELECT id, diameter, start_node_id AS node1,
end_node_id AS node2
FROM gis.pipes
WHERE id = ANY(%s)
""",
(valid_link_ids,),
)
all_links = await cur.fetchall()
# 转换为字典以快速查找
links_dict = {link["id"]: link for link in all_links}
links_dict = {str(link["id"]): link for link in all_links}
# 获取所有需要查询的节点ID
node_ids = set()
app/infra/db/timescaledb/internal_queries.py
+11 -23
View File
@@ -88,16 +88,15 @@ class InternalQueries:
rows = ScadaRepository.get_scada_by_ids_time_range_sync(
conn, device_ids, start_time, end_time
)
# 处理结果，返回每个 device_id 的第一个值
result = {}
for device_id in device_ids:
device_rows = [
row for row in rows if row["device_id"] == device_id
]
if device_rows:
result[device_id] = device_rows[0]["monitored_value"]
else:
result[device_id] = None
# Rows are ordered by device/time; retain the first sample
# for each requested device in one pass.
result = {device_id: None for device_id in device_ids}
seen: set[str] = set()
for row in rows:
device_id = str(row["device_id"])
if device_id in result and device_id not in seen:
result[device_id] = row["monitored_value"]
seen.add(device_id)
return result
except Exception as e:
logger.error(f"查询尝试 {attempt + 1} 失败: {e}")
@@ -135,8 +134,6 @@ class InternalQueries:
result.setdefault(device_id, []).append(
{"time": row["time"].isoformat(), "value": value}
)
for device_id in result:
result[device_id].sort(key=lambda item: item["time"])
return result
except Exception as e:
logger.error(f"查询尝试 {attempt + 1} 失败: {e}")
@@ -307,16 +304,7 @@ class InternalQueries:
def _resolve_simulation_table(element_type: str) -> tuple[str, str, set[str]]:
normalized_type = element_type.lower()
if normalized_type == "node":
return "node_results", "node_id", {"actual_demand", "total_head", "pressure", "quality"}
return "node_results", "node_id", set(RealtimeRepository.NODE_FIELDS)
if normalized_type == "link":
return "link_results", "link_id", {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
return "link_results", "link_id", set(RealtimeRepository.LINK_FIELDS)
raise ValueError(f"Unsupported element_type: {element_type}")
app/infra/db/timescaledb/repositories/realtime.py
+124 -179
View File
@@ -6,6 +6,24 @@ from app.services.time_api import parse_utc_time
class RealtimeRepository:
LINK_FIELDS = frozenset(
{
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
)
NODE_FIELDS = frozenset({"actual_demand", "total_head", "pressure", "quality"})
LINK_RESULT_COLUMNS = (
"time, link_id, flow, friction, headloss, quality, reaction, "
"setting, status, velocity"
)
NODE_RESULT_COLUMNS = "time, node_id, actual_demand, total_head, pressure, quality"
@staticmethod
def _batch_time(data: List[dict]) -> datetime:
@@ -22,6 +40,48 @@ class RealtimeRepository:
# --- Link Simulation ---
@staticmethod
async def _copy_links(cur, data: List[dict], target_time: datetime) -> None:
async with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
def _copy_links_sync(cur, data: List[dict], target_time: datetime) -> None:
with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
async def insert_links_batch(conn: AsyncConnection, data: List[dict]):
"""Batch insert for realtime.link_results using DELETE then COPY."""
@@ -43,25 +103,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
await RealtimeRepository._copy_links(cur, data, target_time)
@staticmethod
def insert_links_batch_sync(conn: Connection, data: List[dict]):
@@ -84,25 +126,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
RealtimeRepository._copy_links_sync(cur, data, target_time)
@staticmethod
async def get_link_by_time_range(
@@ -110,7 +134,8 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.LINK_RESULT_COLUMNS} "
"FROM realtime.link_results WHERE time >= %s AND time <= %s "
"AND link_id = %s ORDER BY time",
(start_time, end_time, link_id),
)
@@ -124,46 +149,13 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.LINK_RESULT_COLUMNS} "
"FROM realtime.link_results WHERE time >= %s AND time <= %s "
"ORDER BY time, link_id",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@staticmethod
async def get_link_field_by_time_range(
conn: AsyncConnection,
start_time: datetime,
end_time: datetime,
link_id: str,
field: str,
) -> List[Dict[str, Any]]:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.link_results WHERE time >= %s "
"AND time <= %s AND link_id = %s ORDER BY time"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, link_id))
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_link_fields_by_ids_time_range(
conn: AsyncConnection,
@@ -172,11 +164,7 @@ class RealtimeRepository:
link_ids: list[str],
field: str,
) -> dict[str, list[dict[str, Any]]]:
valid_fields = {
"flow", "friction", "headloss", "quality", "reaction",
"setting", "status", "velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
result = {link_id: [] for link_id in link_ids}
if not link_ids:
@@ -202,17 +190,7 @@ class RealtimeRepository:
field: str,
) -> dict:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -238,17 +216,7 @@ class RealtimeRepository:
field: str,
value: Any,
):
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -270,6 +238,40 @@ class RealtimeRepository:
# --- Node Simulation ---
@staticmethod
async def _copy_nodes(cur, data: List[dict], target_time: datetime) -> None:
async with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
def _copy_nodes_sync(cur, data: List[dict], target_time: datetime) -> None:
with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
async def insert_nodes_batch(conn: AsyncConnection, data: List[dict]):
if not data:
@@ -290,21 +292,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
await RealtimeRepository._copy_nodes(cur, data, target_time)
@staticmethod
def insert_nodes_batch_sync(conn: Connection, data: List[dict]):
@@ -326,21 +314,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
RealtimeRepository._copy_nodes_sync(cur, data, target_time)
@staticmethod
async def get_node_by_time_range(
@@ -348,7 +322,8 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.NODE_RESULT_COLUMNS} "
"FROM realtime.node_results WHERE time >= %s AND time <= %s "
"AND node_id = %s ORDER BY time",
(start_time, end_time, node_id),
)
@@ -362,36 +337,13 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.NODE_RESULT_COLUMNS} "
"FROM realtime.node_results WHERE time >= %s AND time <= %s "
"ORDER BY time, node_id",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@staticmethod
async def get_node_field_by_time_range(
conn: AsyncConnection,
start_time: datetime,
end_time: datetime,
node_id: str,
field: str,
) -> List[Dict[str, Any]]:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.node_results WHERE time >= %s "
"AND time <= %s AND node_id = %s ORDER BY time"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, node_id))
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_node_fields_by_ids_time_range(
conn: AsyncConnection,
@@ -400,8 +352,7 @@ class RealtimeRepository:
node_ids: list[str],
field: str,
) -> dict[str, list[dict[str, Any]]]:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
result = {node_id: [] for node_id in node_ids}
if not node_ids:
@@ -423,8 +374,7 @@ class RealtimeRepository:
async def get_nodes_field_by_time_range(
conn: AsyncConnection, start_time: datetime, end_time: datetime, field: str
) -> dict:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -450,8 +400,7 @@ class RealtimeRepository:
field: str,
value: Any,
):
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -529,9 +478,8 @@ class RealtimeRepository:
}
)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
# Keep node and link replacement atomic with one lock and one delete per
# table. Copy-only helpers avoid repeating replacement SQL.
async with conn.transaction():
async with conn.cursor() as cur:
await cur.execute(
@@ -546,11 +494,10 @@ class RealtimeRepository:
"DELETE FROM realtime.link_results WHERE time = %s",
(simulation_time,),
)
if node_data:
await RealtimeRepository.insert_nodes_batch(conn, node_data)
if link_data:
await RealtimeRepository.insert_links_batch(conn, link_data)
if node_data:
await RealtimeRepository._copy_nodes(cur, node_data, simulation_time)
if link_data:
await RealtimeRepository._copy_links(cur, link_data, simulation_time)
@staticmethod
def store_realtime_simulation_result_sync(
@@ -608,9 +555,8 @@ class RealtimeRepository:
}
)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
# Keep node and link replacement atomic with one lock and one delete per
# table. Copy-only helpers avoid repeating replacement SQL.
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
@@ -625,11 +571,10 @@ class RealtimeRepository:
"DELETE FROM realtime.link_results WHERE time = %s",
(simulation_time,),
)
if node_data:
RealtimeRepository.insert_nodes_batch_sync(conn, node_data)
if link_data:
RealtimeRepository.insert_links_batch_sync(conn, link_data)
if node_data:
RealtimeRepository._copy_nodes_sync(cur, node_data, simulation_time)
if link_data:
RealtimeRepository._copy_links_sync(cur, link_data, simulation_time)
@staticmethod
async def query_all_record_by_time_property(
app/infra/db/timescaledb/repositories/scada.py
+9 -8
View File
@@ -6,6 +6,8 @@ from psycopg.rows import dict_row
class ScadaRepository:
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
@staticmethod
async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
@@ -35,7 +37,8 @@ class ScadaRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM scada.measurements WHERE device_id = ANY(%s) "
f"SELECT {ScadaRepository.RESULT_COLUMNS} FROM scada.measurements "
"WHERE device_id = ANY(%s) "
"AND time >= %s AND time <= %s ORDER BY device_id, time",
(device_ids, start_time, end_time),
)
@@ -50,7 +53,8 @@ class ScadaRepository:
) -> List[dict]:
with conn.cursor(row_factory=dict_row) as cur:
cur.execute(
"SELECT * FROM scada.measurements WHERE device_id = ANY(%s) "
f"SELECT {ScadaRepository.RESULT_COLUMNS} FROM scada.measurements "
"WHERE device_id = ANY(%s) "
"AND time >= %s AND time <= %s ORDER BY device_id, time",
(device_ids, start_time, end_time),
)
@@ -85,8 +89,7 @@ class ScadaRepository:
end_time: datetime,
field: str,
) -> dict:
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -110,8 +113,7 @@ class ScadaRepository:
async def update_scada_field(
conn: AsyncConnection, time: datetime, device_id: str, field: str, value: Any
):
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
update_query = sql.SQL(
@@ -133,8 +135,7 @@ class ScadaRepository:
field: str,
) -> int:
"""Update existing SCADA samples in one set-based statement."""
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
if not rows:
return 0