OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(db)!: clean up business SQL access

Browse Source 
- make realtime replacement and analysis result writes transactional\n- consolidate SCADA repositories and remove process-global project state\n- validate SCADA batches and use indexed GIS-backed business queries\n\nBREAKING CHANGE: remove the public analysis result writer and the pipeline-health network_name query parameter.
This commit is contained in:
jiang
2026-08-28 11:37:36 +08:00
parent b74799a39d
commit 9b095c7439
34 changed files with 859 additions and 921 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/timescaledb/repositories/realtime.py
+124 -179
View File
@@ -6,6 +6,24 @@ from app.services.time_api import parse_utc_time
class RealtimeRepository:
LINK_FIELDS = frozenset(
{
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
)
NODE_FIELDS = frozenset({"actual_demand", "total_head", "pressure", "quality"})
LINK_RESULT_COLUMNS = (
"time, link_id, flow, friction, headloss, quality, reaction, "
"setting, status, velocity"
)
NODE_RESULT_COLUMNS = "time, node_id, actual_demand, total_head, pressure, quality"
@staticmethod
def _batch_time(data: List[dict]) -> datetime:
@@ -22,6 +40,48 @@ class RealtimeRepository:
# --- Link Simulation ---
@staticmethod
async def _copy_links(cur, data: List[dict], target_time: datetime) -> None:
async with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
def _copy_links_sync(cur, data: List[dict], target_time: datetime) -> None:
with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
async def insert_links_batch(conn: AsyncConnection, data: List[dict]):
"""Batch insert for realtime.link_results using DELETE then COPY."""
@@ -43,25 +103,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
await RealtimeRepository._copy_links(cur, data, target_time)
@staticmethod
def insert_links_batch_sync(conn: Connection, data: List[dict]):
@@ -84,25 +126,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
RealtimeRepository._copy_links_sync(cur, data, target_time)
@staticmethod
async def get_link_by_time_range(
@@ -110,7 +134,8 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.LINK_RESULT_COLUMNS} "
"FROM realtime.link_results WHERE time >= %s AND time <= %s "
"AND link_id = %s ORDER BY time",
(start_time, end_time, link_id),
)
@@ -124,46 +149,13 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.LINK_RESULT_COLUMNS} "
"FROM realtime.link_results WHERE time >= %s AND time <= %s "
"ORDER BY time, link_id",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@staticmethod
async def get_link_field_by_time_range(
conn: AsyncConnection,
start_time: datetime,
end_time: datetime,
link_id: str,
field: str,
) -> List[Dict[str, Any]]:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.link_results WHERE time >= %s "
"AND time <= %s AND link_id = %s ORDER BY time"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, link_id))
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_link_fields_by_ids_time_range(
conn: AsyncConnection,
@@ -172,11 +164,7 @@ class RealtimeRepository:
link_ids: list[str],
field: str,
) -> dict[str, list[dict[str, Any]]]:
valid_fields = {
"flow", "friction", "headloss", "quality", "reaction",
"setting", "status", "velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
result = {link_id: [] for link_id in link_ids}
if not link_ids:
@@ -202,17 +190,7 @@ class RealtimeRepository:
field: str,
) -> dict:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -238,17 +216,7 @@ class RealtimeRepository:
field: str,
value: Any,
):
valid_fields = {
"flow",
"friction",
"headloss",
"quality",
"reaction",
"setting",
"status",
"velocity",
}
if field not in valid_fields:
if field not in RealtimeRepository.LINK_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -270,6 +238,40 @@ class RealtimeRepository:
# --- Node Simulation ---
@staticmethod
async def _copy_nodes(cur, data: List[dict], target_time: datetime) -> None:
async with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
def _copy_nodes_sync(cur, data: List[dict], target_time: datetime) -> None:
with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
async def insert_nodes_batch(conn: AsyncConnection, data: List[dict]):
if not data:
@@ -290,21 +292,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
await RealtimeRepository._copy_nodes(cur, data, target_time)
@staticmethod
def insert_nodes_batch_sync(conn: Connection, data: List[dict]):
@@ -326,21 +314,7 @@ class RealtimeRepository:
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
(
target_time,
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
RealtimeRepository._copy_nodes_sync(cur, data, target_time)
@staticmethod
async def get_node_by_time_range(
@@ -348,7 +322,8 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.NODE_RESULT_COLUMNS} "
"FROM realtime.node_results WHERE time >= %s AND time <= %s "
"AND node_id = %s ORDER BY time",
(start_time, end_time, node_id),
)
@@ -362,36 +337,13 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s "
f"SELECT {RealtimeRepository.NODE_RESULT_COLUMNS} "
"FROM realtime.node_results WHERE time >= %s AND time <= %s "
"ORDER BY time, node_id",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@staticmethod
async def get_node_field_by_time_range(
conn: AsyncConnection,
start_time: datetime,
end_time: datetime,
node_id: str,
field: str,
) -> List[Dict[str, Any]]:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.node_results WHERE time >= %s "
"AND time <= %s AND node_id = %s ORDER BY time"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, node_id))
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_node_fields_by_ids_time_range(
conn: AsyncConnection,
@@ -400,8 +352,7 @@ class RealtimeRepository:
node_ids: list[str],
field: str,
) -> dict[str, list[dict[str, Any]]]:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
result = {node_id: [] for node_id in node_ids}
if not node_ids:
@@ -423,8 +374,7 @@ class RealtimeRepository:
async def get_nodes_field_by_time_range(
conn: AsyncConnection, start_time: datetime, end_time: datetime, field: str
) -> dict:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -450,8 +400,7 @@ class RealtimeRepository:
field: str,
value: Any,
):
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
if field not in RealtimeRepository.NODE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -529,9 +478,8 @@ class RealtimeRepository:
}
)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
# Keep node and link replacement atomic with one lock and one delete per
# table. Copy-only helpers avoid repeating replacement SQL.
async with conn.transaction():
async with conn.cursor() as cur:
await cur.execute(
@@ -546,11 +494,10 @@ class RealtimeRepository:
"DELETE FROM realtime.link_results WHERE time = %s",
(simulation_time,),
)
if node_data:
await RealtimeRepository.insert_nodes_batch(conn, node_data)
if link_data:
await RealtimeRepository.insert_links_batch(conn, link_data)
if node_data:
await RealtimeRepository._copy_nodes(cur, node_data, simulation_time)
if link_data:
await RealtimeRepository._copy_links(cur, link_data, simulation_time)
@staticmethod
def store_realtime_simulation_result_sync(
@@ -608,9 +555,8 @@ class RealtimeRepository:
}
)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
# Keep node and link replacement atomic with one lock and one delete per
# table. Copy-only helpers avoid repeating replacement SQL.
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
@@ -625,11 +571,10 @@ class RealtimeRepository:
"DELETE FROM realtime.link_results WHERE time = %s",
(simulation_time,),
)
if node_data:
RealtimeRepository.insert_nodes_batch_sync(conn, node_data)
if link_data:
RealtimeRepository.insert_links_batch_sync(conn, link_data)
if node_data:
RealtimeRepository._copy_nodes_sync(cur, node_data, simulation_time)
if link_data:
RealtimeRepository._copy_links_sync(cur, link_data, simulation_time)
@staticmethod
async def query_all_record_by_time_property(
app/infra/db/timescaledb/repositories/scada.py
+9 -8
View File
@@ -6,6 +6,8 @@ from psycopg.rows import dict_row
class ScadaRepository:
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
@staticmethod
async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
@@ -35,7 +37,8 @@ class ScadaRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM scada.measurements WHERE device_id = ANY(%s) "
f"SELECT {ScadaRepository.RESULT_COLUMNS} FROM scada.measurements "
"WHERE device_id = ANY(%s) "
"AND time >= %s AND time <= %s ORDER BY device_id, time",
(device_ids, start_time, end_time),
)
@@ -50,7 +53,8 @@ class ScadaRepository:
) -> List[dict]:
with conn.cursor(row_factory=dict_row) as cur:
cur.execute(
"SELECT * FROM scada.measurements WHERE device_id = ANY(%s) "
f"SELECT {ScadaRepository.RESULT_COLUMNS} FROM scada.measurements "
"WHERE device_id = ANY(%s) "
"AND time >= %s AND time <= %s ORDER BY device_id, time",
(device_ids, start_time, end_time),
)
@@ -85,8 +89,7 @@ class ScadaRepository:
end_time: datetime,
field: str,
) -> dict:
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
@@ -110,8 +113,7 @@ class ScadaRepository:
async def update_scada_field(
conn: AsyncConnection, time: datetime, device_id: str, field: str, value: Any
):
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
update_query = sql.SQL(
@@ -133,8 +135,7 @@ class ScadaRepository:
field: str,
) -> int:
"""Update existing SCADA samples in one set-based statement."""
valid_fields = {"monitored_value", "cleaned_value"}
if field not in valid_fields:
if field not in ScadaRepository.VALUE_FIELDS:
raise ValueError(f"Invalid field: {field}")
if not rows:
return 0