OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(db)!: adopt project-routed pooled databases

Browse Source 
Reorganize WNDB by responsibility and remove legacy scheme endpoints.\n\nRoute analysis and time-series access through project pools, preserve transactional realtime replacement, and refresh GIS materialized views after writes.\n\nAdd database architecture documentation, live pooling coverage, API contract updates, and executable container verification.\n\nBREAKING CHANGE: legacy scheme APIs and flat app.native.wndb module imports are removed.
This commit is contained in:
jiang
2026-08-25 18:35:05 +08:00
parent fdbcc5c033
commit fa188af0b1
181 changed files with 8446 additions and 33546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/timescaledb/repositories/realtime.py
+56 -34
View File
@@ -11,7 +11,7 @@ class RealtimeRepository:
@staticmethod
async def insert_links_batch(conn: AsyncConnection, data: List[dict]):
"""Batch insert for realtime.link_simulation using DELETE then COPY for performance."""
"""Batch insert for realtime.link_results using DELETE then COPY."""
if not data:
return
@@ -21,15 +21,19 @@ class RealtimeRepository:
# 使用事务确保原子性
async with conn.transaction():
async with conn.cursor() as cur:
await cur.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))",
(target_time,),
)
# 1. 先删除该时间点的旧数据
await cur.execute(
"DELETE FROM realtime.link_simulation WHERE time = %s",
"DELETE FROM realtime.link_results WHERE time = %s",
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.link_simulation (time, id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
@@ -49,7 +53,7 @@ class RealtimeRepository:
@staticmethod
def insert_links_batch_sync(conn: Connection, data: List[dict]):
"""Batch insert for realtime.link_simulation using DELETE then COPY for performance (sync version)."""
"""Synchronous batch insert for realtime.link_results."""
if not data:
return
@@ -59,15 +63,19 @@ class RealtimeRepository:
# 使用事务确保原子性
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))",
(target_time,),
)
# 1. 先删除该时间点的旧数据
cur.execute(
"DELETE FROM realtime.link_simulation WHERE time = %s",
"DELETE FROM realtime.link_results WHERE time = %s",
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.link_simulation (time, id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
"COPY realtime.link_results (time, link_id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
@@ -91,7 +99,7 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_simulation WHERE time >= %s AND time <= %s AND id = %s",
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s AND link_id = %s",
(start_time, end_time, link_id),
)
return await cur.fetchall()
@@ -104,7 +112,7 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.link_simulation WHERE time >= %s AND time <= %s",
"SELECT * FROM realtime.link_results WHERE time >= %s AND time <= %s",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@@ -132,7 +140,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s AND id = %s"
"SELECT time, {} FROM realtime.link_results WHERE time >= %s AND time <= %s AND link_id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -164,7 +172,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT id, time, {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s"
"SELECT link_id, time, {} FROM realtime.link_results WHERE time >= %s AND time <= %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -172,7 +180,7 @@ class RealtimeRepository:
rows = await cur.fetchall()
result = defaultdict(list)
for row in rows:
result[row["id"]].append(
result[row["link_id"]].append(
{"time": row["time"].isoformat(), "value": row[field]}
)
return dict(result)
@@ -199,7 +207,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"UPDATE realtime.link_simulation SET {} = %s WHERE time = %s AND id = %s"
"UPDATE realtime.link_results SET {} = %s WHERE time = %s AND link_id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -211,7 +219,7 @@ class RealtimeRepository:
):
async with conn.cursor() as cur:
await cur.execute(
"DELETE FROM realtime.link_simulation WHERE time >= %s AND time <= %s",
"DELETE FROM realtime.link_results WHERE time >= %s AND time <= %s",
(start_time, end_time),
)
@@ -228,15 +236,19 @@ class RealtimeRepository:
# 使用事务确保原子性
async with conn.transaction():
async with conn.cursor() as cur:
await cur.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))",
(target_time,),
)
# 1. 先删除该时间点的旧数据
await cur.execute(
"DELETE FROM realtime.node_simulation WHERE time = %s",
"DELETE FROM realtime.node_results WHERE time = %s",
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
async with cur.copy(
"COPY realtime.node_simulation (time, id, actual_demand, total_head, pressure, quality) FROM STDIN"
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row(
@@ -261,15 +273,19 @@ class RealtimeRepository:
# 使用事务确保原子性
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s::text, 1))",
(target_time,),
)
# 1. 先删除该时间点的旧数据
cur.execute(
"DELETE FROM realtime.node_simulation WHERE time = %s",
"DELETE FROM realtime.node_results WHERE time = %s",
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
with cur.copy(
"COPY realtime.node_simulation (time, id, actual_demand, total_head, pressure, quality) FROM STDIN"
"COPY realtime.node_results (time, node_id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row(
@@ -289,7 +305,7 @@ class RealtimeRepository:
) -> List[dict]:
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_simulation WHERE time >= %s AND time <= %s AND id = %s",
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s AND node_id = %s",
(start_time, end_time, node_id),
)
return await cur.fetchall()
@@ -302,7 +318,7 @@ class RealtimeRepository:
normalized_end_time = parse_utc_time(end_time, field_name="end_time")
async with conn.cursor() as cur:
await cur.execute(
"SELECT * FROM realtime.node_simulation WHERE time >= %s AND time <= %s",
"SELECT * FROM realtime.node_results WHERE time >= %s AND time <= %s",
(normalized_start_time, normalized_end_time),
)
return await cur.fetchall()
@@ -320,7 +336,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT time, {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s AND id = %s"
"SELECT time, {} FROM realtime.node_results WHERE time >= %s AND time <= %s AND node_id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -339,7 +355,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT id, time, {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s"
"SELECT node_id, time, {} FROM realtime.node_results WHERE time >= %s AND time <= %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -347,7 +363,7 @@ class RealtimeRepository:
rows = await cur.fetchall()
result = defaultdict(list)
for row in rows:
result[row["id"]].append(
result[row["node_id"]].append(
{"time": row["time"].isoformat(), "value": row[field]}
)
return dict(result)
@@ -365,7 +381,7 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"UPDATE realtime.node_simulation SET {} = %s WHERE time = %s AND id = %s"
"UPDATE realtime.node_results SET {} = %s WHERE time = %s AND node_id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
@@ -377,7 +393,7 @@ class RealtimeRepository:
):
async with conn.cursor() as cur:
await cur.execute(
"DELETE FROM realtime.node_simulation WHERE time >= %s AND time <= %s",
"DELETE FROM realtime.node_results WHERE time >= %s AND time <= %s",
(start_time, end_time),
)
@@ -439,12 +455,15 @@ class RealtimeRepository:
}
)
# Insert data using batch methods
if node_data:
await RealtimeRepository.insert_nodes_batch(conn, node_data)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
async with conn.transaction():
if node_data:
await RealtimeRepository.insert_nodes_batch(conn, node_data)
if link_data:
await RealtimeRepository.insert_links_batch(conn, link_data)
if link_data:
await RealtimeRepository.insert_links_batch(conn, link_data)
@staticmethod
def store_realtime_simulation_result_sync(
@@ -502,12 +521,15 @@ class RealtimeRepository:
}
)
# Insert data using batch methods
if node_data:
RealtimeRepository.insert_nodes_batch_sync(conn, node_data)
# Keep node and link replacement atomic. The batch helpers use nested
# transactions (savepoints), while this outer transaction guarantees
# that a link write failure also rolls back the node replacement.
with conn.transaction():
if node_data:
RealtimeRepository.insert_nodes_batch_sync(conn, node_data)
if link_data:
RealtimeRepository.insert_links_batch_sync(conn, link_data)
if link_data:
RealtimeRepository.insert_links_batch_sync(conn, link_data)
@staticmethod
async def query_all_record_by_time_property(