OrgTJWater/TJWaterServerBinary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增清洗 scada 数据方法，更新数据返回格式

Browse Source
This commit is contained in:
JIANG
2025-12-10 18:04:44 +08:00
parent d40ecfc7c7
commit eb330dda4c
5 changed files with 275 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
timescaledb/schemas/realtime.py
View File

@@ -1,5 +1,6 @@
from typing import List, Any, Dict
from datetime import datetime, timedelta, timezone
from collections import defaultdict
from psycopg import AsyncConnection, Connection, sql
# 定义UTC+8时区
@@ -15,7 +16,7 @@ class RealtimeRepository:
"""Batch insert for realtime.link_simulation using DELETE then COPY for performance."""
if not data:
return
# 假设同一批次的数据时间是相同的
target_time = data[0]["time"]
@@ -25,7 +26,7 @@ class RealtimeRepository:
# 1. 先删除该时间点的旧数据
await cur.execute(
"DELETE FROM realtime.link_simulation WHERE time = %s",
(target_time,)
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
@@ -33,25 +34,27 @@ class RealtimeRepository:
"COPY realtime.link_simulation (time, id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
await copy.write_row((
item["time"],
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
))
await copy.write_row(
(
item["time"],
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
def insert_links_batch_sync(conn: Connection, data: List[dict]):
"""Batch insert for realtime.link_simulation using DELETE then COPY for performance (sync version)."""
if not data:
return
# 假设同一批次的数据时间是相同的
target_time = data[0]["time"]
@@ -61,7 +64,7 @@ class RealtimeRepository:
# 1. 先删除该时间点的旧数据
cur.execute(
"DELETE FROM realtime.link_simulation WHERE time = %s",
(target_time,)
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
@@ -69,18 +72,20 @@ class RealtimeRepository:
"COPY realtime.link_simulation (time, id, flow, friction, headloss, quality, reaction, setting, status, velocity) FROM STDIN"
) as copy:
for item in data:
copy.write_row((
item["time"],
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
))
copy.write_row(
(
item["time"],
item["id"],
item.get("flow"),
item.get("friction"),
item.get("headloss"),
item.get("quality"),
item.get("reaction"),
item.get("setting"),
item.get("status"),
item.get("velocity"),
)
)
@staticmethod
async def get_link_by_time_range(
@@ -111,7 +116,7 @@ class RealtimeRepository:
end_time: datetime,
link_id: str,
field: str,
) -> Any:
) -> List[Dict[str, Any]]:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
@@ -127,13 +132,15 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s AND id = %s"
"SELECT time, {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s AND id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, link_id))
row = await cur.fetchone()
return row[field] if row else None
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_links_field_by_time_range(
@@ -141,7 +148,7 @@ class RealtimeRepository:
start_time: datetime,
end_time: datetime,
field: str,
) -> Any:
) -> dict:
# Validate field name to prevent SQL injection
valid_fields = {
"flow",
@@ -157,13 +164,18 @@ class RealtimeRepository:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT id, {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s"
"SELECT id, time, {} FROM realtime.link_simulation WHERE time >= %s AND time <= %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time))
rows = await cur.fetchall()
return [{"id": row["id"], "value": row[field]} for row in rows]
result = defaultdict(list)
for row in rows:
result[row["id"]].append(
{"time": row["time"].isoformat(), "value": row[field]}
)
return dict(result)
@staticmethod
async def update_link_field(
@@ -209,7 +221,7 @@ class RealtimeRepository:
async def insert_nodes_batch(conn: AsyncConnection, data: List[dict]):
if not data:
return
# 假设同一批次的数据时间是相同的
target_time = data[0]["time"]
@@ -219,7 +231,7 @@ class RealtimeRepository:
# 1. 先删除该时间点的旧数据
await cur.execute(
"DELETE FROM realtime.node_simulation WHERE time = %s",
(target_time,)
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
@@ -227,20 +239,22 @@ class RealtimeRepository:
"COPY realtime.node_simulation (time, id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
await copy.write_row((
item["time"],
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
))
await copy.write_row(
(
item["time"],
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
def insert_nodes_batch_sync(conn: Connection, data: List[dict]):
if not data:
return
# 假设同一批次的数据时间是相同的
target_time = data[0]["time"]
@@ -250,7 +264,7 @@ class RealtimeRepository:
# 1. 先删除该时间点的旧数据
cur.execute(
"DELETE FROM realtime.node_simulation WHERE time = %s",
(target_time,)
(target_time,),
)
# 2. 使用 COPY 快速写入新数据
@@ -258,14 +272,16 @@ class RealtimeRepository:
"COPY realtime.node_simulation (time, id, actual_demand, total_head, pressure, quality) FROM STDIN"
) as copy:
for item in data:
copy.write_row((
item["time"],
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
))
copy.write_row(
(
item["time"],
item["id"],
item.get("actual_demand"),
item.get("total_head"),
item.get("pressure"),
item.get("quality"),
)
)
@staticmethod
async def get_node_by_time_range(
@@ -296,36 +312,43 @@ class RealtimeRepository:
end_time: datetime,
node_id: str,
field: str,
) -> Any:
) -> List[Dict[str, Any]]:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s AND id = %s"
"SELECT time, {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s AND id = %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time, node_id))
row = await cur.fetchone()
return row[field] if row else None
rows = await cur.fetchall()
return [
{"time": row["time"].isoformat(), "value": row[field]} for row in rows
]
@staticmethod
async def get_nodes_field_by_time_range(
conn: AsyncConnection, start_time: datetime, end_time: datetime, field: str
) -> Any:
) -> dict:
valid_fields = {"actual_demand", "total_head", "pressure", "quality"}
if field not in valid_fields:
raise ValueError(f"Invalid field: {field}")
query = sql.SQL(
"SELECT id, {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s"
"SELECT id, time, {} FROM realtime.node_simulation WHERE time >= %s AND time <= %s"
).format(sql.Identifier(field))
async with conn.cursor() as cur:
await cur.execute(query, (start_time, end_time))
rows = await cur.fetchall()
return [{"id": row["id"], "value": row[field]} for row in rows]
result = defaultdict(list)
for row in rows:
result[row["id"]].append(
{"time": row["time"].isoformat(), "value": row[field]}
)
return dict(result)
@staticmethod
async def update_node_field(