refactor: simplify task and history services
Remove retired query paths and repository compatibility branches, and share task test fixtures. Cancel pending questions and resolve their notifications when recovering expired runs.
This commit is contained in:
@@ -1,11 +1,10 @@
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from psycopg import AsyncConnection
|
||||
from uuid import UUID
|
||||
|
||||
from app.algorithms.pipe_health_prediction.survival_predictor import (
|
||||
PipeHealthSurvivalPredictor,
|
||||
@@ -15,14 +14,11 @@ from app.algorithms.scada_cleaning.pressure_series import clean_pressure_data_df
|
||||
from app.infra.db.postgresql.network_assets import NetworkAssetRepository
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
|
||||
from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
|
||||
|
||||
class TimeseriesAnalysisService:
|
||||
"""
|
||||
复合查询类，提供跨表查询功能
|
||||
"""
|
||||
"""SCADA 清洗与管道健康预测；历史查询由 TimeseriesHistoryService 负责。"""
|
||||
|
||||
@staticmethod
|
||||
async def _get_project_scada_index(
|
||||
@@ -31,294 +27,6 @@ class TimeseriesAnalysisService:
|
||||
scadas = await ScadaInfoRepository.get_scadas(postgres_conn)
|
||||
return {scada["device_id"]: scada for scada in scadas}
|
||||
|
||||
@staticmethod
|
||||
async def get_scada_associated_realtime_simulation_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
postgres_conn: AsyncConnection,
|
||||
device_ids: List[str],
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
获取 SCADA 关联的 link/node 模拟值
|
||||
|
||||
根据传入的 SCADA device_ids，找到关联的 link/node，
|
||||
并根据对应的 type，查询对应的模拟数据
|
||||
|
||||
Args:
|
||||
timescale_conn: TimescaleDB 异步连接
|
||||
postgres_conn: PostgreSQL 异步连接
|
||||
device_ids: SCADA 设备ID列表
|
||||
start_time: 开始时间
|
||||
end_time: 结束时间
|
||||
|
||||
Returns:
|
||||
模拟数据字典，以 device_id 为键，值为数据列表，每个数据包含 time, value 和 scada_id
|
||||
|
||||
Raises:
|
||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||
"""
|
||||
scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn)
|
||||
link_devices: dict[str, str] = {}
|
||||
node_devices: dict[str, str] = {}
|
||||
for device_id in device_ids:
|
||||
target_scada = scada_by_id.get(device_id)
|
||||
if not target_scada:
|
||||
raise ValueError(f"SCADA device {device_id} not found")
|
||||
scada_type = target_scada["device_type"]
|
||||
if scada_type in {"pipe_flow", "flow"}:
|
||||
link_devices[device_id] = target_scada["link_id"]
|
||||
elif scada_type == "pressure":
|
||||
node_devices[device_id] = target_scada["node_id"]
|
||||
else:
|
||||
raise ValueError(f"Unknown SCADA type: {scada_type}")
|
||||
|
||||
link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
|
||||
timescale_conn, start_time, end_time,
|
||||
list(dict.fromkeys(link_devices.values())), "flow",
|
||||
)
|
||||
node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
|
||||
timescale_conn, start_time, end_time,
|
||||
list(dict.fromkeys(node_devices.values())), "pressure",
|
||||
)
|
||||
return {
|
||||
device_id: [
|
||||
{**item, "scada_id": device_id}
|
||||
for item in (
|
||||
link_series.get(element_id, [])
|
||||
if device_id in link_devices
|
||||
else node_series.get(element_id, [])
|
||||
)
|
||||
]
|
||||
for device_id, element_id in (link_devices | node_devices).items()
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def get_scada_associated_analysis_simulation_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
postgres_conn: AsyncConnection,
|
||||
device_ids: List[str],
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
run_id: UUID,
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
获取 SCADA 关联的 link/node 分析模拟值
|
||||
|
||||
根据传入的 SCADA device_ids，找到关联的 link/node，
|
||||
并根据对应的 type，查询对应的模拟数据
|
||||
|
||||
Args:
|
||||
timescale_conn: TimescaleDB 异步连接
|
||||
postgres_conn: PostgreSQL 异步连接
|
||||
device_ids: SCADA 设备ID列表
|
||||
start_time: 开始时间
|
||||
end_time: 结束时间
|
||||
|
||||
Returns:
|
||||
模拟数据字典，以 device_id 为键，值为数据列表，每个数据包含 time, value 和 scada_id
|
||||
|
||||
Raises:
|
||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||
"""
|
||||
scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn)
|
||||
link_devices: dict[str, str] = {}
|
||||
node_devices: dict[str, str] = {}
|
||||
for device_id in device_ids:
|
||||
target_scada = scada_by_id.get(device_id)
|
||||
if not target_scada:
|
||||
raise ValueError(f"SCADA device {device_id} not found")
|
||||
|
||||
scada_type = target_scada["device_type"]
|
||||
if scada_type in {"pipe_flow", "flow"}:
|
||||
link_devices[device_id] = target_scada["link_id"]
|
||||
elif scada_type == "pressure":
|
||||
node_devices[device_id] = target_scada["node_id"]
|
||||
else:
|
||||
raise ValueError(f"Unknown SCADA type: {scada_type}")
|
||||
|
||||
link_series = await AnalysisResultsRepository.get_series_by_ids(
|
||||
timescale_conn, run_id, "link",
|
||||
list(dict.fromkeys(link_devices.values())), start_time, end_time, "flow",
|
||||
)
|
||||
node_series = await AnalysisResultsRepository.get_series_by_ids(
|
||||
timescale_conn, run_id, "node",
|
||||
list(dict.fromkeys(node_devices.values())), start_time, end_time, "pressure",
|
||||
)
|
||||
return {
|
||||
device_id: [
|
||||
{**item, "scada_id": device_id}
|
||||
for item in (
|
||||
link_series.get(element_id, [])
|
||||
if device_id in link_devices
|
||||
else node_series.get(element_id, [])
|
||||
)
|
||||
]
|
||||
for device_id, element_id in (link_devices | node_devices).items()
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def get_realtime_simulation_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
feature_infos: List[Tuple[str, str]],
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
获取 link/node 模拟值
|
||||
|
||||
根据传入的 feature_infos，找到关联的 link/node，
|
||||
并根据对应的 type，查询对应的模拟数据
|
||||
|
||||
Args:
|
||||
timescale_conn: TimescaleDB 异步连接
|
||||
feature_infos: 传入的 feature 信息列表，包含 (element_id, type)
|
||||
start_time: 开始时间
|
||||
end_time: 结束时间
|
||||
|
||||
Returns:
|
||||
模拟数据字典，以 feature_id 为键，值为数据列表，每个数据包含 time, value 和 feature_id
|
||||
|
||||
Raises:
|
||||
ValueError: 当 SCADA 设备未找到或字段无效时
|
||||
"""
|
||||
pipe_ids: list[str] = []
|
||||
junction_ids: list[str] = []
|
||||
for feature_id, feature_type in feature_infos:
|
||||
if feature_type.lower() == "pipe":
|
||||
pipe_ids.append(feature_id)
|
||||
elif feature_type.lower() == "junction":
|
||||
junction_ids.append(feature_id)
|
||||
else:
|
||||
raise ValueError(f"Unknown type: {feature_type}")
|
||||
link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
|
||||
timescale_conn, start_time, end_time, list(dict.fromkeys(pipe_ids)), "flow"
|
||||
)
|
||||
node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
|
||||
timescale_conn, start_time, end_time,
|
||||
list(dict.fromkeys(junction_ids)), "pressure",
|
||||
)
|
||||
return {
|
||||
feature_id: [
|
||||
{**item, "feature_id": feature_id}
|
||||
for item in (
|
||||
link_series.get(feature_id, [])
|
||||
if feature_type.lower() == "pipe"
|
||||
else node_series.get(feature_id, [])
|
||||
)
|
||||
]
|
||||
for feature_id, feature_type in feature_infos
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def get_analysis_simulation_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
feature_infos: List[Tuple[str, str]],
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
run_id: UUID,
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
获取 link/node 分析模拟值
|
||||
|
||||
根据传入的 feature_infos，找到关联的 link/node，
|
||||
并根据对应的 type，查询对应的模拟数据
|
||||
|
||||
Args:
|
||||
timescale_conn: TimescaleDB 异步连接
|
||||
feature_infos: 传入的 feature 信息列表，包含 (element_id, type)
|
||||
start_time: 开始时间
|
||||
end_time: 结束时间
|
||||
run_id: 分析运行 ID
|
||||
|
||||
Returns:
|
||||
模拟数据字典，以 feature_id 为键，值为数据列表，每个数据包含 time, value 和 feature_id
|
||||
|
||||
Raises:
|
||||
ValueError: 当类型无效时
|
||||
"""
|
||||
pipe_ids: list[str] = []
|
||||
junction_ids: list[str] = []
|
||||
for feature_id, feature_type in feature_infos:
|
||||
if feature_type.lower() == "pipe":
|
||||
pipe_ids.append(feature_id)
|
||||
elif feature_type.lower() == "junction":
|
||||
junction_ids.append(feature_id)
|
||||
else:
|
||||
raise ValueError(f"Unknown type: {feature_type}")
|
||||
link_series = await AnalysisResultsRepository.get_series_by_ids(
|
||||
timescale_conn, run_id, "link", list(dict.fromkeys(pipe_ids)),
|
||||
start_time, end_time, "flow",
|
||||
)
|
||||
node_series = await AnalysisResultsRepository.get_series_by_ids(
|
||||
timescale_conn, run_id, "node", list(dict.fromkeys(junction_ids)),
|
||||
start_time, end_time, "pressure",
|
||||
)
|
||||
return {
|
||||
feature_id: [
|
||||
{**item, "feature_id": feature_id}
|
||||
for item in (
|
||||
link_series.get(feature_id, [])
|
||||
if feature_type.lower() == "pipe"
|
||||
else node_series.get(feature_id, [])
|
||||
)
|
||||
]
|
||||
for feature_id, feature_type in feature_infos
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def get_element_associated_scada_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
postgres_conn: AsyncConnection,
|
||||
element_id: str,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
use_cleaned: bool = False,
|
||||
) -> Optional[Any]:
|
||||
"""
|
||||
获取 link/node 关联的 SCADA 监测值
|
||||
|
||||
根据传入的 link/node id，匹配 SCADA 信息，
|
||||
如果存在关联的 SCADA device_id，获取实际的监测数据
|
||||
|
||||
Args:
|
||||
timescale_conn: TimescaleDB 异步连接
|
||||
postgres_conn: PostgreSQL 异步连接
|
||||
element_id: link 或 node 的 ID
|
||||
start_time: 开始时间
|
||||
end_time: 结束时间
|
||||
use_cleaned: 是否使用清洗后的数据 (True: "cleaned_value", False: "monitored_value")
|
||||
|
||||
Returns:
|
||||
SCADA 监测数据值，如果没有找到则返回 None
|
||||
|
||||
Raises:
|
||||
ValueError: 当元素类型无效时
|
||||
"""
|
||||
|
||||
scada_by_id = await TimeseriesAnalysisService._get_project_scada_index(postgres_conn)
|
||||
associated_scada = next(
|
||||
(
|
||||
scada
|
||||
for scada in scada_by_id.values()
|
||||
if (scada.get("node_id") or scada.get("link_id")) == element_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if not associated_scada:
|
||||
return None
|
||||
|
||||
device_id = associated_scada["device_id"]
|
||||
|
||||
data_field = "cleaned_value" if use_cleaned else "monitored_value"
|
||||
|
||||
res = await ScadaRepository.get_scada_field_by_id_time_range(
|
||||
timescale_conn, [device_id], start_time, end_time, data_field
|
||||
)
|
||||
|
||||
return {element_id: res.get(device_id, [])}
|
||||
|
||||
@staticmethod
|
||||
async def clean_scada_data(
|
||||
timescale_conn: AsyncConnection,
|
||||
|
||||
Reference in New Issue
Block a user