refactor(db)!: finalize pooled WNDB v2 migration

2026-08-27 17:26:22 +08:00
parent fa188af0b1
commit b74799a39d
105 changed files with 4988 additions and 5565 deletions
@@ -56,38 +56,40 @@ class CompositeQueries:
        Raises:
            ValueError: 当 SCADA 设备未找到或字段无效时
        """
-        result = {}
        scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
-
+        link_devices: dict[str, str] = {}
+        node_devices: dict[str, str] = {}
        for device_id in device_ids:
            target_scada = scada_by_id.get(device_id)
            if not target_scada:
                raise ValueError(f"SCADA device {device_id} not found")
-
            scada_type = target_scada["device_type"]
-            element_id = (
-                target_scada["link_id"]
-                if scada_type in {"pipe_flow", "flow"}
-                else target_scada["node_id"]
-            )
-
-            if scada_type == "pipe_flow":
-                # 查询 link 模拟数据
-                res = await RealtimeRepository.get_link_field_by_time_range(
-                    timescale_conn, start_time, end_time, element_id, "flow"
-                )
+            if scada_type in {"pipe_flow", "flow"}:
+                link_devices[device_id] = target_scada["link_id"]
            elif scada_type == "pressure":
-                # 查询 node 模拟数据
-                res = await RealtimeRepository.get_node_field_by_time_range(
-                    timescale_conn, start_time, end_time, element_id, "pressure"
-                )
+                node_devices[device_id] = target_scada["node_id"]
            else:
                raise ValueError(f"Unknown SCADA type: {scada_type}")
-            # 添加 scada_id 到每个数据项
-            for item in res:
-                item["scada_id"] = device_id
-            result[device_id] = res
-        return result
+
+        link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
+            timescale_conn, start_time, end_time,
+            list(dict.fromkeys(link_devices.values())), "flow",
+        )
+        node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
+            timescale_conn, start_time, end_time,
+            list(dict.fromkeys(node_devices.values())), "pressure",
+        )
+        return {
+            device_id: [
+                {**item, "scada_id": device_id}
+                for item in (
+                    link_series.get(element_id, [])
+                    if device_id in link_devices
+                    else node_series.get(element_id, [])
+                )
+            ]
+            for device_id, element_id in (link_devices | node_devices).items()
+        }

    @staticmethod
    async def get_scada_associated_analysis_simulation_data(
@@ -117,38 +119,41 @@ class CompositeQueries:
        Raises:
            ValueError: 当 SCADA 设备未找到或字段无效时
        """
-        result = {}
        scada_by_id = await CompositeQueries._get_project_scada_index(postgres_conn)
-
+        link_devices: dict[str, str] = {}
+        node_devices: dict[str, str] = {}
        for device_id in device_ids:
            target_scada = scada_by_id.get(device_id)
            if not target_scada:
                raise ValueError(f"SCADA device {device_id} not found")

            scada_type = target_scada["device_type"]
-            element_id = (
-                target_scada["link_id"]
-                if scada_type in {"pipe_flow", "flow"}
-                else target_scada["node_id"]
-            )
-
-            if scada_type == "pipe_flow":
-                # 查询 link 模拟数据
-                res = await AnalysisResultsRepository.get_link_series(
-                    timescale_conn, run_id, element_id, start_time, end_time, "flow"
-                )
+            if scada_type in {"pipe_flow", "flow"}:
+                link_devices[device_id] = target_scada["link_id"]
            elif scada_type == "pressure":
-                # 查询 node 模拟数据
-                res = await AnalysisResultsRepository.get_node_series(
-                    timescale_conn, run_id, element_id, start_time, end_time, "pressure"
-                )
+                node_devices[device_id] = target_scada["node_id"]
            else:
                raise ValueError(f"Unknown SCADA type: {scada_type}")
-            # 添加 scada_id 到每个数据项
-            for item in res:
-                item["scada_id"] = device_id
-            result[device_id] = res
-        return result
+
+        link_series = await AnalysisResultsRepository.get_series_by_ids(
+            timescale_conn, run_id, "link",
+            list(dict.fromkeys(link_devices.values())), start_time, end_time, "flow",
+        )
+        node_series = await AnalysisResultsRepository.get_series_by_ids(
+            timescale_conn, run_id, "node",
+            list(dict.fromkeys(node_devices.values())), start_time, end_time, "pressure",
+        )
+        return {
+            device_id: [
+                {**item, "scada_id": device_id}
+                for item in (
+                    link_series.get(element_id, [])
+                    if device_id in link_devices
+                    else node_series.get(element_id, [])
+                )
+            ]
+            for device_id, element_id in (link_devices | node_devices).items()
+        }

    @staticmethod
    async def get_realtime_simulation_data(
@@ -175,26 +180,33 @@ class CompositeQueries:
        Raises:
            ValueError: 当 SCADA 设备未找到或字段无效时
        """
-        result = {}
+        pipe_ids: list[str] = []
+        junction_ids: list[str] = []
        for feature_id, feature_type in feature_infos:
-
            if feature_type.lower() == "pipe":
-                # 查询 link 模拟数据
-                res = await RealtimeRepository.get_link_field_by_time_range(
-                    timescale_conn, start_time, end_time, feature_id, "flow"
-                )
+                pipe_ids.append(feature_id)
            elif feature_type.lower() == "junction":
-                # 查询 node 模拟数据
-                res = await RealtimeRepository.get_node_field_by_time_range(
-                    timescale_conn, start_time, end_time, feature_id, "pressure"
-                )
+                junction_ids.append(feature_id)
            else:
                raise ValueError(f"Unknown type: {feature_type}")
-            # 添加 scada_id 到每个数据项
-            for item in res:
-                item["feature_id"] = feature_id
-            result[feature_id] = res
-        return result
+        link_series = await RealtimeRepository.get_link_fields_by_ids_time_range(
+            timescale_conn, start_time, end_time, list(dict.fromkeys(pipe_ids)), "flow"
+        )
+        node_series = await RealtimeRepository.get_node_fields_by_ids_time_range(
+            timescale_conn, start_time, end_time,
+            list(dict.fromkeys(junction_ids)), "pressure",
+        )
+        return {
+            feature_id: [
+                {**item, "feature_id": feature_id}
+                for item in (
+                    link_series.get(feature_id, [])
+                    if feature_type.lower() == "pipe"
+                    else node_series.get(feature_id, [])
+                )
+            ]
+            for feature_id, feature_type in feature_infos
+        }

    @staticmethod
    async def get_analysis_simulation_data(
@@ -223,25 +235,34 @@ class CompositeQueries:
        Raises:
            ValueError: 当类型无效时
        """
-        result = {}
+        pipe_ids: list[str] = []
+        junction_ids: list[str] = []
        for feature_id, feature_type in feature_infos:
            if feature_type.lower() == "pipe":
-                # 查询 link 模拟数据
-                res = await AnalysisResultsRepository.get_link_series(
-                    timescale_conn, run_id, feature_id, start_time, end_time, "flow"
-                )
+                pipe_ids.append(feature_id)
            elif feature_type.lower() == "junction":
-                # 查询 node 模拟数据
-                res = await AnalysisResultsRepository.get_node_series(
-                    timescale_conn, run_id, feature_id, start_time, end_time, "pressure"
-                )
+                junction_ids.append(feature_id)
            else:
                raise ValueError(f"Unknown type: {feature_type}")
-            # 添加 feature_id 到每个数据项
-            for item in res:
-                item["feature_id"] = feature_id
-            result[feature_id] = res
-        return result
+        link_series = await AnalysisResultsRepository.get_series_by_ids(
+            timescale_conn, run_id, "link", list(dict.fromkeys(pipe_ids)),
+            start_time, end_time, "flow",
+        )
+        node_series = await AnalysisResultsRepository.get_series_by_ids(
+            timescale_conn, run_id, "node", list(dict.fromkeys(junction_ids)),
+            start_time, end_time, "pressure",
+        )
+        return {
+            feature_id: [
+                {**item, "feature_id": feature_id}
+                for item in (
+                    link_series.get(feature_id, [])
+                    if feature_type.lower() == "pipe"
+                    else node_series.get(feature_id, [])
+                )
+            ]
+            for feature_id, feature_type in feature_infos
+        }

    @staticmethod
    async def get_element_associated_scada_data(
@@ -399,7 +420,7 @@ class CompositeQueries:
            if scada_by_id[device_id]["device_type"] in {"pipe_flow", "flow"}
        ]

-        updated_rows = 0
+        cleaned_rows: list[tuple[datetime, str, float | None]] = []
        for grouped_ids, cleaning_function in (
            (pressure_ids, clean_pressure_data_df_km),
            (flow_ids, clean_flow_data_df_kf),
@@ -422,18 +443,25 @@ class CompositeQueries:
                        if isinstance(time_value, datetime)
                        else datetime.fromisoformat(str(time_value))
                    )
-                    await ScadaRepository.update_scada_field(
-                        timescale_conn,
-                        time_dt,
-                        device_id,
-                        "cleaned_value",
-                        value,
+                    cleaned_rows.append(
+                        (
+                            time_dt,
+                            device_id,
+                            None if pd.isna(value) else float(value),
+                        )
                    )
-                    updated_rows += 1

-        if updated_rows == 0:
+        if not cleaned_rows:
            raise ValueError("SCADA 数据清洗未产生任何数据库更新")

+        updated_rows = await ScadaRepository.update_scada_field_batch(
+            timescale_conn,
+            cleaned_rows,
+            "cleaned_value",
+        )
+        if updated_rows == 0:
+            raise ValueError("SCADA 清洗结果未匹配任何已有监测数据")
+
        return "success"

    @staticmethod