新增清洗 scada 数据方法，更新数据返回格式

timescaledb/composite_queries.py

@@ -1,6 +1,8 @@
-from typing import List, Optional, Dict, Any
+from typing import List, Optional, Any
 from datetime import datetime
 from psycopg import AsyncConnection
+import pandas as pd
+import api_ex
 
 from postgresql.scada_info import ScadaRepository as PostgreScadaRepository
 from timescaledb.schemas.realtime import RealtimeRepository
@@ -204,3 +206,127 @@ class CompositeQueries:
         return await ScadaRepository.get_scada_field_by_id_time_range(
             timescale_conn, device_id, start_time, end_time, data_field
         )
+
+    @staticmethod
+    async def clean_scada_data(
+        timescale_conn: AsyncConnection,
+        postgres_conn: AsyncConnection,
+        device_ids: List[str],
+        start_time: datetime,
+        end_time: datetime,
+    ) -> str:
+        """
+        清洗 SCADA 数据
+
+        根据 device_ids 查询 monitored_value，清洗后更新 cleaned_value
+
+        Args:
+            timescale_conn: TimescaleDB 连接
+            postgres_conn: PostgreSQL 连接
+            device_ids: 设备 ID 列表
+            start_time: 开始时间
+            end_time: 结束时间
+
+        Returns:
+            "success" 或错误信息
+        """
+        try:
+            # 获取所有 SCADA 信息
+            scada_infos = await PostgreScadaRepository.get_scadas(postgres_conn)
+            # 将列表转换为字典，以 device_id 为键
+            scada_device_info_dict = {info["id"]: info for info in scada_infos}
+
+            # 按设备类型分组设备
+            type_groups = {}
+            for device_id in device_ids:
+                device_info = scada_device_info_dict.get(device_id, {})
+                device_type = device_info.get("type", "unknown")
+                if device_type not in type_groups:
+                    type_groups[device_type] = []
+                type_groups[device_type].append(device_id)
+
+            # 批量处理每种类型的设备
+            for device_type, ids in type_groups.items():
+                if device_type not in ["pressure", "pipe_flow"]:
+                    continue  # 跳过未知类型
+
+                # 查询 monitored_value 数据
+                data = await ScadaRepository.get_scada_field_by_id_time_range(
+                    timescale_conn, ids, start_time, end_time, "monitored_value"
+                )
+
+                if not data:
+                    continue
+
+                # 将嵌套字典转换为 DataFrame，使用 time 作为索引
+                # data 格式: {device_id: [{"time": "...", "value": ...}, ...]}
+                all_records = []
+                for device_id, records in data.items():
+                    for record in records:
+                        all_records.append(
+                            {
+                                "time": record["time"],
+                                "device_id": device_id,
+                                "value": record["value"],
+                            }
+                        )
+
+                if not all_records:
+                    continue
+
+                # 创建 DataFrame 并透视，使 device_id 成为列
+                df_long = pd.DataFrame(all_records)
+                df = df_long.pivot(index="time", columns="device_id", values="value")
+
+                # 确保所有请求的设备都在列中（即使没有数据）
+                for device_id in ids:
+                    if device_id not in df.columns:
+                        df[device_id] = None
+
+                # 只保留请求的设备列
+                df = df[ids]
+
+                # 重置索引，将 time 变为普通列
+                df = df.reset_index()
+
+                # 移除 time 列，准备输入给清洗方法
+                value_df = df.drop(columns=["time"])
+
+                # 调用清洗方法
+                if device_type == "pressure":
+                    cleaned_dict = api_ex.Pdataclean.clean_pressure_data_dict_km(
+                        value_df.to_dict(orient="list")
+                    )
+                elif device_type == "pipe_flow":
+                    cleaned_dict = api_ex.Fdataclean.clean_flow_data_dict(
+                        value_df.to_dict(orient="list")
+                    )
+                else:
+                    continue
+
+                # 将字典转换为 DataFrame（字典键为设备ID，值为值列表）
+                cleaned_value_df = pd.DataFrame(cleaned_dict)
+
+                # 添加 time 列到首列
+                cleaned_df = pd.concat([df["time"], cleaned_value_df], axis=1)
+
+                # 将清洗后的数据写回数据库
+                for device_id in ids:
+                    if device_id in cleaned_df.columns:
+                        cleaned_values = cleaned_df[device_id].tolist()
+                        time_values = cleaned_df["time"].tolist()
+                        for i, time_str in enumerate(time_values):
+                            # time_str 已经是 ISO 格式字符串
+                            time_dt = datetime.fromisoformat(time_str)
+                            value = cleaned_values[i]
+                            await ScadaRepository.update_scada_field(
+                                timescale_conn,
+                                time_dt,
+                                device_id,
+                                "cleaned_value",
+                                value,
+                            )
+
+            return "success"
+        except Exception as e:
+            return f"error: {str(e)}"