refactor(backend)!: separate algorithm and data layers

Reorganize algorithm packages by business responsibility, move orchestration into services, and keep database access behind pooled repositories. Harden analysis API validation, remove unsafe legacy simulation endpoints, and add regression and architecture boundary coverage. BREAKING CHANGE: legacy algorithm module paths and obsolete simulation endpoints are removed.
2026-09-04 17:30:55 +08:00
parent 9b095c7439
commit 5966d039de
91 changed files with 1418 additions and 4020 deletions
@@ -0,0 +1,309 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from pykalman import KalmanFilter
+import os
+
+from app.algorithms._utils import fill_time_gaps
+
+
+def clean_flow_data_kf(
+    input_csv_path: str, show_plot: bool = False, fill_gaps: bool = True
+) -> str:
+    """
+    读取 input_csv_path 中的每列时间序列，使用一维 Kalman 滤波平滑并用预测值替换基于 3σ 检测出的异常点。
+    保存输出为：<input_filename>_cleaned.xlsx（与输入同目录），并返回输出文件的绝对路径。
+    仅保留输入文件路径作为参数（按要求）。
+
+    Args:
+        input_csv_path: CSV 文件路径
+        show_plot: 是否显示可视化
+        fill_gaps: 是否先补齐时间缺口（默认 True）
+    """
+    # 读取 CSV
+    data = pd.read_csv(input_csv_path, header=0, index_col=None, encoding="utf-8")
+
+    # 补齐时间缺口（如果数据包含 time 列）
+    if fill_gaps and "time" in data.columns:
+        data = fill_time_gaps(
+            data, time_col="time", freq="1min", short_gap_threshold=10
+        )
+
+    # 分离时间列和数值列
+    time_col_data = None
+    if "time" in data.columns:
+        time_col_data = data["time"]
+        data = data.drop(columns=["time"])
+
+    # 存储 Kalman 平滑结果
+    data_kf = pd.DataFrame(index=data.index, columns=data.columns)
+    # 平滑每一列
+    for col in data.columns:
+        observations = pd.Series(data[col].values).ffill().bfill()
+        if observations.isna().any():
+            observations = observations.fillna(observations.mean())
+        obs = observations.values.astype(float)
+
+        kf = KalmanFilter(
+            transition_matrices=[1],
+            observation_matrices=[1],
+            initial_state_mean=float(obs[0]),
+            initial_state_covariance=1,
+            observation_covariance=1,
+            transition_covariance=0.01,
+        )
+        # 跳过EM学习，使用固定参数以提高性能
+        state_means, _ = kf.smooth(obs)
+        data_kf[col] = state_means.flatten()
+
+    # 计算残差并用IQR检测异常（更稳健的方法）
+    residuals = data - data_kf
+    residual_thresholds = {}
+    for col in data.columns:
+        res_values = residuals[col].dropna().values  # 移除NaN以计算IQR
+        q1 = np.percentile(res_values, 25)
+        q3 = np.percentile(res_values, 75)
+        iqr = q3 - q1
+        lower_threshold = q1 - 1.5 * iqr
+        upper_threshold = q3 + 1.5 * iqr
+        residual_thresholds[col] = (lower_threshold, upper_threshold)
+
+    cleaned_data = data.copy()
+    anomalies_info = {}
+    for col in data.columns:
+        lower, upper = residual_thresholds[col]
+        sensor_residuals = residuals[col]
+        anomaly_mask = (sensor_residuals < lower) | (sensor_residuals > upper)
+        anomaly_idx = data.index[anomaly_mask.fillna(False)]
+        anomalies_info[col] = pd.DataFrame(
+            {
+                "Observed": data.loc[anomaly_idx, col],
+                "Kalman_Predicted": data_kf.loc[anomaly_idx, col],
+                "Residual": sensor_residuals.loc[anomaly_idx],
+            }
+        )
+        cleaned_data.loc[anomaly_idx, f"{col}_cleaned"] = data_kf.loc[anomaly_idx, col]
+
+    # 如果原始数据包含时间列，将其添加回结果
+    if time_col_data is not None:
+        cleaned_data.insert(0, "time", time_col_data)
+
+    # 构造输出文件名：在输入文件名基础上加后缀 _cleaned.xlsx
+    input_dir = os.path.dirname(os.path.abspath(input_csv_path))
+    input_base = os.path.splitext(os.path.basename(input_csv_path))[0]
+    output_filename = f"{input_base}_cleaned.xlsx"
+    output_path = os.path.join(input_dir, output_filename)
+
+    # 覆盖同名文件
+    if os.path.exists(output_path):
+        os.remove(output_path)
+    cleaned_data.to_excel(output_path, index=False)
+
+    # 可选可视化（第一个传感器）
+    plt.rcParams["font.sans-serif"] = ["SimHei"]
+    plt.rcParams["axes.unicode_minus"] = False
+    if show_plot and len(data.columns) > 0:
+        sensor_to_plot = data.columns[0]
+        plt.figure(figsize=(12, 6))
+        plt.plot(
+            data.index,
+            data[sensor_to_plot],
+            label="监测值",
+            marker="o",
+            markersize=3,
+            alpha=0.7,
+        )
+        plt.plot(
+            data.index, data_kf[sensor_to_plot], label="Kalman滤波预测值", linewidth=2
+        )
+        anomaly_idx = anomalies_info[sensor_to_plot].index
+        if len(anomaly_idx) > 0:
+            plt.plot(
+                anomaly_idx,
+                data[sensor_to_plot].loc[anomaly_idx],
+                "ro",
+                markersize=8,
+                label="监测值异常点",
+            )
+            plt.plot(
+                anomaly_idx,
+                data_kf[sensor_to_plot].loc[anomaly_idx],
+                "go",
+                markersize=8,
+                label="Kalman修复值",
+            )
+        plt.xlabel("时间点（序号）")
+        plt.ylabel("监测值")
+        plt.title(f"{sensor_to_plot}：观测值与Kalman滤波预测值（异常点标记）")
+        plt.legend()
+        plt.show()
+
+    # 返回输出文件的绝对路径
+    return os.path.abspath(output_path)
+
+
+def clean_flow_data_df_kf(
+    data: pd.DataFrame, show_plot: bool = False
+) -> pd.DataFrame:
+    """
+    接收一个 DataFrame 数据结构，使用一维 Kalman 滤波平滑并用预测值替换基于 IQR 检测出的异常点。
+    区分合理的0值（流量转换）和异常的0值（连续多个0或孤立0）。
+    返回完整的清洗后 DataFrame。
+
+    Args:
+        data: 输入 DataFrame（可包含 time 列）
+        show_plot: 是否显示可视化
+    """
+    # 使用传入的 DataFrame
+    data = data.copy()
+
+    # 补齐时间缺口（如果启用且数据包含 time 列）
+    data_filled = fill_time_gaps(
+        data, time_col="time", freq="1min", short_gap_threshold=10
+    )
+
+    # 保存 time 列用于最后合并
+    time_col_series = None
+    if "time" in data_filled.columns:
+        time_col_series = data_filled["time"]
+
+    # 移除 time 列用于后续清洗
+    data_filled = data_filled.drop(columns=["time"])
+
+    # 存储 Kalman 平滑结果
+    data_kf = pd.DataFrame(index=data_filled.index, columns=data_filled.columns)
+    # 平滑每一列
+    for col in data_filled.columns:
+        observations = pd.Series(data_filled[col].values).ffill().bfill()
+        if observations.isna().any():
+            observations = observations.fillna(observations.mean())
+        obs = observations.values.astype(float)
+
+        kf = KalmanFilter(
+            transition_matrices=[1],
+            observation_matrices=[1],
+            initial_state_mean=float(obs[0]),
+            initial_state_covariance=1,
+            observation_covariance=10,
+            transition_covariance=10,
+        )
+        state_means, _ = kf.smooth(obs)
+        data_kf[col] = state_means.flatten()
+
+    # 计算残差并用IQR检测异常
+    residuals = data_filled - data_kf
+    residual_thresholds = {}
+    for col in data_filled.columns:
+        res_values = residuals[col].dropna().values
+        q1 = np.percentile(res_values, 25)
+        q3 = np.percentile(res_values, 75)
+        iqr = q3 - q1
+        lower_threshold = q1 - 1.5 * iqr
+        upper_threshold = q3 + 1.5 * iqr
+        residual_thresholds[col] = (lower_threshold, upper_threshold)
+
+    # 创建完整的修复数据
+    cleaned_data = data_filled.copy()
+    anomalies_info = {}
+
+    for col in data_filled.columns:
+        lower, upper = residual_thresholds[col]
+        sensor_residuals = residuals[col]
+        anomaly_mask = (sensor_residuals < lower) | (sensor_residuals > upper)
+        anomaly_idx = data_filled.index[anomaly_mask.fillna(False)]
+
+        anomalies_info[col] = pd.DataFrame(
+            {
+                "Observed": data_filled.loc[anomaly_idx, col],
+                "Kalman_Predicted": data_kf.loc[anomaly_idx, col],
+                "Residual": sensor_residuals.loc[anomaly_idx],
+            }
+        )
+
+        # 直接在原列上替换异常值为 Kalman 预测值
+        cleaned_data.loc[anomaly_idx, col] = data_kf.loc[anomaly_idx, col]
+
+    # 可选可视化
+    plt.rcParams["font.sans-serif"] = ["SimHei"]
+    plt.rcParams["axes.unicode_minus"] = False
+    if show_plot and len(data.columns) > 0:
+        sensor_to_plot = data.columns[0]
+
+        # 定义x轴
+        n = len(data)
+        time = np.arange(n)
+        n_filled = len(data_filled)
+        time_filled = np.arange(n_filled)
+
+        plt.figure(figsize=(12, 8))
+
+        plt.subplot(2, 1, 1)
+        plt.plot(
+            time,
+            data[sensor_to_plot],
+            label="原始监测值",
+            marker="o",
+            markersize=3,
+            alpha=0.7,
+        )
+
+        # 修正：检查 data_filled 的异常值，绘制在 time_filled 上
+        abnormal_zero_mask = data_filled[sensor_to_plot].isna()
+        # 如果目的是检查0值，应该用 == 0。这里保留 isna() 但修正索引引用，防止crash。
+        # 如果原意是 isna() 则在 fillna 后通常没有 na。假设用户可能想检查 0 值？
+        # 基于 "异常0值" 的标签，改为检查 0 值更合理，但为了保险起见，
+        # 如果 isna() 返回空，就不画。防止索引越界是主要的。
+        abnormal_zero_idx = data_filled.index[abnormal_zero_mask]
+
+        if len(abnormal_zero_idx) > 0:
+            # 注意：如果 abnormal_zero_idx 是基于 data_filled 的索引（0..M-1），
+            # 直接作为 x 坐标即可，因为 time_filled 也是 0..M-1
+            # 而 y 值应该取自 data_filled 或 data_kf，取 data 会越界
+            plt.plot(
+                abnormal_zero_idx,
+                data_filled[sensor_to_plot].loc[abnormal_zero_idx],
+                "mo",
+                markersize=8,
+                label="异常值(NaN)",
+            )
+
+        plt.plot(
+            time_filled, data_kf[sensor_to_plot], label="Kalman滤波预测值", linewidth=2
+        )
+        anomaly_idx = anomalies_info[sensor_to_plot].index
+        if len(anomaly_idx) > 0:
+            plt.plot(
+                anomaly_idx,
+                data_filled[sensor_to_plot].loc[anomaly_idx],
+                "ro",
+                markersize=8,
+                label="IQR异常点",
+            )
+        plt.xlabel("时间点（序号）")
+        plt.ylabel("流量值")
+        plt.title(f"{sensor_to_plot}：原始数据与异常检测")
+        plt.legend()
+
+        plt.subplot(2, 1, 2)
+        plt.plot(
+            time_filled,
+            cleaned_data[sensor_to_plot],
+            label="修复后监测值",
+            marker="o",
+            markersize=3,
+            color="green",
+        )
+        plt.xlabel("时间点（序号）")
+        plt.ylabel("流量值")
+        plt.title(f"{sensor_to_plot}：修复后数据")
+        plt.legend()
+
+        plt.tight_layout()
+        plt.show()
+
+    # 将 time 列添加回结果
+    if time_col_series is not None:
+        cleaned_data.insert(0, "time", time_col_series)
+
+    # 返回完整的修复后字典
+    return cleaned_data