jiang
5966d039de
Reorganize algorithm packages by business responsibility, move orchestration into services, and keep database access behind pooled repositories. Harden analysis API validation, remove unsafe legacy simulation endpoints, and add regression and architecture boundary coverage. BREAKING CHANGE: legacy algorithm module paths and obsolete simulation endpoints are removed.
100 lines
3.0 KiB
Python
100 lines
3.0 KiB
Python
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from app.algorithms.scada_cleaning import pressure_series as pressure_cleaning
|
|
|
|
|
|
DATA_DIR = Path(__file__).resolve().parents[3] / "data"
|
|
RAW_DATA_PATH = DATA_DIR / "node_simulation.csv"
|
|
NOISY_DATA_PATH = DATA_DIR / "node_simulation_noisy.csv"
|
|
REQUIRES_PRESSURE_SAMPLES = pytest.mark.skipif(
|
|
not RAW_DATA_PATH.exists() or not NOISY_DATA_PATH.exists(),
|
|
reason="pressure cleaning sample CSV files are not available",
|
|
)
|
|
|
|
@REQUIRES_PRESSURE_SAMPLES
|
|
def test_clean_pressure_data_df_km_repairs_long_form_pressure_series():
|
|
raw_df = pd.read_csv(RAW_DATA_PATH)
|
|
noisy_df = pd.read_csv(NOISY_DATA_PATH)
|
|
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(noisy_df)
|
|
|
|
for df in (raw_df, noisy_df, cleaned_df):
|
|
df["time"] = pd.to_datetime(df["time"])
|
|
|
|
assert len(cleaned_df) == len(raw_df)
|
|
assert set(cleaned_df.columns) == {"time", "id", "pressure"}
|
|
assert cleaned_df["pressure"].isna().sum() == 0
|
|
|
|
noisy_joined = raw_df.merge(
|
|
noisy_df,
|
|
on=["time", "id"],
|
|
how="inner",
|
|
suffixes=("_raw", "_noisy"),
|
|
)
|
|
cleaned_joined = raw_df.merge(
|
|
cleaned_df,
|
|
on=["time", "id"],
|
|
how="inner",
|
|
suffixes=("_raw", "_clean"),
|
|
)
|
|
|
|
noisy_rmse = float(
|
|
np.sqrt(
|
|
np.mean(
|
|
(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"])
|
|
** 2
|
|
)
|
|
)
|
|
)
|
|
cleaned_rmse = float(
|
|
np.sqrt(
|
|
np.mean(
|
|
(cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"])
|
|
** 2
|
|
)
|
|
)
|
|
)
|
|
noisy_mae = float(
|
|
np.mean(np.abs(noisy_joined["pressure_raw"] - noisy_joined["pressure_noisy"]))
|
|
)
|
|
cleaned_mae = float(
|
|
np.mean(np.abs(cleaned_joined["pressure_raw"] - cleaned_joined["pressure_clean"]))
|
|
)
|
|
|
|
assert cleaned_rmse < 0.35
|
|
assert cleaned_rmse < noisy_rmse * 0.5
|
|
assert cleaned_mae < noisy_mae
|
|
|
|
repaired_gap = cleaned_df[
|
|
(cleaned_df["id"] == 170490)
|
|
& (cleaned_df["time"] == pd.Timestamp("2026-01-01T05:00:00+08:00"))
|
|
]["pressure"].iloc[0]
|
|
assert abs(repaired_gap - 30.62433433532715) < 1.0
|
|
|
|
spike_row = cleaned_df[
|
|
(cleaned_df["id"] == 42563)
|
|
& (cleaned_df["time"] == pd.Timestamp("2026-01-01T03:45:00+08:00"))
|
|
]["pressure"].iloc[0]
|
|
assert abs(spike_row - 28.018701553344727) < 2.0
|
|
|
|
|
|
@REQUIRES_PRESSURE_SAMPLES
|
|
def test_clean_pressure_data_df_km_accepts_single_sensor_wide_frame_with_utc_strings():
|
|
noisy_df = pd.read_csv(NOISY_DATA_PATH)
|
|
single_sensor = (
|
|
noisy_df[noisy_df["id"] == 170490][["time", "pressure"]]
|
|
.rename(columns={"pressure": "170490"})
|
|
.copy()
|
|
)
|
|
single_sensor["time"] = (
|
|
pd.to_datetime(single_sensor["time"], utc=True).dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
)
|
|
|
|
cleaned_df = pressure_cleaning.clean_pressure_data_df_km(single_sensor)
|
|
|
|
assert len(cleaned_df) == 192
|
|
assert cleaned_df["170490"].isna().sum() == 0
|