jiang
5966d039de
Reorganize algorithm packages by business responsibility, move orchestration into services, and keep database access behind pooled repositories. Harden analysis API validation, remove unsafe legacy simulation endpoints, and add regression and architecture boundary coverage. BREAKING CHANGE: legacy algorithm module paths and obsolete simulation endpoints are removed.
736 lines
25 KiB
Python
736 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from datetime import datetime, timedelta
|
|
from typing import Any
|
|
from uuid import UUID
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from app.algorithms.burst_detection.pressure_anomaly import PressureAnomalyDetector
|
|
from app.infra.db.postgresql.scada import get_all_scada_info
|
|
from app.infra.db.timescaledb.internal_queries import InternalQueries
|
|
from app.services.scheme_management import (
|
|
store_scheme_info,
|
|
)
|
|
from app.domain.time import parse_utc_time, utc_now
|
|
|
|
|
|
TARGET_DAY_COUNT = 15
|
|
DEFAULT_SAMPLE_INTERVAL_MINUTES = 15
|
|
TARGET_MU = 1
|
|
TARGET_N_ESTIMATORS = 50
|
|
TARGET_RANDOM_STATE = 42
|
|
TARGET_SCORE_THRESHOLD = -0.04
|
|
MIN_COMPLETE_SENSORS = 5
|
|
|
|
|
|
def run_burst_detection(
|
|
*,
|
|
network: str,
|
|
username: str,
|
|
observed_pressure_data: (
|
|
pd.DataFrame
|
|
| dict[str, list[Any]]
|
|
| list[dict[str, Any]]
|
|
| list[list[Any]]
|
|
| None
|
|
) = None,
|
|
points_per_day: int = 1440,
|
|
mu: int = 100,
|
|
iforest_params: dict[str, Any] | None = None,
|
|
target_time: datetime | str | None = None,
|
|
sampling_interval_minutes: int | None = None,
|
|
scada_start: datetime | str | None = None,
|
|
scada_end: datetime | str | None = None,
|
|
sensor_nodes: list[str] | None = None,
|
|
scheme_name: str | None = None,
|
|
data_source: str = "monitoring",
|
|
simulation_run_id: UUID | None = None,
|
|
) -> dict[str, Any]:
|
|
"""
|
|
运行爆管侦测服务入口。
|
|
|
|
调用方式三选一：
|
|
- 不传数据时间窗，自动侦测最近完整时刻；可用 `target_time` 回放历史时刻
|
|
- 直接传 `observed_pressure_data`
|
|
- 或传 `scada_start/scada_end` 让后端自动查询 SCADA 压力数据
|
|
|
|
`observed_pressure_data` 支持格式：
|
|
- `pd.DataFrame`
|
|
行表示时间点，列表示传感器；列名应为传感器/节点 ID。
|
|
- `dict[str, list[Any]]`
|
|
键为传感器/节点 ID，值为按时间顺序排列的压力序列。
|
|
例如：`{"J1": [101.2, 101.0], "J2": [99.8, 99.7]}`。
|
|
- `list[dict[str, Any]]`
|
|
每个元素代表一个时间点的多传感器观测。
|
|
例如：`[{"J1": 101.2, "J2": 99.8}, {"J1": 101.0, "J2": 99.7}]`。
|
|
- `list[list[Any]]`
|
|
二维数组式 JSON，格式为 `(时间点数, 传感器数)`。
|
|
这是最接近原始 `burst_detector` 示例代码的调用方式。
|
|
|
|
数据约束：
|
|
- 统一要求“行=时间点，列=传感器”。
|
|
- 总样本点数必须能被 `points_per_day` 整除。
|
|
- 至少要有 2 天数据，即 `sample_count >= 2 * points_per_day`。
|
|
- 若传入 `sensor_nodes`，输入数据必须包含这些列；SCADA 模式下也会只按这些节点取数。
|
|
"""
|
|
if not network:
|
|
raise ValueError("network is required.")
|
|
|
|
selected_sensor_nodes = (
|
|
list(dict.fromkeys([node for node in (sensor_nodes or []) if node]))
|
|
if sensor_nodes
|
|
else None
|
|
)
|
|
use_scada_source = scada_start is not None or scada_end is not None
|
|
use_target_mode = (
|
|
observed_pressure_data is None
|
|
and not use_scada_source
|
|
and data_source != "simulation"
|
|
) or target_time is not None
|
|
|
|
resolved_target_time: datetime | None = None
|
|
requested_target_time: datetime | None = None
|
|
excluded_sensors: list[dict[str, str]] = []
|
|
daily_times: list[datetime] | None = None
|
|
resolved_sampling_interval_minutes: int | None = None
|
|
|
|
if use_target_mode:
|
|
if observed_pressure_data is not None or use_scada_source:
|
|
raise ValueError(
|
|
"target_time 不能与 observed_pressure_data 或 scada_start/scada_end 同时使用。"
|
|
)
|
|
scada_sensor_nodes = (
|
|
selected_sensor_nodes
|
|
if selected_sensor_nodes is not None
|
|
else _get_pressure_sensor_nodes(network)
|
|
)
|
|
requested_target_time = (
|
|
_to_datetime(target_time) if target_time is not None else None
|
|
)
|
|
resolved_sampling_interval_minutes = _resolve_sampling_interval_minutes(
|
|
network=network,
|
|
sensor_nodes=scada_sensor_nodes,
|
|
requested_interval=sampling_interval_minutes,
|
|
)
|
|
target_points_per_day = 1440 // resolved_sampling_interval_minutes
|
|
(
|
|
observed_input,
|
|
resolved_target_time,
|
|
excluded_sensors,
|
|
) = _build_target_pressure_from_scada(
|
|
network=network,
|
|
sensor_nodes=scada_sensor_nodes,
|
|
requested_target_time=requested_target_time,
|
|
sampling_interval_minutes=resolved_sampling_interval_minutes,
|
|
points_per_day=target_points_per_day,
|
|
)
|
|
selected_sensor_nodes = list(observed_input.columns)
|
|
observed_source = (
|
|
"latest_monitoring" if target_time is None else "historical_monitoring"
|
|
)
|
|
points_per_day = target_points_per_day
|
|
mu = TARGET_MU
|
|
iforest_params = {
|
|
"n_estimators": TARGET_N_ESTIMATORS,
|
|
"random_state": TARGET_RANDOM_STATE,
|
|
"contamination": "auto",
|
|
}
|
|
daily_times = [
|
|
resolved_target_time - timedelta(days=offset)
|
|
for offset in range(TARGET_DAY_COUNT - 1, -1, -1)
|
|
]
|
|
|
|
elif use_scada_source:
|
|
scada_sensor_nodes = (
|
|
selected_sensor_nodes
|
|
if selected_sensor_nodes is not None
|
|
else _get_pressure_sensor_nodes(network)
|
|
)
|
|
if data_source == "simulation":
|
|
if not simulation_run_id:
|
|
raise ValueError("模拟数据模式必须提供 simulation_run_id。")
|
|
observed_df = _build_observed_pressure_from_simulation(
|
|
network=network,
|
|
sensor_nodes=scada_sensor_nodes,
|
|
scada_start=scada_start,
|
|
scada_end=scada_end,
|
|
simulation_run_id=simulation_run_id,
|
|
)
|
|
observed_input = observed_df
|
|
observed_source = "analysis_run_timerange"
|
|
else:
|
|
observed_df = _build_observed_pressure_from_scada(
|
|
network=network,
|
|
sensor_nodes=scada_sensor_nodes,
|
|
scada_start=scada_start,
|
|
scada_end=scada_end,
|
|
)
|
|
observed_input = observed_df
|
|
observed_source = "backend_timerange"
|
|
else:
|
|
if observed_pressure_data is None:
|
|
raise ValueError(
|
|
"未提供 observed_pressure_data，且未提供 scada_start/scada_end。"
|
|
)
|
|
observed_input = observed_pressure_data
|
|
observed_source = "request_payload"
|
|
|
|
detector = PressureAnomalyDetector(
|
|
mu=mu,
|
|
points_per_day=points_per_day,
|
|
iforest_params=iforest_params,
|
|
)
|
|
result_df = detector.run_detection(
|
|
observed_input,
|
|
sensor_nodes=selected_sensor_nodes,
|
|
)
|
|
resolved_sensor_nodes = list(result_df.attrs.get("sensor_nodes", []))
|
|
rows = _serialize_result_rows(
|
|
result_df,
|
|
daily_times=daily_times,
|
|
target_only=use_target_mode,
|
|
)
|
|
summary = _build_detection_summary(
|
|
result_df,
|
|
daily_times=daily_times,
|
|
target_only=use_target_mode,
|
|
)
|
|
payload: dict[str, Any] = {
|
|
"network": network,
|
|
"sensor_nodes": resolved_sensor_nodes,
|
|
"observed_source": observed_source,
|
|
"sample_count": int(result_df.attrs.get("sample_count", 0)),
|
|
"points_per_day": int(result_df.attrs.get("points_per_day", points_per_day)),
|
|
"day_count": int(result_df.attrs.get("day_count", len(result_df))),
|
|
"rows": rows,
|
|
"summary": summary,
|
|
"algorithm_params": {
|
|
"mu": mu,
|
|
"points_per_day": points_per_day,
|
|
"iforest_params": detector.iforest_params,
|
|
**(
|
|
{"score_threshold": TARGET_SCORE_THRESHOLD}
|
|
if use_target_mode
|
|
else {}
|
|
),
|
|
},
|
|
}
|
|
if data_source == "simulation":
|
|
payload["data_source"] = "simulation"
|
|
payload["simulation_run"] = {
|
|
"run_id": str(simulation_run_id),
|
|
}
|
|
else:
|
|
payload["data_source"] = "monitoring"
|
|
|
|
if (
|
|
use_target_mode
|
|
and resolved_target_time is not None
|
|
and resolved_sampling_interval_minutes is not None
|
|
):
|
|
sample_start = resolved_target_time - timedelta(
|
|
days=TARGET_DAY_COUNT,
|
|
minutes=-resolved_sampling_interval_minutes,
|
|
)
|
|
payload.update(
|
|
{
|
|
"requested_target_time": (
|
|
requested_target_time.isoformat()
|
|
if requested_target_time is not None
|
|
else None
|
|
),
|
|
"target_time": resolved_target_time.isoformat(),
|
|
"reference_window": {
|
|
"start": (resolved_target_time - timedelta(days=14)).isoformat(),
|
|
"end": (resolved_target_time - timedelta(days=1)).isoformat(),
|
|
"day_count": 14,
|
|
},
|
|
"sampling_interval_minutes": resolved_sampling_interval_minutes,
|
|
"daily_scores": [
|
|
{
|
|
"timestamp": row["Timestamp"],
|
|
"role": row["Role"],
|
|
"score": row["Score"],
|
|
"raw_prediction": row["Prediction"],
|
|
}
|
|
for row in rows
|
|
],
|
|
"data_quality": {
|
|
"included_sensors": resolved_sensor_nodes,
|
|
"excluded_sensors": excluded_sensors,
|
|
"minimum_required_sensors": MIN_COMPLETE_SENSORS,
|
|
},
|
|
"scada_window": {
|
|
"start": sample_start.isoformat(),
|
|
"end": resolved_target_time.isoformat(),
|
|
},
|
|
}
|
|
)
|
|
elif use_scada_source:
|
|
payload["scada_window"] = {
|
|
"start": _to_datetime(scada_start).isoformat(),
|
|
"end": _to_datetime(scada_end).isoformat(),
|
|
}
|
|
if scheme_name:
|
|
_store_burst_detection_scheme(
|
|
network=network,
|
|
scheme_name=scheme_name,
|
|
username=username,
|
|
payload=payload,
|
|
mu=mu,
|
|
points_per_day=points_per_day,
|
|
iforest_params=detector.iforest_params,
|
|
)
|
|
payload["scheme_name"] = scheme_name
|
|
return payload
|
|
|
|
|
|
def _build_observed_pressure_from_simulation(
|
|
*,
|
|
network: str,
|
|
sensor_nodes: list[str],
|
|
scada_start: datetime | str | None,
|
|
scada_end: datetime | str | None,
|
|
simulation_run_id: UUID,
|
|
) -> pd.DataFrame:
|
|
if scada_start is None or scada_end is None:
|
|
raise ValueError("使用分析模拟数据时必须同时提供 scada_start 与 scada_end。")
|
|
|
|
start_dt = _to_datetime(scada_start)
|
|
end_dt = _to_datetime(scada_end)
|
|
if start_dt >= end_dt:
|
|
raise ValueError("SCADA 时间窗非法：scada_start 必须早于 scada_end。")
|
|
|
|
# Reuse burst_location logic partially here or call internal queries directly
|
|
# For burst detection, we need time series for all sensor nodes.
|
|
|
|
# Check for missing nodes in simulation result if needed, but InternalQueries handles some of it.
|
|
# We assume sensor_nodes are valid pressure nodes.
|
|
|
|
simulation_data = InternalQueries.query_analysis_simulation_by_ids_timerange(
|
|
db_name=network,
|
|
run_id=simulation_run_id,
|
|
element_ids=sensor_nodes,
|
|
start_time=start_dt.isoformat(),
|
|
end_time=end_dt.isoformat(),
|
|
element_type="node",
|
|
field="pressure",
|
|
)
|
|
|
|
# simulation_data is {sensor_id: [{time, value}, ...]}
|
|
# Convert to DataFrame: index=time, columns=sensor_ids
|
|
|
|
data_dict = {}
|
|
timestamps = set()
|
|
|
|
for sensor_id in sensor_nodes:
|
|
records = simulation_data.get(sensor_id, [])
|
|
if not records:
|
|
continue
|
|
|
|
# Convert records to Series with time index
|
|
ts_values = []
|
|
ts_index = []
|
|
for r in records:
|
|
if r.get("value") is not None:
|
|
ts_values.append(float(r["value"]))
|
|
ts_index.append(pd.to_datetime(r["time"]))
|
|
|
|
if ts_values:
|
|
s = pd.Series(ts_values, index=ts_index)
|
|
data_dict[sensor_id] = s
|
|
timestamps.update(ts_index)
|
|
|
|
if not data_dict:
|
|
raise ValueError("指定时间窗内未查询到模拟压力数据。")
|
|
|
|
observation_df = pd.DataFrame(data_dict)
|
|
|
|
# Handle missing timestamps if any (though simulation usually has uniform steps)
|
|
# Forward fill or interpolate might be needed if steps differ, but typically they align.
|
|
observation_df = observation_df.sort_index()
|
|
|
|
# Fill NaN if any missing points for some sensors
|
|
observation_df = observation_df.fillna(method="ffill").fillna(method="bfill")
|
|
|
|
if observation_df.empty:
|
|
raise ValueError("模拟压力数据无法构建观测矩阵。")
|
|
|
|
return observation_df
|
|
|
|
|
|
def _store_burst_detection_scheme(
|
|
*,
|
|
network: str,
|
|
scheme_name: str,
|
|
username: str,
|
|
payload: dict[str, Any],
|
|
mu: int,
|
|
points_per_day: int,
|
|
iforest_params: dict[str, Any],
|
|
) -> None:
|
|
now_iso = utc_now().isoformat()
|
|
scheme_detail = {
|
|
"network": network,
|
|
"sensor_nodes": payload.get("sensor_nodes", []),
|
|
"observed_source": payload.get("observed_source"),
|
|
"scada_window": payload.get("scada_window"),
|
|
"algorithm_params": {
|
|
"mu": mu,
|
|
"points_per_day": points_per_day,
|
|
"iforest_params": iforest_params,
|
|
},
|
|
"result_summary": payload.get("summary", {}),
|
|
"result_payload": payload,
|
|
}
|
|
store_scheme_info(
|
|
name=network,
|
|
scheme_name=scheme_name,
|
|
scheme_type="burst_detection",
|
|
username=username,
|
|
scheme_start_time=now_iso,
|
|
scheme_detail=scheme_detail,
|
|
)
|
|
|
|
|
|
def _serialize_result_rows(
|
|
result_df: pd.DataFrame,
|
|
*,
|
|
daily_times: list[datetime] | None = None,
|
|
target_only: bool = False,
|
|
) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
raw_rows = result_df.to_dict(orient="records")
|
|
for index, row in enumerate(raw_rows):
|
|
is_target = index == len(raw_rows) - 1
|
|
is_burst = bool(row["IsBurst"])
|
|
if target_only:
|
|
is_burst = is_target and float(row["Score"]) <= TARGET_SCORE_THRESHOLD
|
|
rows.append(
|
|
{
|
|
"Day": int(row["Day"]),
|
|
"Score": float(row["Score"]),
|
|
"Prediction": int(row["Prediction"]),
|
|
"IsBurst": is_burst,
|
|
**(
|
|
{
|
|
"Timestamp": daily_times[index].isoformat(),
|
|
"Role": "target" if is_target else "reference",
|
|
}
|
|
if daily_times is not None
|
|
else {}
|
|
),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def _build_detection_summary(
|
|
result_df: pd.DataFrame,
|
|
*,
|
|
daily_times: list[datetime] | None = None,
|
|
target_only: bool = False,
|
|
) -> dict[str, Any]:
|
|
rows = _serialize_result_rows(
|
|
result_df,
|
|
daily_times=daily_times,
|
|
target_only=target_only,
|
|
)
|
|
if not rows:
|
|
raise ValueError("爆管侦测结果为空。")
|
|
|
|
score_series = result_df["Score"]
|
|
most_anomalous_index = int(score_series.idxmin())
|
|
latest_row = rows[-1]
|
|
anomaly_days = [row["Day"] for row in rows if row["IsBurst"]]
|
|
|
|
summary = {
|
|
"burst_detected": bool(latest_row["IsBurst"]),
|
|
"latest_day": latest_row,
|
|
"most_anomalous_day": int(result_df.iloc[most_anomalous_index]["Day"]),
|
|
"anomaly_days": anomaly_days,
|
|
"anomaly_day_count": len(anomaly_days),
|
|
"latest_sensor_rankings": _build_latest_sensor_rankings(result_df),
|
|
}
|
|
if target_only:
|
|
target_score = float(latest_row["Score"])
|
|
summary.update(
|
|
{
|
|
"target_score": target_score,
|
|
"score_threshold": TARGET_SCORE_THRESHOLD,
|
|
"target_rank": int(result_df["Score"].rank(method="min").iloc[-1]),
|
|
"target_time": latest_row.get("Timestamp"),
|
|
"reference_day_count": TARGET_DAY_COUNT - 1,
|
|
}
|
|
)
|
|
return summary
|
|
|
|
|
|
def _build_latest_sensor_rankings(result_df: pd.DataFrame) -> list[dict[str, Any]]:
|
|
feature_matrix = result_df.attrs.get("high_freq_features")
|
|
sensor_nodes = list(result_df.attrs.get("sensor_nodes", []))
|
|
if feature_matrix is None or len(sensor_nodes) == 0:
|
|
return []
|
|
|
|
latest_values = np.asarray(feature_matrix[-1], dtype=float)
|
|
history = np.asarray(feature_matrix[:-1], dtype=float)
|
|
history_means = history.mean(axis=0)
|
|
history_stds = history.std(axis=0)
|
|
safe_stds = np.where(history_stds > 1e-9, history_stds, 1e-9)
|
|
deviations = (latest_values - history_means) / safe_stds
|
|
ranking = sorted(
|
|
zip(
|
|
sensor_nodes,
|
|
latest_values,
|
|
history_means,
|
|
history_stds,
|
|
deviations,
|
|
strict=False,
|
|
),
|
|
key=lambda item: item[4],
|
|
)
|
|
return [
|
|
{
|
|
"sensor_node": sensor_id,
|
|
"latest_high_frequency_value": float(value),
|
|
"historical_mean": float(history_mean),
|
|
"historical_std": float(history_std),
|
|
"standardized_deviation": float(deviation),
|
|
}
|
|
for sensor_id, value, history_mean, history_std, deviation in ranking[
|
|
: min(10, len(ranking))
|
|
]
|
|
]
|
|
|
|
|
|
def _build_target_pressure_from_scada(
|
|
*,
|
|
network: str,
|
|
sensor_nodes: list[str],
|
|
requested_target_time: datetime | None,
|
|
sampling_interval_minutes: int,
|
|
points_per_day: int,
|
|
) -> tuple[pd.DataFrame, datetime, list[dict[str, str]]]:
|
|
node_query_id = _get_pressure_sensor_mapping(network)
|
|
mapped_nodes = [node for node in sensor_nodes if node in node_query_id]
|
|
excluded_without_mapping = [
|
|
{"sensor_node": node, "reason": "missing_api_query_id"}
|
|
for node in sensor_nodes
|
|
if node not in node_query_id
|
|
]
|
|
if len(mapped_nodes) < MIN_COMPLETE_SENSORS:
|
|
raise ValueError(
|
|
f"可查询的压力测点少于 {MIN_COMPLETE_SENSORS} 个，无法执行爆管侦测。"
|
|
)
|
|
|
|
query_ids = [node_query_id[node] for node in mapped_nodes]
|
|
candidate_before = requested_target_time
|
|
last_excluded: list[dict[str, str]] = excluded_without_mapping
|
|
|
|
for _ in range(4):
|
|
resolved_target = InternalQueries.query_latest_scada_time(
|
|
db_name=network,
|
|
device_ids=query_ids,
|
|
before_time=candidate_before,
|
|
)
|
|
if resolved_target is None:
|
|
break
|
|
|
|
sample_start = resolved_target - timedelta(
|
|
days=TARGET_DAY_COUNT,
|
|
minutes=-sampling_interval_minutes,
|
|
)
|
|
expected_index = pd.date_range(
|
|
start=sample_start,
|
|
end=resolved_target,
|
|
freq=f"{sampling_interval_minutes}min",
|
|
)
|
|
scada_data = InternalQueries.query_scada_by_ids_timerange(
|
|
db_name=network,
|
|
device_ids=query_ids,
|
|
start_time=sample_start,
|
|
end_time=resolved_target,
|
|
)
|
|
|
|
complete_columns: dict[str, pd.Series] = {}
|
|
excluded = list(excluded_without_mapping)
|
|
for node_id in mapped_nodes:
|
|
query_id = node_query_id[node_id]
|
|
records = scada_data.get(query_id, [])
|
|
if not records:
|
|
excluded.append({"sensor_node": node_id, "reason": "no_data"})
|
|
continue
|
|
|
|
record_frame = pd.DataFrame.from_records(records)
|
|
record_frame["time"] = pd.to_datetime(record_frame["time"], utc=True)
|
|
record_frame["value"] = pd.to_numeric(
|
|
record_frame["value"], errors="coerce"
|
|
)
|
|
series = (
|
|
record_frame.drop_duplicates(subset="time", keep="last")
|
|
.set_index("time")["value"]
|
|
.reindex(expected_index)
|
|
)
|
|
if len(series) != TARGET_DAY_COUNT * points_per_day:
|
|
excluded.append(
|
|
{"sensor_node": node_id, "reason": "unexpected_sample_count"}
|
|
)
|
|
continue
|
|
if series.isna().any():
|
|
excluded.append(
|
|
{"sensor_node": node_id, "reason": "missing_or_invalid_samples"}
|
|
)
|
|
continue
|
|
complete_columns[node_id] = series
|
|
|
|
if len(complete_columns) >= MIN_COMPLETE_SENSORS:
|
|
observation_df = pd.DataFrame(complete_columns, index=expected_index)
|
|
return observation_df, resolved_target, excluded
|
|
|
|
last_excluded = excluded
|
|
candidate_before = resolved_target - timedelta(microseconds=1)
|
|
|
|
excluded_preview = ", ".join(
|
|
item["sensor_node"] for item in last_excluded[:10]
|
|
)
|
|
raise ValueError(
|
|
f"最近数据中完整压力测点少于 {MIN_COMPLETE_SENSORS} 个；"
|
|
f"请检查 15 天数据完整性。排除测点: {excluded_preview or '无'}"
|
|
)
|
|
|
|
|
|
def _resolve_sampling_interval_minutes(
|
|
*,
|
|
network: str,
|
|
sensor_nodes: list[str],
|
|
requested_interval: int | None,
|
|
) -> int:
|
|
if requested_interval is not None:
|
|
interval = int(requested_interval)
|
|
else:
|
|
selected_nodes = set(sensor_nodes)
|
|
inferred_intervals = [
|
|
parsed
|
|
for item in get_all_scada_info(network)
|
|
if str(item.get("device_type", "")).lower() == "pressure"
|
|
and str(item.get("node_id", "")) in selected_nodes
|
|
and (
|
|
parsed := _parse_sampling_interval_minutes(
|
|
item.get("transmission_frequency")
|
|
)
|
|
)
|
|
is not None
|
|
]
|
|
interval = (
|
|
Counter(inferred_intervals).most_common(1)[0][0]
|
|
if inferred_intervals
|
|
else DEFAULT_SAMPLE_INTERVAL_MINUTES
|
|
)
|
|
|
|
if interval <= 0 or 1440 % interval != 0:
|
|
raise ValueError("采样间隔必须是能整除 1440 分钟的正整数。")
|
|
return interval
|
|
|
|
|
|
def _parse_sampling_interval_minutes(value: Any) -> int | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, (int, float)):
|
|
minutes = float(value)
|
|
else:
|
|
try:
|
|
minutes = pd.to_timedelta(str(value)).total_seconds() / 60
|
|
except (TypeError, ValueError):
|
|
return None
|
|
rounded = round(minutes)
|
|
if minutes <= 0 or abs(minutes - rounded) > 1e-6:
|
|
return None
|
|
return int(rounded)
|
|
|
|
|
|
def _get_pressure_sensor_mapping(network: str) -> dict[str, str]:
|
|
node_query_id: dict[str, str] = {}
|
|
for item in get_all_scada_info(network):
|
|
if str(item.get("device_type", "")).lower() != "pressure":
|
|
continue
|
|
node_id = item.get("node_id")
|
|
query_id = item.get("api_query_id")
|
|
if node_id and query_id is not None:
|
|
node_query_id[str(node_id)] = str(query_id)
|
|
return node_query_id
|
|
|
|
|
|
def _get_pressure_sensor_nodes(network: str) -> list[str]:
|
|
sensor_nodes: list[str] = []
|
|
for item in get_all_scada_info(network):
|
|
if str(item.get("device_type", "")).lower() != "pressure":
|
|
continue
|
|
node_id = item.get("node_id")
|
|
if isinstance(node_id, str) and node_id:
|
|
sensor_nodes.append(node_id)
|
|
sensor_nodes = list(dict.fromkeys(sensor_nodes))
|
|
if not sensor_nodes:
|
|
raise ValueError(
|
|
"未找到压力传感器对应节点（asset.scada_devices.device_type=pressure）。"
|
|
)
|
|
return sensor_nodes
|
|
|
|
|
|
def _build_observed_pressure_from_scada(
|
|
*,
|
|
network: str,
|
|
sensor_nodes: list[str],
|
|
scada_start: datetime | str | None,
|
|
scada_end: datetime | str | None,
|
|
) -> pd.DataFrame:
|
|
if scada_start is None or scada_end is None:
|
|
raise ValueError("使用后端 SCADA 查询时必须同时提供 scada_start 与 scada_end。")
|
|
|
|
start_dt = _to_datetime(scada_start)
|
|
end_dt = _to_datetime(scada_end)
|
|
if start_dt >= end_dt:
|
|
raise ValueError("SCADA 时间窗非法：scada_start 必须早于 scada_end。")
|
|
|
|
node_query_id = _get_pressure_sensor_mapping(network)
|
|
|
|
missing_nodes = [node_id for node_id in sensor_nodes if node_id not in node_query_id]
|
|
if missing_nodes:
|
|
preview = ", ".join(missing_nodes[:10])
|
|
raise ValueError(f"未找到可用于压力观测的 SCADA api_query_id: {preview}")
|
|
|
|
query_ids = [node_query_id[node_id] for node_id in sensor_nodes]
|
|
scada_data = InternalQueries.query_scada_by_ids_timerange(
|
|
db_name=network,
|
|
device_ids=query_ids,
|
|
start_time=start_dt.isoformat(),
|
|
end_time=end_dt.isoformat(),
|
|
)
|
|
|
|
available_lengths = [
|
|
len(scada_data.get(query_id, []))
|
|
for query_id in query_ids
|
|
if len(scada_data.get(query_id, [])) > 0
|
|
]
|
|
if not available_lengths:
|
|
raise ValueError("指定时间窗内未查询到压力 SCADA 数据。")
|
|
|
|
min_len = min(available_lengths)
|
|
observation_df = pd.DataFrame()
|
|
for node_id in sensor_nodes:
|
|
query_id = node_query_id[node_id]
|
|
records = scada_data.get(query_id, [])[:min_len]
|
|
if len(records) < min_len:
|
|
continue
|
|
observation_df[node_id] = [float(item["value"]) for item in records]
|
|
|
|
if observation_df.empty:
|
|
raise ValueError("SCADA 压力数据无法构建观测矩阵。")
|
|
return observation_df
|
|
|
|
|
|
def _to_datetime(value: datetime | str) -> datetime:
|
|
return parse_utc_time(value)
|