OrgTJWater/pipeline-lifetime
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate RC1 fusion model routing

Browse Source
This commit is contained in:
jiang
2026-07-20 14:42:18 +08:00
parent a20339d35e
commit 72b6b63154
12 changed files with 232 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/prediction.py
+144 -130
View File
@@ -1,18 +1,16 @@
from __future__ import annotations
import logging
import os
import sys
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import joblib
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib import font_manager, rcParams
from werkzeug.datastructures import FileStorage
@@ -59,9 +57,12 @@ FEATURES = [
ID_COLUMN = "管道编号"
PIPE_AGE_COLUMN = "管龄（年）"
LEGACY_PIPE_AGE_COLUMN = "管龄"
STATUS_COLUMN = "状态"
EVENT_AGE_COLUMN = "事件/观察管龄（年）"
COLUMN_ALIASES = {
"ID": ID_COLUMN,
"管段ID": ID_COLUMN,
"管段编号": ID_COLUMN,
"当前管龄": PIPE_AGE_COLUMN,
"当前管龄（年）": PIPE_AGE_COLUMN,
"管径（mm）": "管径",
"管径（mm)": "管径",
"流速（m/s）": "流速",
@@ -70,48 +71,42 @@ COLUMN_ALIASES = {
"温度（℃）": "温度",
"年均降雨量（mm）": "降雨量",
"降雨量（mm）": "降雨量",
"结构性缺陷": "结构缺陷",
"功能性缺陷": "功能缺陷",
}
INPUT_COLUMNS = [
ID_COLUMN,
PIPE_AGE_COLUMN,
STATUS_COLUMN,
EVENT_AGE_COLUMN,
*FEATURES,
]
REQUIRED_INPUT_COLUMNS = [ID_COLUMN, PIPE_AGE_COLUMN, "管材", "管径"]
CHART_DISPLAY_LIMIT = 10
MATERIAL_COLUMN = "管材"
MATERIAL_CODE_OPTIONS = [
(1, "镀锌"),
(2, "钢塑"),
(3, "铝塑"),
(4, "PPR"),
(5, "PE"),
(6, "UPVC"),
(7, "铸铁"),
(8, "预应力"),
(9, "自应力"),
(10, "玻璃钢夹砂"),
(11, "钢管"),
(12, "钢套混凝土管"),
(13, "球墨铸铁"),
(14, "其他"),
]
MATERIAL_ALIAS_TO_CODE = {
str(code): code
for code, _ in MATERIAL_CODE_OPTIONS
MATERIAL_CODE_TO_NAME = {
1: "镀锌",
2: "钢塑",
3: "铝塑",
4: "PPR",
5: "PE",
6: "UPVC",
7: "铸铁",
11: "钢管",
}
MATERIAL_ALIAS_TO_CODE.update(
{
name.casefold(): code
for code, name in MATERIAL_CODE_OPTIONS
}
)
MATERIAL_ALIAS_TO_CODE.update(
{
f"{code}-{name}".casefold(): code
for code, name in MATERIAL_CODE_OPTIONS
}
)
LOCATION_CODE_TO_NAME = {
1: "绿化带",
2: "行人道",
3: "非机动车道",
4: "小区内",
6: "市政道路",
7: "桥管",
}
MATERIAL_ALIASES = {str(code): name for code, name in MATERIAL_CODE_TO_NAME.items()}
MATERIAL_ALIASES.update({name.casefold(): name for name in MATERIAL_CODE_TO_NAME.values()})
MATERIAL_ALIASES.update({f"{code}-{name}".casefold(): name for code, name in MATERIAL_CODE_TO_NAME.items()})
LOCATION_ALIASES = {str(code): name for code, name in LOCATION_CODE_TO_NAME.items()}
LOCATION_ALIASES.update({name.casefold(): name for name in LOCATION_CODE_TO_NAME.values()})
LOCATION_ALIASES.update({f"{code}-{name}".casefold(): name for code, name in LOCATION_CODE_TO_NAME.items()})
LOCATION_ALIASES["人行道"] = "行人道"
SUPPORTED_EXTENSIONS = {".csv", ".xls", ".xlsx"}
CHINESE_FONT_PROP = None
DEFECT_GRADE_VALUES = {"无": 0.0, "轻度": 1.0, "中度": 3.0, "严重": 5.0}
@@ -134,30 +129,7 @@ class PredictionArtifacts:
sample_count: int
summary_rows: list[dict[str, Any]]
analysis_text: str
class ModelBundleAdapter:
"""Expose a bundled preprocessor and survival model as one predictor."""
def __init__(self, preprocessor, model) -> None:
self.preprocessor = preprocessor
self.model = model
def _transform(self, frame: pd.DataFrame):
values = frame[FEATURES].copy()
values[MATERIAL_COLUMN] = values[MATERIAL_COLUMN].map(
lambda value: f"M{int(value)}" if pd.notna(value) else value
)
values["位置"] = values["位置"].map(
lambda value: f"L{int(value)}" if pd.notna(value) else value
)
return self.preprocessor.transform(values.to_numpy(dtype=object)).astype(np.float32)
def predict_survival_function(self, frame: pd.DataFrame):
return self.model.predict_survival_function(self._transform(frame))
def predict(self, frame: pd.DataFrame):
return self.model.predict(self._transform(frame))
model_version: str
def configure_matplotlib_fonts():
@@ -190,24 +162,18 @@ def chinese_font_kwargs() -> dict[str, Any]:
return {"fontproperties": CHINESE_FONT_PROP} if CHINESE_FONT_PROP else {}
def load_model(model_path: str):
if not os.path.exists(model_path):
raise FileNotFoundError(f"未找到模型文件: {model_path}")
model = joblib.load(model_path)
if isinstance(model, dict):
if not {"preprocessor", "model"}.issubset(model):
raise ValueError("模型包缺少 preprocessor 或 model。")
model = ModelBundleAdapter(model["preprocessor"], model["model"])
feature_names = getattr(model, "feature_names_in_", None)
if feature_names is not None and list(feature_names) != FEATURES:
raise ValueError(
"模型输入特征与系统配置不一致: "
f"model={list(feature_names)}, app={FEATURES}"
)
n_features = getattr(model, "n_features_in_", None)
if n_features is not None and int(n_features) != len(FEATURES):
raise ValueError(f"模型特征数量不一致: model={n_features}, app={len(FEATURES)}")
return model
def load_model(core_dir: str):
core_path = Path(core_dir)
if not core_path.exists():
raise FileNotFoundError(f"未找到模型核心目录: {core_path}")
if not (core_path / "frozen_fusion_inference.py").exists():
raise FileNotFoundError(f"模型核心目录缺少 frozen_fusion_inference.py: {core_path}")
if str(core_path) not in sys.path:
sys.path.insert(0, str(core_path))
from frozen_fusion_inference import FrozenFusionPredictor
return FrozenFusionPredictor(core_path)
def safe_unlink(path: Path) -> None:
@@ -339,31 +305,62 @@ def fill_defects_from_detail_sheet(path: Path, df: pd.DataFrame) -> pd.DataFrame
def validate_input_frame(df: pd.DataFrame) -> None:
if df.empty:
raise PredictionError("上传文件没有可预测的数据。")
missing = [col for col in INPUT_COLUMNS if col not in df.columns]
missing = [col for col in REQUIRED_INPUT_COLUMNS if col not in df.columns]
if missing:
raise PredictionError(f"缺少必要字段: {', '.join(missing)}")
def normalize_material_code(value: Any) -> int:
def normalize_category_value(
value: Any,
aliases: dict[str, str],
label: str,
*,
required: bool = False,
) -> str | pd.NA:
if pd.isna(value):
raise PredictionError("管材不能为空。")
if isinstance(value, str):
key = value.strip().casefold()
else:
if required:
raise PredictionError(f"{label}不能为空。")
return pd.NA
text = str(value).strip()
if text == "":
if required:
raise PredictionError(f"{label}不能为空。")
return pd.NA
key = text.casefold()
try:
numeric_value = float(value)
if not numeric_value.is_integer():
raise PredictionError(f"管材编码无效: {value}")
raise ValueError
key = str(int(numeric_value))
code = MATERIAL_ALIAS_TO_CODE.get(key)
if code is None:
raise PredictionError(f"管材编码无效: {value}")
return code
except (TypeError, ValueError):
pass
normalized = aliases.get(key)
if normalized is None:
raise PredictionError(f"{label}超出RC1支持范围: {value}")
return normalized
def prepare_model_features(df: pd.DataFrame) -> pd.DataFrame:
x_test = df[FEATURES].copy()
x_test[MATERIAL_COLUMN] = x_test[MATERIAL_COLUMN].map(normalize_material_code)
return x_test
result = pd.DataFrame(index=df.index)
result["ID"] = df[ID_COLUMN]
result["Current Age"] = df[PIPE_AGE_COLUMN]
result["Material"] = df[MATERIAL_COLUMN].map(
lambda value: normalize_category_value(value, MATERIAL_ALIASES, "管材", required=True)
)
result["Diameter"] = df["管径"]
result["Flow Velocity"] = df["流速"] if "流速" in df.columns else pd.NA
result["Pressure"] = df["压力"] if "压力" in df.columns else pd.NA
result["Temperature"] = df["温度"] if "温度" in df.columns else pd.NA
result["Precipitation"] = df["降雨量"] if "降雨量" in df.columns else pd.NA
if "位置" in df.columns:
result["Location"] = df["位置"].map(
lambda value: normalize_category_value(value, LOCATION_ALIASES, "位置")
)
else:
result["Location"] = pd.NA
result["Structural Defects"] = df["结构缺陷"] if "结构缺陷" in df.columns else pd.NA
result["Functional Defects"] = df["功能缺陷"] if "功能缺陷" in df.columns else pd.NA
return result
def grade_info(probability: float) -> tuple[str, str, str]:
@@ -389,22 +386,22 @@ def interpolate_probability(times: list[float], probs: list[float], target: floa
return float(probs[-1])
def estimate_remaining_life(times: list[float], probs: list[float]) -> float:
for t, p in zip(times, probs):
if p <= 0.5:
return float(t)
return float(times[-1]) if times else 0.0
def estimate_remaining_life(times: list[float], health_states: list[float], current_age: float = 0.0) -> float | None:
for age, health_state in zip(times, health_states):
if age >= current_age and health_state <= 0.5:
return max(float(age) - current_age, 0.0)
return None
def make_analysis_text(summary_rows: list[dict[str, Any]]) -> str:
if not summary_rows:
return "当前结果为空，暂无可供解释的样本。"
worst = min(summary_rows, key=lambda x: x["health_probability"])
best = max(summary_rows, key=lambda x: x["health_probability"])
worst = min(summary_rows, key=lambda x: x["health_state"])
best = max(summary_rows, key=lambda x: x["health_state"])
return (
"阶梯状曲线表示模型对不同管道随时间推移维持在安全健康状态概率的动态预测。"
"曲线表示模型对不同管道随管龄变化的健康状态动态预测。"
f"当前样本中风险最高管道为 {worst['pipe_id']}（{worst['grade_label']}），"
f"健康概率最高管道为 {best['pipe_id']}（{best['health_probability']:.1%}）。"
f"当前健康状态最高管道为 {best['pipe_id']}（{best['health_state']:.1%}）。"
)
@@ -427,18 +424,21 @@ def run_prediction(uploaded: FileStorage, user_id: int, model) -> PredictionArti
validate_input_frame(df)
x_test = prepare_model_features(df)
try:
curves = model.predict_survival_function(x_test)
predictions = model.predict(x_test, variant="defect_sensitive")
except ValueError as exc:
logging.exception("预测输入校验失败: %s", exc)
raise PredictionError(str(exc))
except Exception as exc:
logging.exception("预测失败: %s", exc)
raise PredictionError("模型预测失败，请检查输入字段类型是否正确。", 500)
image_filename = f"plot_{user_id}_{run_id}.png"
image_path = IMAGE_DIR / image_filename
summary_rows, summary_sheet_rows = render_survival_chart(df, curves, image_path)
summary_rows, summary_sheet_rows = render_survival_chart(predictions, image_path)
safe_stem = Path(saved_filename).stem
excel_path = user_dir / f"{safe_stem}_pre.xlsx"
write_prediction_workbook(excel_path, curves, summary_rows, summary_sheet_rows)
write_prediction_workbook(excel_path, predictions, summary_rows, summary_sheet_rows)
return PredictionArtifacts(
original_filename=original_filename,
saved_path=original_path,
@@ -448,36 +448,45 @@ def run_prediction(uploaded: FileStorage, user_id: int, model) -> PredictionArti
sample_count=len(summary_rows),
summary_rows=summary_rows,
analysis_text=make_analysis_text(summary_rows),
model_version=str(getattr(model, "config", {}).get("version", "")),
)
except PredictionError:
safe_unlink(original_path)
raise
def render_survival_chart(df: pd.DataFrame, curves, image_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
def float_list(values: Any) -> list[float]:
return [float(value) for value in list(values)]
def format_years(value: float | None, current_age: float, max_age: float) -> str | float:
if value is None:
return f">{max(max_age - current_age, 0.0):g}"
return round(float(value), 3)
def render_survival_chart(predictions: list[dict[str, Any]], image_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
plt.figure(figsize=(10, 5.6))
summary_rows: list[dict[str, Any]] = []
summary_sheet_rows: list[dict[str, Any]] = []
display_note = f"说明：图中仅展示前{CHART_DISPLAY_LIMIT}条管道的示例数据，不足{CHART_DISPLAY_LIMIT}条则全部展示；完整结果请下载电子表格。"
for i, curve in enumerate(curves):
times = [float(x) for x in list(curve.x)]
probs = [float(y) for y in list(curve.y)]
pipe_id = normalize_id_value(df.iloc[i][ID_COLUMN], f"Pipe_{i+1:03d}")
raw_pipe_age = df.iloc[i][PIPE_AGE_COLUMN] if PIPE_AGE_COLUMN in df.columns else None
pipe_age_value = float(raw_pipe_age) if pd.notna(raw_pipe_age) else 0.0
pipe_age = f"{raw_pipe_age} 年" if pd.notna(raw_pipe_age) else "-"
health_probability = interpolate_probability(times, probs, pipe_age_value)
health_risk = 1.0 - health_probability
remaining_life = estimate_remaining_life(times, probs)
grade_label, grade_desc, grade_class = grade_info(health_probability)
for i, prediction in enumerate(predictions):
times = float_list(prediction["pipe_age"])
health_states = float_list(prediction["health_state"])
pipe_id = normalize_id_value(prediction.get("ID"), f"Pipe_{i+1:03d}")
current_age = float(prediction["current_age"])
pipe_age = f"{current_age:g} 年"
current_health_state = float(prediction["current_health_state"])
remaining_life = estimate_remaining_life(times, health_states, current_age)
grade_label, grade_desc, grade_class = grade_info(current_health_state)
max_age = max(times) if times else current_age
summary_rows.append(
{
"pipe_id": pipe_id,
"pipe_age": pipe_age,
"health_probability": health_probability,
"health_risk": health_risk,
"health_state": current_health_state,
"remaining_life": remaining_life,
"grade_label": grade_label,
"grade_desc": grade_desc,
@@ -488,16 +497,18 @@ def render_survival_chart(df: pd.DataFrame, curves, image_path: Path) -> tuple[l
{
ID_COLUMN: pipe_id,
PIPE_AGE_COLUMN: pipe_age,
"健康风险值": health_risk,
"当前健康状态": current_health_state,
"健康等级": grade_label,
"预计剩余寿命（年）": format_years(remaining_life, current_age, max_age),
}
)
if i < CHART_DISPLAY_LIMIT:
plt.step(times, probs, where="post", linewidth=2, label=pipe_id)
plt.step(times, health_states, where="post", linewidth=2, label=pipe_id)
font_kwargs = chinese_font_kwargs()
plt.xlabel("管龄（年）", **font_kwargs)
plt.ylabel("健康风险", **font_kwargs)
plt.title("管道的剩余寿命分析图", **font_kwargs)
plt.ylabel("管道健康状态", **font_kwargs)
plt.title("供水管道健康状态曲线", **font_kwargs)
plt.figtext(0.5, 0.02, display_note, ha="center", fontsize=9, color="#475569", **font_kwargs)
plt.grid(alpha=0.18)
if min(len(summary_rows), CHART_DISPLAY_LIMIT) <= 12:
@@ -514,18 +525,18 @@ def render_survival_chart(df: pd.DataFrame, curves, image_path: Path) -> tuple[l
def write_prediction_workbook(
excel_path: Path,
curves,
predictions: list[dict[str, Any]],
summary_rows: list[dict[str, Any]],
summary_sheet_rows: list[dict[str, Any]],
) -> None:
chart_times: set[float] = set()
chart_series: list[tuple[str, dict[float, float]]] = []
for i, curve in enumerate(curves):
times = [float(x) for x in list(curve.x)]
probs = [float(y) for y in list(curve.y)]
for i, prediction in enumerate(predictions):
times = float_list(prediction["pipe_age"])
health_states = float_list(prediction["health_state"])
pipe_id = summary_rows[i]["pipe_id"]
chart_times.update(times)
chart_series.append((pipe_id, dict(zip(times, probs))))
chart_series.append((pipe_id, dict(zip(times, health_states))))
sorted_times = sorted(chart_times)
sample_columns = ["管龄（年）", *[pipe_id for pipe_id, _ in chart_series]]
@@ -535,7 +546,10 @@ def write_prediction_workbook(
]
with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:
summary_df = pd.DataFrame(summary_sheet_rows, columns=[ID_COLUMN, PIPE_AGE_COLUMN, "健康风险值"])
summary_df = pd.DataFrame(
summary_sheet_rows,
columns=[ID_COLUMN, PIPE_AGE_COLUMN, "当前健康状态", "健康等级", "预计剩余寿命（年）"],
)
sample_df = pd.DataFrame(sample_data_rows, columns=sample_columns)
if ID_COLUMN in summary_df.columns:
summary_df[ID_COLUMN] = summary_df[ID_COLUMN].astype("string")