381 lines
14 KiB
Python
381 lines
14 KiB
Python
from __future__ import annotations
|
||
|
||
import logging
|
||
import os
|
||
import uuid
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import joblib
|
||
import matplotlib
|
||
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
import pandas as pd
|
||
from matplotlib import font_manager, rcParams
|
||
from werkzeug.datastructures import FileStorage
|
||
from werkzeug.utils import secure_filename
|
||
|
||
from .config import IMAGE_DIR, UPLOAD_DIR
|
||
|
||
CHINESE_FONT_CANDIDATES = [
|
||
"Noto Sans CJK SC",
|
||
"Noto Sans CJK JP",
|
||
"Noto Sans CJK TC",
|
||
"Source Han Sans SC",
|
||
"WenQuanYi Micro Hei",
|
||
"SimHei",
|
||
"Microsoft YaHei",
|
||
"Arial Unicode MS",
|
||
]
|
||
|
||
FEATURES = [
|
||
"管材",
|
||
"管径",
|
||
"流速",
|
||
"压力",
|
||
"温度",
|
||
"降雨量",
|
||
"位置",
|
||
"结构缺陷",
|
||
"功能缺陷",
|
||
]
|
||
|
||
ID_COLUMN = "管道编号"
|
||
PIPE_AGE_COLUMN = "管龄"
|
||
SUPPORTED_EXTENSIONS = {".csv", ".xls", ".xlsx"}
|
||
|
||
|
||
class PredictionError(Exception):
|
||
def __init__(self, message: str, status_code: int = 400) -> None:
|
||
super().__init__(message)
|
||
self.message = message
|
||
self.status_code = status_code
|
||
|
||
|
||
@dataclass
|
||
class PredictionArtifacts:
|
||
original_filename: str
|
||
saved_path: Path
|
||
excel_path: Path
|
||
image_path: Path
|
||
image_filename: str
|
||
importance_filename: str | None
|
||
sample_count: int
|
||
summary_rows: list[dict[str, Any]]
|
||
analysis_text: str
|
||
|
||
|
||
def configure_matplotlib_fonts() -> None:
|
||
available_fonts = {font.name for font in font_manager.fontManager.ttflist}
|
||
selected_fonts = [font for font in CHINESE_FONT_CANDIDATES if font in available_fonts]
|
||
rcParams["font.sans-serif"] = selected_fonts + ["DejaVu Sans"]
|
||
rcParams["axes.unicode_minus"] = False
|
||
|
||
|
||
configure_matplotlib_fonts()
|
||
|
||
|
||
def load_model(model_path: str):
|
||
if not os.path.exists(model_path):
|
||
raise FileNotFoundError(f"未找到模型文件: {model_path}")
|
||
model = joblib.load(model_path)
|
||
feature_names = getattr(model, "feature_names_in_", None)
|
||
if feature_names is not None and list(feature_names) != FEATURES:
|
||
raise ValueError(
|
||
"模型输入特征与系统配置不一致: "
|
||
f"model={list(feature_names)}, app={FEATURES}"
|
||
)
|
||
n_features = getattr(model, "n_features_in_", None)
|
||
if n_features is not None and int(n_features) != len(FEATURES):
|
||
raise ValueError(f"模型特征数量不一致: model={n_features}, app={len(FEATURES)}")
|
||
return model
|
||
|
||
|
||
def safe_unlink(path: Path) -> None:
|
||
try:
|
||
path.unlink(missing_ok=True)
|
||
except OSError as exc:
|
||
logging.warning("删除文件失败 %s: %s", path, exc)
|
||
|
||
|
||
def secure_upload_name(original_filename: str, run_id: str) -> tuple[str, str]:
|
||
suffix = Path(original_filename).suffix.lower()
|
||
if suffix not in SUPPORTED_EXTENSIONS:
|
||
raise PredictionError("不支持的格式，仅支持 CSV / XLS / XLSX")
|
||
|
||
safe_full_name = secure_filename(original_filename)
|
||
safe_stem = Path(safe_full_name).stem if safe_full_name else ""
|
||
if not safe_stem:
|
||
safe_stem = "upload"
|
||
return f"{safe_stem}_{run_id}{suffix}", suffix
|
||
|
||
|
||
def read_input_file(path: Path, suffix: str) -> pd.DataFrame:
|
||
try:
|
||
if suffix == ".csv":
|
||
return pd.read_csv(path)
|
||
return pd.read_excel(path)
|
||
except Exception as exc:
|
||
logging.exception("文件解析失败: %s", exc)
|
||
raise PredictionError("文件解析失败，请检查编码或表格格式。")
|
||
|
||
|
||
def validate_input_frame(df: pd.DataFrame) -> None:
|
||
if df.empty:
|
||
raise PredictionError("上传文件没有可预测的数据。")
|
||
required_columns = [ID_COLUMN, *FEATURES]
|
||
missing = [col for col in required_columns if col not in df.columns]
|
||
if missing:
|
||
raise PredictionError(f"缺少必要字段: {', '.join(missing)}")
|
||
|
||
|
||
def grade_info(probability: float) -> tuple[str, str, str]:
|
||
if probability <= 0.2:
|
||
return ("I级", "管道安全风险十分严重，需立刻进行抢修或更新改造", "bg-dangerSoft text-dangerText")
|
||
if probability <= 0.4:
|
||
return ("II级", "管道安全风险较为严重，需尽快安排检修及加频巡检", "bg-orange-50 text-orange-600")
|
||
if probability <= 0.6:
|
||
return ("III级", "管道安全风险较低，需安排定期巡检", "bg-amber-50 text-amber-600")
|
||
if probability <= 0.8:
|
||
return ("IV级", "管道安全风险较小，维持常规巡视", "bg-blue-50 text-blue-600")
|
||
return ("V级", "管道安全，维持常规巡视", "bg-blueSoft text-primary")
|
||
|
||
|
||
def interpolate_probability(times: list[float], probs: list[float], target: float) -> float:
|
||
if not times:
|
||
return 0.0
|
||
if target <= times[0]:
|
||
return float(probs[0])
|
||
for idx in range(1, len(times)):
|
||
if times[idx] >= target:
|
||
return float(probs[idx])
|
||
return float(probs[-1])
|
||
|
||
|
||
def estimate_remaining_life(times: list[float], probs: list[float]) -> float:
|
||
for t, p in zip(times, probs):
|
||
if p <= 0.5:
|
||
return float(t)
|
||
return float(times[-1]) if times else 0.0
|
||
|
||
|
||
def make_analysis_text(summary_rows: list[dict[str, Any]]) -> str:
|
||
if not summary_rows:
|
||
return "当前结果为空，暂无可供解释的样本。"
|
||
worst = min(summary_rows, key=lambda x: x["health_probability"])
|
||
best = max(summary_rows, key=lambda x: x["health_probability"])
|
||
return (
|
||
"阶梯状曲线表示模型对不同管道随时间推移维持在安全健康状态概率的动态预测。"
|
||
f"当前样本中风险最高管道为 {worst['pipe_id']}（{worst['grade_label']}），"
|
||
f"健康概率最高管道为 {best['pipe_id']}（{best['health_probability']:.1%}）。"
|
||
)
|
||
|
||
|
||
def compute_feature_importance(model, x_test: pd.DataFrame) -> np.ndarray | None:
|
||
try:
|
||
importances = model.feature_importances_
|
||
except Exception:
|
||
importances = None
|
||
|
||
if importances is not None and len(importances) == len(FEATURES):
|
||
vals = np.asarray(importances, dtype=float)
|
||
else:
|
||
try:
|
||
baseline = np.asarray(model.predict(x_test), dtype=float)
|
||
except Exception as exc:
|
||
logging.exception("特征重要性基线预测失败: %s", exc)
|
||
return None
|
||
|
||
rng = np.random.default_rng(42)
|
||
vals = np.zeros(len(FEATURES), dtype=float)
|
||
for j, feat in enumerate(FEATURES):
|
||
diffs = []
|
||
for _ in range(5):
|
||
x_perm = x_test.copy()
|
||
x_perm[feat] = rng.permutation(x_perm[feat].to_numpy())
|
||
try:
|
||
perm_pred = np.asarray(model.predict(x_perm), dtype=float)
|
||
except Exception:
|
||
perm_pred = baseline
|
||
diffs.append(float(np.mean(np.abs(perm_pred - baseline))))
|
||
vals[j] = float(np.mean(diffs)) if diffs else 0.0
|
||
|
||
vals = np.clip(vals, a_min=0.0, a_max=None)
|
||
total = float(vals.sum())
|
||
if total > 0:
|
||
vals = vals / total
|
||
return vals
|
||
|
||
|
||
def render_importance_chart(values: np.ndarray, save_path: Path) -> None:
|
||
from matplotlib.colors import LinearSegmentedColormap
|
||
|
||
keep = values >= 5e-4
|
||
if not bool(np.any(keep)):
|
||
keep = np.ones_like(values, dtype=bool)
|
||
kept_feats = [FEATURES[i] for i in range(len(FEATURES)) if keep[i]]
|
||
kept_vals = values[keep]
|
||
|
||
order = np.argsort(kept_vals)
|
||
sorted_feats = [kept_feats[k] for k in order]
|
||
sorted_vals = kept_vals[order]
|
||
n = len(sorted_vals)
|
||
|
||
fig, ax = plt.subplots(figsize=(9, max(3.0, 0.62 * n + 1.6)))
|
||
cmap = LinearSegmentedColormap.from_list("brand", ["#7fb2e6", "#005EB8", "#0c4188"])
|
||
colors = cmap(np.linspace(0.15, 1.0, n)) if n else None
|
||
bars = ax.barh(sorted_feats, sorted_vals, color=colors, height=0.66, edgecolor="white", linewidth=0.8, zorder=3)
|
||
|
||
ax.set_xlabel("相对重要性", fontsize=11, color="#475569")
|
||
ax.set_title("模型输入因素重要性排序", fontsize=15, fontweight="bold", color="#0f172a", pad=14)
|
||
ax.grid(axis="x", color="#e2e8f0", linewidth=1, zorder=0)
|
||
ax.set_axisbelow(True)
|
||
for spine in ("top", "right", "left"):
|
||
ax.spines[spine].set_visible(False)
|
||
ax.spines["bottom"].set_color("#cbd5e1")
|
||
ax.tick_params(axis="y", length=0, labelsize=11)
|
||
ax.tick_params(axis="x", colors="#94a3b8", labelsize=9)
|
||
|
||
max_val = float(sorted_vals.max()) if n else 0.0
|
||
for bar, value in zip(bars, sorted_vals):
|
||
ax.text(
|
||
bar.get_width() + max_val * 0.012,
|
||
bar.get_y() + bar.get_height() / 2,
|
||
f"{value:.1%}",
|
||
va="center",
|
||
ha="left",
|
||
fontsize=10,
|
||
fontweight="bold",
|
||
color="#1e293b",
|
||
)
|
||
if max_val > 0:
|
||
ax.set_xlim(0, max_val * 1.18)
|
||
|
||
fig.tight_layout()
|
||
fig.savefig(save_path, dpi=160, bbox_inches="tight")
|
||
plt.close(fig)
|
||
|
||
|
||
def run_prediction(uploaded: FileStorage, user_id: int, model) -> PredictionArtifacts:
|
||
original_filename = uploaded.filename or ""
|
||
if not original_filename:
|
||
raise PredictionError("未选择文件")
|
||
|
||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||
run_id = f"{timestamp}_{uuid.uuid4().hex[:8]}"
|
||
saved_filename, suffix = secure_upload_name(original_filename, run_id)
|
||
|
||
user_dir = UPLOAD_DIR / f"user_{user_id}"
|
||
user_dir.mkdir(parents=True, exist_ok=True)
|
||
original_path = user_dir / saved_filename
|
||
uploaded.save(original_path)
|
||
|
||
try:
|
||
df = read_input_file(original_path, suffix)
|
||
validate_input_frame(df)
|
||
x_test = df[FEATURES].copy()
|
||
try:
|
||
curves = model.predict_survival_function(x_test)
|
||
except Exception as exc:
|
||
logging.exception("预测失败: %s", exc)
|
||
raise PredictionError("模型预测失败，请检查输入字段类型是否正确。", 500)
|
||
|
||
image_filename = f"plot_{user_id}_{run_id}.png"
|
||
image_path = IMAGE_DIR / image_filename
|
||
summary_rows, summary_sheet_rows = render_survival_chart(df, curves, image_path)
|
||
|
||
importance_filename = None
|
||
try:
|
||
importance_values = compute_feature_importance(model, x_test)
|
||
if importance_values is not None:
|
||
importance_filename = f"importance_{user_id}_{run_id}.png"
|
||
render_importance_chart(importance_values, IMAGE_DIR / importance_filename)
|
||
except Exception as exc:
|
||
logging.exception("生成特征重要性图失败: %s", exc)
|
||
|
||
safe_stem = Path(saved_filename).stem
|
||
excel_path = user_dir / f"{safe_stem}_pre.xlsx"
|
||
write_prediction_workbook(excel_path, curves, summary_rows, summary_sheet_rows)
|
||
return PredictionArtifacts(
|
||
original_filename=original_filename,
|
||
saved_path=original_path,
|
||
excel_path=excel_path,
|
||
image_path=image_path,
|
||
image_filename=image_filename,
|
||
importance_filename=importance_filename,
|
||
sample_count=len(summary_rows),
|
||
summary_rows=summary_rows,
|
||
analysis_text=make_analysis_text(summary_rows),
|
||
)
|
||
except PredictionError:
|
||
safe_unlink(original_path)
|
||
raise
|
||
|
||
|
||
def render_survival_chart(df: pd.DataFrame, curves, image_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||
plt.figure(figsize=(10, 5.6))
|
||
summary_rows: list[dict[str, Any]] = []
|
||
summary_sheet_rows: list[dict[str, Any]] = []
|
||
|
||
for i, curve in enumerate(curves):
|
||
times = [float(x) for x in list(curve.x)]
|
||
probs = [float(y) for y in list(curve.y)]
|
||
pipe_id = str(df.iloc[i][ID_COLUMN]) if pd.notna(df.iloc[i][ID_COLUMN]) else f"Pipe_{i+1:03d}"
|
||
pipe_age = f"{df.iloc[i][PIPE_AGE_COLUMN]} 年" if PIPE_AGE_COLUMN in df.columns and pd.notna(df.iloc[i][PIPE_AGE_COLUMN]) else "-"
|
||
health_probability = interpolate_probability(times, probs, 10)
|
||
remaining_life = estimate_remaining_life(times, probs)
|
||
grade_label, grade_desc, grade_class = grade_info(health_probability)
|
||
|
||
summary_rows.append(
|
||
{
|
||
"pipe_id": pipe_id,
|
||
"pipe_age": pipe_age,
|
||
"health_probability": health_probability,
|
||
"remaining_life": remaining_life,
|
||
"grade_label": grade_label,
|
||
"grade_desc": grade_desc,
|
||
"grade_class": grade_class,
|
||
}
|
||
)
|
||
summary_sheet_rows.append(
|
||
{
|
||
"管道编号": pipe_id,
|
||
"管龄": pipe_age,
|
||
"健康概率": health_probability,
|
||
"预计剩余寿命": remaining_life,
|
||
"健康等级": grade_label,
|
||
}
|
||
)
|
||
plt.step(times, probs, where="post", linewidth=2, label=pipe_id)
|
||
|
||
plt.xlabel("预测时间轴（年）")
|
||
plt.ylabel("生存概率")
|
||
plt.title("预测分析图")
|
||
plt.grid(alpha=0.18)
|
||
if len(summary_rows) <= 12:
|
||
plt.legend(loc="best", fontsize=8)
|
||
plt.tight_layout()
|
||
plt.savefig(image_path, dpi=160, bbox_inches="tight")
|
||
plt.close()
|
||
return summary_rows, summary_sheet_rows
|
||
|
||
|
||
def write_prediction_workbook(
|
||
excel_path: Path,
|
||
curves,
|
||
summary_rows: list[dict[str, Any]],
|
||
summary_sheet_rows: list[dict[str, Any]],
|
||
) -> None:
|
||
with pd.ExcelWriter(excel_path, engine="xlsxwriter") as writer:
|
||
pd.DataFrame(summary_sheet_rows).to_excel(writer, sheet_name="结果摘要", index=False)
|
||
for i, curve in enumerate(curves):
|
||
times = [float(x) for x in list(curve.x)]
|
||
probs = [float(y) for y in list(curve.y)]
|
||
pipe_id = summary_rows[i]["pipe_id"]
|
||
out_df = pd.DataFrame({"时间（年）": times, f"{pipe_id}生存概率": probs})
|
||
out_df.to_excel(writer, sheet_name=f"样本{i+1}", index=False)
|