from __future__ import annotations import logging import os import uuid from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any import joblib import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pandas as pd from matplotlib import font_manager, rcParams from werkzeug.datastructures import FileStorage from werkzeug.utils import secure_filename from .config import IMAGE_DIR, UPLOAD_DIR CHINESE_FONT_CANDIDATES = [ "Noto Sans CJK SC", "Noto Sans SC", "Noto Sans CJK JP", "Noto Sans CJK TC", "Source Han Sans SC", "WenQuanYi Micro Hei", "SimHei", "Microsoft YaHei", "Arial Unicode MS", ] CHINESE_FONT_FILES = [ "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", "/usr/share/fonts/truetype/noto/NotoSansSC-Regular.ttf", "/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc", "/usr/local/share/fonts/NotoSansCJK-Regular.ttc", "/mnt/c/Windows/Fonts/NotoSansSC-VF.ttf", "/mnt/c/Windows/Fonts/msyh.ttc", "/mnt/c/Windows/Fonts/simhei.ttf", "/mnt/c/Windows/Fonts/simsun.ttc", ] FEATURES = [ "管材", "管径", "流速", "压力", "温度", "降雨量", "位置", "结构缺陷", "功能缺陷", ] ID_COLUMN = "管道编号" PIPE_AGE_COLUMN = "管龄（年）" LEGACY_PIPE_AGE_COLUMN = "管龄" STATUS_COLUMN = "状态" EVENT_AGE_COLUMN = "事件/观察管龄（年）" INPUT_COLUMNS = [ ID_COLUMN, PIPE_AGE_COLUMN, STATUS_COLUMN, EVENT_AGE_COLUMN, *FEATURES, ] MATERIAL_COLUMN = "管材" MATERIAL_CODE_OPTIONS = [ (1, "镀锌"), (2, "钢塑"), (3, "铝塑"), (4, "PPR"), (5, "PE"), (6, "UPVC"), (7, "铸铁"), (8, "预应力"), (9, "自应力"), (10, "玻璃钢夹砂"), (11, "钢管"), (12, "钢套混凝土管"), (13, "球墨铸铁"), (14, "其他"), ] MATERIAL_ALIAS_TO_CODE = { str(code): code for code, _ in MATERIAL_CODE_OPTIONS } MATERIAL_ALIAS_TO_CODE.update( { name.casefold(): code for code, name in MATERIAL_CODE_OPTIONS } ) MATERIAL_ALIAS_TO_CODE.update( { f"{code}-{name}".casefold(): code for code, name in MATERIAL_CODE_OPTIONS } ) SUPPORTED_EXTENSIONS = {".csv", ".xls", ".xlsx"} CHINESE_FONT_PROP = None class PredictionError(Exception): def __init__(self, message: str, status_code: int = 400) -> None: super().__init__(message) self.message = message self.status_code = status_code @dataclass class PredictionArtifacts: original_filename: str saved_path: Path excel_path: Path image_path: Path image_filename: str importance_filename: str | None sample_count: int summary_rows: list[dict[str, Any]] analysis_text: str def configure_matplotlib_fonts(): for font_path in CHINESE_FONT_FILES: path = Path(font_path) if path.exists(): font_manager.fontManager.addfont(str(path)) prop = font_manager.FontProperties(fname=str(path)) rcParams["font.family"] = [prop.get_name(), "sans-serif"] rcParams["font.sans-serif"] = [prop.get_name(), *CHINESE_FONT_CANDIDATES, "DejaVu Sans"] rcParams["axes.unicode_minus"] = False return prop available_fonts = {font.name for font in font_manager.fontManager.ttflist} selected_fonts = [font for font in CHINESE_FONT_CANDIDATES if font in available_fonts] if selected_fonts: rcParams["font.family"] = [selected_fonts[0], "sans-serif"] rcParams["font.sans-serif"] = selected_fonts + ["DejaVu Sans"] rcParams["axes.unicode_minus"] = False if selected_fonts: return font_manager.FontProperties(family=selected_fonts[0]) logging.warning("未找到中文字体，生成的图表中文可能无法显示。") return None CHINESE_FONT_PROP = configure_matplotlib_fonts() def chinese_font_kwargs() -> dict[str, Any]: return {"fontproperties": CHINESE_FONT_PROP} if CHINESE_FONT_PROP else {} def load_model(model_path: str): if not os.path.exists(model_path): raise FileNotFoundError(f"未找到模型文件: {model_path}") model = joblib.load(model_path) feature_names = getattr(model, "feature_names_in_", None) if feature_names is not None and list(feature_names) != FEATURES: raise ValueError( "模型输入特征与系统配置不一致: " f"model={list(feature_names)}, app={FEATURES}" ) n_features = getattr(model, "n_features_in_", None) if n_features is not None and int(n_features) != len(FEATURES): raise ValueError(f"模型特征数量不一致: model={n_features}, app={len(FEATURES)}") return model def safe_unlink(path: Path) -> None: try: path.unlink(missing_ok=True) except OSError as exc: logging.warning("删除文件失败 %s: %s", path, exc) def secure_upload_name(original_filename: str, run_id: str) -> tuple[str, str]: suffix = Path(original_filename).suffix.lower() if suffix not in SUPPORTED_EXTENSIONS: raise PredictionError("不支持的格式，仅支持逗号分隔值文件或电子表格文件") safe_full_name = secure_filename(original_filename) safe_stem = Path(safe_full_name).stem if safe_full_name else "" if not safe_stem: safe_stem = "upload" return f"{safe_stem}_{run_id}{suffix}", suffix def read_input_file(path: Path, suffix: str) -> pd.DataFrame: try: if suffix == ".csv": df = pd.read_csv(path, dtype={ID_COLUMN: "string"}) else: df = pd.read_excel(path, dtype={ID_COLUMN: "string"}) except Exception as exc: logging.exception("文件解析失败: %s", exc) raise PredictionError("文件解析失败，请检查编码或表格格式。") return normalize_input_columns(df) def normalize_input_columns(df: pd.DataFrame) -> pd.DataFrame: if PIPE_AGE_COLUMN not in df.columns and LEGACY_PIPE_AGE_COLUMN in df.columns: return df.rename(columns={LEGACY_PIPE_AGE_COLUMN: PIPE_AGE_COLUMN}) return df def normalize_id_value(value: Any, fallback: str) -> str: if pd.isna(value): return fallback if isinstance(value, str): return value.strip() return str(value) def validate_input_frame(df: pd.DataFrame) -> None: if df.empty: raise PredictionError("上传文件没有可预测的数据。") missing = [col for col in INPUT_COLUMNS if col not in df.columns] if missing: raise PredictionError(f"缺少必要字段: {', '.join(missing)}") def normalize_material_code(value: Any) -> int: if pd.isna(value): raise PredictionError("管材不能为空。") if isinstance(value, str): key = value.strip().casefold() else: numeric_value = float(value) if not numeric_value.is_integer(): raise PredictionError(f"管材编码无效: {value}") key = str(int(numeric_value)) code = MATERIAL_ALIAS_TO_CODE.get(key) if code is None: raise PredictionError(f"管材编码无效: {value}") return code def prepare_model_features(df: pd.DataFrame) -> pd.DataFrame: x_test = df[FEATURES].copy() x_test[MATERIAL_COLUMN] = x_test[MATERIAL_COLUMN].map(normalize_material_code) return x_test def grade_info(probability: float) -> tuple[str, str, str]: if probability <= 0.2: return ("I级", "管道安全风险十分严重，需立刻进行抢修或更新改造", "bg-dangerSoft text-dangerText") if probability <= 0.4: return ("II级", "管道安全风险较为严重，需尽快安排检修及加频巡检", "bg-orange-50 text-orange-600") if probability <= 0.6: return ("III级", "管道安全风险较低，需安排定期巡检", "bg-amber-50 text-amber-600") if probability <= 0.8: return ("IV级", "管道安全风险较小，维持常规巡视", "bg-blue-50 text-blue-600") return ("V级", "管道安全，维持常规巡视", "bg-blueSoft text-primary") def interpolate_probability(times: list[float], probs: list[float], target: float) -> float: if not times: return 0.0 if target <= times[0]: return float(probs[0]) for idx in range(1, len(times)): if times[idx] >= target: return float(probs[idx]) return float(probs[-1]) def estimate_remaining_life(times: list[float], probs: list[float]) -> float: for t, p in zip(times, probs): if p <= 0.5: return float(t) return float(times[-1]) if times else 0.0 def make_analysis_text(summary_rows: list[dict[str, Any]]) -> str: if not summary_rows: return "当前结果为空，暂无可供解释的样本。" worst = min(summary_rows, key=lambda x: x["health_probability"]) best = max(summary_rows, key=lambda x: x["health_probability"]) return ( "阶梯状曲线表示模型对不同管道随时间推移维持在安全健康状态概率的动态预测。" f"当前样本中风险最高管道为 {worst['pipe_id']}（{worst['grade_label']}），" f"健康概率最高管道为 {best['pipe_id']}（{best['health_probability']:.1%}）。" ) def compute_feature_importance(model, x_test: pd.DataFrame) -> np.ndarray | None: try: importances = model.feature_importances_ except Exception: importances = None if importances is not None and len(importances) == len(FEATURES): vals = np.asarray(importances, dtype=float) else: try: baseline = np.asarray(model.predict(x_test), dtype=float) except Exception as exc: logging.exception("特征重要性基线预测失败: %s", exc) return None rng = np.random.default_rng(42) vals = np.zeros(len(FEATURES), dtype=float) for j, feat in enumerate(FEATURES): diffs = [] for _ in range(5): x_perm = x_test.copy() x_perm[feat] = rng.permutation(x_perm[feat].to_numpy()) try: perm_pred = np.asarray(model.predict(x_perm), dtype=float) except Exception: perm_pred = baseline diffs.append(float(np.mean(np.abs(perm_pred - baseline)))) vals[j] = float(np.mean(diffs)) if diffs else 0.0 vals = np.clip(vals, a_min=0.0, a_max=None) total = float(vals.sum()) if total > 0: vals = vals / total return vals def render_importance_chart(values: np.ndarray, save_path: Path) -> None: from matplotlib.colors import LinearSegmentedColormap keep = values >= 5e-4 if not bool(np.any(keep)): keep = np.ones_like(values, dtype=bool) kept_feats = [FEATURES[i] for i in range(len(FEATURES)) if keep[i]] kept_vals = values[keep] order = np.argsort(kept_vals) sorted_feats = [kept_feats[k] for k in order] sorted_vals = kept_vals[order] n = len(sorted_vals) fig, ax = plt.subplots(figsize=(9, max(3.0, 0.62 * n + 1.6))) cmap = LinearSegmentedColormap.from_list("brand", ["#7fb2e6", "#005EB8", "#0c4188"]) colors = cmap(np.linspace(0.15, 1.0, n)) if n else None y_pos = np.arange(n) bars = ax.barh(y_pos, sorted_vals, color=colors, height=0.66, edgecolor="white", linewidth=0.8, zorder=3) font_kwargs = chinese_font_kwargs() ax.set_xlabel("相对重要性", fontsize=11, color="#475569", **font_kwargs) ax.set_title("模型输入因素重要性排序", fontsize=15, fontweight="bold", color="#0f172a", pad=14, **font_kwargs) ax.set_yticks(y_pos) ax.set_yticklabels(sorted_feats, **font_kwargs) ax.grid(axis="x", color="#e2e8f0", linewidth=1, zorder=0) ax.set_axisbelow(True) for spine in ("top", "right", "left"): ax.spines[spine].set_visible(False) ax.spines["bottom"].set_color("#cbd5e1") ax.tick_params(axis="y", length=0, labelsize=11) ax.tick_params(axis="x", colors="#94a3b8", labelsize=9) if CHINESE_FONT_PROP: for label in [*ax.get_yticklabels(), *ax.get_xticklabels()]: label.set_fontproperties(CHINESE_FONT_PROP) max_val = float(sorted_vals.max()) if n else 0.0 for bar, value in zip(bars, sorted_vals): ax.text( bar.get_width() + max_val * 0.012, bar.get_y() + bar.get_height() / 2, f"{value:.1%}", va="center", ha="left", fontsize=10, fontweight="bold", color="#1e293b", **font_kwargs, ) if max_val > 0: ax.set_xlim(0, max_val * 1.18) fig.tight_layout() fig.savefig(save_path, dpi=160, bbox_inches="tight") plt.close(fig) def run_prediction(uploaded: FileStorage, user_id: int, model) -> PredictionArtifacts: original_filename = uploaded.filename or "" if not original_filename: raise PredictionError("未选择文件") timestamp = datetime.now().strftime("%Y%m%d%H%M%S") run_id = f"{timestamp}_{uuid.uuid4().hex[:8]}" saved_filename, suffix = secure_upload_name(original_filename, run_id) user_dir = UPLOAD_DIR / f"user_{user_id}" user_dir.mkdir(parents=True, exist_ok=True) original_path = user_dir / saved_filename uploaded.save(original_path) try: df = read_input_file(original_path, suffix) validate_input_frame(df) x_test = prepare_model_features(df) try: curves = model.predict_survival_function(x_test) except Exception as exc: logging.exception("预测失败: %s", exc) raise PredictionError("模型预测失败，请检查输入字段类型是否正确。", 500) image_filename = f"plot_{user_id}_{run_id}.png" image_path = IMAGE_DIR / image_filename summary_rows, summary_sheet_rows = render_survival_chart(df, curves, image_path) importance_filename = None try: importance_values = compute_feature_importance(model, x_test) if importance_values is not None: importance_filename = f"importance_{user_id}_{run_id}.png" render_importance_chart(importance_values, IMAGE_DIR / importance_filename) except Exception as exc: logging.exception("生成特征重要性图失败: %s", exc) safe_stem = Path(saved_filename).stem excel_path = user_dir / f"{safe_stem}_pre.xlsx" write_prediction_workbook(excel_path, curves, summary_rows, summary_sheet_rows) return PredictionArtifacts( original_filename=original_filename, saved_path=original_path, excel_path=excel_path, image_path=image_path, image_filename=image_filename, importance_filename=importance_filename, sample_count=len(summary_rows), summary_rows=summary_rows, analysis_text=make_analysis_text(summary_rows), ) except PredictionError: safe_unlink(original_path) raise def render_survival_chart(df: pd.DataFrame, curves, image_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: plt.figure(figsize=(10, 5.6)) summary_rows: list[dict[str, Any]] = [] summary_sheet_rows: list[dict[str, Any]] = [] for i, curve in enumerate(curves): times = [float(x) for x in list(curve.x)] probs = [float(y) for y in list(curve.y)] pipe_id = normalize_id_value(df.iloc[i][ID_COLUMN], f"Pipe_{i+1:03d}") pipe_age = f"{df.iloc[i][PIPE_AGE_COLUMN]} 年" if PIPE_AGE_COLUMN in df.columns and pd.notna(df.iloc[i][PIPE_AGE_COLUMN]) else "-" health_probability = interpolate_probability(times, probs, 10) remaining_life = estimate_remaining_life(times, probs) grade_label, grade_desc, grade_class = grade_info(health_probability) summary_rows.append( { "pipe_id": pipe_id, "pipe_age": pipe_age, "health_probability": health_probability, "remaining_life": remaining_life, "grade_label": grade_label, "grade_desc": grade_desc, "grade_class": grade_class, } ) summary_sheet_rows.append( { ID_COLUMN: pipe_id, PIPE_AGE_COLUMN: pipe_age, "健康等级": grade_label, } ) plt.step(times, probs, where="post", linewidth=2, label=pipe_id) font_kwargs = chinese_font_kwargs() plt.xlabel("管龄（年）", **font_kwargs) plt.ylabel("健康风险", **font_kwargs) plt.title("预测分析图", **font_kwargs) plt.grid(alpha=0.18) if len(summary_rows) <= 12: plt.legend(loc="best", fontsize=8, prop=CHINESE_FONT_PROP) ax = plt.gca() if CHINESE_FONT_PROP: for label in [*ax.get_xticklabels(), *ax.get_yticklabels()]: label.set_fontproperties(CHINESE_FONT_PROP) plt.tight_layout() plt.savefig(image_path, dpi=160, bbox_inches="tight") plt.close() return summary_rows, summary_sheet_rows def write_prediction_workbook( excel_path: Path, curves, summary_rows: list[dict[str, Any]], summary_sheet_rows: list[dict[str, Any]], ) -> None: chart_times: set[float] = set() chart_series: list[tuple[str, dict[float, float]]] = [] for i, curve in enumerate(curves): times = [float(x) for x in list(curve.x)] probs = [float(y) for y in list(curve.y)] pipe_id = summary_rows[i]["pipe_id"] chart_times.update(times) chart_series.append((pipe_id, dict(zip(times, probs)))) sorted_times = sorted(chart_times) sample_columns = ["管龄（年）", *[pipe_id for pipe_id, _ in chart_series]] sample_data_rows = [ [time, *[series.get(time) for _, series in chart_series]] for time in sorted_times ] with pd.ExcelWriter(excel_path, engine="openpyxl") as writer: summary_df = pd.DataFrame(summary_sheet_rows, columns=[ID_COLUMN, PIPE_AGE_COLUMN, "健康等级"]) sample_df = pd.DataFrame(sample_data_rows, columns=sample_columns) if ID_COLUMN in summary_df.columns: summary_df[ID_COLUMN] = summary_df[ID_COLUMN].astype("string") summary_df.to_excel(writer, sheet_name="结果摘要", index=False) sample_df.to_excel(writer, sheet_name="样本数据", index=False) sample_worksheet = writer.book["样本数据"] sample_worksheet.freeze_panes = "A2" sample_worksheet.auto_filter.ref = sample_worksheet.dimensions sample_worksheet.column_dimensions["A"].width = 12 for column_cells in sample_worksheet.iter_cols(min_col=2, max_col=sample_worksheet.max_column): header_cell = column_cells[0] sample_worksheet.column_dimensions[header_cell.column_letter].width = max(12, len(str(header_cell.value)) + 2) for cell in column_cells[1:]: cell.number_format = "0.0%" for sheet_name in ("结果摘要", "样本数据"): worksheet = writer.book[sheet_name] header_cells = next(worksheet.iter_rows(min_row=1, max_row=1), []) id_column_index = None for cell in header_cells: if cell.value == ID_COLUMN: id_column_index = cell.column break if id_column_index is None: continue for cell in worksheet.iter_cols( min_col=id_column_index, max_col=id_column_index, min_row=2, max_row=worksheet.max_row, ): for item in cell: if item.value is not None: item.value = str(item.value) item.number_format = "@"