feat(projects): automate project infrastructure provisioning

app/services/burst_location.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 from datetime import datetime, timedelta
 from typing import Any
 from uuid import UUID
@@ -10,7 +9,7 @@ import pandas as pd
 from app.algorithms.burst_localization import run_burst_location
 from app.infra.db.postgresql.scada import get_all_scada_info
 from app.infra.db.timescaledb.internal_queries import InternalQueries
-from app.native.wndb.inp.exporter import dump_inp
+from app.services.project_inp import temporary_project_inp
 from app.services.scheme_management import (
     get_analysis_run,
     store_scheme_info,
@@ -300,20 +299,20 @@ def run_burst_location_by_network(
         burst_flow_samples = 1 if burst_flow_series is not None else 0
         normal_flow_samples = 1 if normal_flow_series is not None else 0
 
-    inp_path = _prepare_burst_inp(network)
-    result = run_burst_location(
-        wn_inp_path=inp_path,
-        pressure_scada_ids=selected_pressure_ids,
-        burst_pressure=burst_pressure_series,
-        normal_pressure=normal_pressure_series,
-        burst_leakage=burst_leakage,
-        flow_scada_ids=selected_flow_ids,
-        burst_flow=burst_flow_series,
-        normal_flow=normal_flow_series,
-        min_dpressure=min_dpressure,
-        basic_pressure=basic_pressure,
-        visualize_partition=False,
-    )
+    with temporary_project_inp(network, purpose="burst-location") as inp_path:
+        result = run_burst_location(
+            wn_inp_path=str(inp_path),
+            pressure_scada_ids=selected_pressure_ids,
+            burst_pressure=burst_pressure_series,
+            normal_pressure=normal_pressure_series,
+            burst_leakage=burst_leakage,
+            flow_scada_ids=selected_flow_ids,
+            burst_flow=burst_flow_series,
+            normal_flow=normal_flow_series,
+            min_dpressure=min_dpressure,
+            basic_pressure=basic_pressure,
+            visualize_partition=False,
+        )
 
     payload: dict[str, Any] = {
         **result,
@@ -775,16 +774,3 @@ def _normalize_timeseries_by_id(
 
 def _to_datetime(value: datetime | str) -> datetime:
     return parse_utc_time(value)
-
-
-def _prepare_burst_inp(network: str) -> str:
-    project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
-    db_inp_dir = os.path.join(project_root, "db_inp")
-    os.makedirs(db_inp_dir, exist_ok=True)
-    inp_path = os.path.join(db_inp_dir, f"{network}.burst.inp")
-    if os.path.isfile(inp_path) and os.path.getsize(inp_path) > 0:
-        return inp_path
-    dump_inp(network, inp_path, "2")
-    if not os.path.isfile(inp_path) or os.path.getsize(inp_path) <= 0:
-        raise ValueError(f"爆管定位 INP 文件无效: {inp_path}")
-    return inp_path

app/services/dma_leakage_estimation.py

@@ -14,7 +14,7 @@ from app.native.wndb.gis.network_views import (
     get_network_link_nodes,
     get_network_node_coords,
 )
-from app.native.wndb.inp.exporter import dump_inp
+from app.services.project_inp import temporary_project_inp
 from app.services.scheme_management import store_analysis_run_with_result
 from app.domain.time import parse_utc_time, utc_now
 
@@ -42,8 +42,6 @@ def run_leakage_identification(
     sensor_nodes: list[str] | None = None,
     scheme_name: str | None = None,
 ) -> dict[str, Any]:
-    inp_path = _prepare_leakage_inp(network)
-
     selected_sensor_nodes = (
         list(dict.fromkeys([node for node in (sensor_nodes or []) if node]))
         if sensor_nodes
@@ -52,7 +50,7 @@ def run_leakage_identification(
     if not selected_sensor_nodes:
         raise ValueError("未提供有效传感器节点，且系统未识别到可用压力传感器。")
 
-    area_map, areas, node_coords = _build_area_map_by_topology(
+    area_map, areas, _ = _build_area_map_by_topology(
         network, selected_sensor_nodes, dma_count
     )
 
@@ -73,26 +71,25 @@ def run_leakage_identification(
         observed_df = observed_pressure_data
 
     q_sum_m3s = DmaLeakageOptimizer._flow_to_m3s(q_sum, q_sum_unit)
-    identifier = DmaLeakageOptimizer(
-        inp_path=inp_path,
-        sensor_nodes=selected_sensor_nodes,
-        area_map=area_map,
-        start_time=start_time,
-        duration=duration,
-        timestep=timestep,
-        q_sum=q_sum_m3s,
-    )
-    result_df = identifier.run_identification(
-        observed_pressure_data=observed_df,
-        pop_size=pop_size,
-        max_gen=max_gen,
-        n_workers=n_workers,
-        output_flow_unit=output_flow_unit,
-        save_result=False,
-    )
+    with temporary_project_inp(network, purpose="dma-leakage") as inp_path:
+        identifier = DmaLeakageOptimizer(
+            inp_path=str(inp_path),
+            sensor_nodes=selected_sensor_nodes,
+            area_map=area_map,
+            start_time=start_time,
+            duration=duration,
+            timestep=timestep,
+            q_sum=q_sum_m3s,
+        )
+        result_df = identifier.run_identification(
+            observed_pressure_data=observed_df,
+            pop_size=pop_size,
+            max_gen=max_gen,
+            n_workers=n_workers,
+            output_flow_unit=output_flow_unit,
+            save_result=False,
+        )
     rows = result_df.to_dict(orient="records")
-    # node_visual_payload = _build_node_visual_payload(area_map, node_coords, rows)
-    # drawing_payload = _build_drawing_payload(node_visual_payload)
     payload = {
         "result_path": result_df.attrs.get("result_path"),
         "sensor_nodes": selected_sensor_nodes,
@@ -100,8 +97,6 @@ def run_leakage_identification(
         "area_count": len(set(area_map.values())),
         "node_area_map": area_map,
         "areas": areas,
-        # "node_visual_payload": node_visual_payload,
-        # "drawing_payload": drawing_payload,
         "rows": rows,
     }
     if scheme_name:
@@ -326,16 +321,3 @@ def _build_observed_pressure_from_scada(
 
 def _to_datetime(value: datetime | str) -> datetime:
     return parse_utc_time(value)
-
-
-def _prepare_leakage_inp(network: str) -> str:
-    project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
-    db_inp_dir = os.path.join(project_root, "db_inp")
-    os.makedirs(db_inp_dir, exist_ok=True)
-    inp_path = os.path.join(db_inp_dir, f"{network}.leakage.inp")
-    if os.path.isfile(inp_path) and os.path.getsize(inp_path) > 0:
-        return inp_path
-    dump_inp(network, inp_path, "2")
-    if not os.path.isfile(inp_path) or os.path.getsize(inp_path) <= 0:
-        raise ValueError(f"漏损识别 INP 文件无效: {inp_path}")
-    return inp_path

app/services/project_inp.py

@@ -0,0 +1,35 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from app.native.wndb.inp.exporter import dump_inp
+
+
+PROJECT_INP_DIRECTORY = Path("db_inp")
+
+
+@contextmanager
+def temporary_project_inp(
+    project_code: str,
+    *,
+    purpose: str,
+    version: str = "2",
+) -> Iterator[Path]:
+    """Export one project model to an isolated INP and remove it afterwards."""
+    PROJECT_INP_DIRECTORY.mkdir(parents=True, exist_ok=True)
+    with NamedTemporaryFile(
+        dir=PROJECT_INP_DIRECTORY,
+        prefix=f"{purpose}_",
+        suffix=".inp",
+        delete=False,
+    ) as temporary_file:
+        path = Path(temporary_file.name).resolve()
+
+    try:
+        dump_inp(project_code, str(path), version)
+        if not path.is_file() or path.stat().st_size == 0:
+            raise ValueError(f"项目 {project_code!r} 的 INP 导出失败")
+        yield path
+    finally:
+        path.unlink(missing_ok=True)

app/services/project_provisioning.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+from pathlib import Path
+import re
+from urllib.parse import quote
+
+from app.core.config import settings
+from app.infra.db.timescaledb.lifecycle import (
+    create_timescale_database,
+    delete_timescale_database,
+    require_timescale_schema_template,
+    timescale_database_exists,
+)
+from app.infra.db.postgresql.project_spatial import get_project_map_bbox
+from app.infra.geoserver.client import GeoServerAdminClient
+from app.native.wndb.core.project_templates import (
+    create_project_model_template,
+    delete_project_model_template,
+    ensure_replication_worker_capacity,
+)
+from app.native.wndb.core.projects import create_project, delete_project, have_project
+from app.services.network_import import network_update
+
+
+logger = logging.getLogger(__name__)
+_PROJECT_CODE = re.compile(r"^[a-z][a-z0-9_]{0,49}$")
+
+
+class ProjectProvisioningError(RuntimeError):
+    def __init__(self, stage: str, cause: Exception, cleanup_errors: list[str]) -> None:
+        self.stage = stage
+        self.cause = cause
+        self.cleanup_errors = cleanup_errors
+        cleanup_suffix = (
+            f"; cleanup failures: {', '.join(cleanup_errors)}"
+            if cleanup_errors
+            else ""
+        )
+        super().__init__(f"Project provisioning failed at {stage}: {cause}{cleanup_suffix}")
+
+
+def validate_project_code(code: str) -> str:
+    if not _PROJECT_CODE.fullmatch(code):
+        raise ValueError(
+            "Project code must start with a lowercase letter and contain only "
+            "lowercase letters, digits, and underscores"
+        )
+    if code.endswith("_template"):
+        raise ValueError("Project code must not end with '_template'")
+    if code in {
+        "postgres",
+        "template0",
+        "template1",
+        settings.METADATA_DB_NAME.casefold(),
+    }:
+        raise ValueError(f"Project code {code!r} is reserved")
+    return code
+
+
+def _database_url(*, timescale: bool, database_name: str) -> str:
+    if timescale:
+        host = settings.TIMESCALEDB_DB_HOST
+        port = settings.TIMESCALEDB_DB_PORT
+        user = settings.TIMESCALEDB_DB_USER
+        password = settings.TIMESCALEDB_DB_PASSWORD
+    else:
+        host = settings.DB_HOST
+        port = settings.DB_PORT
+        user = settings.DB_USER
+        password = settings.DB_PASSWORD
+    return (
+        f"postgresql://{quote(user, safe='')}:{quote(password, safe='')}"
+        f"@{host}:{port}/{database_name}"
+    )
+
+
+@dataclass
+class ProvisionedProjectInfrastructure:
+    code: str
+    workspace: str
+    model_template: str
+    map_bbox: tuple[float, float, float, float]
+    layers: tuple[str, ...]
+
+    @property
+    def business_dsn(self) -> str:
+        return _database_url(timescale=False, database_name=self.code)
+
+    @property
+    def timescale_dsn(self) -> str:
+        return _database_url(timescale=True, database_name=self.code)
+
+    def cleanup(self) -> list[str]:
+        errors: list[str] = []
+        cleanup_steps = (
+            ("geoserver", self._delete_geoserver),
+            ("timescaledb", lambda: delete_timescale_database(self.code)),
+            ("model_template", lambda: delete_project_model_template(self.code)),
+            ("business_database", lambda: delete_project(self.code)),
+        )
+        for name, cleanup in cleanup_steps:
+            try:
+                cleanup()
+            except Exception as exc:  # preserve every cleanup attempt
+                logger.exception("Project provisioning cleanup failed at %s", name)
+                errors.append(name)
+        return errors
+
+    def _delete_geoserver(self) -> None:
+        with GeoServerAdminClient() as geoserver:
+            geoserver.delete_workspace(self.workspace)
+
+
+def _preflight(code: str, workspace: str, geoserver: GeoServerAdminClient) -> None:
+    validate_project_code(code)
+    if not _PROJECT_CODE.fullmatch(workspace):
+        raise ValueError(
+            "GeoServer workspace must start with a lowercase letter and contain "
+            "only lowercase letters, digits, and underscores"
+        )
+    if have_project(settings.WNDB_SCHEMA_TEMPLATE_DB_NAME) is False:
+        raise RuntimeError(
+            f"Business schema template {settings.WNDB_SCHEMA_TEMPLATE_DB_NAME!r} does not exist"
+        )
+    require_timescale_schema_template()
+    ensure_replication_worker_capacity()
+    if have_project(code):
+        raise ValueError(f"Business database {code!r} already exists")
+    model_template = f"{code}_template"
+    if have_project(model_template):
+        raise ValueError(f"Project model template {model_template!r} already exists")
+    if timescale_database_exists(code):
+        raise ValueError(f"TimescaleDB database {code!r} already exists")
+    geoserver.check_ready()
+    if geoserver.workspace_exists(workspace):
+        raise ValueError(f"GeoServer workspace {workspace!r} already exists")
+
+
+def provision_project_infrastructure(
+    *,
+    code: str,
+    workspace: str,
+    inp_path: str | Path,
+) -> ProvisionedProjectInfrastructure:
+    stage = "preflight"
+    business_created = False
+    template_attempted = False
+    timescale_created = False
+    workspace_attempted = False
+    map_bbox: tuple[float, float, float, float] | None = None
+    layers: tuple[str, ...] = ()
+    try:
+        with GeoServerAdminClient() as geoserver:
+            _preflight(code, workspace, geoserver)
+            stage = "business_database"
+            create_project(code)
+            business_created = True
+
+            stage = "model_import"
+            network_update(str(inp_path), code)
+            map_bbox = get_project_map_bbox(code)
+
+            stage = "model_template"
+            template_attempted = True
+            model_template = create_project_model_template(code)
+
+            stage = "timescaledb"
+            create_timescale_database(code)
+            timescale_created = True
+
+            stage = "geoserver"
+            workspace_attempted = True
+            layers = geoserver.create_project_workspace(
+                workspace=workspace,
+                database_name=code,
+                map_bbox=map_bbox,
+            )
+    except Exception as exc:
+        cleanup_errors: list[str] = []
+        if workspace_attempted:
+            try:
+                with GeoServerAdminClient() as geoserver:
+                    geoserver.delete_workspace(workspace)
+            except Exception:
+                logger.exception("Failed to remove GeoServer workspace %s", workspace)
+                cleanup_errors.append("geoserver")
+        if timescale_created:
+            try:
+                delete_timescale_database(code)
+            except Exception:
+                logger.exception("Failed to remove TimescaleDB database %s", code)
+                cleanup_errors.append("timescaledb")
+        if template_attempted:
+            try:
+                delete_project_model_template(code)
+            except Exception:
+                logger.exception("Failed to remove project template for %s", code)
+                cleanup_errors.append("model_template")
+        if business_created:
+            try:
+                delete_project(code)
+            except Exception:
+                logger.exception("Failed to remove business database %s", code)
+                cleanup_errors.append("business_database")
+        raise ProjectProvisioningError(stage, exc, cleanup_errors) from exc
+
+    assert map_bbox is not None
+    return ProvisionedProjectInfrastructure(
+        code=code,
+        workspace=workspace,
+        model_template=f"{code}_template",
+        map_bbox=map_bbox,
+        layers=layers,
+    )

app/services/sensor_placement.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime
 import fcntl
@@ -16,7 +17,7 @@ import wntr
 from app.algorithms.pressure_sensor_placement import kmeans_placement
 from app.algorithms.pressure_sensor_placement import sensitivity_placement
 from app.infra.db.postgresql import sensor_placement as sensor_placement_repository
-from app.native.wndb.inp.exporter import dump_inp
+from app.services.project_inp import temporary_project_inp
 
 
 class SensorPlacementNotFoundError(LookupError):
@@ -31,7 +32,7 @@ class SensorPlacementConflictError(RuntimeError):
     pass
 
 
-def _sensor_inp_path(project_code: str) -> Path:
+def _sensor_lock_path(project_code: str) -> Path:
     if (
         not project_code
         or project_code in {".", ".."}
@@ -40,14 +41,13 @@ def _sensor_inp_path(project_code: str) -> Path:
         or "\x00" in project_code
     ):
         raise SensorPlacementValidationError("管网名称不是有效的项目标识")
-    return Path("db_inp") / f"{project_code}.db.inp"
+    return Path("db_inp") / f"{project_code}.sensor.lock"
 
 
 @contextmanager
-def _sensor_inp_lock(project_code: str):
-    inp_path = _sensor_inp_path(project_code)
-    inp_path.parent.mkdir(parents=True, exist_ok=True)
-    lock_path = inp_path.with_suffix(".sensor.lock")
+def _sensor_run_lock(project_code: str) -> Iterator[None]:
+    lock_path = _sensor_lock_path(project_code)
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
     with lock_path.open("w", encoding="utf-8") as lock_file:
         try:
             fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -56,11 +56,23 @@ def _sensor_inp_lock(project_code: str):
                 "当前项目已有监测点优化任务正在运行，请稍后重试"
             ) from exc
         try:
-            yield inp_path
+            yield
         finally:
             fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
 
+@contextmanager
+def _sensor_network_model(
+    project_code: str,
+) -> Iterator[wntr.network.WaterNetworkModel]:
+    with _sensor_run_lock(project_code):
+        with temporary_project_inp(
+            project_code,
+            purpose="sensor-placement",
+        ) as inp_path:
+            yield wntr.network.WaterNetworkModel(str(inp_path))
+
+
 def _create_validated_placement(
     project_code: str,
     *,
@@ -88,10 +100,7 @@ def optimize_sensor_placement_by_sensitivity(
 ) -> dict[str, Any]:
     """Run sensitivity placement and persist the validated result."""
 
-    with _sensor_inp_lock(project_code):
-        inp_path = _sensor_inp_path(project_code)
-        dump_inp(project_code, str(inp_path), "2")
-        network_model = wntr.network.WaterNetworkModel(str(inp_path))
+    with _sensor_network_model(project_code) as network_model:
         sensor_locations = sensitivity_placement.optimize_sensor_placement(
             network_model,
             sensor_num=sensor_count,
@@ -115,9 +124,7 @@ def optimize_sensor_placement_by_kmeans(
 ) -> dict[str, Any]:
     """Export the model, run K-means placement, and persist the result."""
 
-    with _sensor_inp_lock(project_code) as inp_path:
-        dump_inp(project_code, str(inp_path), "2")
-        network_model = wntr.network.WaterNetworkModel(str(inp_path))
+    with _sensor_network_model(project_code) as network_model:
         sensor_locations = kmeans_placement.optimize_sensor_placement(
             network_model,
             sensor_count=sensor_count,