feat(scada): add calibrated monitoring and incident recovery
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare replay-rule thresholds against frozen hydraulic outputs without rerunning hydraulics."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--calibration-dir", required=True, type=Path)
|
||||
parser.add_argument("--thresholds", nargs="+", type=float, default=[3, 5, 10, 20, 30])
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
from scripts.benchmark.calibration import digest, pressure_preview, save_json
|
||||
from scripts.benchmark.pressure_thresholds import learn_pressure_thresholds
|
||||
from scripts.benchmark.simulation_design import BurstSimulationDesign
|
||||
|
||||
folder = args.calibration_dir.resolve()
|
||||
source = folder / "calibration.json"
|
||||
report = json.loads(source.read_text())
|
||||
if report.get("status") != "completed" or any(not 0 < value <= 100 for value in args.thresholds):
|
||||
raise ValueError("a completed calibration and thresholds in (0,100] are required")
|
||||
design = BurstSimulationDesign.model_validate(report["design"])
|
||||
pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
|
||||
cases = {case.case_id: case for case in design.cases}
|
||||
index_by_time = {at: index for index, at in enumerate(design.event_times)}
|
||||
output = {"version": "burst-threshold-sensitivity@1", "calibration_sha256": digest(source),
|
||||
"scope": "pure cleaner/replay-rule sensitivity, not Agent or worker acceptance",
|
||||
"thresholds_percent": args.thresholds, "rows": [], "summary": [],
|
||||
"adaptive_fit": learn_pressure_thresholds([], ["calibration-history"], design.event_times[0]),
|
||||
"adaptive_note": "this package has four warmup samples; fitting needs 1440 pre-event normal samples per device",
|
||||
"absolute_drop_gate_applied": False}
|
||||
cache = {}
|
||||
controls = {}
|
||||
for entry in report["cases"]:
|
||||
case = cases[entry["case_id"]]
|
||||
pipe = pipes[case.pipe_id]
|
||||
index = index_by_time[case.occurred_at]
|
||||
if index not in cache:
|
||||
cache[index] = (json.loads((folder / f"normal-{index}.json").read_text()),
|
||||
json.loads((folder / f"warmup-{index}.json").read_text()))
|
||||
normal, warmup = cache[index]
|
||||
burst = json.loads((folder / f"{case.case_id}-{entry['severity']}.json").read_text())
|
||||
for threshold in args.thresholds:
|
||||
rule = SimpleNamespace(primary_pressure_drop_percent=threshold,
|
||||
consecutive_samples=design.selection_rule.consecutive_samples)
|
||||
key = (pipe.pipe_id, index, threshold)
|
||||
if key not in controls:
|
||||
controls[key] = pressure_preview(pipe, case.occurred_at, report["duration_seconds"], 900,
|
||||
warmup["nodes"], normal["nodes"], rule)
|
||||
event = pressure_preview(pipe, case.occurred_at, report["duration_seconds"], 900,
|
||||
warmup["nodes"], burst["nodes"], rule)
|
||||
at = event["first_signal_at"]
|
||||
output["rows"].append({"case_id": case.case_id, "pipe_id": pipe.pipe_id,
|
||||
"severity": entry["severity"], "threshold_percent": threshold,
|
||||
"event_signal": event["raised"], "first_signal_at": at,
|
||||
"delay_minutes": (datetime.fromisoformat(at) - case.occurred_at).total_seconds() / 60 if at else None,
|
||||
"normal_false_positive": controls[key]["raised"]})
|
||||
for threshold in args.thresholds:
|
||||
rows = [row for row in output["rows"] if row["threshold_percent"] == threshold]
|
||||
normal = [value for key, value in controls.items() if key[2] == threshold]
|
||||
output["summary"].append({"threshold_percent": threshold, "event_cases": len(rows),
|
||||
"event_signals": sum(row["event_signal"] for row in rows),
|
||||
"unique_normal_controls": len(normal),
|
||||
"normal_false_positives": sum(row["raised"] for row in normal)})
|
||||
path = folder / "threshold-sensitivity.json"
|
||||
save_json(path, output)
|
||||
for summary in output["summary"]:
|
||||
print(summary)
|
||||
print(f"threshold sensitivity complete: {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1 @@
|
||||
"""Offline benchmark preparation; never imported by the application runtime."""
|
||||
@@ -0,0 +1,226 @@
|
||||
"""File-isolated hydraulic calibration; database connections are read-only."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from scripts.benchmark.simulation_design import PipeCalibration, SelectionRule
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def save_json(path: Path, value: dict) -> None:
|
||||
temporary = path.with_suffix(".pending")
|
||||
with temporary.open("x", encoding="utf-8") as output:
|
||||
os.chmod(temporary, 0o600)
|
||||
json.dump(value, output, ensure_ascii=False, indent=2, allow_nan=False, default=str)
|
||||
output.write("\n")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def select_calibrated_area(
|
||||
pipe: PipeCalibration, passed_levels: list[str], case_results: list[dict], retain_weak: bool,
|
||||
) -> dict:
|
||||
selected = passed_levels[-1] if passed_levels else None
|
||||
soft_blockers = {"insufficient_consecutive_pressure_and_corroboration", "event_rule_did_not_trigger"}
|
||||
physically_valid = bool(case_results) and all(
|
||||
not set(case["blockers"]) - soft_blockers for case in case_results
|
||||
)
|
||||
status = "calibrated_candidate" if selected else (
|
||||
"weak_signal_candidate" if retain_weak and physically_valid else "unsuitable"
|
||||
)
|
||||
return {"status": status, "severity": selected,
|
||||
"area_cm2": pipe.candidate_orifice_areas_cm2[selected] if selected else None}
|
||||
|
||||
|
||||
def configure_inp(
|
||||
source: str,
|
||||
start: datetime,
|
||||
duration: int,
|
||||
*,
|
||||
emitter: tuple[str, float] | None = None,
|
||||
pda_options: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Apply the burst service's options to a file, including the zero-leak control."""
|
||||
if start.tzinfo is None:
|
||||
raise ValueError("simulation start must include a timezone")
|
||||
start = start.astimezone(ZoneInfo("Asia/Shanghai"))
|
||||
updates = {
|
||||
"TIMES": {"DURATION": f"{duration // 3600}:{duration % 3600 // 60:02d}:00",
|
||||
"PATTERN START": start.strftime("%H:%M:%S"),
|
||||
"REPORT START": "0:00:00", "STATISTIC": "NONE"},
|
||||
"OPTIONS": {**(pda_options or {}), "DEMAND MODEL": "PDA", "REQUIRED PRESSURE": "10.0000"},
|
||||
"REPORT": {"NODES": "NONE", "LINKS": "NONE"},
|
||||
}
|
||||
section = None
|
||||
output = []
|
||||
pending = {}
|
||||
for line in source.splitlines():
|
||||
match = re.fullmatch(r"\s*\[([^]]+)\]\s*", line)
|
||||
if match:
|
||||
output.extend(f"{key} {value}" for key, value in pending.items())
|
||||
section = match[1].upper()
|
||||
pending = dict(updates.get(section, {}))
|
||||
output.append(line)
|
||||
if section == "EMITTERS" and emitter:
|
||||
node, area = emitter
|
||||
coefficient = 0.65 * area / 10000 * math.sqrt(19.6) * 1000
|
||||
output.append(f"{node} {coefficient:.12g}")
|
||||
continue
|
||||
tokens = line.split(";", 1)[0].upper().split()
|
||||
if any(tokens[:len(key.split())] == key.split() for key in updates.get(section, {})):
|
||||
continue
|
||||
output.append(line)
|
||||
output.extend(f"{key} {value}" for key, value in pending.items())
|
||||
if "[EMITTERS]" not in source.upper() and emitter:
|
||||
raise ValueError("exported model has no emitter section")
|
||||
return "\n".join(output) + "\n"
|
||||
|
||||
|
||||
def run_file(inp: Path, node_ids: list[str], link_ids: list[str], *, timeout: int) -> dict:
|
||||
from app.infra.epanet.epanet import Output
|
||||
|
||||
executable = Path(__file__).resolve().parents[2] / "app/infra/epanet/linux/runepanet"
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = f"{executable.parent}:{env.get('LD_LIBRARY_PATH', '')}"
|
||||
report_path, binary = inp.with_suffix(".rpt"), inp.with_suffix(".bin")
|
||||
result = subprocess.run(
|
||||
[str(executable), str(inp), str(report_path), str(binary)],
|
||||
env=env, capture_output=True, text=True, timeout=timeout, check=False,
|
||||
)
|
||||
report = report_path.read_text(encoding="utf-8", errors="replace") if report_path.exists() else ""
|
||||
warnings = [line.strip() for line in report.splitlines()
|
||||
if re.search(r"^\s*(?:warning\b|error\s+\d+\b)|system\s+(?:is\s+)?unbalanced|cannot solve hydraulic",
|
||||
line, re.IGNORECASE)]
|
||||
summary = {"exit_code": result.returncode, "warnings": warnings,
|
||||
"inp_sha256": digest(inp), "report_sha256": digest(report_path) if report_path.exists() else None}
|
||||
if result.returncode or not binary.exists():
|
||||
return {**summary, "status": "failed"}
|
||||
reader = Output(str(binary))
|
||||
names = reader.element_name()
|
||||
summary.update(status="completed", units=reader.units(), times=reader.times(), nodes={}, links={})
|
||||
for kind, selected, attributes, function in (
|
||||
("nodes", node_ids, ["demand", "head", "pressure", "quality"], reader._lib.ENR_getNodeResult),
|
||||
("links", link_ids, ["flow", "velocity", "headloss", "quality", "status", "setting", "reaction", "friction"],
|
||||
reader._lib.ENR_getLinkResult),
|
||||
):
|
||||
indices = {name: index + 1 for index, name in enumerate(names[kind])}
|
||||
for name in selected:
|
||||
index = indices.get(name)
|
||||
if index is None:
|
||||
raise ValueError("requested hydraulic output is missing")
|
||||
series = []
|
||||
for period in range(summary["times"]["num_periods"]):
|
||||
values = ctypes.POINTER(ctypes.c_float)()
|
||||
length = ctypes.c_int()
|
||||
reader._check(function(reader._handle, period, index, ctypes.byref(values), ctypes.byref(length)))
|
||||
try:
|
||||
if length.value != len(attributes):
|
||||
raise ValueError("unexpected hydraulic result attributes")
|
||||
row = {key: float(values[i]) for i, key in enumerate(attributes)}
|
||||
if not all(math.isfinite(value) for value in row.values()):
|
||||
raise ValueError("non-finite hydraulic result")
|
||||
series.append(row)
|
||||
finally:
|
||||
reader._lib.ENR_free(ctypes.byref(values))
|
||||
summary[kind][name] = series
|
||||
return summary
|
||||
|
||||
|
||||
def pressure_preview(
|
||||
pipe: PipeCalibration, start: datetime, duration: int, step: int,
|
||||
history: dict, current: dict, rule: SelectionRule,
|
||||
) -> dict:
|
||||
"""Exercise the actual replay cleaner/detector without database mutations."""
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
||||
|
||||
ids = [binding.device_id for binding in pipe.pressure_bindings]
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, f"calibration:{pipe.pipe_id}:{start.isoformat()}"),
|
||||
start_time=start, end_time=start + timedelta(seconds=duration + step), device_ids=ids,
|
||||
rules=[{"rule_id": "calibration-pressure", "device_ids": ids,
|
||||
"kind": "relative_change", "direction": "decrease", "unit": "m",
|
||||
"threshold": rule.primary_pressure_drop_percent,
|
||||
"window_samples": 4, "minimum_samples": 4,
|
||||
"consecutive_samples": rule.consecutive_samples, "minimum_devices": 1}],
|
||||
)
|
||||
document = {"replay_id": str(config.request_id), "devices": [
|
||||
{"device_id": key, "transmission_frequency": "0:15:00"} for key in ids],
|
||||
"histories": {binding.device_id: [row["pressure"] for row in history[binding.node_id]]
|
||||
for binding in pipe.pressure_bindings}}
|
||||
raised = []
|
||||
for index in range(duration // step + 1):
|
||||
at = start + timedelta(seconds=index * step)
|
||||
rows = [{"device_id": binding.device_id, "time": at,
|
||||
"monitored_value": current[binding.node_id][index]["pressure"],
|
||||
"source_hash": hashlib.sha256(f"{binding.device_id}:{at}".encode()).hexdigest()}
|
||||
for binding in pipe.pressure_bindings]
|
||||
document, changes = detect_frame(document, prepare_cleaning(rows, config), config, at)
|
||||
raised.extend(change for change in changes
|
||||
if change["change"] == "raised" and change.get("category") != "data_quality")
|
||||
return {"raised": bool(raised), "first_signal_at": raised[0]["occurred_at"] if raised else None,
|
||||
"scope": "pure replay rule preview; not monitoring worker or Agent acceptance"}
|
||||
|
||||
|
||||
def assess(
|
||||
pipe: PipeCalibration, normal: dict, burst: dict, rule: SelectionRule,
|
||||
*, duration: int, step: int,
|
||||
) -> dict:
|
||||
expected_times = {"report_start": 0, "report_step": step,
|
||||
"sim_duration": duration, "num_periods": duration // step + 1}
|
||||
blockers = []
|
||||
for label, result in (("normal", normal), ("burst", burst)):
|
||||
if result.get("status") != "completed" or result.get("times") != expected_times:
|
||||
blockers.append(f"{label}_incomplete_or_failed")
|
||||
if result.get("warnings"):
|
||||
blockers.append(f"{label}_solver_warning")
|
||||
if result.get("units") != {"flow": "LPS", "pressure": "MTR", "quality": "NONE"}:
|
||||
blockers.append(f"{label}_unexpected_units")
|
||||
if blockers:
|
||||
return {"passed": False, "blockers": blockers}
|
||||
pressures, flows = {}, {}
|
||||
for binding in pipe.pressure_bindings:
|
||||
before = [row["pressure"] for row in normal["nodes"][binding.node_id]]
|
||||
after = [row["pressure"] for row in burst["nodes"][binding.node_id]]
|
||||
if min(before) <= rule.relative_change_epsilon:
|
||||
blockers.append(f"nonpositive_normal_pressure:{binding.device_id}")
|
||||
if min(after) < 0:
|
||||
blockers.append(f"negative_monitor_pressure:{binding.device_id}")
|
||||
pressures[binding.device_id] = [100 * (a - b) / max(abs(a), rule.relative_change_epsilon)
|
||||
for a, b in zip(before, after)]
|
||||
for binding in pipe.flow_bindings:
|
||||
flows[binding.device_id] = [100 * abs(a["flow"] - b["flow"]) /
|
||||
max(abs(a["flow"]), rule.relative_change_epsilon)
|
||||
for a, b in zip(normal["links"][binding.link_id], burst["links"][binding.link_id])]
|
||||
witnesses = []
|
||||
count = duration // step + 1
|
||||
for primary, drops in pressures.items():
|
||||
for start in range(count - rule.consecutive_samples + 1):
|
||||
window = range(start, start + rule.consecutive_samples)
|
||||
if not all(drops[i] >= rule.primary_pressure_drop_percent for i in window):
|
||||
continue
|
||||
second = [key for key, values in pressures.items() if key != primary and
|
||||
all(values[i] >= rule.corroborating_pressure_drop_percent for i in window)]
|
||||
flow = [key for key, values in flows.items() if
|
||||
all(values[i] >= rule.flow_change_percent for i in window) and
|
||||
all(abs(normal["links"][next(b.link_id for b in pipe.flow_bindings if b.device_id == key)][i]["flow"])
|
||||
> rule.relative_change_epsilon for i in window)]
|
||||
if second or flow:
|
||||
witnesses.append({"primary_device": primary, "second_pressure_devices": second,
|
||||
"flow_devices": flow, "start_offset_seconds": start * step})
|
||||
if not witnesses:
|
||||
blockers.append("insufficient_consecutive_pressure_and_corroboration")
|
||||
return {"passed": not blockers, "blockers": blockers, "witnesses": witnesses,
|
||||
"pressure_drop_percent": pressures, "flow_change_percent": flows}
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Versioned burst cases and deterministic preparation checks."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import defaultdict, deque
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
|
||||
class BurstCase(StrictModel):
|
||||
case_id: str = Field(pattern=r"^B[0-9]{2}$")
|
||||
pipe_id: str = Field(pattern=r"^[0-9]+$")
|
||||
occurred_at: datetime
|
||||
reference_analysis: str = Field(min_length=1, max_length=200)
|
||||
observation_provenance: Literal["declared_historical_replay", "requires_generation"]
|
||||
provenance_note: str = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def aware_time(self):
|
||||
if self.occurred_at.tzinfo is None:
|
||||
raise ValueError("case time must include a timezone")
|
||||
return self
|
||||
|
||||
|
||||
class TriggerRule(StrictModel):
|
||||
pressure_drop_percent: float = Field(gt=0, le=100)
|
||||
baseline_samples: int = Field(ge=3, le=96)
|
||||
consecutive_samples: int = Field(ge=1, le=20)
|
||||
minimum_devices: int = Field(ge=1, le=500)
|
||||
|
||||
|
||||
class BurstSuite(StrictModel):
|
||||
version: Literal["burst-benchmark@1"]
|
||||
suite_id: Literal["tjwater-v2-burst"]
|
||||
project_code: Literal["tjwater_v2"]
|
||||
project_id: UUID
|
||||
sampling_interval_minutes: int = Field(ge=1, le=60)
|
||||
event_minutes: int = Field(ge=15, le=1440)
|
||||
detector_days: Literal[15]
|
||||
trigger_rule: TriggerRule
|
||||
score_weights: dict[str, int]
|
||||
cases: list[BurstCase] = Field(min_length=1, max_length=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_suite(self):
|
||||
if 1440 % self.sampling_interval_minutes or self.event_minutes % self.sampling_interval_minutes:
|
||||
raise ValueError("sample interval must divide a day and the event window")
|
||||
if len({case.case_id for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("case ids must be unique")
|
||||
if set(self.score_weights) != {"outcome", "tools", "plan", "efficiency"}:
|
||||
raise ValueError("all four score dimensions are required")
|
||||
if any(value <= 0 for value in self.score_weights.values()) or sum(self.score_weights.values()) != 100:
|
||||
raise ValueError("positive score weights must sum to 100")
|
||||
for case in self.cases:
|
||||
if case.occurred_at.timestamp() % (60 * self.sampling_interval_minutes):
|
||||
raise ValueError("event times must align with the sampling grid")
|
||||
return self
|
||||
|
||||
def select(self, case_ids: list[str]) -> list[BurstCase]:
|
||||
if not case_ids:
|
||||
return self.cases
|
||||
unknown = set(case_ids) - {case.case_id for case in self.cases}
|
||||
if unknown:
|
||||
raise ValueError("unknown case ids: " + ", ".join(sorted(unknown)))
|
||||
return [case for case in self.cases if case.case_id in case_ids]
|
||||
|
||||
|
||||
def load_suite(path: Path) -> BurstSuite:
|
||||
return BurstSuite.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def canonical_json(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
||||
allow_nan=False, default=str)
|
||||
|
||||
|
||||
def fingerprint(value) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode()).hexdigest()
|
||||
|
||||
|
||||
def complete_devices(rows: list[dict], start: datetime, end: datetime, interval: int) -> list[str]:
|
||||
"""Counts alone are insufficient: require finite values on the exact grid."""
|
||||
step = timedelta(minutes=interval)
|
||||
expected = {start + step * index for index in range(int((end - start) / step) + 1)}
|
||||
values: dict[str, dict[datetime, float]] = defaultdict(dict)
|
||||
invalid: set[str] = set()
|
||||
for row in rows:
|
||||
device = row["device_id"]
|
||||
at = row["time"]
|
||||
if not start <= at <= end:
|
||||
continue
|
||||
value = row["monitored_value"]
|
||||
if at not in expected or at in values[device] or value is None or not math.isfinite(float(value)):
|
||||
invalid.add(device)
|
||||
else:
|
||||
values[device][at] = float(value)
|
||||
return sorted(device for device, series in values.items()
|
||||
if device not in invalid and set(series) == expected)
|
||||
|
||||
|
||||
def isolation_reference(links: list[dict], source_nodes: list[str], pipe_id: str) -> dict:
|
||||
"""Independent graph check, separate from the production isolation algorithm."""
|
||||
graph: dict[str, list[str]] = defaultdict(list)
|
||||
target = None
|
||||
valves = []
|
||||
for link in links:
|
||||
if link["id"] == pipe_id:
|
||||
target = (link["start_node_id"], link["end_node_id"])
|
||||
if link["kind"].lower() == "valve":
|
||||
valves.append(link["id"])
|
||||
continue
|
||||
if str(link["status"]).lower() == "closed":
|
||||
continue
|
||||
left, right = link["start_node_id"], link["end_node_id"]
|
||||
graph[left].append(right)
|
||||
graph[right].append(left)
|
||||
if target is None:
|
||||
return {"status": "target_missing"}
|
||||
if not source_nodes:
|
||||
return {"status": "sources_missing"}
|
||||
reached = set(source_nodes)
|
||||
pending = deque(source_nodes)
|
||||
while pending:
|
||||
for node in graph[pending.popleft()]:
|
||||
if node not in reached:
|
||||
reached.add(node)
|
||||
pending.append(node)
|
||||
connected = any(node in reached for node in target)
|
||||
return {
|
||||
"status": "not_isolatable" if connected else "requires_boundary_verification",
|
||||
"model_valve_count": len(valves),
|
||||
"source_count": len(source_nodes),
|
||||
"source_reachable_after_closing_all_model_valves": connected,
|
||||
"scope": "undirected model topology; not hydraulic or field acceptance",
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
"""Isolated database preparation and deterministic burst-dataset scheduling."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from psycopg import connect, sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
|
||||
def canonical_hash(value) -> str:
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def deterministic_standard_normal(*, seed: str, at: datetime, device_id: str, metric: str) -> float:
|
||||
"""Return order-independent deterministic Gaussian noise for one observation."""
|
||||
material = f"{seed}\0{at.isoformat()}\0{device_id}\0{metric}".encode()
|
||||
digest = hashlib.sha256(material).digest()
|
||||
first = (int.from_bytes(digest[:8], "big") + 1) / (2**64 + 1)
|
||||
second = (int.from_bytes(digest[8:16], "big") + 1) / (2**64 + 1)
|
||||
return math.sqrt(-2.0 * math.log(first)) * math.cos(2.0 * math.pi * second)
|
||||
|
||||
|
||||
def noisy_scada_observation(
|
||||
*,
|
||||
raw_value: float,
|
||||
device_type: str,
|
||||
at: datetime,
|
||||
device_id: str,
|
||||
seed: str,
|
||||
pressure_sigma_percent: float = 0.2,
|
||||
pressure_sigma_floor_m: float = 0.02,
|
||||
flow_sigma_percent: float = 0.5,
|
||||
clip_sigma: float = 3.0,
|
||||
) -> float:
|
||||
"""Convert model units and add reproducible zero-mean telemetry noise."""
|
||||
if not math.isfinite(raw_value):
|
||||
raise ValueError("model observation must be finite")
|
||||
if min(pressure_sigma_percent, pressure_sigma_floor_m, flow_sigma_percent, clip_sigma) < 0:
|
||||
raise ValueError("noise parameters must be non-negative")
|
||||
z = max(-clip_sigma, min(clip_sigma, deterministic_standard_normal(
|
||||
seed=seed, at=at, device_id=device_id, metric=device_type,
|
||||
)))
|
||||
if device_type == "pressure":
|
||||
converted = raw_value
|
||||
sigma = max(abs(converted) * pressure_sigma_percent / 100.0, pressure_sigma_floor_m)
|
||||
return max(0.0, converted + z * sigma)
|
||||
if device_type == "pipe_flow":
|
||||
converted = raw_value * 3.6 # L/s -> m3/h
|
||||
sigma = abs(converted) * flow_sigma_percent / 100.0
|
||||
return converted + z * sigma
|
||||
raise ValueError(f"unsupported SCADA device type: {device_type}")
|
||||
|
||||
|
||||
def write_realtime_scada_day(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
start: datetime,
|
||||
interval_seconds: int = 900,
|
||||
duration_seconds: int = 86400,
|
||||
noise_seed: str,
|
||||
pressure_sigma_percent: float = 0.2,
|
||||
pressure_sigma_floor_m: float = 0.02,
|
||||
flow_sigma_percent: float = 0.5,
|
||||
clip_sigma: float = 3.0,
|
||||
) -> dict:
|
||||
"""Map one online-model day into noisy SCADA observations."""
|
||||
if start.tzinfo is None or duration_seconds <= 0 or interval_seconds <= 0:
|
||||
raise ValueError("an aware start and positive duration/interval are required")
|
||||
if duration_seconds % interval_seconds:
|
||||
raise ValueError("duration must be divisible by the sampling interval")
|
||||
expected_times = [
|
||||
start + timedelta(seconds=offset)
|
||||
for offset in range(0, duration_seconds, interval_seconds)
|
||||
]
|
||||
end = start + timedelta(seconds=duration_seconds)
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow')
|
||||
ORDER BY device_id"""
|
||||
).fetchall()
|
||||
if not devices:
|
||||
raise ValueError("no pressure or pipe-flow SCADA bindings are available")
|
||||
records: list[tuple] = []
|
||||
raw_fingerprint_rows: list[tuple] = []
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
for device in devices:
|
||||
if device["device_type"] == "pressure":
|
||||
if not device["node_id"] or device["measurement_unit"] != "m":
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
rows = timescale.execute(
|
||||
"""SELECT time,pressure AS value FROM realtime.node_results
|
||||
WHERE node_id=%s AND time>=%s AND time<%s ORDER BY time""",
|
||||
(device["node_id"], start, end),
|
||||
).fetchall()
|
||||
else:
|
||||
if not device["link_id"] or device["measurement_unit"] != "m3/h":
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
rows = timescale.execute(
|
||||
"""SELECT time,flow AS value FROM realtime.link_results
|
||||
WHERE link_id=%s AND time>=%s AND time<%s ORDER BY time""",
|
||||
(device["link_id"], start, end),
|
||||
).fetchall()
|
||||
actual_times = [row["time"] for row in rows]
|
||||
if actual_times != expected_times:
|
||||
raise ValueError(f"incomplete realtime samples for device {device['device_id']}")
|
||||
for row in rows:
|
||||
raw_value = float(row["value"])
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw_value,
|
||||
device_type=device["device_type"],
|
||||
at=row["time"],
|
||||
device_id=device["device_id"],
|
||||
seed=noise_seed,
|
||||
pressure_sigma_percent=pressure_sigma_percent,
|
||||
pressure_sigma_floor_m=pressure_sigma_floor_m,
|
||||
flow_sigma_percent=flow_sigma_percent,
|
||||
clip_sigma=clip_sigma,
|
||||
)
|
||||
raw_fingerprint_rows.append((row["time"].isoformat(), device["device_id"], raw_value))
|
||||
records.append((row["time"], device["device_id"], value, None))
|
||||
collision = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if collision:
|
||||
raise ValueError("SCADA target day is not empty")
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in records:
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if persisted != len(records):
|
||||
raise ValueError("SCADA normal-day read-back count does not match generated records")
|
||||
return {
|
||||
"start": start.isoformat(),
|
||||
"end_exclusive": end.isoformat(),
|
||||
"devices": len(devices),
|
||||
"samples_per_device": len(expected_times),
|
||||
"rows": persisted,
|
||||
"units": {"pressure": "m", "pipe_flow": "m3/h", "model_flow": "L/s"},
|
||||
"noise": {
|
||||
"seed": noise_seed,
|
||||
"distribution": "deterministic_clipped_gaussian",
|
||||
"pressure_sigma_percent": pressure_sigma_percent,
|
||||
"pressure_sigma_floor_m": pressure_sigma_floor_m,
|
||||
"flow_sigma_percent": flow_sigma_percent,
|
||||
"clip_sigma": clip_sigma,
|
||||
},
|
||||
"raw_sha256": canonical_hash(raw_fingerprint_rows),
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in records]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_demo_scada_window(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
normal_source_start: datetime,
|
||||
target_start: datetime,
|
||||
day_count: int,
|
||||
event_start: datetime,
|
||||
event_run_id: str,
|
||||
noise_seed: str,
|
||||
interval_seconds: int = 900,
|
||||
) -> dict:
|
||||
"""Repeat one normal online day and replace one five-hour event window."""
|
||||
if any(value.tzinfo is None for value in (normal_source_start, target_start, event_start)):
|
||||
raise ValueError("normal, target, and event times must be timezone-aware")
|
||||
if day_count < 15 or interval_seconds <= 0 or 86400 % interval_seconds:
|
||||
raise ValueError("demo windows require at least 15 complete aligned days")
|
||||
target_end = target_start + timedelta(days=day_count)
|
||||
if not target_start <= event_start < target_end:
|
||||
raise ValueError("event must fall inside the demo window")
|
||||
run_uuid = UUID(str(event_run_id))
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow') ORDER BY device_id"""
|
||||
).fetchall()
|
||||
pressure_devices = [row for row in devices if row["device_type"] == "pressure"]
|
||||
flow_devices = [row for row in devices if row["device_type"] == "pipe_flow"]
|
||||
if any(not row["node_id"] or row["measurement_unit"] != "m" for row in pressure_devices):
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
if any(not row["link_id"] or row["measurement_unit"] != "m3/h" for row in flow_devices):
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
normal_end = normal_source_start + timedelta(days=1)
|
||||
expected_normal_times = [
|
||||
normal_source_start + timedelta(seconds=offset)
|
||||
for offset in range(0, 86400, interval_seconds)
|
||||
]
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
normal_nodes = timescale.execute(
|
||||
"""SELECT time,node_id,pressure AS value FROM realtime.node_results
|
||||
WHERE time>=%s AND time<%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(normal_source_start, normal_end, [row["node_id"] for row in pressure_devices]),
|
||||
).fetchall()
|
||||
normal_links = timescale.execute(
|
||||
"""SELECT time,link_id,flow AS value FROM realtime.link_results
|
||||
WHERE time>=%s AND time<%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(normal_source_start, normal_end, [row["link_id"] for row in flow_devices]),
|
||||
).fetchall()
|
||||
event_nodes = timescale.execute(
|
||||
"""SELECT time,node_id,pressure AS value FROM analysis.node_results
|
||||
WHERE run_id=%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(run_uuid, [row["node_id"] for row in pressure_devices]),
|
||||
).fetchall()
|
||||
event_links = timescale.execute(
|
||||
"""SELECT time,link_id,flow AS value FROM analysis.link_results
|
||||
WHERE run_id=%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(run_uuid, [row["link_id"] for row in flow_devices]),
|
||||
).fetchall()
|
||||
|
||||
normal_node_values = {(row["time"], row["node_id"]): float(row["value"]) for row in normal_nodes}
|
||||
normal_link_values = {(row["time"], row["link_id"]): float(row["value"]) for row in normal_links}
|
||||
for device in devices:
|
||||
values = normal_node_values if device["device_type"] == "pressure" else normal_link_values
|
||||
element = device["node_id"] if device["device_type"] == "pressure" else device["link_id"]
|
||||
if any((at, element) not in values for at in expected_normal_times):
|
||||
raise ValueError(f"normal source day is incomplete for device {device['device_id']}")
|
||||
|
||||
records: dict[tuple[datetime, str], tuple] = {}
|
||||
for day_offset in range(day_count):
|
||||
day_start = target_start + timedelta(days=day_offset)
|
||||
for sample_offset, source_at in enumerate(expected_normal_times):
|
||||
target_at = day_start + timedelta(seconds=sample_offset * interval_seconds)
|
||||
for device in pressure_devices:
|
||||
raw = normal_node_values[(source_at, device["node_id"])]
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw, device_type="pressure", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
for device in flow_devices:
|
||||
raw = normal_link_values[(source_at, device["link_id"])]
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw, device_type="pipe_flow", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
|
||||
if not event_nodes or not event_links:
|
||||
raise ValueError("event analysis has no bound pressure or flow results")
|
||||
source_event_start = min(row["time"] for row in event_nodes + event_links)
|
||||
event_times = sorted({row["time"] for row in event_nodes + event_links})
|
||||
if len(event_times) != 21 or any(
|
||||
right - left != timedelta(seconds=interval_seconds)
|
||||
for left, right in zip(event_times, event_times[1:])
|
||||
):
|
||||
raise ValueError("event analysis must contain 21 aligned fifteen-minute frames")
|
||||
event_node_values = {(row["time"], row["node_id"]): float(row["value"]) for row in event_nodes}
|
||||
event_link_values = {(row["time"], row["link_id"]): float(row["value"]) for row in event_links}
|
||||
for source_at in event_times:
|
||||
target_at = event_start + (source_at - source_event_start)
|
||||
if target_at >= target_end:
|
||||
raise ValueError("event results extend beyond the demo window")
|
||||
for device in pressure_devices:
|
||||
key = (source_at, device["node_id"])
|
||||
if key not in event_node_values:
|
||||
raise ValueError(f"event pressure is incomplete for device {device['device_id']}")
|
||||
value = noisy_scada_observation(
|
||||
raw_value=event_node_values[key], device_type="pressure", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
for device in flow_devices:
|
||||
key = (source_at, device["link_id"])
|
||||
if key not in event_link_values:
|
||||
raise ValueError(f"event flow is incomplete for device {device['device_id']}")
|
||||
value = noisy_scada_observation(
|
||||
raw_value=event_link_values[key], device_type="pipe_flow", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
|
||||
ordered = [records[key] for key in sorted(records)]
|
||||
expected_count = day_count * len(expected_normal_times) * len(devices)
|
||||
if len(ordered) != expected_count:
|
||||
raise ValueError("demo SCADA window has an unexpected observation count")
|
||||
timescale.execute(
|
||||
"DELETE FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(target_start, target_end),
|
||||
)
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in ordered:
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"""SELECT count(*) count,count(DISTINCT device_id) devices,
|
||||
count(DISTINCT time) frames,count(cleaned_value) cleaned
|
||||
FROM scada.measurements WHERE time>=%s AND time<%s""",
|
||||
(target_start, target_end),
|
||||
).fetchone()
|
||||
if persisted != {
|
||||
"count": expected_count,
|
||||
"devices": len(devices),
|
||||
"frames": day_count * len(expected_normal_times),
|
||||
"cleaned": 0,
|
||||
}:
|
||||
raise ValueError("demo SCADA read-back verification failed")
|
||||
return {
|
||||
"start": target_start.isoformat(),
|
||||
"end_exclusive": target_end.isoformat(),
|
||||
"day_count": day_count,
|
||||
"event_start": event_start.isoformat(),
|
||||
"event_end": (event_start + timedelta(seconds=20 * interval_seconds)).isoformat(),
|
||||
"event_run_id": str(run_uuid),
|
||||
"devices": len(devices),
|
||||
"frames": day_count * len(expected_normal_times),
|
||||
"rows": expected_count,
|
||||
"cleaned_rows": 0,
|
||||
"noise_seed": noise_seed,
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in ordered]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def create_empty_database(source_dsn: str, target: str, schema_template: str) -> None:
|
||||
if not target.startswith("tjw_bench_") or target.endswith("_template"):
|
||||
raise ValueError("benchmark targets must use the owned tjw_bench_ prefix")
|
||||
with connect(make_conninfo(source_dsn, dbname="postgres"), autocommit=True, row_factory=dict_row) as connection:
|
||||
if connection.execute("SELECT 1 FROM pg_database WHERE datname=%s", (target,)).fetchone():
|
||||
raise ValueError("target database already exists; explicit state is required for resumption")
|
||||
# Templates are intentionally connection-disabled; clone without altering that state.
|
||||
connection.execute(sql.SQL("CREATE DATABASE {} TEMPLATE {}").format(sql.Identifier(target), sql.Identifier(schema_template)))
|
||||
|
||||
|
||||
def copy_project_inputs(source_dsn: str, target_dsn: str) -> dict:
|
||||
from scripts.check_burst_simulation_design import readonly
|
||||
from app.native.wndb.core.model_replace import _copy_order, _copy_table, _model_tables, _table_columns
|
||||
|
||||
summaries = {}
|
||||
with readonly(source_dsn) as source, connect(target_dsn, row_factory=dict_row) as target:
|
||||
source.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
tables = _model_tables(source)
|
||||
ordered = _copy_order(source, tables)
|
||||
for schema, table in reversed(ordered):
|
||||
target.execute(sql.SQL("DELETE FROM {}").format(sql.Identifier(schema, table)))
|
||||
for schema, table in ordered + [("asset", "scada_devices")]:
|
||||
columns = _table_columns(source, schema, table)
|
||||
if _table_columns(target, schema, table) != columns:
|
||||
raise ValueError(f"schema differs for {schema}.{table}")
|
||||
_copy_table(source, target, schema, table, columns)
|
||||
statement = sql.SQL("SELECT count(*) count FROM {}").format(sql.Identifier(schema, table))
|
||||
before, after = source.execute(statement).fetchone()["count"], target.execute(statement).fetchone()["count"]
|
||||
if before != after:
|
||||
raise ValueError("isolated model copy count mismatch")
|
||||
summaries[f"{schema}.{table}"] = before
|
||||
target.commit()
|
||||
return summaries
|
||||
|
||||
|
||||
def apply_project_runtime_migrations(target_dsn: str, repository_root: Path) -> list[str]:
|
||||
"""Install the additive MCP and calculation queue tables in an owned target."""
|
||||
migrations = (
|
||||
"025_mcp_query_contexts.sql",
|
||||
"026_calculation_jobs.sql",
|
||||
"027_calculation_job_lifecycle.sql",
|
||||
"030_remove_calculation_queue_expiry.sql",
|
||||
)
|
||||
applied = []
|
||||
with connect(target_dsn) as connection:
|
||||
for filename in migrations:
|
||||
path = repository_root / "resources" / "sql" / filename
|
||||
connection.execute(path.read_text(encoding="utf-8"))
|
||||
applied.append(filename)
|
||||
connection.commit()
|
||||
return applied
|
||||
|
||||
|
||||
async def register_benchmark_project(
|
||||
*,
|
||||
source_project_id: str,
|
||||
target: str,
|
||||
target_biz_dsn: str,
|
||||
target_iot_dsn: str,
|
||||
):
|
||||
"""Register owned databases and mirror source membership without role expansion."""
|
||||
from uuid import UUID
|
||||
|
||||
from app.infra.db.metadb.database import SessionLocal
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
|
||||
source_id = UUID(str(source_project_id))
|
||||
async with SessionLocal() as session:
|
||||
repository = MetadataRepository(session)
|
||||
source = await repository.get_project_by_id(source_id)
|
||||
if source is None:
|
||||
raise ValueError("source project metadata is unavailable")
|
||||
project = await repository.get_project_by_code(target)
|
||||
marker = {
|
||||
"benchmark_source_project_id": str(source_id),
|
||||
"benchmark_dataset": "burst-v1",
|
||||
}
|
||||
if project is None:
|
||||
project = await repository.create_project(
|
||||
name=f"{source.name} 爆管 Benchmark",
|
||||
code=target,
|
||||
description="隔离的爆管 benchmark 模拟与 SCADA 回放数据集",
|
||||
gs_workspace=target,
|
||||
map_extent=source.map_extent,
|
||||
status="active",
|
||||
map_config=source.map_config or {},
|
||||
settings={**(source.settings or {}), **marker},
|
||||
)
|
||||
elif any((project.settings or {}).get(key) != value for key, value in marker.items()):
|
||||
raise ValueError("existing target project is not owned by this benchmark dataset")
|
||||
await repository.upsert_project_database_config(
|
||||
project.id,
|
||||
db_role="biz_data",
|
||||
db_type="postgresql",
|
||||
dsn=target_biz_dsn,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
)
|
||||
await repository.upsert_project_database_config(
|
||||
project.id,
|
||||
db_role="iot_data",
|
||||
db_type="timescaledb",
|
||||
dsn=target_iot_dsn,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
)
|
||||
existing = {member.user_id for member in await repository.list_project_members(project.id)}
|
||||
for member in await repository.list_project_members(source_id):
|
||||
if member.user_id not in existing:
|
||||
await repository.add_project_member(
|
||||
project.id, member.user_id, member.project_role
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def write_analysis_scada(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
run_id: str,
|
||||
) -> dict:
|
||||
"""Materialize one completed hydraulic run as bound SCADA observations."""
|
||||
run_uuid = UUID(str(run_id))
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow')
|
||||
ORDER BY device_id"""
|
||||
).fetchall()
|
||||
pressure = [row for row in devices if row["device_type"] == "pressure"]
|
||||
flows = [row for row in devices if row["device_type"] == "pipe_flow"]
|
||||
if any(not row["node_id"] or row["measurement_unit"] != "m" for row in pressure):
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
if any(not row["link_id"] or row["measurement_unit"] != "m3/h" for row in flows):
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
|
||||
pressure_by_node: dict[str, list[str]] = {}
|
||||
flow_by_link: dict[str, list[str]] = {}
|
||||
for row in pressure:
|
||||
pressure_by_node.setdefault(row["node_id"], []).append(row["device_id"])
|
||||
for row in flows:
|
||||
flow_by_link.setdefault(row["link_id"], []).append(row["device_id"])
|
||||
|
||||
records: list[tuple] = []
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
node_rows = timescale.execute(
|
||||
"""SELECT time,node_id,pressure FROM analysis.node_results
|
||||
WHERE run_id=%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(run_uuid, list(pressure_by_node)),
|
||||
).fetchall()
|
||||
link_rows = timescale.execute(
|
||||
"""SELECT time,link_id,flow FROM analysis.link_results
|
||||
WHERE run_id=%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(run_uuid, list(flow_by_link)),
|
||||
).fetchall()
|
||||
for row in node_rows:
|
||||
records.extend(
|
||||
(row["time"], device_id, float(row["pressure"]), None)
|
||||
for device_id in pressure_by_node[row["node_id"]]
|
||||
)
|
||||
for row in link_rows:
|
||||
records.extend(
|
||||
(row["time"], device_id, float(row["flow"]) * 3.6, None)
|
||||
for device_id in flow_by_link[row["link_id"]]
|
||||
)
|
||||
if not records:
|
||||
raise ValueError("analysis run has no bound SCADA results")
|
||||
start = min(row[0] for row in records)
|
||||
end = max(row[0] for row in records)
|
||||
collision = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time BETWEEN %s AND %s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if collision:
|
||||
raise ValueError("SCADA target interval is not empty")
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in sorted(records):
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time BETWEEN %s AND %s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if persisted != len(records):
|
||||
raise ValueError("SCADA read-back count does not match generated records")
|
||||
samples_per_device: dict[str, int] = {}
|
||||
for _, device_id, _, _ in records:
|
||||
samples_per_device[device_id] = samples_per_device.get(device_id, 0) + 1
|
||||
if len(set(samples_per_device.values())) != 1:
|
||||
raise ValueError("bound SCADA devices have inconsistent sample counts")
|
||||
return {
|
||||
"run_id": str(run_uuid),
|
||||
"start": start.isoformat(),
|
||||
"end": end.isoformat(),
|
||||
"devices": len(samples_per_device),
|
||||
"samples_per_device": next(iter(samples_per_device.values())),
|
||||
"rows": persisted,
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in sorted(records)]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def make_schedule(design) -> list[dict]:
|
||||
"""Seventeen-day slots keep fifteen clean training days plus a recovery buffer."""
|
||||
local = ZoneInfo("Asia/Shanghai")
|
||||
first_date = datetime(2024, 1, 16, tzinfo=local)
|
||||
rows = []
|
||||
pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
|
||||
for case in design.cases:
|
||||
for level in ("light", "medium", "severe"):
|
||||
source = case.occurred_at.astimezone(local)
|
||||
at = first_date + timedelta(days=17 * len(rows), hours=source.hour, minutes=source.minute)
|
||||
rows.append({"case_key": f"{case.case_id}-{level}", "case_id": case.case_id,
|
||||
"severity": level, "pipe_id": case.pipe_id,
|
||||
"area_cm2": pipes[case.pipe_id].candidate_orifice_areas_cm2[level],
|
||||
"event_at": at.isoformat(), "history_start": (at - timedelta(days=15)).isoformat(),
|
||||
"event_end": (at + timedelta(seconds=design.duration_seconds)).isoformat(),
|
||||
"reserved_end": (at + timedelta(seconds=design.duration_seconds, days=1)).isoformat()})
|
||||
previous = None
|
||||
for row in rows:
|
||||
start = datetime.fromisoformat(row["history_start"])
|
||||
if previous is not None and start <= previous:
|
||||
raise ValueError("dataset date reservations overlap")
|
||||
previous = datetime.fromisoformat(row["reserved_end"])
|
||||
if previous is not None and previous >= datetime.now(timezone.utc):
|
||||
raise ValueError("benchmark slots must remain historical")
|
||||
return rows
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Learn pressure thresholds from pre-event normal history, never burst truth."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime, timedelta
|
||||
from statistics import median
|
||||
|
||||
|
||||
def learn_pressure_thresholds(
|
||||
rows: list[dict], device_ids: list[str], cutoff: datetime, *,
|
||||
history_days: int = 15, interval_seconds: int = 900,
|
||||
relative_floor_percent: float = 3.0, noise_multiplier: float = 3.0,
|
||||
absolute_floor_m: float = 0.5,
|
||||
) -> dict:
|
||||
"""Estimate noise after removing each daily slot's median operating pattern."""
|
||||
if (cutoff.tzinfo is None or history_days < 1 or interval_seconds <= 0
|
||||
or 86400 % interval_seconds or not device_ids or len(set(device_ids)) != len(device_ids)):
|
||||
raise ValueError("an aware cutoff and whole-day sampling grid are required")
|
||||
if not all(math.isfinite(value) and value > 0 for value in (relative_floor_percent, noise_multiplier, absolute_floor_m)):
|
||||
raise ValueError("threshold floors and noise multiplier must be positive")
|
||||
count = history_days * 86400 // interval_seconds
|
||||
start = cutoff - timedelta(days=history_days)
|
||||
expected = {start + timedelta(seconds=interval_seconds * index) for index in range(count)}
|
||||
data = {device: {} for device in device_ids}
|
||||
for row in rows:
|
||||
device = row["device_id"]
|
||||
if device not in data:
|
||||
continue
|
||||
at = row["time"]
|
||||
if at.tzinfo is None or at >= cutoff:
|
||||
raise ValueError("adaptive fitting must not consume event or future observations")
|
||||
if at < start:
|
||||
continue
|
||||
value = row["monitored_value"]
|
||||
if at not in expected or at in data[device] or value is None or not math.isfinite(float(value)) or float(value) <= 0:
|
||||
raise ValueError("adaptive fitting requires distinct positive finite pressure samples on the grid")
|
||||
data[device][at] = float(value)
|
||||
insufficient = {device: len(values) for device, values in data.items() if set(values) != expected}
|
||||
if insufficient:
|
||||
return {"status": "history_not_ready", "required_samples_per_device": count,
|
||||
"available_samples": insufficient, "knowledge_cutoff": cutoff.isoformat(), "devices": {}}
|
||||
fitted = {}
|
||||
for device, values in data.items():
|
||||
slots = {}
|
||||
for at, value in values.items():
|
||||
slot = int((at - start).total_seconds()) // interval_seconds % (86400 // interval_seconds)
|
||||
slots.setdefault(slot, []).append(value)
|
||||
references = {slot: median(group) for slot, group in slots.items()}
|
||||
residuals = []
|
||||
for at, value in values.items():
|
||||
slot = int((at - start).total_seconds()) // interval_seconds % (86400 // interval_seconds)
|
||||
residuals.append(100 * (value / references[slot] - 1))
|
||||
center = median(residuals)
|
||||
robust_sigma = 1.4826 * median(abs(value - center) for value in residuals)
|
||||
fitted[device] = {"relative_drop_percent": max(relative_floor_percent, noise_multiplier * robust_sigma),
|
||||
"absolute_drop_m": absolute_floor_m, "robust_noise_percent": robust_sigma,
|
||||
"reference_pressure_m_by_slot": {str(slot): value for slot, value in references.items()}}
|
||||
return {"status": "ready", "required_samples_per_device": count,
|
||||
"knowledge_cutoff": cutoff.isoformat(), "devices": fitted,
|
||||
"reference_grid_start": start.isoformat(), "interval_seconds": interval_seconds,
|
||||
"method": "daily-slot median removal; 1.4826*MAD; thresholds frozen before event"}
|
||||
|
||||
|
||||
def evaluate_pressure_thresholds(rows: list[dict], fitted: dict, *, consecutive_samples: int = 2) -> dict:
|
||||
"""Experimental adaptive evaluator; this is not the production replay worker."""
|
||||
if fitted.get("status") != "ready" or consecutive_samples < 1:
|
||||
raise ValueError("a fitted rule and positive consecutive sample count are required")
|
||||
start = datetime.fromisoformat(fitted["reference_grid_start"])
|
||||
cutoff = datetime.fromisoformat(fitted["knowledge_cutoff"])
|
||||
step = fitted["interval_seconds"]
|
||||
states = {}
|
||||
signals = []
|
||||
for row in sorted(rows, key=lambda item: (item["time"], item["device_id"])):
|
||||
device = row["device_id"]
|
||||
if device not in fitted["devices"]:
|
||||
continue
|
||||
at, value = row["time"], row["monitored_value"]
|
||||
if at.tzinfo is None or at < cutoff or (at - start).total_seconds() % step:
|
||||
raise ValueError("evaluation samples must follow the fit cutoff on the sampling grid")
|
||||
previous, hits, signaled = states.get(device, (None, 0, False))
|
||||
if previous is not None and at <= previous:
|
||||
raise ValueError("evaluation samples must be distinct")
|
||||
if previous is None or at - previous != timedelta(seconds=step):
|
||||
hits = 0
|
||||
rule = fitted["devices"][device]
|
||||
slot = int((at - start).total_seconds()) // step % (86400 // step)
|
||||
expected = rule["reference_pressure_m_by_slot"][str(slot)]
|
||||
valid = value is not None and math.isfinite(float(value)) and float(value) > 0
|
||||
drop = expected - float(value) if valid else 0
|
||||
hit = valid and drop >= rule["absolute_drop_m"] and 100 * drop / expected >= rule["relative_drop_percent"]
|
||||
hits = hits + 1 if hit else 0
|
||||
if hits >= consecutive_samples and not signaled:
|
||||
signals.append({"device_id": device, "first_signal_at": at.isoformat(),
|
||||
"reference_pressure_m": expected, "absolute_drop_m": drop,
|
||||
"relative_drop_percent": 100 * drop / expected})
|
||||
signaled = True
|
||||
states[device] = (at, hits, signaled)
|
||||
return {"raised": bool(signals), "signals": signals,
|
||||
"scope": "experimental adaptive pressure rule only; not a worker or Agent result"}
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Read project data through metadata routing, in repeatable read-only transactions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from scripts.benchmark.contracts import (
|
||||
BurstCase, BurstSuite, canonical_json, complete_devices, fingerprint, isolation_reference,
|
||||
)
|
||||
|
||||
|
||||
async def resolve_sources(suite: BurstSuite) -> tuple[dict, dict[str, str]]:
|
||||
# Import lazily: validating a suite must not load environment credentials.
|
||||
from sqlalchemy import text
|
||||
from app.infra.db.metadb.database import SessionLocal, engine
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY"))
|
||||
repo = MetadataRepository(session)
|
||||
project = await repo.get_project_by_code(suite.project_code)
|
||||
if project is None or project.id != suite.project_id or project.status != "active":
|
||||
raise ValueError("benchmark project identity or active status does not match")
|
||||
routes = {}
|
||||
for role in ("biz_data", "iot_data"):
|
||||
route = await repo.get_project_db_routing(project.id, role)
|
||||
if route is None:
|
||||
raise ValueError("benchmark project database routing is incomplete")
|
||||
routes[role] = route.dsn
|
||||
identity = {"project_id": str(project.id), "project_code": project.code, "name": project.name}
|
||||
await session.rollback()
|
||||
return identity, routes
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def readonly_source(dsn: str):
|
||||
# Replace inherited options: never rely on application credentials being read-only.
|
||||
safe_dsn = make_conninfo(dsn, options=(
|
||||
"-c default_transaction_read_only=on -c statement_timeout=20000 "
|
||||
"-c lock_timeout=2000 -c idle_in_transaction_session_timeout=600000"
|
||||
), application_name="tjwater-benchmark-source-readonly", connect_timeout="5")
|
||||
with psycopg.connect(safe_dsn, row_factory=dict_row) as connection:
|
||||
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
if connection.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] != "on":
|
||||
raise ValueError("source transaction is not read-only")
|
||||
try:
|
||||
yield connection
|
||||
finally:
|
||||
connection.rollback()
|
||||
|
||||
|
||||
def source_identity(connection, dsn: str) -> dict:
|
||||
values = conninfo_to_dict(dsn)
|
||||
row = connection.execute(
|
||||
"SELECT current_database() AS database, current_setting('server_version_num') AS server_version, "
|
||||
"transaction_timestamp() AS observed_at"
|
||||
).fetchone()
|
||||
if row["database"] != values.get("dbname"):
|
||||
raise ValueError("connected source differs from the metadata route")
|
||||
return {
|
||||
**row,
|
||||
"read_only": True,
|
||||
"isolation": "repeatable read",
|
||||
# Equal database names on different servers are not equal routes.
|
||||
"route_sha256": fingerprint({key: values.get(key) for key in ("host", "port", "dbname")}),
|
||||
}
|
||||
|
||||
|
||||
def read_model(connection, cases: list[BurstCase]) -> dict:
|
||||
links = connection.execute(
|
||||
"SELECT l.id, l.link_type::text AS kind, l.start_node_id, l.end_node_id, "
|
||||
"COALESCE(s.status::text,p.status::text) AS status "
|
||||
"FROM network.links l LEFT JOIN network.pipes p ON p.link_id=l.id "
|
||||
"LEFT JOIN network.link_initial_settings s ON s.link_id=l.id ORDER BY l.id"
|
||||
).fetchall()
|
||||
for link in links:
|
||||
for key in ("id", "start_node_id", "end_node_id"):
|
||||
link[key] = str(link[key])
|
||||
sources = [str(row["id"]) for row in connection.execute(
|
||||
"SELECT id FROM network.nodes WHERE node_type::text IN ('reservoir','tank') ORDER BY id"
|
||||
).fetchall()]
|
||||
devices = connection.execute(
|
||||
"SELECT device_id,device_type,node_id,link_id,api_query_id,measurement_unit, "
|
||||
"transmission_frequency::text AS transmission_frequency FROM asset.scada_devices "
|
||||
"WHERE device_type::text IN ('pressure','pipe_flow','flow') ORDER BY device_id"
|
||||
).fetchall()
|
||||
for device in devices:
|
||||
for key in ("device_id", "node_id", "link_id"):
|
||||
if device[key] is not None:
|
||||
device[key] = str(device[key])
|
||||
analyses = connection.execute(
|
||||
"SELECT run_id,name,run_type,status,parameters FROM analysis.runs WHERE name=ANY(%s) "
|
||||
"ORDER BY name,created_at,run_id", ([case.reference_analysis for case in cases],),
|
||||
).fetchall()
|
||||
return {"links": links, "source_nodes": sources, "devices": devices, "reference_analyses": analyses}
|
||||
|
||||
|
||||
def readings(connection, device_ids: list[str], start: datetime, end: datetime) -> list[dict]:
|
||||
return connection.execute(
|
||||
"SELECT time,device_id,monitored_value FROM scada.measurements "
|
||||
"WHERE device_id=ANY(%s) AND time>=%s AND time<=%s ORDER BY time,device_id",
|
||||
(device_ids, start, end),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def long_history_capability(connection, suite: BurstSuite, case: BurstCase, devices: list[dict]) -> dict:
|
||||
interval = suite.sampling_interval_minutes
|
||||
expected = suite.detector_days * 1440 // interval
|
||||
start = case.occurred_at - timedelta(days=suite.detector_days) + timedelta(minutes=interval)
|
||||
mapped = {device["device_id"]: device["node_id"] for device in devices
|
||||
if device["device_type"] == "pressure" and device["node_id"] and device["api_query_id"]}
|
||||
rows = connection.execute(
|
||||
"SELECT device_id,count(*) AS count,count(DISTINCT time) AS unique_count, "
|
||||
"count(*) FILTER (WHERE monitored_value IS NOT NULL "
|
||||
"AND monitored_value::text NOT IN ('NaN','Infinity','-Infinity') "
|
||||
"AND mod(extract(epoch FROM (time-%s::timestamptz)),%s)=0) AS valid_count "
|
||||
"FROM scada.measurements WHERE device_id=ANY(%s) AND time>=%s AND time<=%s "
|
||||
"GROUP BY device_id ORDER BY device_id",
|
||||
(case.occurred_at, interval * 60, list(mapped), start, case.occurred_at),
|
||||
).fetchall()
|
||||
eligible = sorted({mapped[row["device_id"]] for row in rows
|
||||
if row["count"] == row["unique_count"] == row["valid_count"] == expected})
|
||||
return {"available": len(eligible) >= 5, "expected_samples_per_device": expected,
|
||||
"complete_pressure_nodes": eligible, "tool_node_limit": 20,
|
||||
"required_for": "optional run_burst_detection, not the short-window SCADA trigger"}
|
||||
|
||||
|
||||
def trigger_preview(suite: BurstSuite, case: BurstCase, rows: list[dict], devices: list[dict], *, normal=False) -> dict:
|
||||
"""Exercise the current pure detector; do not tick replay, clean TSDB or create tasks."""
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
||||
|
||||
start = case.occurred_at - (timedelta(days=1) if normal else timedelta())
|
||||
pressure = [device for device in devices
|
||||
if device["device_type"] == "pressure" and device["measurement_unit"] == "m"]
|
||||
ids = {device["device_id"] for device in pressure}
|
||||
rule = suite.trigger_rule
|
||||
if len(ids) < rule.minimum_devices:
|
||||
return {"raised": False, "reason": "insufficient pressure devices with unit m"}
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, f"benchmark:{case.case_id}:{normal}"),
|
||||
start_time=start, end_time=start + timedelta(minutes=suite.event_minutes + suite.sampling_interval_minutes),
|
||||
device_ids=sorted(ids), rules=[{
|
||||
"rule_id": "benchmark-pressure-drop", "device_ids": sorted(ids),
|
||||
"kind": "relative_change", "direction": "decrease", "unit": "m",
|
||||
"threshold": rule.pressure_drop_percent, "window_samples": rule.baseline_samples,
|
||||
"minimum_samples": rule.baseline_samples, "consecutive_samples": rule.consecutive_samples,
|
||||
"minimum_devices": rule.minimum_devices,
|
||||
}],
|
||||
)
|
||||
history: dict[str, list[float]] = {device: [] for device in ids}
|
||||
for row in rows:
|
||||
value = row["monitored_value"]
|
||||
if row["device_id"] in ids and row["time"] < start and value is not None and math.isfinite(float(value)):
|
||||
history[row["device_id"]].append(float(value))
|
||||
document = {"replay_id": str(config.request_id), "devices": pressure,
|
||||
"histories": {key: values[-rule.baseline_samples:] for key, values in history.items()}}
|
||||
changes = []
|
||||
for at, group in itertools.groupby(rows, key=lambda row: row["time"]):
|
||||
if at < start:
|
||||
continue
|
||||
frame = [{**row, "source_hash": fingerprint([row["time"], row["device_id"], row["monitored_value"]])}
|
||||
for row in group if row["device_id"] in ids]
|
||||
document, events = detect_frame(document, prepare_cleaning(frame, config), config, at)
|
||||
changes.extend(event for event in events if event.get("category") != "data_quality")
|
||||
raised = [event for event in changes if event["change"] == "raised"]
|
||||
return {"raised": bool(raised), "first_signal_at": raised[0]["occurred_at"] if raised else None,
|
||||
"event_count": len(raised), "warm_pressure_devices": sum(len(v) >= rule.baseline_samples for v in history.values()),
|
||||
"scope": "pure short-window rule preview; not worker/Agent end-to-end acceptance"}
|
||||
|
||||
|
||||
def inspect_case(iot_connection, suite: BurstSuite, case: BurstCase, model: dict) -> dict:
|
||||
devices = model["devices"]
|
||||
ids = [device["device_id"] for device in devices]
|
||||
warmup = timedelta(minutes=suite.trigger_rule.baseline_samples * suite.sampling_interval_minutes)
|
||||
end = case.occurred_at + timedelta(minutes=suite.event_minutes)
|
||||
event_rows = readings(iot_connection, ids, case.occurred_at - warmup, end)
|
||||
normal_rows = readings(iot_connection, ids, case.occurred_at - timedelta(days=1) - warmup,
|
||||
end - timedelta(days=1))
|
||||
complete = set(complete_devices(event_rows, case.occurred_at, end, suite.sampling_interval_minutes))
|
||||
complete &= set(complete_devices(normal_rows, case.occurred_at - timedelta(days=1),
|
||||
end - timedelta(days=1), suite.sampling_interval_minutes))
|
||||
pressure_ids = {device["device_id"] for device in devices if device["device_type"] == "pressure"}
|
||||
matching = [run for run in model["reference_analyses"] if run["name"] == case.reference_analysis
|
||||
and run["run_type"] == "burst_analysis" and run["status"] == "completed"
|
||||
and case.pipe_id in [str(value) for value in run["parameters"].get("burst_ID", [])]]
|
||||
topology = isolation_reference(model["links"], model["source_nodes"], case.pipe_id)
|
||||
blockers = []
|
||||
if case.observation_provenance != "declared_historical_replay":
|
||||
blockers.append("event_observations_require_isolated_generation")
|
||||
if not matching:
|
||||
blockers.append("matching_completed_reference_analysis_missing")
|
||||
if len(complete & pressure_ids) < 5:
|
||||
blockers.append("fewer_than_five_complete_event_and_control_pressure_devices")
|
||||
if topology["status"] in {"target_missing", "sources_missing"}:
|
||||
blockers.append(topology["status"])
|
||||
event_signal = trigger_preview(suite, case, event_rows, devices)
|
||||
normal_signal = trigger_preview(suite, case, normal_rows, devices, normal=True)
|
||||
scada_blockers = list(blockers)
|
||||
if not event_signal["raised"]:
|
||||
scada_blockers.append("short_window_event_trigger_not_raised")
|
||||
if normal_signal["raised"]:
|
||||
scada_blockers.append("normal_control_false_positive")
|
||||
return {
|
||||
"case_id": case.case_id, "pipe_id": case.pipe_id, "occurred_at": case.occurred_at,
|
||||
"observation_provenance": case.observation_provenance,
|
||||
"data_candidate_available": not blockers,
|
||||
"blockers": blockers,
|
||||
"complete_event_and_control_pressure_devices": sorted(complete & pressure_ids),
|
||||
"window_sha256": fingerprint({"event": event_rows, "normal": normal_rows}),
|
||||
"optional_long_history_detection": long_history_capability(iot_connection, suite, case, devices),
|
||||
"event_signal_preview": event_signal, "normal_signal_preview": normal_signal,
|
||||
"topology_reference": topology,
|
||||
"variants": [
|
||||
{"case_id": case.case_id + "-WO", "data_candidate_available": not blockers,
|
||||
"blockers": list(blockers)},
|
||||
{"case_id": case.case_id + "-SCADA", "data_candidate_available": not scada_blockers,
|
||||
"blockers": scada_blockers},
|
||||
],
|
||||
"acceptance_pending": ["numerical_reference_review", "approved_equivalent_candidates",
|
||||
"public_leakage_input_or_declared_location_limitation",
|
||||
"isolated_runtime_and_cutoff_verification"],
|
||||
}
|
||||
|
||||
|
||||
def export_readings(connection, path: Path, suite: BurstSuite, cases: list[BurstCase], devices: list[dict]) -> dict:
|
||||
"""Export only burst-related devices and the selected history/event window."""
|
||||
start = min(case.occurred_at for case in cases) - timedelta(days=suite.detector_days)
|
||||
end = max(case.occurred_at for case in cases) + timedelta(minutes=suite.event_minutes)
|
||||
ids = [device["device_id"] for device in devices]
|
||||
count = 0
|
||||
digest = hashlib.sha256()
|
||||
path.touch(mode=0o600, exist_ok=False)
|
||||
with path.open("wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as output:
|
||||
with connection.cursor(name="benchmark_raw_export") as cursor:
|
||||
cursor.execute(
|
||||
"SELECT time,device_id,monitored_value FROM scada.measurements "
|
||||
"WHERE device_id=ANY(%s) AND time>=%s AND time<=%s ORDER BY time,device_id", (ids, start, end),
|
||||
)
|
||||
for row in cursor:
|
||||
count += 1
|
||||
if count > 1_000_000:
|
||||
raise ValueError("selected observation export exceeds one million records")
|
||||
line = (canonical_json(row) + "\n").encode()
|
||||
digest.update(line)
|
||||
output.write(line)
|
||||
return {"file": str(path.name), "records": count, "content_sha256": digest.hexdigest(),
|
||||
"start": start, "end": end, "origin": "existing_database_readonly_export",
|
||||
"visibility": "private; contains future samples and must not be exposed directly to Agent"}
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Calibrate a read-only model export using file-isolated native EPANET runs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--design", type=Path, default=Path("resources/benchmarks/tjwater2-v2-burst-simulation.json"))
|
||||
parser.add_argument("--output-root", type=Path, default=Path("data/benchmark-calibration"))
|
||||
parser.add_argument("--duration-seconds", type=int, help="Override calibration duration, e.g. 18000 for five hours")
|
||||
parser.add_argument("--timeout-seconds", type=int, default=240)
|
||||
parser.add_argument("--all-severities", action="store_true", help="Retain weak-signal sizes even if the severe level fails")
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
report = None
|
||||
report_path = None
|
||||
try:
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from app.infra.db.project_routing import ActiveProjectRouting, activate_project_routing
|
||||
from app.native.wndb.core.connection import close_all_project_pools, project_transaction
|
||||
from app.native.wndb.inp.exporter import dump_inp
|
||||
from scripts.check_burst_simulation_design import readonly, resolve_project, validate_database, validate_tool_inputs
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
from scripts.benchmark.calibration import assess, configure_inp, digest, pressure_preview, run_file, save_json, select_calibrated_area
|
||||
|
||||
design = load_simulation_design(args.design)
|
||||
validate_tool_inputs(design)
|
||||
duration = args.duration_seconds if args.duration_seconds is not None else design.duration_seconds
|
||||
step = design.model_expectations.report_timestep_seconds
|
||||
if not 3600 <= duration <= 18000 or duration % step or not 1 <= args.timeout_seconds <= 1800:
|
||||
raise ValueError("duration must be whole report steps between one and five hours; timeout 1..1800")
|
||||
biz, _ = asyncio.run(resolve_project(design))
|
||||
with readonly(biz) as connection:
|
||||
validate_database(connection, design, template=False)
|
||||
folder = (args.output_root / (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid4().hex[:8])).resolve()
|
||||
folder.mkdir(parents=True, mode=0o700)
|
||||
snapshot = folder / "source.inp"
|
||||
readonly_dsn = make_conninfo(biz, options="-c default_transaction_read_only=on -c statement_timeout=20000")
|
||||
routing = ActiveProjectRouting(project_code=design.project_code, business_dsn=readonly_dsn)
|
||||
try:
|
||||
with activate_project_routing(routing), project_transaction(design.project_code) as connection:
|
||||
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
# Stateless models permit event-aligned runs and independent warmup history.
|
||||
counts = connection.execute("SELECT (SELECT count(*) FROM network.tanks) tanks, "
|
||||
"(SELECT count(*) FROM network.pumps) pumps, "
|
||||
"(SELECT count(*) FROM network.controls) controls, "
|
||||
"(SELECT count(*) FROM network.rules) rules").fetchone()
|
||||
if any(counts.values()):
|
||||
raise ValueError("stateful model requires an explicit warm-start implementation")
|
||||
pda_options = {row["key"]: row["value"] for row in connection.execute(
|
||||
"SELECT key,value FROM network.simulation_settings WHERE engine_version='legacy' "
|
||||
"AND key=ANY(%s)",
|
||||
(["HEADERROR", "FLOWCHANGE", "MINIMUM PRESSURE", "PRESSURE EXPONENT"],),
|
||||
)}
|
||||
dump_inp(design.project_code, str(snapshot), "2")
|
||||
finally:
|
||||
close_all_project_pools()
|
||||
source = snapshot.read_text(encoding="utf-8")
|
||||
node_ids = sorted({node for pipe in design.pipes for node in pipe.output_node_ids})
|
||||
link_ids = sorted({link for pipe in design.pipes for link in pipe.output_link_ids})
|
||||
report = {"version": "burst-calibration@1", "status": "running", "created_at": datetime.now(timezone.utc),
|
||||
"source_effect": "read-only model export; local file simulations only",
|
||||
"model_sha256": digest(snapshot), "design_sha256": digest(args.design),
|
||||
"design": design.model_dump(mode="json"), "duration_seconds": duration,
|
||||
"pda_options": pda_options,
|
||||
"assessment_scope": "strong-signal calibration, not Agent accuracy or absence of a physical burst",
|
||||
"all_severities": args.all_severities,
|
||||
"engine_sha256": digest(Path("app/infra/epanet/linux/runepanet")),
|
||||
"cases": [], "selection": {}, "controls": {},
|
||||
"limitations": ["No analysis scenario or SCADA rows have been persisted",
|
||||
"No worker, MCP, Agent, or 15-day detection has been executed",
|
||||
"The four-sample pressure replay rule is a calibration preview"]}
|
||||
report_path = folder / "calibration.json"
|
||||
save_json(report_path, report)
|
||||
|
||||
def simulate(label, start, seconds, emitter=None):
|
||||
path = folder / f"{label}.inp"
|
||||
path.write_text(configure_inp(source, start, seconds, emitter=emitter, pda_options=pda_options), encoding="utf-8")
|
||||
result = run_file(path, node_ids, link_ids, timeout=args.timeout_seconds)
|
||||
save_json(folder / f"{label}.json", result)
|
||||
print(f"{label}: {result['status']}, solver warnings={len(result['warnings'])}", flush=True)
|
||||
return result
|
||||
|
||||
controls = {}
|
||||
histories = {}
|
||||
for index, at in enumerate(design.event_times):
|
||||
controls[at] = simulate(f"normal-{index}", at, duration)
|
||||
warmup = simulate(f"warmup-{index}", at - timedelta(seconds=4 * step), 3 * step)
|
||||
if warmup.get("status") != "completed" or warmup.get("warnings") or warmup.get("times") != {
|
||||
"report_start": 0, "report_step": step, "sim_duration": 3 * step, "num_periods": 4
|
||||
}:
|
||||
raise ValueError("normal warmup did not produce four converged samples")
|
||||
histories[at] = warmup["nodes"]
|
||||
report["controls"][at.isoformat()] = {"normal_file": f"normal-{index}.json", "warmup_file": f"warmup-{index}.json"}
|
||||
save_json(report_path, report)
|
||||
for pipe in design.pipes:
|
||||
cases = [case for case in design.cases if case.pipe_id == pipe.pipe_id]
|
||||
passed_levels = []
|
||||
for severity in design.calibration_order:
|
||||
level_passed = True
|
||||
for case in cases:
|
||||
area = pipe.candidate_orifice_areas_cm2[severity]
|
||||
burst = simulate(f"{case.case_id}-{severity}", case.occurred_at, duration, (pipe.emitter_node_id, area))
|
||||
normal = controls[case.occurred_at]
|
||||
result = assess(pipe, normal, burst, design.selection_rule, duration=duration, step=step)
|
||||
if normal.get("status") == burst.get("status") == "completed" and not any(
|
||||
reason.endswith("incomplete_or_failed") for reason in result["blockers"]
|
||||
):
|
||||
for key, current in (("normal_trigger", normal), ("event_trigger", burst)):
|
||||
result[key] = pressure_preview(pipe, case.occurred_at, duration, step,
|
||||
histories[case.occurred_at], current["nodes"], design.selection_rule)
|
||||
if result["normal_trigger"]["raised"]:
|
||||
result["blockers"].append("normal_control_false_positive")
|
||||
if not result["event_trigger"]["raised"]:
|
||||
result["blockers"].append("event_rule_did_not_trigger")
|
||||
coefficient = 0.65 * area / 10000 * (19.6 ** 0.5) * 1000
|
||||
# PDA demand can change with pressure: demand delta alone is not emitter flow.
|
||||
result["emitter_flow_m3_s"] = [coefficient * max(row["pressure"], 0) ** 0.5 / 1000
|
||||
for row in burst["nodes"][pipe.emitter_node_id]]
|
||||
result.update(case_id=case.case_id, severity=severity, orifice_area_cm2=area,
|
||||
passed=not result["blockers"])
|
||||
report["cases"].append(result)
|
||||
level_passed &= result["passed"]
|
||||
save_json(report_path, report)
|
||||
print(f"{case.case_id}-{severity}: physical and rule checks {'PASS' if result['passed'] else 'FAIL'}", flush=True)
|
||||
if level_passed:
|
||||
passed_levels.append(severity)
|
||||
if severity == "severe" and not level_passed and not args.all_severities:
|
||||
break
|
||||
case_ids = {case.case_id for case in cases}
|
||||
report["selection"][pipe.pipe_id] = select_calibrated_area(
|
||||
pipe, passed_levels, [case for case in report["cases"] if case["case_id"] in case_ids], args.all_severities,
|
||||
)
|
||||
save_json(report_path, report)
|
||||
report["status"] = "completed"
|
||||
save_json(report_path, report)
|
||||
print(f"calibration complete; report: {report_path}", flush=True)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
frames = traceback.extract_tb(exc.__traceback__)
|
||||
location = frames[-1] if frames else None
|
||||
suffix = f" at {location.filename}:{location.lineno}" if location else ""
|
||||
if report is not None and report_path is not None:
|
||||
report.update(status="failed", failure_type=type(exc).__name__)
|
||||
save_json(report_path, report)
|
||||
print(f"calibration failed ({type(exc).__name__}){suffix}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate persisted isolated burst scenarios and mapped SCADA observations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--design", type=Path, default=Path("resources/benchmarks/tjwater2-v2-burst-balanced.json"))
|
||||
parser.add_argument("--target", default="tjw_bench_tjwater2_20261105")
|
||||
parser.add_argument("--state-dir", type=Path, default=Path("data/benchmark-generation/tjwater2-20261105"))
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--prepare-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
from psycopg import connect
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
from app.core.config import settings
|
||||
from scripts.benchmark.calibration import save_json, digest
|
||||
from scripts.benchmark.generation import (
|
||||
apply_project_runtime_migrations,
|
||||
create_empty_database,
|
||||
copy_project_inputs,
|
||||
make_schedule,
|
||||
register_benchmark_project,
|
||||
)
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
from scripts.check_burst_simulation_design import resolve_project, readonly, validate_database
|
||||
|
||||
design = load_simulation_design(args.design)
|
||||
schedule = make_schedule(design)
|
||||
if not args.apply:
|
||||
print(f"dry run: {len(schedule)} cases, target={args.target}, dates={schedule[0]['history_start']}..{schedule[-1]['reserved_end']}")
|
||||
return 0
|
||||
folder = args.state_dir.resolve()
|
||||
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
state_path = folder / "state.json"
|
||||
import json
|
||||
state = json.loads(state_path.read_text()) if state_path.exists() else {
|
||||
"version": "burst-dataset-generation@1", "created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"target": args.target, "design_sha256": digest(args.design), "schedule": schedule,
|
||||
"stages": {}, "cases": {}, "source_effect": "read-only", "status": "preparing"}
|
||||
if state["target"] != args.target or state["design_sha256"] != digest(args.design):
|
||||
raise ValueError("state belongs to a different target or design")
|
||||
save_json(state_path, state)
|
||||
biz, iot = asyncio.run(resolve_project(design))
|
||||
with readonly(biz) as connection:
|
||||
validate_database(connection, design, template=False)
|
||||
target_biz, target_iot = make_conninfo(biz, dbname=args.target), make_conninfo(iot, dbname=args.target)
|
||||
for role, source, template in (("biz", biz, settings.WNDB_SCHEMA_TEMPLATE_DB_NAME),
|
||||
("iot", iot, settings.TIMESCALEDB_SCHEMA_TEMPLATE_DB_NAME)):
|
||||
if not state["stages"].get(f"{role}_created"):
|
||||
create_empty_database(source, args.target, template)
|
||||
state["stages"][f"{role}_created"] = True
|
||||
save_json(state_path, state)
|
||||
print(f"created isolated {role} database: {args.target}", flush=True)
|
||||
if not state["stages"].get("model_copied"):
|
||||
state["model_counts"] = copy_project_inputs(biz, target_biz)
|
||||
state["stages"]["model_copied"] = True
|
||||
save_json(state_path, state)
|
||||
print("copied model and device bindings from a read-only source snapshot", flush=True)
|
||||
required_migrations = {
|
||||
"025_mcp_query_contexts.sql",
|
||||
"026_calculation_jobs.sql",
|
||||
"027_calculation_job_lifecycle.sql",
|
||||
"030_remove_calculation_queue_expiry.sql",
|
||||
}
|
||||
if not required_migrations.issubset(state.get("runtime_migrations", [])):
|
||||
state["runtime_migrations"] = apply_project_runtime_migrations(
|
||||
target_biz, Path(__file__).resolve().parents[1]
|
||||
)
|
||||
state["stages"]["runtime_migrations_applied"] = True
|
||||
save_json(state_path, state)
|
||||
print("installed isolated MCP and calculation queue storage", flush=True)
|
||||
if not state["stages"].get("model_template_created"):
|
||||
from app.native.wndb.core.project_templates import (
|
||||
create_project_model_template,
|
||||
ensure_replication_worker_capacity,
|
||||
)
|
||||
from app.native.wndb.core.projects import have_project
|
||||
|
||||
template = f"{args.target}_template"
|
||||
if not have_project(template):
|
||||
try:
|
||||
ensure_replication_worker_capacity()
|
||||
except RuntimeError as exc:
|
||||
if "no reserved logical-replication worker capacity" not in str(exc):
|
||||
raise
|
||||
state["model_template_mode"] = "snapshot_per_run"
|
||||
else:
|
||||
create_project_model_template(args.target)
|
||||
if have_project(template):
|
||||
state["model_template"] = template
|
||||
state["model_template_mode"] = "logical_replication"
|
||||
state["stages"]["model_template_created"] = True
|
||||
save_json(state_path, state)
|
||||
print(f"configured hydraulic model isolation: {state['model_template_mode']}", flush=True)
|
||||
if not state["stages"].get("metadata_registered"):
|
||||
project = asyncio.run(register_benchmark_project(
|
||||
source_project_id=str(design.project_id),
|
||||
target=args.target,
|
||||
target_biz_dsn=target_biz,
|
||||
target_iot_dsn=target_iot,
|
||||
))
|
||||
state["project_id"] = str(project.id)
|
||||
state["stages"]["metadata_registered"] = True
|
||||
save_json(state_path, state)
|
||||
print(f"registered isolated benchmark project: {project.id}", flush=True)
|
||||
# Inspect isolated schema readiness before invoking backend persistence.
|
||||
with connect(target_biz, row_factory=dict_row) as connection:
|
||||
print("isolated BizDB analysis schema", connection.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_schema='analysis' AND table_name='runs' ORDER BY ordinal_position").fetchall(), flush=True)
|
||||
state["status"] = "prepared"
|
||||
save_json(state_path, state)
|
||||
print(f"isolated preparation complete: {state_path}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Materialize selected burst demo months from one normal day and event runs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--manifest", type=Path,
|
||||
default=Path("resources/benchmarks/tjwater2-v2-burst-demo.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--generation-state", type=Path,
|
||||
default=Path("data/benchmark-generation/tjwater2-20261105/state.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state", type=Path,
|
||||
default=Path("data/benchmark-generation/tjwater2-20261105/demo-scada-state.json"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from scripts.benchmark.calibration import digest, save_json
|
||||
from scripts.benchmark.generation import write_demo_scada_window
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
from scripts.check_burst_simulation_design import resolve_project
|
||||
|
||||
manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
|
||||
generation = json.loads(args.generation_state.read_text(encoding="utf-8"))
|
||||
if manifest["project_code"] != generation["target"] or generation.get("status") != "prepared":
|
||||
raise ValueError("manifest and isolated generation state do not match")
|
||||
design = load_simulation_design(Path("resources/benchmarks/tjwater2-v2-burst-balanced.json"))
|
||||
source_biz, source_iot = asyncio.run(resolve_project(design))
|
||||
business_dsn = make_conninfo(source_biz, dbname=generation["target"])
|
||||
timescale_dsn = make_conninfo(source_iot, dbname=generation["target"])
|
||||
state_path = args.state.resolve()
|
||||
state_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
manifest_sha256 = digest(args.manifest)
|
||||
state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.exists() else {
|
||||
"version": "burst-demo-scada-state@1",
|
||||
"target": generation["target"],
|
||||
"manifest_sha256": manifest_sha256,
|
||||
"cases": {},
|
||||
"status": "materializing",
|
||||
}
|
||||
if state["target"] != generation["target"] or state["manifest_sha256"] != manifest_sha256:
|
||||
raise ValueError("demo state belongs to different inputs")
|
||||
save_json(state_path, state)
|
||||
normal_source_start = datetime.fromisoformat(manifest["normal_source_start"])
|
||||
for case in manifest["cases"]:
|
||||
source_case = generation["cases"].get(case["case_key"])
|
||||
if not source_case or source_case.get("status") != "scada_materialized":
|
||||
raise ValueError(f"event source is unavailable: {case['case_key']}")
|
||||
summary = write_demo_scada_window(
|
||||
business_dsn=business_dsn,
|
||||
timescale_dsn=timescale_dsn,
|
||||
normal_source_start=normal_source_start,
|
||||
target_start=datetime.fromisoformat(case["target_start"]),
|
||||
day_count=int(manifest["day_count"]),
|
||||
event_start=datetime.fromisoformat(case["event_start"]),
|
||||
event_run_id=source_case["analysis_run_id"],
|
||||
noise_seed=manifest["noise_seed"],
|
||||
)
|
||||
state["cases"][case["demo_id"]] = {
|
||||
"case_key": case["case_key"],
|
||||
"entry_mode": case["entry_mode"],
|
||||
**summary,
|
||||
}
|
||||
save_json(state_path, state)
|
||||
print({"demo_id": case["demo_id"], "rows": summary["rows"]}, flush=True)
|
||||
state["status"] = "completed"
|
||||
save_json(state_path, state)
|
||||
print(json.dumps({"status": state["status"], "cases": state["cases"]}, ensure_ascii=False), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a private, read-only preflight report for the tjwater_v2 burst suite."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--suite",
|
||||
type=Path,
|
||||
default=Path("resources/benchmarks/tjwater-v2-burst.json"),
|
||||
help="versioned burst suite manifest",
|
||||
)
|
||||
parser.add_argument("--case", action="append", default=[], help="case ID to inspect; repeatable")
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path("data/benchmark/preflight"),
|
||||
help="private local artifact directory (default: ignored data/benchmark/preflight)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from scripts.benchmark.contracts import fingerprint, load_suite
|
||||
from scripts.benchmark.source import inspect_case, read_model, resolve_sources, source_identity, readonly_source
|
||||
|
||||
try:
|
||||
suite = load_suite(args.suite)
|
||||
cases = suite.select(args.case)
|
||||
identity, routes = asyncio.run(resolve_sources(suite))
|
||||
|
||||
with readonly_source(routes["biz_data"]) as biz_connection:
|
||||
biz_source = source_identity(biz_connection, routes["biz_data"])
|
||||
model = read_model(biz_connection, cases)
|
||||
with readonly_source(routes["iot_data"]) as iot_connection:
|
||||
iot_source = source_identity(iot_connection, routes["iot_data"])
|
||||
case_results = [inspect_case(iot_connection, suite, case, model) for case in cases]
|
||||
|
||||
content_fingerprint = fingerprint({
|
||||
"suite": suite.model_dump(mode="json"),
|
||||
"source_routes": {
|
||||
"biz_data": biz_source["route_sha256"],
|
||||
"iot_data": iot_source["route_sha256"],
|
||||
},
|
||||
"model": model,
|
||||
"cases": [{
|
||||
"case_id": result["case_id"],
|
||||
"window_sha256": result["window_sha256"],
|
||||
"topology_reference": result["topology_reference"],
|
||||
"data_candidate_available": result["data_candidate_available"],
|
||||
"blockers": result["blockers"],
|
||||
} for result in case_results],
|
||||
})
|
||||
report = {
|
||||
"report_version": "burst-preflight@1",
|
||||
"status": "preflight_complete",
|
||||
"readiness": "preflight_only; isolated runtime and Agent execution are not verified",
|
||||
"created_at": datetime.now(timezone.utc),
|
||||
"project": identity,
|
||||
"sources": {"biz_data": biz_source, "iot_data": iot_source},
|
||||
"suite_id": suite.suite_id,
|
||||
"suite_version": suite.version,
|
||||
"content_fingerprint_sha256": content_fingerprint,
|
||||
"source_effect": "read-only transactions; no source database writes requested",
|
||||
"cases": case_results,
|
||||
"limitations": [
|
||||
"preflight does not execute Agent, work-order, SCADA replay worker, or MCP tools",
|
||||
"SCADA event data requiring generation remains blocked until produced in an isolated dataset",
|
||||
"this report is not a frozen evaluation database snapshot",
|
||||
"numeric reference outputs and approved equivalent candidate groups still require review",
|
||||
],
|
||||
}
|
||||
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
output_dir = args.output_root / f"{suite.suite_id}-{stamp}-{uuid4().hex[:8]}"
|
||||
output_dir.mkdir(parents=True, mode=0o700)
|
||||
os.chmod(output_dir, 0o700)
|
||||
output_file = output_dir / "preflight.json"
|
||||
output_file.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, default=str) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(output_file, 0o600)
|
||||
except Exception as exc:
|
||||
# Driver exceptions can include connection details; never echo them into CI logs.
|
||||
frames = traceback.extract_tb(exc.__traceback__)
|
||||
location = frames[-1] if frames else None
|
||||
suffix = f" at {location.filename}:{location.lineno}" if location else ""
|
||||
print(f"benchmark preflight failed ({type(exc).__name__}){suffix}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
work_order_candidates = sum(bool(case["variants"][0]["data_candidate_available"])
|
||||
for case in case_results)
|
||||
scada_candidates = sum(bool(case["variants"][1]["data_candidate_available"])
|
||||
for case in case_results)
|
||||
print(
|
||||
f"preflight complete: {work_order_candidates}/{len(case_results)} WO and "
|
||||
f"{scada_candidates}/{len(case_results)} SCADA data candidates; report: {output_file}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a private multi-region, multi-area and multi-time design from screening evidence."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def require_isolatable_pipe(topology: list[str], pipe_id: str, sources: list[str], closed: list[str]) -> dict:
|
||||
from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis
|
||||
|
||||
result = valve_isolation_analysis(topology, pipe_id, accident_element_type="link",
|
||||
water_source_nodes=sources, closed_links=closed)
|
||||
if not result["isolatable"] or not result["source_check_performed"] or not result["must_close_valves"]:
|
||||
raise ValueError("selected region cannot be isolated from water sources")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--screening", type=Path, required=True)
|
||||
parser.add_argument("--additional-screening", type=Path, action="append", default=[])
|
||||
parser.add_argument("--pipe", action="append", required=True, help="New pipe ID; the existing primary is retained")
|
||||
parser.add_argument("--allow-weak", action="store_true", help="Include weak-signal physical bursts without treating them as no-burst controls")
|
||||
parser.add_argument("--case-start", type=int, default=9)
|
||||
parser.add_argument("--output-root", type=Path, default=Path("data/benchmark-regional"))
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
from scripts.benchmark.calibration import save_json, digest
|
||||
from scripts.benchmark.simulation_design import BurstSimulationDesign, load_simulation_design
|
||||
from scripts.check_burst_simulation_design import readonly, resolve_project, validate_database, validate_tool_inputs
|
||||
|
||||
original = load_simulation_design(Path("resources/benchmarks/tjwater2-v2-burst-simulation.json"))
|
||||
screening = json.loads(args.screening.read_text())
|
||||
candidates = {row["pipe_id"]: row for row in screening["results"]}
|
||||
for path in args.additional_screening:
|
||||
extra = json.loads(path.read_text())
|
||||
if extra["model_sha256"] != screening["model_sha256"]:
|
||||
raise ValueError("screening model fingerprints do not match")
|
||||
candidates.update({row["pipe_id"]: row for row in extra["results"]})
|
||||
requested = list(dict.fromkeys(args.pipe))
|
||||
if not 1 <= len(requested) <= 3 or any(pipe not in candidates for pipe in requested):
|
||||
raise ValueError("select one to three screened pipes")
|
||||
for pipe in requested:
|
||||
if not candidates[pipe].get("possible_candidate") and not args.allow_weak:
|
||||
raise ValueError("selected pipe has not passed single-frame screening")
|
||||
if candidates[pipe].get("status") != "completed":
|
||||
raise ValueError("selected pipe screening simulation did not complete")
|
||||
folder = (args.output_root / (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid4().hex[:8])).resolve()
|
||||
folder.mkdir(parents=True, mode=0o700)
|
||||
biz, _ = asyncio.run(resolve_project(original))
|
||||
with readonly(biz) as connection:
|
||||
links = connection.execute(
|
||||
"SELECT l.id,l.start_node_id,l.end_node_id,l.link_type,"
|
||||
"COALESCE(s.status::text,p.status::text,'OPEN') status FROM network.links l "
|
||||
"LEFT JOIN network.pipes p ON p.link_id=l.id "
|
||||
"LEFT JOIN network.link_initial_settings s ON s.link_id=l.id"
|
||||
).fetchall()
|
||||
sources = [row["id"] for row in connection.execute(
|
||||
"SELECT id FROM network.nodes WHERE node_type IN ('reservoir','tank')"
|
||||
)]
|
||||
devices = connection.execute(
|
||||
"SELECT device_id,device_type,node_id,link_id,measurement_unit FROM asset.scada_devices "
|
||||
"WHERE device_type IN ('pressure','pipe_flow') AND device_id<>'547063'"
|
||||
).fetchall()
|
||||
positions = {row["id"]: row for row in connection.execute(
|
||||
"SELECT l.id,l.start_node_id,l.end_node_id,p.diameter,p.length,"
|
||||
"ST_X(ST_Transform(g.geom,4326)) longitude,ST_Y(ST_Transform(g.geom,4326)) latitude "
|
||||
"FROM network.links l JOIN network.pipes p ON p.link_id=l.id "
|
||||
"JOIN gis.node_geometries g ON g.node_id=l.end_node_id WHERE l.id=ANY(%s)",
|
||||
([original.pipes[0].pipe_id, *requested],),
|
||||
)}
|
||||
pipe_geometries = {row["id"]: json.loads(row["geometry"]) for row in connection.execute(
|
||||
"SELECT id,ST_AsGeoJSON(ST_Transform(geom,4326)) geometry FROM gis.pipes WHERE id=ANY(%s)",
|
||||
([original.pipes[0].pipe_id, *requested],),
|
||||
)}
|
||||
topology = [f"{row['id']}:{row['link_type']}:{row['start_node_id']}:{row['end_node_id']}" for row in links]
|
||||
closed = [row["id"] for row in links if str(row["status"]).upper() == "CLOSED"]
|
||||
by_device = {row["device_id"]: row for row in devices}
|
||||
pipes = [original.pipes[0].model_dump(mode="json")]
|
||||
for pipe_id in requested:
|
||||
candidate = candidates[pipe_id]
|
||||
position = positions[pipe_id]
|
||||
pressure_ids = [key for key in candidate["pressure_drop_percent"]
|
||||
if key in by_device and by_device[key]["device_type"] == "pressure"][:5]
|
||||
# Keep the plant outlets plus the two most responsive other flow meters.
|
||||
flow_ids = ["542017", "486033", "542121"]
|
||||
for key, _ in sorted(candidate["flow_change_percent"].items(), key=lambda pair: pair[1], reverse=True):
|
||||
if key in by_device and key not in flow_ids:
|
||||
flow_ids.append(key)
|
||||
if len(flow_ids) >= 5:
|
||||
break
|
||||
pressure = [{"device_id": key, "node_id": by_device[key]["node_id"]} for key in pressure_ids]
|
||||
flow = [{"device_id": key, "link_id": by_device[key]["link_id"]} for key in flow_ids]
|
||||
area = math.pi * (position["diameter"] / 10) ** 2 / 4
|
||||
pipes.append({
|
||||
"role": "secondary", "pipe_id": pipe_id, "diameter_mm": position["diameter"],
|
||||
"length_m": position["length"], "start_node_id": position["start_node_id"],
|
||||
"emitter_node_id": position["end_node_id"],
|
||||
"longitude": position["longitude"], "latitude": position["latitude"],
|
||||
"candidate_orifice_areas_cm2": {"light": round(area / 32, 6), "medium": round(area / 16, 6), "severe": round(area / 8, 6)},
|
||||
"pressure_bindings": pressure, "flow_bindings": flow,
|
||||
"boundary_valve_ids": [],
|
||||
"output_node_ids": sorted({position["start_node_id"], position["end_node_id"], *[item["node_id"] for item in pressure]}),
|
||||
"output_link_ids": [],
|
||||
})
|
||||
isolations = {}
|
||||
for pipe in pipes:
|
||||
pipe_id = pipe["pipe_id"]
|
||||
result = require_isolatable_pipe(topology, pipe_id, sources, closed)
|
||||
pipe["boundary_valve_ids"] = result["must_close_valves"]
|
||||
pipe["output_link_ids"] = sorted({pipe_id, *result["must_close_valves"],
|
||||
*[item["link_id"] for item in pipe["flow_bindings"]]})
|
||||
pipe.update(longitude=positions[pipe_id]["longitude"], latitude=positions[pipe_id]["latitude"])
|
||||
isolations[pipe_id] = result
|
||||
payload = original.model_dump(mode="json")
|
||||
times = [f"2026-10-08T{hour:02d}:00:00+08:00" for hour in (2, 8, 12, 20)]
|
||||
payload.update(pipes=pipes, duration_seconds=18000, event_times=times)
|
||||
payload["selection_rule"]["expected_sample_count"] = 21
|
||||
if not 1 <= args.case_start <= 84:
|
||||
raise ValueError("case start must leave room for sixteen distinct Bxx cases")
|
||||
payload["cases"] = [{"case_id": f"B{args.case_start + p * len(times) + t:02d}", "pipe_id": pipe["pipe_id"], "occurred_at": at}
|
||||
for p, pipe in enumerate(pipes) for t, at in enumerate(times)]
|
||||
design = BurstSimulationDesign.model_validate(payload)
|
||||
validate_tool_inputs(design)
|
||||
with readonly(biz) as connection:
|
||||
validate_database(connection, design, template=False)
|
||||
path = folder / "design.json"
|
||||
save_json(path, design.model_dump(mode="json"))
|
||||
save_json(folder / "selection.json", {"source_effect": "read-only", "screening_sha256": digest(args.screening),
|
||||
"additional_screening_sha256": [digest(path) for path in args.additional_screening],
|
||||
"weak_signal_cases_included": args.allow_weak,
|
||||
"assessment_note": "a missed pressure rule does not change physical burst ground truth",
|
||||
"model_sha256": screening["model_sha256"], "design_sha256": digest(path),
|
||||
"isolations": isolations,
|
||||
"coordinates": "WGS84 burst endpoint, not pipe midpoint"})
|
||||
valve_ids = sorted({valve for pipe in pipes for valve in pipe["boundary_valve_ids"]})
|
||||
with readonly(biz) as connection:
|
||||
valve_geometries = {row["id"]: json.loads(row["geometry"]) for row in connection.execute(
|
||||
"SELECT l.id,ST_AsGeoJSON(ST_Transform(ST_LineInterpolatePoint(ST_MakeLine(a.geom,b.geom),0.5),4326)) geometry "
|
||||
"FROM network.links l JOIN gis.node_geometries a ON a.node_id=l.start_node_id "
|
||||
"JOIN gis.node_geometries b ON b.node_id=l.end_node_id WHERE l.id=ANY(%s)",
|
||||
(valve_ids,),
|
||||
)}
|
||||
features = []
|
||||
for index, pipe in enumerate(pipes, 1):
|
||||
properties = {"region": f"R{index}", "pipe_id": pipe["pipe_id"], "diameter_mm": pipe["diameter_mm"]}
|
||||
features.append({"type": "Feature", "properties": {**properties, "kind": "burst"},
|
||||
"geometry": {"type": "Point", "coordinates": [pipe["longitude"], pipe["latitude"]]}})
|
||||
features.append({"type": "Feature", "properties": {**properties, "kind": "pipe"},
|
||||
"geometry": pipe_geometries[pipe["pipe_id"]]})
|
||||
for valve in pipe["boundary_valve_ids"]:
|
||||
features.append({"type": "Feature", "properties": {**properties, "kind": "valve", "valve_id": valve},
|
||||
"geometry": valve_geometries[valve]})
|
||||
save_json(folder / "locations.geojson", {"type": "FeatureCollection", "features": features})
|
||||
print(f"regional design ready: {len(pipes)} regions x 3 areas x {len(times)} times = {len(pipes) * 3 * len(times)} physical cases; {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Submit, execute, and materialize isolated burst benchmark cases through MCP."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--design", type=Path, default=Path("resources/benchmarks/tjwater2-v2-burst-balanced.json"))
|
||||
parser.add_argument("--state-dir", type=Path, default=Path("data/benchmark-generation/tjwater2-20261105"))
|
||||
parser.add_argument("--agent-env", type=Path, default=Path("../next-tjwater-agent/.local.env"))
|
||||
parser.add_argument("--limit", type=int, default=1, help="Maximum new cases to complete in this invocation")
|
||||
parser.add_argument("--batches", type=int, default=1, help="Run isolated child processes for this many batches")
|
||||
parser.add_argument("--local-executor-concurrency", type=int, default=1, choices=(1, 2))
|
||||
args = parser.parse_args()
|
||||
if args.limit < 1 or args.limit > args.local_executor_concurrency:
|
||||
parser.error("--limit must be between 1 and local executor concurrency")
|
||||
if args.batches < 1:
|
||||
parser.error("--batches must be positive")
|
||||
if args.batches > 1:
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve()),
|
||||
"--design", str(args.design),
|
||||
"--state-dir", str(args.state_dir),
|
||||
"--agent-env", str(args.agent_env),
|
||||
"--limit", str(args.limit),
|
||||
"--local-executor-concurrency", str(args.local_executor_concurrency),
|
||||
"--batches", "1",
|
||||
]
|
||||
for _ in range(args.batches):
|
||||
subprocess.run(command, check=True)
|
||||
return 0
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from scripts.benchmark.calibration import save_json
|
||||
from scripts.benchmark.generation import write_analysis_scada
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
from scripts.check_burst_simulation_design import resolve_project
|
||||
|
||||
load_dotenv(Path(__file__).resolve().parents[1] / ".env", override=False)
|
||||
state_path = args.state_dir.resolve() / "state.json"
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
project_id = state.get("project_id")
|
||||
if not project_id or state.get("status") != "prepared":
|
||||
raise ValueError("run isolated preparation before MCP generation")
|
||||
design = load_simulation_design(args.design)
|
||||
pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
|
||||
source_biz, source_iot = asyncio.run(resolve_project(design))
|
||||
business_dsn = make_conninfo(source_biz, dbname=state["target"])
|
||||
timescale_dsn = make_conninfo(source_iot, dbname=state["target"])
|
||||
|
||||
pending = [
|
||||
row for row in state["schedule"]
|
||||
if state["cases"].get(row["case_key"], {}).get("status") != "scada_materialized"
|
||||
][:args.limit]
|
||||
for offset in range(0, len(pending), args.local_executor_concurrency):
|
||||
batch = pending[offset:offset + args.local_executor_concurrency]
|
||||
job_ids = submit_batch(
|
||||
batch=batch,
|
||||
pipes=pipes,
|
||||
project_id=project_id,
|
||||
agent_env=args.agent_env,
|
||||
state=state,
|
||||
state_path=state_path,
|
||||
)
|
||||
asyncio.run(execute_batch(project_id, job_ids, args.local_executor_concurrency))
|
||||
for row in batch:
|
||||
case = state["cases"][row["case_key"]]
|
||||
run = find_analysis_run(business_dsn, case["scenario_name"])
|
||||
summary = write_analysis_scada(
|
||||
business_dsn=business_dsn,
|
||||
timescale_dsn=timescale_dsn,
|
||||
run_id=str(run["run_id"]),
|
||||
)
|
||||
case.update({
|
||||
"analysis_run_id": str(run["run_id"]),
|
||||
"status": "scada_materialized",
|
||||
"scada": summary,
|
||||
})
|
||||
save_json(state_path, state)
|
||||
print({"case_key": row["case_key"], "job_id": case["job_id"], "scada_rows": summary["rows"]}, flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
def exchange_token(agent_env: Path) -> str:
|
||||
import httpx
|
||||
from dotenv import dotenv_values
|
||||
from app.mcp.config import McpSettings
|
||||
from scripts.run_operational_worker import CredentialProvider, WorkerSettings
|
||||
|
||||
values = dotenv_values(agent_env)
|
||||
mcp = McpSettings()
|
||||
|
||||
async def exchange() -> str:
|
||||
issuer = (values.get("TJWATER_MCP_DEV_HTTP_ISSUER_URI") or values["TJWATER_MCP_ISSUER_URI"]).rstrip("/")
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
product = await CredentialProvider(WorkerSettings.from_environment()).headers(client)
|
||||
response = await client.post(issuer + "/protocol/openid-connect/token", data={
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:token-exchange",
|
||||
"client_id": values["TJWATER_MCP_CLIENT_ID"],
|
||||
"client_secret": values["TJWATER_MCP_CLIENT_SECRET"],
|
||||
"subject_token": product["Authorization"].removeprefix("Bearer "),
|
||||
"subject_token_type": "urn:ietf:params:oauth:token-type:access_token",
|
||||
"requested_token_type": "urn:ietf:params:oauth:token-type:access_token",
|
||||
"audience": mcp.audience,
|
||||
"resource": mcp.resource_uri,
|
||||
"scope": "mcp.catalog.read mcp.query mcp.context.bind",
|
||||
})
|
||||
response.raise_for_status()
|
||||
return response.json()["access_token"]
|
||||
|
||||
return asyncio.run(exchange())
|
||||
|
||||
|
||||
def mcp_call(client, token: str, project_id: str, name: str, arguments: dict) -> dict:
|
||||
from mcp import types
|
||||
|
||||
method = "tools/call"
|
||||
call_id = str(uuid4())
|
||||
meta = {
|
||||
types.PROTOCOL_VERSION_META_KEY: "2026-07-28",
|
||||
types.CLIENT_CAPABILITIES_META_KEY: {},
|
||||
types.CLIENT_INFO_META_KEY: {"name": "benchmark-generator", "version": "1"},
|
||||
"io.tjwater/callId": call_id,
|
||||
}
|
||||
response = client.post("/mcp", headers={
|
||||
"Authorization": "Bearer " + token,
|
||||
"X-Project-Id": project_id,
|
||||
"MCP-Protocol-Version": "2026-07-28",
|
||||
"MCP-Method": method,
|
||||
"MCP-Name": name,
|
||||
"Accept": "application/json, text/event-stream",
|
||||
}, json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": call_id,
|
||||
"method": method,
|
||||
"params": {"_meta": meta, "name": name, "arguments": arguments},
|
||||
})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
result = payload.get("result") or {}
|
||||
if payload.get("error") or result.get("isError"):
|
||||
raise RuntimeError("MCP tool call failed")
|
||||
return result["structuredContent"]["data"]
|
||||
|
||||
|
||||
def submit_batch(*, batch, pipes, project_id, agent_env, state, state_path) -> list[str]:
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import app
|
||||
from scripts.benchmark.calibration import save_json
|
||||
|
||||
token = exchange_token(agent_env)
|
||||
session_id = "benchmark-tjwater2-burst-v1"
|
||||
jobs = []
|
||||
with TestClient(app, base_url="http://127.0.0.1:8000") as client:
|
||||
common = {"Authorization": "Bearer " + token, "X-Project-Id": project_id}
|
||||
for row in batch:
|
||||
case = state["cases"].setdefault(row["case_key"], {})
|
||||
request_id = case.setdefault("request_id", str(uuid4()))
|
||||
scenario_name = case.setdefault(
|
||||
"scenario_name",
|
||||
f"bench-{row['case_key']}-{row['event_at'][0:16].replace('-', '').replace(':', '').replace('T', '')}",
|
||||
)
|
||||
binding = client.post(
|
||||
"/api/v1/mcp-query-contexts",
|
||||
headers=common,
|
||||
json={"native_session_id": session_id, "purpose": "ordinary_chat"},
|
||||
)
|
||||
binding.raise_for_status()
|
||||
context_handle = binding.json()["context_handle"]
|
||||
pipe = pipes[row["pipe_id"]]
|
||||
parameters = {
|
||||
"start_time": row["event_at"],
|
||||
"duration": 18000,
|
||||
"output_node_ids": pipe.output_node_ids,
|
||||
"output_link_ids": pipe.output_link_ids,
|
||||
"burst_pipe_ids": [row["pipe_id"]],
|
||||
"orifice_areas_cm2": [row["area_cm2"]],
|
||||
}
|
||||
preview = mcp_call(client, token, project_id, "save_simulation_scenario", {
|
||||
"context_handle": context_handle,
|
||||
"timeout_seconds": 240,
|
||||
"request_id": request_id,
|
||||
"dry_run": True,
|
||||
"name": scenario_name,
|
||||
"scenario": "burst",
|
||||
"parameters": parameters,
|
||||
})
|
||||
case["expected_digest"] = preview["expected_digest"]
|
||||
save_json(state_path, state)
|
||||
queued = mcp_call(client, token, project_id, "save_simulation_scenario", {
|
||||
"context_handle": context_handle,
|
||||
"timeout_seconds": 240,
|
||||
"request_id": request_id,
|
||||
"dry_run": False,
|
||||
"expected_digest": preview["expected_digest"],
|
||||
"name": scenario_name,
|
||||
"scenario": "burst",
|
||||
"parameters": parameters,
|
||||
})
|
||||
case.update({"job_id": queued["job_id"], "status": queued["status"]})
|
||||
jobs.append(queued["job_id"])
|
||||
save_json(state_path, state)
|
||||
return jobs
|
||||
|
||||
|
||||
async def execute_batch(project_id: str, expected_job_ids: list[str], concurrency: int) -> None:
|
||||
from app.services.calculation_worker import CalculationWorker, WorkerSettings, queue_repository
|
||||
|
||||
worker = CalculationWorker(WorkerSettings(
|
||||
max_concurrency=concurrency,
|
||||
poll_seconds=1,
|
||||
lease_seconds=30,
|
||||
heartbeat_seconds=5,
|
||||
))
|
||||
rows = []
|
||||
for expected in expected_job_ids:
|
||||
async with queue_repository(project_id) as repository:
|
||||
await repository.recover()
|
||||
row = await repository.claim(worker.owner, worker.settings.lease_seconds)
|
||||
if row is None or str(row["job_id"]) != expected:
|
||||
raise RuntimeError("isolated benchmark queue order changed unexpectedly")
|
||||
rows.append(row)
|
||||
await asyncio.gather(*(worker.run_job(row) for row in rows))
|
||||
for row in rows:
|
||||
async with queue_repository(project_id) as repository:
|
||||
stored = await repository.get(row["job_id"])
|
||||
if stored["status"] != "succeeded":
|
||||
raise RuntimeError(f"benchmark calculation failed with {stored['status']}")
|
||||
|
||||
|
||||
def find_analysis_run(business_dsn: str, scenario_name: str) -> dict:
|
||||
from psycopg import connect
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
with connect(business_dsn, row_factory=dict_row) as connection:
|
||||
rows = connection.execute(
|
||||
"""SELECT run_id,name,status,started_at FROM analysis.runs
|
||||
WHERE name=%s ORDER BY created_at DESC LIMIT 2""",
|
||||
(scenario_name,),
|
||||
).fetchall()
|
||||
if len(rows) != 1 or rows[0]["status"] != "completed":
|
||||
raise ValueError("persisted scenario run is missing, duplicated, or incomplete")
|
||||
return dict(rows[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run one isolated online-model day through MCP and materialize noisy SCADA."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--start", default="2026-05-01T00:00:00+08:00")
|
||||
parser.add_argument(
|
||||
"--generation-state",
|
||||
type=Path,
|
||||
default=Path("data/benchmark-generation/tjwater2-20261105/state.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state",
|
||||
type=Path,
|
||||
default=Path("data/benchmark-generation/tjwater2-20261105/normal-day-state.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--design",
|
||||
type=Path,
|
||||
default=Path("resources/benchmarks/tjwater2-v2-burst-balanced.json"),
|
||||
)
|
||||
parser.add_argument("--agent-env", type=Path, default=Path("../next-tjwater-agent/.local.env"))
|
||||
parser.add_argument("--noise-seed", default="tjwater2-normal-scada-v1")
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
os.umask(0o077)
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from fastapi.testclient import TestClient
|
||||
from psycopg import connect
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.main import app
|
||||
from scripts.benchmark.calibration import save_json
|
||||
from scripts.benchmark.generation import write_realtime_scada_day
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
from scripts.check_burst_simulation_design import resolve_project
|
||||
from scripts.run_burst_benchmark_mcp import exchange_token, execute_batch, mcp_call
|
||||
|
||||
load_dotenv(Path(__file__).resolve().parents[1] / ".env", override=False)
|
||||
start = datetime.fromisoformat(args.start)
|
||||
if start.tzinfo is None or start.minute % 15 or start.second or start.microsecond:
|
||||
raise ValueError("start must be timezone-aware and aligned to 15 minutes")
|
||||
generation = json.loads(args.generation_state.read_text(encoding="utf-8"))
|
||||
if generation.get("status") != "prepared" or not generation.get("project_id"):
|
||||
raise ValueError("isolated benchmark project is not prepared")
|
||||
state_path = args.state.resolve()
|
||||
state_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.exists() else {
|
||||
"version": "normal-scada-day@1",
|
||||
"target": generation["target"],
|
||||
"project_id": generation["project_id"],
|
||||
"start": start.isoformat(),
|
||||
"duration_seconds": 86400,
|
||||
"noise_seed": args.noise_seed,
|
||||
"status": "preparing",
|
||||
}
|
||||
expected = {
|
||||
"target": generation["target"],
|
||||
"project_id": generation["project_id"],
|
||||
"start": start.isoformat(),
|
||||
"noise_seed": args.noise_seed,
|
||||
}
|
||||
if any(state.get(key) != value for key, value in expected.items()):
|
||||
raise ValueError("normal-day state belongs to different inputs")
|
||||
save_json(state_path, state)
|
||||
|
||||
design = load_simulation_design(args.design)
|
||||
source_biz, source_iot = asyncio.run(resolve_project(design))
|
||||
business_dsn = make_conninfo(source_biz, dbname=state["target"])
|
||||
timescale_dsn = make_conninfo(source_iot, dbname=state["target"])
|
||||
with connect(business_dsn, row_factory=dict_row) as connection:
|
||||
node = connection.execute(
|
||||
"""SELECT node_id FROM asset.scada_devices
|
||||
WHERE device_type='pressure' AND node_id IS NOT NULL ORDER BY device_id LIMIT 1"""
|
||||
).fetchone()["node_id"]
|
||||
link = connection.execute(
|
||||
"""SELECT link_id FROM asset.scada_devices
|
||||
WHERE device_type='pipe_flow' AND link_id IS NOT NULL ORDER BY device_id LIMIT 1"""
|
||||
).fetchone()["link_id"]
|
||||
|
||||
if not state.get("job_id"):
|
||||
token = exchange_token(args.agent_env)
|
||||
with TestClient(app, base_url="http://127.0.0.1:8000") as client:
|
||||
headers = {
|
||||
"Authorization": "Bearer " + token,
|
||||
"X-Project-Id": state["project_id"],
|
||||
}
|
||||
binding = client.post(
|
||||
"/api/v1/mcp-query-contexts",
|
||||
headers=headers,
|
||||
json={"native_session_id": "benchmark-normal-scada-v1", "purpose": "ordinary_chat"},
|
||||
)
|
||||
binding.raise_for_status()
|
||||
queued = mcp_call(client, token, state["project_id"], "run_simulation", {
|
||||
"context_handle": binding.json()["context_handle"],
|
||||
"timeout_seconds": 240,
|
||||
"start_time": start.isoformat(),
|
||||
"duration": 86400,
|
||||
"output_node_ids": [node],
|
||||
"output_link_ids": [link],
|
||||
})
|
||||
state.update({"job_id": queued["job_id"], "status": queued["status"]})
|
||||
save_json(state_path, state)
|
||||
print({"job_id": state["job_id"], "status": state["status"]}, flush=True)
|
||||
|
||||
async def stored_job(include_result: bool = False):
|
||||
from app.services.calculation_worker import queue_repository
|
||||
|
||||
async with queue_repository(state["project_id"]) as repository:
|
||||
return await repository.get(state["job_id"], include_result=include_result)
|
||||
|
||||
stored = asyncio.run(stored_job())
|
||||
if stored["status"] == "queued":
|
||||
asyncio.run(execute_batch(state["project_id"], [state["job_id"]], 1))
|
||||
stored = asyncio.run(stored_job())
|
||||
if stored["status"] != "succeeded":
|
||||
raise RuntimeError(f"normal online simulation ended with {stored['status']}")
|
||||
result = asyncio.run(stored_job(include_result=True))["result"]
|
||||
calculation_data = (result or {}).get("data") or {}
|
||||
scenario_result = calculation_data.get("result") or calculation_data
|
||||
units = scenario_result.get("units") or {}
|
||||
if str(units.get("pressure", "")).upper() not in {"MTR", "M", "METERS"}:
|
||||
raise ValueError(f"unexpected model pressure unit: {units.get('pressure')}")
|
||||
if str(units.get("flow", "")).upper() != "LPS":
|
||||
raise ValueError(f"unexpected model flow unit: {units.get('flow')}")
|
||||
if not state.get("scada"):
|
||||
state["status"] = "simulated"
|
||||
save_json(state_path, state)
|
||||
state["scada"] = write_realtime_scada_day(
|
||||
business_dsn=business_dsn,
|
||||
timescale_dsn=timescale_dsn,
|
||||
start=start,
|
||||
noise_seed=args.noise_seed,
|
||||
)
|
||||
state["model_units"] = units
|
||||
state["status"] = "scada_materialized"
|
||||
save_json(state_path, state)
|
||||
print(json.dumps({"status": state["status"], "job_id": state["job_id"], "scada": state["scada"]}, ensure_ascii=False), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -146,6 +146,14 @@ class OperationalWorker:
|
||||
json={"limit": 100, **scope},
|
||||
)
|
||||
recovery.raise_for_status()
|
||||
try:
|
||||
recovery_checks = await self.client.post(
|
||||
f"{server}/api/v1/operational-tasks/recovery/reconcile", headers=headers,
|
||||
json={"limit": 20, **scope},
|
||||
)
|
||||
recovery_checks.raise_for_status()
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning("SCADA recovery reconciliation unavailable: %s", type(exc).__name__)
|
||||
active_run_ids: set[str] = set()
|
||||
for status_value in ("claimed", "running"):
|
||||
response = await self.client.get(
|
||||
|
||||
+44
-8
@@ -1,22 +1,58 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import uvicorn
|
||||
|
||||
# 将项目根目录添加到 python 路径
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
|
||||
def build_server_options(argv: list[str] | None = None) -> dict[str, Any]:
|
||||
parser = argparse.ArgumentParser(description="Run the TJWater API server")
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument("--reload", action="store_true")
|
||||
parser.add_argument("--reload-dir", action="append", default=[])
|
||||
parser.add_argument("--startup-timeout", type=int, default=60)
|
||||
args = parser.parse_args(argv)
|
||||
if args.startup_timeout <= 0:
|
||||
parser.error("--startup-timeout must be positive")
|
||||
return {
|
||||
"app": "app.main:app",
|
||||
"host": args.host,
|
||||
"port": args.port,
|
||||
"reload": args.reload,
|
||||
"reload_dirs": args.reload_dir,
|
||||
"startup_timeout": args.startup_timeout,
|
||||
"loop": "asyncio",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from pathlib import Path
|
||||
from scripts.server_handoff import HandoffSupervisor
|
||||
|
||||
options = build_server_options()
|
||||
reload_enabled = options.pop("reload")
|
||||
reload_dirs = options.pop("reload_dirs")
|
||||
startup_timeout = options.pop("startup_timeout")
|
||||
if reload_enabled:
|
||||
HandoffSupervisor(
|
||||
uvicorn.Config(**options),
|
||||
startup_timeout=startup_timeout,
|
||||
reload_dirs=[Path(directory) for directory in reload_dirs] or None,
|
||||
readiness_check="app.api.health:metadata_database_ready",
|
||||
).run()
|
||||
else:
|
||||
uvicorn.run(**options)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Windows 设置事件循环策略
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
# 用 uvicorn.run 支持 workers 参数
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
# workers=2, # 这里可以设置多进程
|
||||
loop="asyncio",
|
||||
)
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Screen alternative burst pipes near existing pressure meters using a frozen INP."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import heapq
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import os
|
||||
from collections import defaultdict, deque
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def nearest_pressure_distances(
|
||||
graph: dict[str, list[tuple[str, float]]], devices: list[dict], radius_m: float,
|
||||
) -> dict[str, tuple[float, str]]:
|
||||
"""Measure to the nearest pressure meter over the network, not just a selected meter."""
|
||||
distances = {}
|
||||
pending = []
|
||||
for device in devices:
|
||||
node, device_id = device["node_id"], device["device_id"]
|
||||
value = (0.0, device_id)
|
||||
if value < distances.get(node, (math.inf, "")):
|
||||
distances[node] = value
|
||||
heapq.heappush(pending, (0.0, device_id, node))
|
||||
while pending:
|
||||
distance, device_id, node = heapq.heappop(pending)
|
||||
if distances[node] != (distance, device_id):
|
||||
continue
|
||||
for neighbor, length in graph.get(node, []):
|
||||
cost = distance + length
|
||||
value = (cost, device_id)
|
||||
if cost <= radius_m and value < distances.get(neighbor, (math.inf, "")):
|
||||
distances[neighbor] = value
|
||||
heapq.heappush(pending, (cost, device_id, neighbor))
|
||||
return distances
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--calibration-dir", required=True, type=Path)
|
||||
parser.add_argument("--limit", type=int, default=24)
|
||||
parser.add_argument("--offset", type=int, default=0)
|
||||
parser.add_argument("--radius-m", type=float, default=400)
|
||||
parser.add_argument("--min-network-distance-m", type=float, default=0)
|
||||
parser.add_argument("--min-diameter", type=float, default=150)
|
||||
parser.add_argument("--require-isolation", action="store_true")
|
||||
parser.add_argument("--exclude-existing-km", type=float, default=0)
|
||||
parser.add_argument("--skip-screening", type=Path)
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from scripts.benchmark.calibration import configure_inp, digest, run_file, save_json
|
||||
from scripts.benchmark.simulation_design import BurstSimulationDesign
|
||||
from scripts.check_burst_simulation_design import readonly, resolve_project
|
||||
|
||||
if (not 1 <= args.limit <= 100 or args.offset < 0 or not 100 <= args.radius_m <= 5000
|
||||
or not 0 <= args.min_network_distance_m < args.radius_m
|
||||
or not 50 <= args.min_diameter <= 800 or not 0 <= args.exclude_existing_km <= 10):
|
||||
parser.error("invalid screen limit, offset, radius, diameter or exclusion distance")
|
||||
folder = args.calibration_dir.resolve()
|
||||
calibration = json.loads((folder / "calibration.json").read_text())
|
||||
design = BurstSimulationDesign.model_validate(calibration["design"])
|
||||
snapshot = folder / "source.inp"
|
||||
if digest(snapshot) != calibration["model_sha256"]:
|
||||
raise ValueError("model snapshot fingerprint changed")
|
||||
biz, _ = asyncio.run(resolve_project(design))
|
||||
with readonly(biz) as connection:
|
||||
devices = connection.execute(
|
||||
"SELECT device_id,device_type,node_id,link_id FROM asset.scada_devices "
|
||||
"WHERE device_type IN ('pressure','pipe_flow') ORDER BY device_id"
|
||||
).fetchall()
|
||||
pressure = [device for device in devices if device["device_type"] == "pressure"]
|
||||
flow = [device for device in devices if device["device_type"] == "pipe_flow" and device["device_id"] != "547063"]
|
||||
source = snapshot.read_text()
|
||||
graph = defaultdict(list)
|
||||
pipes = {}
|
||||
valve_links = {}
|
||||
source_nodes = set()
|
||||
closed_links = set()
|
||||
junctions = set()
|
||||
existing_emitters = set()
|
||||
section = ""
|
||||
for raw in source.splitlines():
|
||||
line = raw.split(";", 1)[0].strip()
|
||||
if line.startswith("["):
|
||||
section = line.strip("[]").upper()
|
||||
continue
|
||||
tokens = line.split()
|
||||
if not tokens:
|
||||
continue
|
||||
if section == "JUNCTIONS":
|
||||
junctions.add(tokens[0])
|
||||
elif section in {"RESERVOIRS", "TANKS"}:
|
||||
source_nodes.add(tokens[0])
|
||||
elif section == "STATUS" and tokens[-1].upper() == "CLOSED":
|
||||
closed_links.add(tokens[0])
|
||||
elif section == "EMITTERS":
|
||||
existing_emitters.add(tokens[0])
|
||||
elif section in {"PIPES", "VALVES"}:
|
||||
link, left, right = tokens[:3]
|
||||
length = float(tokens[3]) if section == "PIPES" else 0.01
|
||||
if section == "PIPES":
|
||||
if tokens[-1].upper() == "CLOSED":
|
||||
continue
|
||||
pipes[link] = {"pipe_id": link, "start_node_id": left, "emitter_node_id": right,
|
||||
"length_m": length, "diameter_mm": float(tokens[4])}
|
||||
else:
|
||||
valve_links[link] = (left, right)
|
||||
graph[left].append((right, max(length, 0.01)))
|
||||
graph[right].append((left, max(length, 0.01)))
|
||||
# Initial link overrides also apply to pipe and valve connectivity.
|
||||
graph.clear()
|
||||
pipe_graph = defaultdict(list)
|
||||
for link, pipe in pipes.items():
|
||||
if link in closed_links:
|
||||
continue
|
||||
left, right = pipe["start_node_id"], pipe["emitter_node_id"]
|
||||
length = max(pipe["length_m"], 0.01)
|
||||
graph[left].append((right, length))
|
||||
graph[right].append((left, length))
|
||||
pipe_graph[left].append(right)
|
||||
pipe_graph[right].append(left)
|
||||
for link, (left, right) in valve_links.items():
|
||||
if link not in closed_links:
|
||||
graph[left].append((right, 0.01))
|
||||
graph[right].append((left, 0.01))
|
||||
component_by_node = {}
|
||||
component_data = {}
|
||||
|
||||
def isolation(node):
|
||||
if node not in component_by_node:
|
||||
seen = {node}
|
||||
pending = deque([node])
|
||||
while pending:
|
||||
for neighbor in pipe_graph[pending.popleft()]:
|
||||
if neighbor not in seen:
|
||||
seen.add(neighbor)
|
||||
pending.append(neighbor)
|
||||
component_id = min(seen)
|
||||
for member in seen:
|
||||
component_by_node[member] = component_id
|
||||
boundaries = sorted(link for link, (left, right) in valve_links.items()
|
||||
if link not in closed_links and (left in seen) != (right in seen))
|
||||
component_data[component_id] = {"region_id": component_id, "node_count": len(seen),
|
||||
"boundary_valve_ids": boundaries,
|
||||
"source_inside": bool(seen & source_nodes)}
|
||||
return component_data[component_by_node[node]]
|
||||
# Only real pressure-meter model nodes present in this frozen snapshot are used.
|
||||
pressure = [device for device in pressure if device["node_id"] in junctions]
|
||||
flow = [device for device in flow if device["link_id"] in pipes]
|
||||
distances = nearest_pressure_distances(graph, pressure, args.radius_m)
|
||||
candidates = []
|
||||
for pipe in pipes.values():
|
||||
node = pipe["emitter_node_id"]
|
||||
if (args.min_diameter <= pipe["diameter_mm"] <= 800 and pipe["length_m"] >= 50
|
||||
and pipe["pipe_id"] not in closed_links
|
||||
and node in junctions and node not in existing_emitters and node in distances):
|
||||
distance, device = distances[node]
|
||||
if distance < args.min_network_distance_m:
|
||||
continue
|
||||
region = isolation(node)
|
||||
if args.require_isolation and (region["source_inside"] or not 1 <= len(region["boundary_valve_ids"]) <= 12):
|
||||
continue
|
||||
candidates.append({**pipe, "nearest_pressure_device": device, "network_distance_m": distance,
|
||||
"screen_priority": pipe["length_m"] / pipe["diameter_mm"] ** 2,
|
||||
"isolation": region})
|
||||
skipped = set()
|
||||
if args.skip_screening:
|
||||
skipped = {entry["pipe_id"] for entry in json.loads(args.skip_screening.read_text())["results"]}
|
||||
with readonly(biz) as connection:
|
||||
coords = {row["node_id"]: row for row in connection.execute(
|
||||
"SELECT node_id,ST_X(ST_Transform(geom,4326)) longitude,"
|
||||
"ST_Y(ST_Transform(geom,4326)) latitude FROM gis.node_geometries WHERE node_id=ANY(%s)",
|
||||
(list({pipe["emitter_node_id"] for pipe in candidates}),),
|
||||
)}
|
||||
existing_positions = connection.execute(
|
||||
"SELECT ST_X(ST_Transform(geom,4326)) longitude,ST_Y(ST_Transform(geom,4326)) latitude "
|
||||
"FROM gis.node_geometries WHERE node_id=ANY(%s)",
|
||||
([pipe.emitter_node_id for pipe in design.pipes],),
|
||||
).fetchall()
|
||||
filtered = []
|
||||
for candidate in candidates:
|
||||
position = coords.get(candidate["emitter_node_id"])
|
||||
if position is None or candidate["pipe_id"] in skipped:
|
||||
continue
|
||||
distances_km = [111.32 * math.hypot((position["longitude"] - old["longitude"]) *
|
||||
math.cos(math.radians(position["latitude"])),
|
||||
position["latitude"] - old["latitude"])
|
||||
for old in existing_positions]
|
||||
if distances_km and min(distances_km) < args.exclude_existing_km:
|
||||
continue
|
||||
candidate.update(longitude=position["longitude"], latitude=position["latitude"],
|
||||
nearest_existing_km=min(distances_km) if distances_km else None)
|
||||
filtered.append(candidate)
|
||||
candidates = filtered
|
||||
# Cover different meter neighborhoods before taking additional pipes at the same meter.
|
||||
candidates.sort(key=lambda pipe: pipe["screen_priority"], reverse=True)
|
||||
used = set()
|
||||
unique, remaining = [], []
|
||||
for candidate in candidates:
|
||||
device = candidate["nearest_pressure_device"]
|
||||
(remaining if device in used else unique).append(candidate)
|
||||
used.add(device)
|
||||
selected = (unique + remaining)[args.offset:args.offset + args.limit]
|
||||
node_ids = sorted({device["node_id"] for device in pressure} | {pipe["emitter_node_id"] for pipe in selected})
|
||||
link_ids = sorted({device["link_id"] for device in flow})
|
||||
at = max(design.event_times)
|
||||
output = folder / f"screening-{uuid4().hex[:8]}"
|
||||
output.mkdir(mode=0o700)
|
||||
report = {"version": "burst-pipe-screening@1", "model_sha256": digest(snapshot),
|
||||
"at": at.isoformat(), "candidate_count": len(candidates), "results": [],
|
||||
"screen_options": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()},
|
||||
"device_bindings": devices,
|
||||
"scope": "daytime single-frame screening only; requires full paired calibration and isolation review"}
|
||||
|
||||
def simulate(label, emitter=None):
|
||||
path = output / f"{label}.inp"
|
||||
path.write_text(configure_inp(source, at, 0, emitter=emitter, pda_options=calibration["pda_options"]))
|
||||
data = run_file(path, node_ids, link_ids, timeout=240)
|
||||
save_json(output / f"{label}.json", data)
|
||||
return data
|
||||
|
||||
normal = simulate("normal")
|
||||
if normal["status"] != "completed" or normal["warnings"]:
|
||||
raise ValueError("normal screening baseline failed")
|
||||
for index, pipe in enumerate(selected):
|
||||
area = math.pi * (pipe["diameter_mm"] / 10) ** 2 / 4 / 8
|
||||
burst = simulate(f"candidate-{index:02d}", (pipe["emitter_node_id"], area))
|
||||
result = {**pipe, "orifice_area_cm2": area, "status": burst["status"], "output_file": f"candidate-{index:02d}.json"}
|
||||
if burst["status"] == "completed":
|
||||
drops = {device["device_id"]: 100 * (normal["nodes"][device["node_id"]][0]["pressure"] -
|
||||
burst["nodes"][device["node_id"]][0]["pressure"]) /
|
||||
max(abs(normal["nodes"][device["node_id"]][0]["pressure"]), 1e-6) for device in pressure}
|
||||
changes = {device["device_id"]: 100 * abs(normal["links"][device["link_id"]][0]["flow"] -
|
||||
burst["links"][device["link_id"]][0]["flow"]) /
|
||||
abs(normal["links"][device["link_id"]][0]["flow"]) for device in flow
|
||||
if abs(normal["links"][device["link_id"]][0]["flow"]) > 1e-6}
|
||||
ranked = sorted(drops.items(), key=lambda pair: pair[1], reverse=True)
|
||||
negative = any(row[0]["pressure"] < 0 for row in burst["nodes"].values())
|
||||
result.update(pressure_drop_percent=dict(ranked), flow_change_percent=changes,
|
||||
possible_candidate=not burst["warnings"] and not negative and ranked[0][1] >= 30
|
||||
and (ranked[1][1] >= 10 or max(changes.values(), default=0) >= 20))
|
||||
print(f"{index+1}/{len(selected)} {pipe['pipe_id']}: max pressure drop={ranked[0][1]:.2f}%, "
|
||||
f"possible candidate={result['possible_candidate']}", flush=True)
|
||||
report["results"].append(result)
|
||||
save_json(output / "screening.json", report)
|
||||
print(f"screening complete; report: {output / 'screening.json'}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Keep the listening socket alive while replacement ASGI workers become ready."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import multiprocessing
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from multiprocessing.connection import Connection
|
||||
from multiprocessing.process import BaseProcess
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
from uvicorn._subprocess import get_subprocess
|
||||
from uvicorn.importer import import_from_string
|
||||
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
||||
class _ReadinessGate:
|
||||
def __init__(self, app, check):
|
||||
self.app, self.check = app, check
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "lifespan":
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
async def checked_send(message):
|
||||
if message["type"] == "lifespan.startup.complete" and not await self.check():
|
||||
await send({"type": "lifespan.startup.failed", "message": "API readiness check failed"})
|
||||
else:
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, receive, checked_send)
|
||||
|
||||
|
||||
class _ReadyServer(uvicorn.Server):
|
||||
def __init__(self, config: uvicorn.Config, control: Connection):
|
||||
super().__init__(config)
|
||||
self.control = control
|
||||
|
||||
async def startup(self, sockets=None) -> None:
|
||||
await super().startup(sockets=sockets)
|
||||
if self.started and not self.should_exit:
|
||||
self.control.send("ready")
|
||||
|
||||
async def shutdown(self, sockets=None) -> None:
|
||||
# Acknowledge that the old worker has actually stopped accepting. Its
|
||||
# established streams may still run for minutes after this handshake.
|
||||
for listener in self.servers:
|
||||
listener.close()
|
||||
for sock in sockets or []:
|
||||
sock.close()
|
||||
self.control.send("draining")
|
||||
await super().shutdown(sockets=sockets)
|
||||
|
||||
|
||||
def _serve_worker(*, config: uvicorn.Config, control: Connection, readiness_check: str | None, sockets) -> None:
|
||||
config.reload = False
|
||||
config.workers = 1
|
||||
if readiness_check:
|
||||
config.load()
|
||||
config.loaded_app = _ReadinessGate(config.loaded_app, import_from_string(readiness_check))
|
||||
server = _ReadyServer(config, control)
|
||||
|
||||
async def receive_commands() -> None:
|
||||
while True:
|
||||
if control.poll():
|
||||
try:
|
||||
command = control.recv()
|
||||
except EOFError:
|
||||
command = "drain"
|
||||
if command == "drain":
|
||||
server.should_exit = True
|
||||
return
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
async def serve() -> None:
|
||||
receiver = asyncio.create_task(receive_commands())
|
||||
try:
|
||||
await server.serve(sockets=sockets)
|
||||
finally:
|
||||
receiver.cancel()
|
||||
await asyncio.gather(receiver, return_exceptions=True)
|
||||
control.close()
|
||||
|
||||
config.setup_event_loop()
|
||||
asyncio.run(serve())
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
process: BaseProcess
|
||||
control: Connection
|
||||
|
||||
def drain(self, *, wait_timeout: float = 0) -> bool:
|
||||
if self.process.is_alive():
|
||||
try:
|
||||
self.control.send("drain")
|
||||
if wait_timeout:
|
||||
deadline = time.monotonic() + wait_timeout
|
||||
while time.monotonic() < deadline:
|
||||
if self.control.poll(0.05) and self.control.recv() == "draining":
|
||||
return True
|
||||
if not self.process.is_alive():
|
||||
return True
|
||||
return False
|
||||
except (BrokenPipeError, EOFError, OSError):
|
||||
return not self.process.is_alive()
|
||||
return True
|
||||
|
||||
|
||||
class HandoffSupervisor:
|
||||
"""Start the replacement before draining the worker with existing connections."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: uvicorn.Config,
|
||||
*,
|
||||
startup_timeout: float = 60,
|
||||
reload_dirs: list[Path] | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
readiness_check: str | None = None,
|
||||
):
|
||||
self.config = config
|
||||
self.startup_timeout = startup_timeout
|
||||
self.reload_dirs = reload_dirs or [Path("app"), Path("samples/peripheral")]
|
||||
self.poll_interval = poll_interval
|
||||
self.readiness_check = readiness_check
|
||||
self.current: Worker | None = None
|
||||
self.retiring: list[Worker] = []
|
||||
self.socket = config.bind_socket()
|
||||
self.socket.listen(config.backlog)
|
||||
self.should_exit = threading.Event()
|
||||
self._files: dict[Path, tuple[int, int]] = {}
|
||||
|
||||
def _candidate(self) -> Worker | None:
|
||||
parent = child = process = None
|
||||
ready = False
|
||||
try:
|
||||
parent, child = multiprocessing.get_context("spawn").Pipe()
|
||||
process = get_subprocess(
|
||||
config=self.config,
|
||||
target=partial(_serve_worker, config=self.config, control=child, readiness_check=self.readiness_check),
|
||||
sockets=[self.socket],
|
||||
)
|
||||
worker = Worker(process, parent)
|
||||
process.start()
|
||||
child.close()
|
||||
deadline = time.monotonic() + self.startup_timeout
|
||||
while not self.should_exit.is_set() and time.monotonic() < deadline:
|
||||
if parent.poll(0.05):
|
||||
try:
|
||||
ready = parent.recv() == "ready" and process.is_alive()
|
||||
except EOFError:
|
||||
break
|
||||
if ready:
|
||||
return worker
|
||||
if not process.is_alive():
|
||||
break
|
||||
logger.error("Replacement worker did not become ready; keeping the current worker")
|
||||
return None
|
||||
except Exception:
|
||||
logger.exception("Replacement startup or readiness handshake failed; keeping the current worker")
|
||||
return None
|
||||
finally:
|
||||
if child is not None:
|
||||
child.close()
|
||||
if not ready:
|
||||
# Only an unready candidate is terminated. The serving worker
|
||||
# and workers draining established connections are untouched.
|
||||
if process is not None and process.pid is not None:
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join(timeout=2)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
if parent is not None:
|
||||
parent.close()
|
||||
|
||||
def start(self) -> bool:
|
||||
self._files = self._snapshot()
|
||||
self.current = self._candidate()
|
||||
return self.current is not None
|
||||
|
||||
def reload(self) -> bool:
|
||||
self.reap()
|
||||
candidate = self._candidate()
|
||||
if candidate is None:
|
||||
return False
|
||||
old, self.current = self.current, candidate
|
||||
logger.info("Worker %s is ready to accept new connections", candidate.process.pid)
|
||||
if old is not None:
|
||||
acknowledged = old.drain(wait_timeout=2)
|
||||
self.retiring.append(old)
|
||||
if not acknowledged:
|
||||
logger.error("Worker %s has not yet acknowledged draining", old.process.pid)
|
||||
return False
|
||||
logger.info("Worker %s is draining existing connections", old.process.pid)
|
||||
return True
|
||||
|
||||
def reap(self) -> None:
|
||||
remaining = []
|
||||
for worker in self.retiring:
|
||||
if worker.process.is_alive():
|
||||
remaining.append(worker)
|
||||
else:
|
||||
worker.process.join()
|
||||
worker.control.close()
|
||||
self.retiring = remaining
|
||||
|
||||
def _snapshot(self) -> dict[Path, tuple[int, int]]:
|
||||
files = {}
|
||||
for directory in self.reload_dirs:
|
||||
for path in directory.rglob("*.py"):
|
||||
try:
|
||||
stat = path.stat()
|
||||
files[path.resolve()] = (stat.st_mtime_ns, stat.st_size)
|
||||
except OSError:
|
||||
continue
|
||||
return files
|
||||
|
||||
def run(self) -> None:
|
||||
signals = (signal.SIGINT, signal.SIGTERM)
|
||||
original = {sig: signal.signal(sig, lambda *_: self.should_exit.set()) for sig in signals}
|
||||
try:
|
||||
if not self.start():
|
||||
raise RuntimeError("The initial API worker failed to start")
|
||||
pending_reload = False
|
||||
while not self.should_exit.wait(self.poll_interval):
|
||||
self.reap()
|
||||
files = self._snapshot()
|
||||
if files != self._files:
|
||||
self._files = files
|
||||
pending_reload = True
|
||||
active_failed = self.current is None or not self.current.process.is_alive()
|
||||
if active_failed:
|
||||
logger.error("The active worker exited; starting a replacement")
|
||||
pending_reload = True
|
||||
# Coalesce rapid edits while old SSE connections drain, rather
|
||||
# than keeping an unbounded number of application generations.
|
||||
if pending_reload and (active_failed or len(self.retiring) < 2):
|
||||
pending_reload = False
|
||||
self.reload()
|
||||
finally:
|
||||
try:
|
||||
self.close()
|
||||
finally:
|
||||
for sig, handler in original.items():
|
||||
signal.signal(sig, handler)
|
||||
|
||||
def close(self) -> None:
|
||||
self.should_exit.set()
|
||||
workers = [*self.retiring, *([self.current] if self.current else [])]
|
||||
for worker in workers:
|
||||
worker.drain()
|
||||
self.socket.close()
|
||||
for worker in workers:
|
||||
worker.process.join()
|
||||
worker.control.close()
|
||||
self.current = None
|
||||
self.retiring = []
|
||||
@@ -215,14 +215,17 @@ async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, periphe
|
||||
evidence_summary = order["summary"]
|
||||
if notes:
|
||||
evidence_summary += "\n现场反馈：\n" + "\n".join(notes[-10:])
|
||||
field_times = [activity["occurred_at"] for activity in order.get("activity", [])
|
||||
if activity.get("kind") == "feedback" and activity.get("occurred_at")]
|
||||
feedback_time = field_times[-1] if order["status"] == "completed" and field_times else order["updated_at"]
|
||||
feedback = {
|
||||
"request_id": str(uuid5(NAMESPACE_URL, event_id)), "source_event_id": external_id,
|
||||
"source_revision": order["revision"], "status": STATUS[order["status"]],
|
||||
"occurred_at": order["updated_at"],
|
||||
"occurred_at": feedback_time,
|
||||
"detail": f"{source_label} {order['work_order_id']}：{order['status']}",
|
||||
"evidence": {"evidence_id": event_id, "kind": "external_feedback", "summary": evidence_summary,
|
||||
"source_label": source_label, "source_ref": external_id,
|
||||
"observed_at": order["updated_at"]},
|
||||
"observed_at": feedback_time},
|
||||
}
|
||||
# A child order's creation is already represented by its dispatch
|
||||
# receipt. Only a subsequent source revision is new field input.
|
||||
|
||||
Reference in New Issue
Block a user