feat(scada): add calibrated monitoring and incident recovery
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Offline benchmark preparation; never imported by the application runtime."""
|
||||
@@ -0,0 +1,226 @@
|
||||
"""File-isolated hydraulic calibration; database connections are read-only."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from scripts.benchmark.simulation_design import PipeCalibration, SelectionRule
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def save_json(path: Path, value: dict) -> None:
|
||||
temporary = path.with_suffix(".pending")
|
||||
with temporary.open("x", encoding="utf-8") as output:
|
||||
os.chmod(temporary, 0o600)
|
||||
json.dump(value, output, ensure_ascii=False, indent=2, allow_nan=False, default=str)
|
||||
output.write("\n")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def select_calibrated_area(
|
||||
pipe: PipeCalibration, passed_levels: list[str], case_results: list[dict], retain_weak: bool,
|
||||
) -> dict:
|
||||
selected = passed_levels[-1] if passed_levels else None
|
||||
soft_blockers = {"insufficient_consecutive_pressure_and_corroboration", "event_rule_did_not_trigger"}
|
||||
physically_valid = bool(case_results) and all(
|
||||
not set(case["blockers"]) - soft_blockers for case in case_results
|
||||
)
|
||||
status = "calibrated_candidate" if selected else (
|
||||
"weak_signal_candidate" if retain_weak and physically_valid else "unsuitable"
|
||||
)
|
||||
return {"status": status, "severity": selected,
|
||||
"area_cm2": pipe.candidate_orifice_areas_cm2[selected] if selected else None}
|
||||
|
||||
|
||||
def configure_inp(
|
||||
source: str,
|
||||
start: datetime,
|
||||
duration: int,
|
||||
*,
|
||||
emitter: tuple[str, float] | None = None,
|
||||
pda_options: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Apply the burst service's options to a file, including the zero-leak control."""
|
||||
if start.tzinfo is None:
|
||||
raise ValueError("simulation start must include a timezone")
|
||||
start = start.astimezone(ZoneInfo("Asia/Shanghai"))
|
||||
updates = {
|
||||
"TIMES": {"DURATION": f"{duration // 3600}:{duration % 3600 // 60:02d}:00",
|
||||
"PATTERN START": start.strftime("%H:%M:%S"),
|
||||
"REPORT START": "0:00:00", "STATISTIC": "NONE"},
|
||||
"OPTIONS": {**(pda_options or {}), "DEMAND MODEL": "PDA", "REQUIRED PRESSURE": "10.0000"},
|
||||
"REPORT": {"NODES": "NONE", "LINKS": "NONE"},
|
||||
}
|
||||
section = None
|
||||
output = []
|
||||
pending = {}
|
||||
for line in source.splitlines():
|
||||
match = re.fullmatch(r"\s*\[([^]]+)\]\s*", line)
|
||||
if match:
|
||||
output.extend(f"{key} {value}" for key, value in pending.items())
|
||||
section = match[1].upper()
|
||||
pending = dict(updates.get(section, {}))
|
||||
output.append(line)
|
||||
if section == "EMITTERS" and emitter:
|
||||
node, area = emitter
|
||||
coefficient = 0.65 * area / 10000 * math.sqrt(19.6) * 1000
|
||||
output.append(f"{node} {coefficient:.12g}")
|
||||
continue
|
||||
tokens = line.split(";", 1)[0].upper().split()
|
||||
if any(tokens[:len(key.split())] == key.split() for key in updates.get(section, {})):
|
||||
continue
|
||||
output.append(line)
|
||||
output.extend(f"{key} {value}" for key, value in pending.items())
|
||||
if "[EMITTERS]" not in source.upper() and emitter:
|
||||
raise ValueError("exported model has no emitter section")
|
||||
return "\n".join(output) + "\n"
|
||||
|
||||
|
||||
def run_file(inp: Path, node_ids: list[str], link_ids: list[str], *, timeout: int) -> dict:
|
||||
from app.infra.epanet.epanet import Output
|
||||
|
||||
executable = Path(__file__).resolve().parents[2] / "app/infra/epanet/linux/runepanet"
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = f"{executable.parent}:{env.get('LD_LIBRARY_PATH', '')}"
|
||||
report_path, binary = inp.with_suffix(".rpt"), inp.with_suffix(".bin")
|
||||
result = subprocess.run(
|
||||
[str(executable), str(inp), str(report_path), str(binary)],
|
||||
env=env, capture_output=True, text=True, timeout=timeout, check=False,
|
||||
)
|
||||
report = report_path.read_text(encoding="utf-8", errors="replace") if report_path.exists() else ""
|
||||
warnings = [line.strip() for line in report.splitlines()
|
||||
if re.search(r"^\s*(?:warning\b|error\s+\d+\b)|system\s+(?:is\s+)?unbalanced|cannot solve hydraulic",
|
||||
line, re.IGNORECASE)]
|
||||
summary = {"exit_code": result.returncode, "warnings": warnings,
|
||||
"inp_sha256": digest(inp), "report_sha256": digest(report_path) if report_path.exists() else None}
|
||||
if result.returncode or not binary.exists():
|
||||
return {**summary, "status": "failed"}
|
||||
reader = Output(str(binary))
|
||||
names = reader.element_name()
|
||||
summary.update(status="completed", units=reader.units(), times=reader.times(), nodes={}, links={})
|
||||
for kind, selected, attributes, function in (
|
||||
("nodes", node_ids, ["demand", "head", "pressure", "quality"], reader._lib.ENR_getNodeResult),
|
||||
("links", link_ids, ["flow", "velocity", "headloss", "quality", "status", "setting", "reaction", "friction"],
|
||||
reader._lib.ENR_getLinkResult),
|
||||
):
|
||||
indices = {name: index + 1 for index, name in enumerate(names[kind])}
|
||||
for name in selected:
|
||||
index = indices.get(name)
|
||||
if index is None:
|
||||
raise ValueError("requested hydraulic output is missing")
|
||||
series = []
|
||||
for period in range(summary["times"]["num_periods"]):
|
||||
values = ctypes.POINTER(ctypes.c_float)()
|
||||
length = ctypes.c_int()
|
||||
reader._check(function(reader._handle, period, index, ctypes.byref(values), ctypes.byref(length)))
|
||||
try:
|
||||
if length.value != len(attributes):
|
||||
raise ValueError("unexpected hydraulic result attributes")
|
||||
row = {key: float(values[i]) for i, key in enumerate(attributes)}
|
||||
if not all(math.isfinite(value) for value in row.values()):
|
||||
raise ValueError("non-finite hydraulic result")
|
||||
series.append(row)
|
||||
finally:
|
||||
reader._lib.ENR_free(ctypes.byref(values))
|
||||
summary[kind][name] = series
|
||||
return summary
|
||||
|
||||
|
||||
def pressure_preview(
|
||||
pipe: PipeCalibration, start: datetime, duration: int, step: int,
|
||||
history: dict, current: dict, rule: SelectionRule,
|
||||
) -> dict:
|
||||
"""Exercise the actual replay cleaner/detector without database mutations."""
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
||||
|
||||
ids = [binding.device_id for binding in pipe.pressure_bindings]
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, f"calibration:{pipe.pipe_id}:{start.isoformat()}"),
|
||||
start_time=start, end_time=start + timedelta(seconds=duration + step), device_ids=ids,
|
||||
rules=[{"rule_id": "calibration-pressure", "device_ids": ids,
|
||||
"kind": "relative_change", "direction": "decrease", "unit": "m",
|
||||
"threshold": rule.primary_pressure_drop_percent,
|
||||
"window_samples": 4, "minimum_samples": 4,
|
||||
"consecutive_samples": rule.consecutive_samples, "minimum_devices": 1}],
|
||||
)
|
||||
document = {"replay_id": str(config.request_id), "devices": [
|
||||
{"device_id": key, "transmission_frequency": "0:15:00"} for key in ids],
|
||||
"histories": {binding.device_id: [row["pressure"] for row in history[binding.node_id]]
|
||||
for binding in pipe.pressure_bindings}}
|
||||
raised = []
|
||||
for index in range(duration // step + 1):
|
||||
at = start + timedelta(seconds=index * step)
|
||||
rows = [{"device_id": binding.device_id, "time": at,
|
||||
"monitored_value": current[binding.node_id][index]["pressure"],
|
||||
"source_hash": hashlib.sha256(f"{binding.device_id}:{at}".encode()).hexdigest()}
|
||||
for binding in pipe.pressure_bindings]
|
||||
document, changes = detect_frame(document, prepare_cleaning(rows, config), config, at)
|
||||
raised.extend(change for change in changes
|
||||
if change["change"] == "raised" and change.get("category") != "data_quality")
|
||||
return {"raised": bool(raised), "first_signal_at": raised[0]["occurred_at"] if raised else None,
|
||||
"scope": "pure replay rule preview; not monitoring worker or Agent acceptance"}
|
||||
|
||||
|
||||
def assess(
|
||||
pipe: PipeCalibration, normal: dict, burst: dict, rule: SelectionRule,
|
||||
*, duration: int, step: int,
|
||||
) -> dict:
|
||||
expected_times = {"report_start": 0, "report_step": step,
|
||||
"sim_duration": duration, "num_periods": duration // step + 1}
|
||||
blockers = []
|
||||
for label, result in (("normal", normal), ("burst", burst)):
|
||||
if result.get("status") != "completed" or result.get("times") != expected_times:
|
||||
blockers.append(f"{label}_incomplete_or_failed")
|
||||
if result.get("warnings"):
|
||||
blockers.append(f"{label}_solver_warning")
|
||||
if result.get("units") != {"flow": "LPS", "pressure": "MTR", "quality": "NONE"}:
|
||||
blockers.append(f"{label}_unexpected_units")
|
||||
if blockers:
|
||||
return {"passed": False, "blockers": blockers}
|
||||
pressures, flows = {}, {}
|
||||
for binding in pipe.pressure_bindings:
|
||||
before = [row["pressure"] for row in normal["nodes"][binding.node_id]]
|
||||
after = [row["pressure"] for row in burst["nodes"][binding.node_id]]
|
||||
if min(before) <= rule.relative_change_epsilon:
|
||||
blockers.append(f"nonpositive_normal_pressure:{binding.device_id}")
|
||||
if min(after) < 0:
|
||||
blockers.append(f"negative_monitor_pressure:{binding.device_id}")
|
||||
pressures[binding.device_id] = [100 * (a - b) / max(abs(a), rule.relative_change_epsilon)
|
||||
for a, b in zip(before, after)]
|
||||
for binding in pipe.flow_bindings:
|
||||
flows[binding.device_id] = [100 * abs(a["flow"] - b["flow"]) /
|
||||
max(abs(a["flow"]), rule.relative_change_epsilon)
|
||||
for a, b in zip(normal["links"][binding.link_id], burst["links"][binding.link_id])]
|
||||
witnesses = []
|
||||
count = duration // step + 1
|
||||
for primary, drops in pressures.items():
|
||||
for start in range(count - rule.consecutive_samples + 1):
|
||||
window = range(start, start + rule.consecutive_samples)
|
||||
if not all(drops[i] >= rule.primary_pressure_drop_percent for i in window):
|
||||
continue
|
||||
second = [key for key, values in pressures.items() if key != primary and
|
||||
all(values[i] >= rule.corroborating_pressure_drop_percent for i in window)]
|
||||
flow = [key for key, values in flows.items() if
|
||||
all(values[i] >= rule.flow_change_percent for i in window) and
|
||||
all(abs(normal["links"][next(b.link_id for b in pipe.flow_bindings if b.device_id == key)][i]["flow"])
|
||||
> rule.relative_change_epsilon for i in window)]
|
||||
if second or flow:
|
||||
witnesses.append({"primary_device": primary, "second_pressure_devices": second,
|
||||
"flow_devices": flow, "start_offset_seconds": start * step})
|
||||
if not witnesses:
|
||||
blockers.append("insufficient_consecutive_pressure_and_corroboration")
|
||||
return {"passed": not blockers, "blockers": blockers, "witnesses": witnesses,
|
||||
"pressure_drop_percent": pressures, "flow_change_percent": flows}
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Versioned burst cases and deterministic preparation checks."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import defaultdict, deque
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
|
||||
class BurstCase(StrictModel):
|
||||
case_id: str = Field(pattern=r"^B[0-9]{2}$")
|
||||
pipe_id: str = Field(pattern=r"^[0-9]+$")
|
||||
occurred_at: datetime
|
||||
reference_analysis: str = Field(min_length=1, max_length=200)
|
||||
observation_provenance: Literal["declared_historical_replay", "requires_generation"]
|
||||
provenance_note: str = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def aware_time(self):
|
||||
if self.occurred_at.tzinfo is None:
|
||||
raise ValueError("case time must include a timezone")
|
||||
return self
|
||||
|
||||
|
||||
class TriggerRule(StrictModel):
|
||||
pressure_drop_percent: float = Field(gt=0, le=100)
|
||||
baseline_samples: int = Field(ge=3, le=96)
|
||||
consecutive_samples: int = Field(ge=1, le=20)
|
||||
minimum_devices: int = Field(ge=1, le=500)
|
||||
|
||||
|
||||
class BurstSuite(StrictModel):
|
||||
version: Literal["burst-benchmark@1"]
|
||||
suite_id: Literal["tjwater-v2-burst"]
|
||||
project_code: Literal["tjwater_v2"]
|
||||
project_id: UUID
|
||||
sampling_interval_minutes: int = Field(ge=1, le=60)
|
||||
event_minutes: int = Field(ge=15, le=1440)
|
||||
detector_days: Literal[15]
|
||||
trigger_rule: TriggerRule
|
||||
score_weights: dict[str, int]
|
||||
cases: list[BurstCase] = Field(min_length=1, max_length=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_suite(self):
|
||||
if 1440 % self.sampling_interval_minutes or self.event_minutes % self.sampling_interval_minutes:
|
||||
raise ValueError("sample interval must divide a day and the event window")
|
||||
if len({case.case_id for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("case ids must be unique")
|
||||
if set(self.score_weights) != {"outcome", "tools", "plan", "efficiency"}:
|
||||
raise ValueError("all four score dimensions are required")
|
||||
if any(value <= 0 for value in self.score_weights.values()) or sum(self.score_weights.values()) != 100:
|
||||
raise ValueError("positive score weights must sum to 100")
|
||||
for case in self.cases:
|
||||
if case.occurred_at.timestamp() % (60 * self.sampling_interval_minutes):
|
||||
raise ValueError("event times must align with the sampling grid")
|
||||
return self
|
||||
|
||||
def select(self, case_ids: list[str]) -> list[BurstCase]:
|
||||
if not case_ids:
|
||||
return self.cases
|
||||
unknown = set(case_ids) - {case.case_id for case in self.cases}
|
||||
if unknown:
|
||||
raise ValueError("unknown case ids: " + ", ".join(sorted(unknown)))
|
||||
return [case for case in self.cases if case.case_id in case_ids]
|
||||
|
||||
|
||||
def load_suite(path: Path) -> BurstSuite:
|
||||
return BurstSuite.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def canonical_json(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
||||
allow_nan=False, default=str)
|
||||
|
||||
|
||||
def fingerprint(value) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode()).hexdigest()
|
||||
|
||||
|
||||
def complete_devices(rows: list[dict], start: datetime, end: datetime, interval: int) -> list[str]:
|
||||
"""Counts alone are insufficient: require finite values on the exact grid."""
|
||||
step = timedelta(minutes=interval)
|
||||
expected = {start + step * index for index in range(int((end - start) / step) + 1)}
|
||||
values: dict[str, dict[datetime, float]] = defaultdict(dict)
|
||||
invalid: set[str] = set()
|
||||
for row in rows:
|
||||
device = row["device_id"]
|
||||
at = row["time"]
|
||||
if not start <= at <= end:
|
||||
continue
|
||||
value = row["monitored_value"]
|
||||
if at not in expected or at in values[device] or value is None or not math.isfinite(float(value)):
|
||||
invalid.add(device)
|
||||
else:
|
||||
values[device][at] = float(value)
|
||||
return sorted(device for device, series in values.items()
|
||||
if device not in invalid and set(series) == expected)
|
||||
|
||||
|
||||
def isolation_reference(links: list[dict], source_nodes: list[str], pipe_id: str) -> dict:
|
||||
"""Independent graph check, separate from the production isolation algorithm."""
|
||||
graph: dict[str, list[str]] = defaultdict(list)
|
||||
target = None
|
||||
valves = []
|
||||
for link in links:
|
||||
if link["id"] == pipe_id:
|
||||
target = (link["start_node_id"], link["end_node_id"])
|
||||
if link["kind"].lower() == "valve":
|
||||
valves.append(link["id"])
|
||||
continue
|
||||
if str(link["status"]).lower() == "closed":
|
||||
continue
|
||||
left, right = link["start_node_id"], link["end_node_id"]
|
||||
graph[left].append(right)
|
||||
graph[right].append(left)
|
||||
if target is None:
|
||||
return {"status": "target_missing"}
|
||||
if not source_nodes:
|
||||
return {"status": "sources_missing"}
|
||||
reached = set(source_nodes)
|
||||
pending = deque(source_nodes)
|
||||
while pending:
|
||||
for node in graph[pending.popleft()]:
|
||||
if node not in reached:
|
||||
reached.add(node)
|
||||
pending.append(node)
|
||||
connected = any(node in reached for node in target)
|
||||
return {
|
||||
"status": "not_isolatable" if connected else "requires_boundary_verification",
|
||||
"model_valve_count": len(valves),
|
||||
"source_count": len(source_nodes),
|
||||
"source_reachable_after_closing_all_model_valves": connected,
|
||||
"scope": "undirected model topology; not hydraulic or field acceptance",
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
"""Isolated database preparation and deterministic burst-dataset scheduling."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from psycopg import connect, sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
|
||||
def canonical_hash(value) -> str:
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def deterministic_standard_normal(*, seed: str, at: datetime, device_id: str, metric: str) -> float:
|
||||
"""Return order-independent deterministic Gaussian noise for one observation."""
|
||||
material = f"{seed}\0{at.isoformat()}\0{device_id}\0{metric}".encode()
|
||||
digest = hashlib.sha256(material).digest()
|
||||
first = (int.from_bytes(digest[:8], "big") + 1) / (2**64 + 1)
|
||||
second = (int.from_bytes(digest[8:16], "big") + 1) / (2**64 + 1)
|
||||
return math.sqrt(-2.0 * math.log(first)) * math.cos(2.0 * math.pi * second)
|
||||
|
||||
|
||||
def noisy_scada_observation(
|
||||
*,
|
||||
raw_value: float,
|
||||
device_type: str,
|
||||
at: datetime,
|
||||
device_id: str,
|
||||
seed: str,
|
||||
pressure_sigma_percent: float = 0.2,
|
||||
pressure_sigma_floor_m: float = 0.02,
|
||||
flow_sigma_percent: float = 0.5,
|
||||
clip_sigma: float = 3.0,
|
||||
) -> float:
|
||||
"""Convert model units and add reproducible zero-mean telemetry noise."""
|
||||
if not math.isfinite(raw_value):
|
||||
raise ValueError("model observation must be finite")
|
||||
if min(pressure_sigma_percent, pressure_sigma_floor_m, flow_sigma_percent, clip_sigma) < 0:
|
||||
raise ValueError("noise parameters must be non-negative")
|
||||
z = max(-clip_sigma, min(clip_sigma, deterministic_standard_normal(
|
||||
seed=seed, at=at, device_id=device_id, metric=device_type,
|
||||
)))
|
||||
if device_type == "pressure":
|
||||
converted = raw_value
|
||||
sigma = max(abs(converted) * pressure_sigma_percent / 100.0, pressure_sigma_floor_m)
|
||||
return max(0.0, converted + z * sigma)
|
||||
if device_type == "pipe_flow":
|
||||
converted = raw_value * 3.6 # L/s -> m3/h
|
||||
sigma = abs(converted) * flow_sigma_percent / 100.0
|
||||
return converted + z * sigma
|
||||
raise ValueError(f"unsupported SCADA device type: {device_type}")
|
||||
|
||||
|
||||
def write_realtime_scada_day(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
start: datetime,
|
||||
interval_seconds: int = 900,
|
||||
duration_seconds: int = 86400,
|
||||
noise_seed: str,
|
||||
pressure_sigma_percent: float = 0.2,
|
||||
pressure_sigma_floor_m: float = 0.02,
|
||||
flow_sigma_percent: float = 0.5,
|
||||
clip_sigma: float = 3.0,
|
||||
) -> dict:
|
||||
"""Map one online-model day into noisy SCADA observations."""
|
||||
if start.tzinfo is None or duration_seconds <= 0 or interval_seconds <= 0:
|
||||
raise ValueError("an aware start and positive duration/interval are required")
|
||||
if duration_seconds % interval_seconds:
|
||||
raise ValueError("duration must be divisible by the sampling interval")
|
||||
expected_times = [
|
||||
start + timedelta(seconds=offset)
|
||||
for offset in range(0, duration_seconds, interval_seconds)
|
||||
]
|
||||
end = start + timedelta(seconds=duration_seconds)
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow')
|
||||
ORDER BY device_id"""
|
||||
).fetchall()
|
||||
if not devices:
|
||||
raise ValueError("no pressure or pipe-flow SCADA bindings are available")
|
||||
records: list[tuple] = []
|
||||
raw_fingerprint_rows: list[tuple] = []
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
for device in devices:
|
||||
if device["device_type"] == "pressure":
|
||||
if not device["node_id"] or device["measurement_unit"] != "m":
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
rows = timescale.execute(
|
||||
"""SELECT time,pressure AS value FROM realtime.node_results
|
||||
WHERE node_id=%s AND time>=%s AND time<%s ORDER BY time""",
|
||||
(device["node_id"], start, end),
|
||||
).fetchall()
|
||||
else:
|
||||
if not device["link_id"] or device["measurement_unit"] != "m3/h":
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
rows = timescale.execute(
|
||||
"""SELECT time,flow AS value FROM realtime.link_results
|
||||
WHERE link_id=%s AND time>=%s AND time<%s ORDER BY time""",
|
||||
(device["link_id"], start, end),
|
||||
).fetchall()
|
||||
actual_times = [row["time"] for row in rows]
|
||||
if actual_times != expected_times:
|
||||
raise ValueError(f"incomplete realtime samples for device {device['device_id']}")
|
||||
for row in rows:
|
||||
raw_value = float(row["value"])
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw_value,
|
||||
device_type=device["device_type"],
|
||||
at=row["time"],
|
||||
device_id=device["device_id"],
|
||||
seed=noise_seed,
|
||||
pressure_sigma_percent=pressure_sigma_percent,
|
||||
pressure_sigma_floor_m=pressure_sigma_floor_m,
|
||||
flow_sigma_percent=flow_sigma_percent,
|
||||
clip_sigma=clip_sigma,
|
||||
)
|
||||
raw_fingerprint_rows.append((row["time"].isoformat(), device["device_id"], raw_value))
|
||||
records.append((row["time"], device["device_id"], value, None))
|
||||
collision = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if collision:
|
||||
raise ValueError("SCADA target day is not empty")
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in records:
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if persisted != len(records):
|
||||
raise ValueError("SCADA normal-day read-back count does not match generated records")
|
||||
return {
|
||||
"start": start.isoformat(),
|
||||
"end_exclusive": end.isoformat(),
|
||||
"devices": len(devices),
|
||||
"samples_per_device": len(expected_times),
|
||||
"rows": persisted,
|
||||
"units": {"pressure": "m", "pipe_flow": "m3/h", "model_flow": "L/s"},
|
||||
"noise": {
|
||||
"seed": noise_seed,
|
||||
"distribution": "deterministic_clipped_gaussian",
|
||||
"pressure_sigma_percent": pressure_sigma_percent,
|
||||
"pressure_sigma_floor_m": pressure_sigma_floor_m,
|
||||
"flow_sigma_percent": flow_sigma_percent,
|
||||
"clip_sigma": clip_sigma,
|
||||
},
|
||||
"raw_sha256": canonical_hash(raw_fingerprint_rows),
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in records]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_demo_scada_window(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
normal_source_start: datetime,
|
||||
target_start: datetime,
|
||||
day_count: int,
|
||||
event_start: datetime,
|
||||
event_run_id: str,
|
||||
noise_seed: str,
|
||||
interval_seconds: int = 900,
|
||||
) -> dict:
|
||||
"""Repeat one normal online day and replace one five-hour event window."""
|
||||
if any(value.tzinfo is None for value in (normal_source_start, target_start, event_start)):
|
||||
raise ValueError("normal, target, and event times must be timezone-aware")
|
||||
if day_count < 15 or interval_seconds <= 0 or 86400 % interval_seconds:
|
||||
raise ValueError("demo windows require at least 15 complete aligned days")
|
||||
target_end = target_start + timedelta(days=day_count)
|
||||
if not target_start <= event_start < target_end:
|
||||
raise ValueError("event must fall inside the demo window")
|
||||
run_uuid = UUID(str(event_run_id))
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow') ORDER BY device_id"""
|
||||
).fetchall()
|
||||
pressure_devices = [row for row in devices if row["device_type"] == "pressure"]
|
||||
flow_devices = [row for row in devices if row["device_type"] == "pipe_flow"]
|
||||
if any(not row["node_id"] or row["measurement_unit"] != "m" for row in pressure_devices):
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
if any(not row["link_id"] or row["measurement_unit"] != "m3/h" for row in flow_devices):
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
normal_end = normal_source_start + timedelta(days=1)
|
||||
expected_normal_times = [
|
||||
normal_source_start + timedelta(seconds=offset)
|
||||
for offset in range(0, 86400, interval_seconds)
|
||||
]
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
normal_nodes = timescale.execute(
|
||||
"""SELECT time,node_id,pressure AS value FROM realtime.node_results
|
||||
WHERE time>=%s AND time<%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(normal_source_start, normal_end, [row["node_id"] for row in pressure_devices]),
|
||||
).fetchall()
|
||||
normal_links = timescale.execute(
|
||||
"""SELECT time,link_id,flow AS value FROM realtime.link_results
|
||||
WHERE time>=%s AND time<%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(normal_source_start, normal_end, [row["link_id"] for row in flow_devices]),
|
||||
).fetchall()
|
||||
event_nodes = timescale.execute(
|
||||
"""SELECT time,node_id,pressure AS value FROM analysis.node_results
|
||||
WHERE run_id=%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(run_uuid, [row["node_id"] for row in pressure_devices]),
|
||||
).fetchall()
|
||||
event_links = timescale.execute(
|
||||
"""SELECT time,link_id,flow AS value FROM analysis.link_results
|
||||
WHERE run_id=%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(run_uuid, [row["link_id"] for row in flow_devices]),
|
||||
).fetchall()
|
||||
|
||||
normal_node_values = {(row["time"], row["node_id"]): float(row["value"]) for row in normal_nodes}
|
||||
normal_link_values = {(row["time"], row["link_id"]): float(row["value"]) for row in normal_links}
|
||||
for device in devices:
|
||||
values = normal_node_values if device["device_type"] == "pressure" else normal_link_values
|
||||
element = device["node_id"] if device["device_type"] == "pressure" else device["link_id"]
|
||||
if any((at, element) not in values for at in expected_normal_times):
|
||||
raise ValueError(f"normal source day is incomplete for device {device['device_id']}")
|
||||
|
||||
records: dict[tuple[datetime, str], tuple] = {}
|
||||
for day_offset in range(day_count):
|
||||
day_start = target_start + timedelta(days=day_offset)
|
||||
for sample_offset, source_at in enumerate(expected_normal_times):
|
||||
target_at = day_start + timedelta(seconds=sample_offset * interval_seconds)
|
||||
for device in pressure_devices:
|
||||
raw = normal_node_values[(source_at, device["node_id"])]
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw, device_type="pressure", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
for device in flow_devices:
|
||||
raw = normal_link_values[(source_at, device["link_id"])]
|
||||
value = noisy_scada_observation(
|
||||
raw_value=raw, device_type="pipe_flow", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
|
||||
if not event_nodes or not event_links:
|
||||
raise ValueError("event analysis has no bound pressure or flow results")
|
||||
source_event_start = min(row["time"] for row in event_nodes + event_links)
|
||||
event_times = sorted({row["time"] for row in event_nodes + event_links})
|
||||
if len(event_times) != 21 or any(
|
||||
right - left != timedelta(seconds=interval_seconds)
|
||||
for left, right in zip(event_times, event_times[1:])
|
||||
):
|
||||
raise ValueError("event analysis must contain 21 aligned fifteen-minute frames")
|
||||
event_node_values = {(row["time"], row["node_id"]): float(row["value"]) for row in event_nodes}
|
||||
event_link_values = {(row["time"], row["link_id"]): float(row["value"]) for row in event_links}
|
||||
for source_at in event_times:
|
||||
target_at = event_start + (source_at - source_event_start)
|
||||
if target_at >= target_end:
|
||||
raise ValueError("event results extend beyond the demo window")
|
||||
for device in pressure_devices:
|
||||
key = (source_at, device["node_id"])
|
||||
if key not in event_node_values:
|
||||
raise ValueError(f"event pressure is incomplete for device {device['device_id']}")
|
||||
value = noisy_scada_observation(
|
||||
raw_value=event_node_values[key], device_type="pressure", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
for device in flow_devices:
|
||||
key = (source_at, device["link_id"])
|
||||
if key not in event_link_values:
|
||||
raise ValueError(f"event flow is incomplete for device {device['device_id']}")
|
||||
value = noisy_scada_observation(
|
||||
raw_value=event_link_values[key], device_type="pipe_flow", at=target_at,
|
||||
device_id=device["device_id"], seed=noise_seed,
|
||||
)
|
||||
records[(target_at, device["device_id"])] = (
|
||||
target_at, device["device_id"], value, None,
|
||||
)
|
||||
|
||||
ordered = [records[key] for key in sorted(records)]
|
||||
expected_count = day_count * len(expected_normal_times) * len(devices)
|
||||
if len(ordered) != expected_count:
|
||||
raise ValueError("demo SCADA window has an unexpected observation count")
|
||||
timescale.execute(
|
||||
"DELETE FROM scada.measurements WHERE time>=%s AND time<%s",
|
||||
(target_start, target_end),
|
||||
)
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in ordered:
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"""SELECT count(*) count,count(DISTINCT device_id) devices,
|
||||
count(DISTINCT time) frames,count(cleaned_value) cleaned
|
||||
FROM scada.measurements WHERE time>=%s AND time<%s""",
|
||||
(target_start, target_end),
|
||||
).fetchone()
|
||||
if persisted != {
|
||||
"count": expected_count,
|
||||
"devices": len(devices),
|
||||
"frames": day_count * len(expected_normal_times),
|
||||
"cleaned": 0,
|
||||
}:
|
||||
raise ValueError("demo SCADA read-back verification failed")
|
||||
return {
|
||||
"start": target_start.isoformat(),
|
||||
"end_exclusive": target_end.isoformat(),
|
||||
"day_count": day_count,
|
||||
"event_start": event_start.isoformat(),
|
||||
"event_end": (event_start + timedelta(seconds=20 * interval_seconds)).isoformat(),
|
||||
"event_run_id": str(run_uuid),
|
||||
"devices": len(devices),
|
||||
"frames": day_count * len(expected_normal_times),
|
||||
"rows": expected_count,
|
||||
"cleaned_rows": 0,
|
||||
"noise_seed": noise_seed,
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in ordered]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def create_empty_database(source_dsn: str, target: str, schema_template: str) -> None:
|
||||
if not target.startswith("tjw_bench_") or target.endswith("_template"):
|
||||
raise ValueError("benchmark targets must use the owned tjw_bench_ prefix")
|
||||
with connect(make_conninfo(source_dsn, dbname="postgres"), autocommit=True, row_factory=dict_row) as connection:
|
||||
if connection.execute("SELECT 1 FROM pg_database WHERE datname=%s", (target,)).fetchone():
|
||||
raise ValueError("target database already exists; explicit state is required for resumption")
|
||||
# Templates are intentionally connection-disabled; clone without altering that state.
|
||||
connection.execute(sql.SQL("CREATE DATABASE {} TEMPLATE {}").format(sql.Identifier(target), sql.Identifier(schema_template)))
|
||||
|
||||
|
||||
def copy_project_inputs(source_dsn: str, target_dsn: str) -> dict:
|
||||
from scripts.check_burst_simulation_design import readonly
|
||||
from app.native.wndb.core.model_replace import _copy_order, _copy_table, _model_tables, _table_columns
|
||||
|
||||
summaries = {}
|
||||
with readonly(source_dsn) as source, connect(target_dsn, row_factory=dict_row) as target:
|
||||
source.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
tables = _model_tables(source)
|
||||
ordered = _copy_order(source, tables)
|
||||
for schema, table in reversed(ordered):
|
||||
target.execute(sql.SQL("DELETE FROM {}").format(sql.Identifier(schema, table)))
|
||||
for schema, table in ordered + [("asset", "scada_devices")]:
|
||||
columns = _table_columns(source, schema, table)
|
||||
if _table_columns(target, schema, table) != columns:
|
||||
raise ValueError(f"schema differs for {schema}.{table}")
|
||||
_copy_table(source, target, schema, table, columns)
|
||||
statement = sql.SQL("SELECT count(*) count FROM {}").format(sql.Identifier(schema, table))
|
||||
before, after = source.execute(statement).fetchone()["count"], target.execute(statement).fetchone()["count"]
|
||||
if before != after:
|
||||
raise ValueError("isolated model copy count mismatch")
|
||||
summaries[f"{schema}.{table}"] = before
|
||||
target.commit()
|
||||
return summaries
|
||||
|
||||
|
||||
def apply_project_runtime_migrations(target_dsn: str, repository_root: Path) -> list[str]:
|
||||
"""Install the additive MCP and calculation queue tables in an owned target."""
|
||||
migrations = (
|
||||
"025_mcp_query_contexts.sql",
|
||||
"026_calculation_jobs.sql",
|
||||
"027_calculation_job_lifecycle.sql",
|
||||
"030_remove_calculation_queue_expiry.sql",
|
||||
)
|
||||
applied = []
|
||||
with connect(target_dsn) as connection:
|
||||
for filename in migrations:
|
||||
path = repository_root / "resources" / "sql" / filename
|
||||
connection.execute(path.read_text(encoding="utf-8"))
|
||||
applied.append(filename)
|
||||
connection.commit()
|
||||
return applied
|
||||
|
||||
|
||||
async def register_benchmark_project(
|
||||
*,
|
||||
source_project_id: str,
|
||||
target: str,
|
||||
target_biz_dsn: str,
|
||||
target_iot_dsn: str,
|
||||
):
|
||||
"""Register owned databases and mirror source membership without role expansion."""
|
||||
from uuid import UUID
|
||||
|
||||
from app.infra.db.metadb.database import SessionLocal
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
|
||||
source_id = UUID(str(source_project_id))
|
||||
async with SessionLocal() as session:
|
||||
repository = MetadataRepository(session)
|
||||
source = await repository.get_project_by_id(source_id)
|
||||
if source is None:
|
||||
raise ValueError("source project metadata is unavailable")
|
||||
project = await repository.get_project_by_code(target)
|
||||
marker = {
|
||||
"benchmark_source_project_id": str(source_id),
|
||||
"benchmark_dataset": "burst-v1",
|
||||
}
|
||||
if project is None:
|
||||
project = await repository.create_project(
|
||||
name=f"{source.name} 爆管 Benchmark",
|
||||
code=target,
|
||||
description="隔离的爆管 benchmark 模拟与 SCADA 回放数据集",
|
||||
gs_workspace=target,
|
||||
map_extent=source.map_extent,
|
||||
status="active",
|
||||
map_config=source.map_config or {},
|
||||
settings={**(source.settings or {}), **marker},
|
||||
)
|
||||
elif any((project.settings or {}).get(key) != value for key, value in marker.items()):
|
||||
raise ValueError("existing target project is not owned by this benchmark dataset")
|
||||
await repository.upsert_project_database_config(
|
||||
project.id,
|
||||
db_role="biz_data",
|
||||
db_type="postgresql",
|
||||
dsn=target_biz_dsn,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
)
|
||||
await repository.upsert_project_database_config(
|
||||
project.id,
|
||||
db_role="iot_data",
|
||||
db_type="timescaledb",
|
||||
dsn=target_iot_dsn,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
)
|
||||
existing = {member.user_id for member in await repository.list_project_members(project.id)}
|
||||
for member in await repository.list_project_members(source_id):
|
||||
if member.user_id not in existing:
|
||||
await repository.add_project_member(
|
||||
project.id, member.user_id, member.project_role
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def write_analysis_scada(
|
||||
*,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
run_id: str,
|
||||
) -> dict:
|
||||
"""Materialize one completed hydraulic run as bound SCADA observations."""
|
||||
run_uuid = UUID(str(run_id))
|
||||
with connect(business_dsn, row_factory=dict_row) as business:
|
||||
devices = business.execute(
|
||||
"""SELECT device_id,device_type,node_id,link_id,measurement_unit
|
||||
FROM asset.scada_devices
|
||||
WHERE device_type IN ('pressure','pipe_flow')
|
||||
ORDER BY device_id"""
|
||||
).fetchall()
|
||||
pressure = [row for row in devices if row["device_type"] == "pressure"]
|
||||
flows = [row for row in devices if row["device_type"] == "pipe_flow"]
|
||||
if any(not row["node_id"] or row["measurement_unit"] != "m" for row in pressure):
|
||||
raise ValueError("pressure devices must bind nodes and use metres")
|
||||
if any(not row["link_id"] or row["measurement_unit"] != "m3/h" for row in flows):
|
||||
raise ValueError("flow devices must bind links and use m3/h")
|
||||
|
||||
pressure_by_node: dict[str, list[str]] = {}
|
||||
flow_by_link: dict[str, list[str]] = {}
|
||||
for row in pressure:
|
||||
pressure_by_node.setdefault(row["node_id"], []).append(row["device_id"])
|
||||
for row in flows:
|
||||
flow_by_link.setdefault(row["link_id"], []).append(row["device_id"])
|
||||
|
||||
records: list[tuple] = []
|
||||
with connect(timescale_dsn, row_factory=dict_row) as timescale:
|
||||
node_rows = timescale.execute(
|
||||
"""SELECT time,node_id,pressure FROM analysis.node_results
|
||||
WHERE run_id=%s AND node_id=ANY(%s) ORDER BY time,node_id""",
|
||||
(run_uuid, list(pressure_by_node)),
|
||||
).fetchall()
|
||||
link_rows = timescale.execute(
|
||||
"""SELECT time,link_id,flow FROM analysis.link_results
|
||||
WHERE run_id=%s AND link_id=ANY(%s) ORDER BY time,link_id""",
|
||||
(run_uuid, list(flow_by_link)),
|
||||
).fetchall()
|
||||
for row in node_rows:
|
||||
records.extend(
|
||||
(row["time"], device_id, float(row["pressure"]), None)
|
||||
for device_id in pressure_by_node[row["node_id"]]
|
||||
)
|
||||
for row in link_rows:
|
||||
records.extend(
|
||||
(row["time"], device_id, float(row["flow"]) * 3.6, None)
|
||||
for device_id in flow_by_link[row["link_id"]]
|
||||
)
|
||||
if not records:
|
||||
raise ValueError("analysis run has no bound SCADA results")
|
||||
start = min(row[0] for row in records)
|
||||
end = max(row[0] for row in records)
|
||||
collision = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time BETWEEN %s AND %s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if collision:
|
||||
raise ValueError("SCADA target interval is not empty")
|
||||
with timescale.cursor().copy(
|
||||
"COPY scada.measurements (time,device_id,monitored_value,cleaned_value) FROM STDIN"
|
||||
) as copy:
|
||||
for record in sorted(records):
|
||||
copy.write_row(record)
|
||||
timescale.commit()
|
||||
persisted = timescale.execute(
|
||||
"SELECT count(*) count FROM scada.measurements WHERE time BETWEEN %s AND %s",
|
||||
(start, end),
|
||||
).fetchone()["count"]
|
||||
if persisted != len(records):
|
||||
raise ValueError("SCADA read-back count does not match generated records")
|
||||
samples_per_device: dict[str, int] = {}
|
||||
for _, device_id, _, _ in records:
|
||||
samples_per_device[device_id] = samples_per_device.get(device_id, 0) + 1
|
||||
if len(set(samples_per_device.values())) != 1:
|
||||
raise ValueError("bound SCADA devices have inconsistent sample counts")
|
||||
return {
|
||||
"run_id": str(run_uuid),
|
||||
"start": start.isoformat(),
|
||||
"end": end.isoformat(),
|
||||
"devices": len(samples_per_device),
|
||||
"samples_per_device": next(iter(samples_per_device.values())),
|
||||
"rows": persisted,
|
||||
"value_sha256": canonical_hash(
|
||||
[(time.isoformat(), device_id, value) for time, device_id, value, _ in sorted(records)]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def make_schedule(design) -> list[dict]:
|
||||
"""Seventeen-day slots keep fifteen clean training days plus a recovery buffer."""
|
||||
local = ZoneInfo("Asia/Shanghai")
|
||||
first_date = datetime(2024, 1, 16, tzinfo=local)
|
||||
rows = []
|
||||
pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
|
||||
for case in design.cases:
|
||||
for level in ("light", "medium", "severe"):
|
||||
source = case.occurred_at.astimezone(local)
|
||||
at = first_date + timedelta(days=17 * len(rows), hours=source.hour, minutes=source.minute)
|
||||
rows.append({"case_key": f"{case.case_id}-{level}", "case_id": case.case_id,
|
||||
"severity": level, "pipe_id": case.pipe_id,
|
||||
"area_cm2": pipes[case.pipe_id].candidate_orifice_areas_cm2[level],
|
||||
"event_at": at.isoformat(), "history_start": (at - timedelta(days=15)).isoformat(),
|
||||
"event_end": (at + timedelta(seconds=design.duration_seconds)).isoformat(),
|
||||
"reserved_end": (at + timedelta(seconds=design.duration_seconds, days=1)).isoformat()})
|
||||
previous = None
|
||||
for row in rows:
|
||||
start = datetime.fromisoformat(row["history_start"])
|
||||
if previous is not None and start <= previous:
|
||||
raise ValueError("dataset date reservations overlap")
|
||||
previous = datetime.fromisoformat(row["reserved_end"])
|
||||
if previous is not None and previous >= datetime.now(timezone.utc):
|
||||
raise ValueError("benchmark slots must remain historical")
|
||||
return rows
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Learn pressure thresholds from pre-event normal history, never burst truth."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime, timedelta
|
||||
from statistics import median
|
||||
|
||||
|
||||
def learn_pressure_thresholds(
|
||||
rows: list[dict], device_ids: list[str], cutoff: datetime, *,
|
||||
history_days: int = 15, interval_seconds: int = 900,
|
||||
relative_floor_percent: float = 3.0, noise_multiplier: float = 3.0,
|
||||
absolute_floor_m: float = 0.5,
|
||||
) -> dict:
|
||||
"""Estimate noise after removing each daily slot's median operating pattern."""
|
||||
if (cutoff.tzinfo is None or history_days < 1 or interval_seconds <= 0
|
||||
or 86400 % interval_seconds or not device_ids or len(set(device_ids)) != len(device_ids)):
|
||||
raise ValueError("an aware cutoff and whole-day sampling grid are required")
|
||||
if not all(math.isfinite(value) and value > 0 for value in (relative_floor_percent, noise_multiplier, absolute_floor_m)):
|
||||
raise ValueError("threshold floors and noise multiplier must be positive")
|
||||
count = history_days * 86400 // interval_seconds
|
||||
start = cutoff - timedelta(days=history_days)
|
||||
expected = {start + timedelta(seconds=interval_seconds * index) for index in range(count)}
|
||||
data = {device: {} for device in device_ids}
|
||||
for row in rows:
|
||||
device = row["device_id"]
|
||||
if device not in data:
|
||||
continue
|
||||
at = row["time"]
|
||||
if at.tzinfo is None or at >= cutoff:
|
||||
raise ValueError("adaptive fitting must not consume event or future observations")
|
||||
if at < start:
|
||||
continue
|
||||
value = row["monitored_value"]
|
||||
if at not in expected or at in data[device] or value is None or not math.isfinite(float(value)) or float(value) <= 0:
|
||||
raise ValueError("adaptive fitting requires distinct positive finite pressure samples on the grid")
|
||||
data[device][at] = float(value)
|
||||
insufficient = {device: len(values) for device, values in data.items() if set(values) != expected}
|
||||
if insufficient:
|
||||
return {"status": "history_not_ready", "required_samples_per_device": count,
|
||||
"available_samples": insufficient, "knowledge_cutoff": cutoff.isoformat(), "devices": {}}
|
||||
fitted = {}
|
||||
for device, values in data.items():
|
||||
slots = {}
|
||||
for at, value in values.items():
|
||||
slot = int((at - start).total_seconds()) // interval_seconds % (86400 // interval_seconds)
|
||||
slots.setdefault(slot, []).append(value)
|
||||
references = {slot: median(group) for slot, group in slots.items()}
|
||||
residuals = []
|
||||
for at, value in values.items():
|
||||
slot = int((at - start).total_seconds()) // interval_seconds % (86400 // interval_seconds)
|
||||
residuals.append(100 * (value / references[slot] - 1))
|
||||
center = median(residuals)
|
||||
robust_sigma = 1.4826 * median(abs(value - center) for value in residuals)
|
||||
fitted[device] = {"relative_drop_percent": max(relative_floor_percent, noise_multiplier * robust_sigma),
|
||||
"absolute_drop_m": absolute_floor_m, "robust_noise_percent": robust_sigma,
|
||||
"reference_pressure_m_by_slot": {str(slot): value for slot, value in references.items()}}
|
||||
return {"status": "ready", "required_samples_per_device": count,
|
||||
"knowledge_cutoff": cutoff.isoformat(), "devices": fitted,
|
||||
"reference_grid_start": start.isoformat(), "interval_seconds": interval_seconds,
|
||||
"method": "daily-slot median removal; 1.4826*MAD; thresholds frozen before event"}
|
||||
|
||||
|
||||
def evaluate_pressure_thresholds(rows: list[dict], fitted: dict, *, consecutive_samples: int = 2) -> dict:
|
||||
"""Experimental adaptive evaluator; this is not the production replay worker."""
|
||||
if fitted.get("status") != "ready" or consecutive_samples < 1:
|
||||
raise ValueError("a fitted rule and positive consecutive sample count are required")
|
||||
start = datetime.fromisoformat(fitted["reference_grid_start"])
|
||||
cutoff = datetime.fromisoformat(fitted["knowledge_cutoff"])
|
||||
step = fitted["interval_seconds"]
|
||||
states = {}
|
||||
signals = []
|
||||
for row in sorted(rows, key=lambda item: (item["time"], item["device_id"])):
|
||||
device = row["device_id"]
|
||||
if device not in fitted["devices"]:
|
||||
continue
|
||||
at, value = row["time"], row["monitored_value"]
|
||||
if at.tzinfo is None or at < cutoff or (at - start).total_seconds() % step:
|
||||
raise ValueError("evaluation samples must follow the fit cutoff on the sampling grid")
|
||||
previous, hits, signaled = states.get(device, (None, 0, False))
|
||||
if previous is not None and at <= previous:
|
||||
raise ValueError("evaluation samples must be distinct")
|
||||
if previous is None or at - previous != timedelta(seconds=step):
|
||||
hits = 0
|
||||
rule = fitted["devices"][device]
|
||||
slot = int((at - start).total_seconds()) // step % (86400 // step)
|
||||
expected = rule["reference_pressure_m_by_slot"][str(slot)]
|
||||
valid = value is not None and math.isfinite(float(value)) and float(value) > 0
|
||||
drop = expected - float(value) if valid else 0
|
||||
hit = valid and drop >= rule["absolute_drop_m"] and 100 * drop / expected >= rule["relative_drop_percent"]
|
||||
hits = hits + 1 if hit else 0
|
||||
if hits >= consecutive_samples and not signaled:
|
||||
signals.append({"device_id": device, "first_signal_at": at.isoformat(),
|
||||
"reference_pressure_m": expected, "absolute_drop_m": drop,
|
||||
"relative_drop_percent": 100 * drop / expected})
|
||||
signaled = True
|
||||
states[device] = (at, hits, signaled)
|
||||
return {"raised": bool(signals), "signals": signals,
|
||||
"scope": "experimental adaptive pressure rule only; not a worker or Agent result"}
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Read project data through metadata routing, in repeatable read-only transactions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from scripts.benchmark.contracts import (
|
||||
BurstCase, BurstSuite, canonical_json, complete_devices, fingerprint, isolation_reference,
|
||||
)
|
||||
|
||||
|
||||
async def resolve_sources(suite: BurstSuite) -> tuple[dict, dict[str, str]]:
|
||||
# Import lazily: validating a suite must not load environment credentials.
|
||||
from sqlalchemy import text
|
||||
from app.infra.db.metadb.database import SessionLocal, engine
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY"))
|
||||
repo = MetadataRepository(session)
|
||||
project = await repo.get_project_by_code(suite.project_code)
|
||||
if project is None or project.id != suite.project_id or project.status != "active":
|
||||
raise ValueError("benchmark project identity or active status does not match")
|
||||
routes = {}
|
||||
for role in ("biz_data", "iot_data"):
|
||||
route = await repo.get_project_db_routing(project.id, role)
|
||||
if route is None:
|
||||
raise ValueError("benchmark project database routing is incomplete")
|
||||
routes[role] = route.dsn
|
||||
identity = {"project_id": str(project.id), "project_code": project.code, "name": project.name}
|
||||
await session.rollback()
|
||||
return identity, routes
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def readonly_source(dsn: str):
|
||||
# Replace inherited options: never rely on application credentials being read-only.
|
||||
safe_dsn = make_conninfo(dsn, options=(
|
||||
"-c default_transaction_read_only=on -c statement_timeout=20000 "
|
||||
"-c lock_timeout=2000 -c idle_in_transaction_session_timeout=600000"
|
||||
), application_name="tjwater-benchmark-source-readonly", connect_timeout="5")
|
||||
with psycopg.connect(safe_dsn, row_factory=dict_row) as connection:
|
||||
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
if connection.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] != "on":
|
||||
raise ValueError("source transaction is not read-only")
|
||||
try:
|
||||
yield connection
|
||||
finally:
|
||||
connection.rollback()
|
||||
|
||||
|
||||
def source_identity(connection, dsn: str) -> dict:
|
||||
values = conninfo_to_dict(dsn)
|
||||
row = connection.execute(
|
||||
"SELECT current_database() AS database, current_setting('server_version_num') AS server_version, "
|
||||
"transaction_timestamp() AS observed_at"
|
||||
).fetchone()
|
||||
if row["database"] != values.get("dbname"):
|
||||
raise ValueError("connected source differs from the metadata route")
|
||||
return {
|
||||
**row,
|
||||
"read_only": True,
|
||||
"isolation": "repeatable read",
|
||||
# Equal database names on different servers are not equal routes.
|
||||
"route_sha256": fingerprint({key: values.get(key) for key in ("host", "port", "dbname")}),
|
||||
}
|
||||
|
||||
|
||||
def read_model(connection, cases: list[BurstCase]) -> dict:
|
||||
links = connection.execute(
|
||||
"SELECT l.id, l.link_type::text AS kind, l.start_node_id, l.end_node_id, "
|
||||
"COALESCE(s.status::text,p.status::text) AS status "
|
||||
"FROM network.links l LEFT JOIN network.pipes p ON p.link_id=l.id "
|
||||
"LEFT JOIN network.link_initial_settings s ON s.link_id=l.id ORDER BY l.id"
|
||||
).fetchall()
|
||||
for link in links:
|
||||
for key in ("id", "start_node_id", "end_node_id"):
|
||||
link[key] = str(link[key])
|
||||
sources = [str(row["id"]) for row in connection.execute(
|
||||
"SELECT id FROM network.nodes WHERE node_type::text IN ('reservoir','tank') ORDER BY id"
|
||||
).fetchall()]
|
||||
devices = connection.execute(
|
||||
"SELECT device_id,device_type,node_id,link_id,api_query_id,measurement_unit, "
|
||||
"transmission_frequency::text AS transmission_frequency FROM asset.scada_devices "
|
||||
"WHERE device_type::text IN ('pressure','pipe_flow','flow') ORDER BY device_id"
|
||||
).fetchall()
|
||||
for device in devices:
|
||||
for key in ("device_id", "node_id", "link_id"):
|
||||
if device[key] is not None:
|
||||
device[key] = str(device[key])
|
||||
analyses = connection.execute(
|
||||
"SELECT run_id,name,run_type,status,parameters FROM analysis.runs WHERE name=ANY(%s) "
|
||||
"ORDER BY name,created_at,run_id", ([case.reference_analysis for case in cases],),
|
||||
).fetchall()
|
||||
return {"links": links, "source_nodes": sources, "devices": devices, "reference_analyses": analyses}
|
||||
|
||||
|
||||
def readings(connection, device_ids: list[str], start: datetime, end: datetime) -> list[dict]:
|
||||
return connection.execute(
|
||||
"SELECT time,device_id,monitored_value FROM scada.measurements "
|
||||
"WHERE device_id=ANY(%s) AND time>=%s AND time<=%s ORDER BY time,device_id",
|
||||
(device_ids, start, end),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def long_history_capability(connection, suite: BurstSuite, case: BurstCase, devices: list[dict]) -> dict:
|
||||
interval = suite.sampling_interval_minutes
|
||||
expected = suite.detector_days * 1440 // interval
|
||||
start = case.occurred_at - timedelta(days=suite.detector_days) + timedelta(minutes=interval)
|
||||
mapped = {device["device_id"]: device["node_id"] for device in devices
|
||||
if device["device_type"] == "pressure" and device["node_id"] and device["api_query_id"]}
|
||||
rows = connection.execute(
|
||||
"SELECT device_id,count(*) AS count,count(DISTINCT time) AS unique_count, "
|
||||
"count(*) FILTER (WHERE monitored_value IS NOT NULL "
|
||||
"AND monitored_value::text NOT IN ('NaN','Infinity','-Infinity') "
|
||||
"AND mod(extract(epoch FROM (time-%s::timestamptz)),%s)=0) AS valid_count "
|
||||
"FROM scada.measurements WHERE device_id=ANY(%s) AND time>=%s AND time<=%s "
|
||||
"GROUP BY device_id ORDER BY device_id",
|
||||
(case.occurred_at, interval * 60, list(mapped), start, case.occurred_at),
|
||||
).fetchall()
|
||||
eligible = sorted({mapped[row["device_id"]] for row in rows
|
||||
if row["count"] == row["unique_count"] == row["valid_count"] == expected})
|
||||
return {"available": len(eligible) >= 5, "expected_samples_per_device": expected,
|
||||
"complete_pressure_nodes": eligible, "tool_node_limit": 20,
|
||||
"required_for": "optional run_burst_detection, not the short-window SCADA trigger"}
|
||||
|
||||
|
||||
def trigger_preview(suite: BurstSuite, case: BurstCase, rows: list[dict], devices: list[dict], *, normal=False) -> dict:
|
||||
"""Exercise the current pure detector; do not tick replay, clean TSDB or create tasks."""
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import detect_frame, prepare_cleaning
|
||||
|
||||
start = case.occurred_at - (timedelta(days=1) if normal else timedelta())
|
||||
pressure = [device for device in devices
|
||||
if device["device_type"] == "pressure" and device["measurement_unit"] == "m"]
|
||||
ids = {device["device_id"] for device in pressure}
|
||||
rule = suite.trigger_rule
|
||||
if len(ids) < rule.minimum_devices:
|
||||
return {"raised": False, "reason": "insufficient pressure devices with unit m"}
|
||||
config = ScadaReplayCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, f"benchmark:{case.case_id}:{normal}"),
|
||||
start_time=start, end_time=start + timedelta(minutes=suite.event_minutes + suite.sampling_interval_minutes),
|
||||
device_ids=sorted(ids), rules=[{
|
||||
"rule_id": "benchmark-pressure-drop", "device_ids": sorted(ids),
|
||||
"kind": "relative_change", "direction": "decrease", "unit": "m",
|
||||
"threshold": rule.pressure_drop_percent, "window_samples": rule.baseline_samples,
|
||||
"minimum_samples": rule.baseline_samples, "consecutive_samples": rule.consecutive_samples,
|
||||
"minimum_devices": rule.minimum_devices,
|
||||
}],
|
||||
)
|
||||
history: dict[str, list[float]] = {device: [] for device in ids}
|
||||
for row in rows:
|
||||
value = row["monitored_value"]
|
||||
if row["device_id"] in ids and row["time"] < start and value is not None and math.isfinite(float(value)):
|
||||
history[row["device_id"]].append(float(value))
|
||||
document = {"replay_id": str(config.request_id), "devices": pressure,
|
||||
"histories": {key: values[-rule.baseline_samples:] for key, values in history.items()}}
|
||||
changes = []
|
||||
for at, group in itertools.groupby(rows, key=lambda row: row["time"]):
|
||||
if at < start:
|
||||
continue
|
||||
frame = [{**row, "source_hash": fingerprint([row["time"], row["device_id"], row["monitored_value"]])}
|
||||
for row in group if row["device_id"] in ids]
|
||||
document, events = detect_frame(document, prepare_cleaning(frame, config), config, at)
|
||||
changes.extend(event for event in events if event.get("category") != "data_quality")
|
||||
raised = [event for event in changes if event["change"] == "raised"]
|
||||
return {"raised": bool(raised), "first_signal_at": raised[0]["occurred_at"] if raised else None,
|
||||
"event_count": len(raised), "warm_pressure_devices": sum(len(v) >= rule.baseline_samples for v in history.values()),
|
||||
"scope": "pure short-window rule preview; not worker/Agent end-to-end acceptance"}
|
||||
|
||||
|
||||
def inspect_case(iot_connection, suite: BurstSuite, case: BurstCase, model: dict) -> dict:
|
||||
devices = model["devices"]
|
||||
ids = [device["device_id"] for device in devices]
|
||||
warmup = timedelta(minutes=suite.trigger_rule.baseline_samples * suite.sampling_interval_minutes)
|
||||
end = case.occurred_at + timedelta(minutes=suite.event_minutes)
|
||||
event_rows = readings(iot_connection, ids, case.occurred_at - warmup, end)
|
||||
normal_rows = readings(iot_connection, ids, case.occurred_at - timedelta(days=1) - warmup,
|
||||
end - timedelta(days=1))
|
||||
complete = set(complete_devices(event_rows, case.occurred_at, end, suite.sampling_interval_minutes))
|
||||
complete &= set(complete_devices(normal_rows, case.occurred_at - timedelta(days=1),
|
||||
end - timedelta(days=1), suite.sampling_interval_minutes))
|
||||
pressure_ids = {device["device_id"] for device in devices if device["device_type"] == "pressure"}
|
||||
matching = [run for run in model["reference_analyses"] if run["name"] == case.reference_analysis
|
||||
and run["run_type"] == "burst_analysis" and run["status"] == "completed"
|
||||
and case.pipe_id in [str(value) for value in run["parameters"].get("burst_ID", [])]]
|
||||
topology = isolation_reference(model["links"], model["source_nodes"], case.pipe_id)
|
||||
blockers = []
|
||||
if case.observation_provenance != "declared_historical_replay":
|
||||
blockers.append("event_observations_require_isolated_generation")
|
||||
if not matching:
|
||||
blockers.append("matching_completed_reference_analysis_missing")
|
||||
if len(complete & pressure_ids) < 5:
|
||||
blockers.append("fewer_than_five_complete_event_and_control_pressure_devices")
|
||||
if topology["status"] in {"target_missing", "sources_missing"}:
|
||||
blockers.append(topology["status"])
|
||||
event_signal = trigger_preview(suite, case, event_rows, devices)
|
||||
normal_signal = trigger_preview(suite, case, normal_rows, devices, normal=True)
|
||||
scada_blockers = list(blockers)
|
||||
if not event_signal["raised"]:
|
||||
scada_blockers.append("short_window_event_trigger_not_raised")
|
||||
if normal_signal["raised"]:
|
||||
scada_blockers.append("normal_control_false_positive")
|
||||
return {
|
||||
"case_id": case.case_id, "pipe_id": case.pipe_id, "occurred_at": case.occurred_at,
|
||||
"observation_provenance": case.observation_provenance,
|
||||
"data_candidate_available": not blockers,
|
||||
"blockers": blockers,
|
||||
"complete_event_and_control_pressure_devices": sorted(complete & pressure_ids),
|
||||
"window_sha256": fingerprint({"event": event_rows, "normal": normal_rows}),
|
||||
"optional_long_history_detection": long_history_capability(iot_connection, suite, case, devices),
|
||||
"event_signal_preview": event_signal, "normal_signal_preview": normal_signal,
|
||||
"topology_reference": topology,
|
||||
"variants": [
|
||||
{"case_id": case.case_id + "-WO", "data_candidate_available": not blockers,
|
||||
"blockers": list(blockers)},
|
||||
{"case_id": case.case_id + "-SCADA", "data_candidate_available": not scada_blockers,
|
||||
"blockers": scada_blockers},
|
||||
],
|
||||
"acceptance_pending": ["numerical_reference_review", "approved_equivalent_candidates",
|
||||
"public_leakage_input_or_declared_location_limitation",
|
||||
"isolated_runtime_and_cutoff_verification"],
|
||||
}
|
||||
|
||||
|
||||
def export_readings(connection, path: Path, suite: BurstSuite, cases: list[BurstCase], devices: list[dict]) -> dict:
|
||||
"""Export only burst-related devices and the selected history/event window."""
|
||||
start = min(case.occurred_at for case in cases) - timedelta(days=suite.detector_days)
|
||||
end = max(case.occurred_at for case in cases) + timedelta(minutes=suite.event_minutes)
|
||||
ids = [device["device_id"] for device in devices]
|
||||
count = 0
|
||||
digest = hashlib.sha256()
|
||||
path.touch(mode=0o600, exist_ok=False)
|
||||
with path.open("wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as output:
|
||||
with connection.cursor(name="benchmark_raw_export") as cursor:
|
||||
cursor.execute(
|
||||
"SELECT time,device_id,monitored_value FROM scada.measurements "
|
||||
"WHERE device_id=ANY(%s) AND time>=%s AND time<=%s ORDER BY time,device_id", (ids, start, end),
|
||||
)
|
||||
for row in cursor:
|
||||
count += 1
|
||||
if count > 1_000_000:
|
||||
raise ValueError("selected observation export exceeds one million records")
|
||||
line = (canonical_json(row) + "\n").encode()
|
||||
digest.update(line)
|
||||
output.write(line)
|
||||
return {"file": str(path.name), "records": count, "content_sha256": digest.hexdigest(),
|
||||
"start": start, "end": end, "origin": "existing_database_readonly_export",
|
||||
"visibility": "private; contains future samples and must not be exposed directly to Agent"}
|
||||
Reference in New Issue
Block a user