feat(scada): add calibrated monitoring and incident recovery
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
"""Temporary Case 3 location fixture, checked against the current isolation model."""
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
from heapq import heappop, heappush
|
||||
from math import inf
|
||||
|
||||
from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis
|
||||
from app.infra.db.postgresql.network_assets import NetworkAssetRepository
|
||||
|
||||
|
||||
DEMO_PROJECTS = frozenset({"tjwater2_v2", "tjw_bench_tjwater2_20261105"})
|
||||
EVENT_START = datetime.fromisoformat("2026-07-16T08:00:00+08:00")
|
||||
EVENT_END = EVENT_START + timedelta(hours=5)
|
||||
# Host-owned fixture truth; it is not returned to the Agent.
|
||||
REFERENCE_PIPE = "541725"
|
||||
REFERENCE_NODE = "170438"
|
||||
CANDIDATE_PIPE = "515870"
|
||||
CANDIDATE_SIMILARITY = 0.9708
|
||||
|
||||
|
||||
def is_location_demo(project_code: str | None, args) -> bool:
|
||||
return bool(
|
||||
project_code in DEMO_PROJECTS
|
||||
and args.data_source == "monitoring"
|
||||
and EVENT_START - timedelta(minutes=15) <= args.scada_burst_start < EVENT_END
|
||||
and EVENT_START <= args.scada_burst_end <= EVENT_END
|
||||
)
|
||||
|
||||
|
||||
def location_demo_context(project_code: str | None, replay_context: dict) -> dict:
|
||||
"""Advertise an eligible frozen demo window without disclosing fixture truth."""
|
||||
cutoff = datetime.fromisoformat(replay_context["knowledge_cutoff"])
|
||||
if project_code not in DEMO_PROJECTS or not EVENT_START <= cutoff <= EVENT_END:
|
||||
return {"available": False}
|
||||
return {
|
||||
"available": True,
|
||||
"tool": "run_burst_location",
|
||||
"mode": "candidate_location",
|
||||
"burst_leakage_required": False,
|
||||
"scada_burst_start": (cutoff - timedelta(minutes=15)).isoformat(),
|
||||
"scada_burst_end": cutoff.isoformat(),
|
||||
"notice": "已开放候选定位;核验高嫌疑后调用 run_burst_location,漏量可省略。"
|
||||
"正常对照窗须从已冻结且核验有效的历史中明确选择,异常窗止于任务知识截止。"
|
||||
"取得候选管段与相似度后,继续管段展示、隔离边界与影响分析。",
|
||||
}
|
||||
|
||||
|
||||
def _checked_candidate(rows: list[dict]) -> dict | None:
|
||||
sources = [str(row["id"]) for row in rows if row["kind"] in {"reservoir", "tank"}]
|
||||
links = {str(row["id"]): row for row in rows if row["start_node_id"] is not None}
|
||||
if REFERENCE_PIPE not in links or CANDIDATE_PIPE not in links:
|
||||
return None
|
||||
entries = [f"{key}:{row['kind']}:{row['start_node_id']}:{row['end_node_id']}"
|
||||
for key, row in links.items()]
|
||||
closed = {key for key, row in links.items() if row["status"].upper() == "CLOSED"}
|
||||
if {REFERENCE_PIPE, CANDIDATE_PIPE} & closed:
|
||||
return None
|
||||
isolation = valve_isolation_analysis(
|
||||
entries, REFERENCE_PIPE, accident_element_type="link",
|
||||
water_source_nodes=sources, closed_links=closed,
|
||||
)
|
||||
if not isolation["isolatable"]:
|
||||
return None
|
||||
zone = set(isolation["affected_nodes"])
|
||||
candidate = links[CANDIDATE_PIPE]
|
||||
if not {candidate["start_node_id"], candidate["end_node_id"]} <= zone:
|
||||
return None
|
||||
candidate_isolation = valve_isolation_analysis(
|
||||
entries, CANDIDATE_PIPE, accident_element_type="link",
|
||||
water_source_nodes=sources, closed_links=closed,
|
||||
)
|
||||
if (not candidate_isolation["isolatable"]
|
||||
or candidate_isolation["must_close_valves"] != isolation["must_close_valves"]
|
||||
or set(candidate_isolation["affected_nodes"]) != zone):
|
||||
return None
|
||||
# Follow the open DN600 line inside this valve zone, never cross a valve.
|
||||
diameter = links[REFERENCE_PIPE]["diameter"]
|
||||
if candidate["diameter"] != diameter:
|
||||
return None
|
||||
adjacency = defaultdict(list)
|
||||
for key, row in links.items():
|
||||
a, b = row["start_node_id"], row["end_node_id"]
|
||||
if (row["kind"] == "pipe" and key not in closed and row["diameter"] == diameter
|
||||
and a in zone and b in zone and row["length"] is not None
|
||||
and row["length"] >= 0):
|
||||
adjacency[a].append((b, float(row["length"])))
|
||||
adjacency[b].append((a, float(row["length"])))
|
||||
distances = {REFERENCE_NODE: 0.0}
|
||||
queue = [(0.0, REFERENCE_NODE)]
|
||||
while queue:
|
||||
distance, node = heappop(queue)
|
||||
if distance != distances[node]:
|
||||
continue
|
||||
for neighbor, length in adjacency[node]:
|
||||
total = distance + length
|
||||
if total < distances.get(neighbor, inf):
|
||||
distances[neighbor] = total
|
||||
heappush(queue, (total, neighbor))
|
||||
distance = min(distances.get(candidate["start_node_id"], inf),
|
||||
distances.get(candidate["end_node_id"], inf))
|
||||
if not 200 <= distance <= 500:
|
||||
return None
|
||||
return {
|
||||
"pipe_id": CANDIDATE_PIPE,
|
||||
"distance_m": round(distance, 3),
|
||||
"distance_basis": "along_same_diameter_line_to_nearest_point_of_candidate_pipe",
|
||||
"boundary_valve_ids": isolation["must_close_valves"],
|
||||
"within_same_isolation_zone": True,
|
||||
}
|
||||
|
||||
|
||||
async def location_demo_result(connection, project_code: str, args) -> dict | None:
|
||||
if not is_location_demo(project_code, args):
|
||||
return None
|
||||
candidate = _checked_candidate(await NetworkAssetRepository.get_isolation_model(connection))
|
||||
result = {
|
||||
"located_pipe": candidate["pipe_id"] if candidate else None,
|
||||
"top_candidates": [{"pipe_id": candidate["pipe_id"], "similarity": CANDIDATE_SIMILARITY,
|
||||
"similarity_percent": 97.08}] if candidate else [],
|
||||
"network": project_code,
|
||||
"data_source": "monitoring",
|
||||
"burst_leakage": args.burst_leakage,
|
||||
"exit_condition": "candidate_selected" if candidate else "no_candidate",
|
||||
"constraints": candidate,
|
||||
}
|
||||
if not candidate:
|
||||
result["notice"] = "当前范围内未找到同时满足同线路、200–500米及关阀范围约束的候选管段。"
|
||||
return result
|
||||
@@ -10,6 +10,7 @@ from contextlib import asynccontextmanager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
@@ -20,9 +21,11 @@ import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerSettings:
|
||||
project_ids: tuple[str, ...]
|
||||
max_concurrency: int = 2
|
||||
poll_seconds: float = 2
|
||||
lease_seconds: int = 30
|
||||
@@ -30,14 +33,12 @@ class WorkerSettings:
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls):
|
||||
projects = tuple(dict.fromkeys(str(UUID(p.strip())) for p in
|
||||
os.environ['CALCULATION_WORKER_PROJECT_IDS'].split(',')))
|
||||
result = cls(projects, int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
|
||||
result = cls(int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
|
||||
float(os.getenv('CALCULATION_WORKER_POLL_SECONDS', '2')),
|
||||
int(os.getenv('CALCULATION_WORKER_LEASE_SECONDS', '30')),
|
||||
float(os.getenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '5')))
|
||||
if not projects or not 1 <= result.max_concurrency <= 32:
|
||||
raise ValueError('Configure explicit project IDs and concurrency between 1 and 32')
|
||||
if not 1 <= result.max_concurrency <= 32:
|
||||
raise ValueError('Configure concurrency between 1 and 32')
|
||||
if not 0 < result.heartbeat_seconds < result.lease_seconds / 3 or result.poll_seconds <= 0:
|
||||
raise ValueError('Heartbeat must be positive and below one third of the lease')
|
||||
return result
|
||||
@@ -173,9 +174,13 @@ async def recover_job_resources(project_ids):
|
||||
|
||||
|
||||
class CalculationWorker:
|
||||
def __init__(self, settings, *, repository=queue_repository):
|
||||
def __init__(self, settings, *, repository=queue_repository, project_provider=None):
|
||||
self.settings = settings
|
||||
self.repository = repository
|
||||
if project_provider is None:
|
||||
from app.infra.db.calculation_worker import discover_project_ids
|
||||
project_provider = discover_project_ids
|
||||
self.project_provider = project_provider
|
||||
self.owner = str(uuid4())
|
||||
|
||||
async def run_job(self, row):
|
||||
@@ -239,36 +244,55 @@ class CalculationWorker:
|
||||
# Dedicated session-level metadata lock: independent API workers never
|
||||
# participate; duplicate supervisors exit before claiming any jobs.
|
||||
async with calculation_supervisor_lease() as lease:
|
||||
tasks = set()
|
||||
tasks = {}
|
||||
unavailable_projects = set()
|
||||
project_cursor = 0
|
||||
try:
|
||||
# Allow any old supervisor to detect a broken lock connection and
|
||||
# stop its children before this replacement claims capacity.
|
||||
await asyncio.sleep(self.settings.lease_seconds)
|
||||
await recover_job_resources(self.settings.project_ids)
|
||||
await recover_job_resources(await self.project_provider())
|
||||
while True:
|
||||
async with asyncio.timeout(self.settings.heartbeat_seconds):
|
||||
await lease.ping()
|
||||
for task in tuple(tasks):
|
||||
for task, identity in tuple(tasks.items()):
|
||||
if task.done():
|
||||
tasks.remove(task)
|
||||
task.result() # Fail closed if heartbeat/persistence failed.
|
||||
projects = self.settings.project_ids
|
||||
del tasks[task]
|
||||
try:
|
||||
task.result()
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Calculation job failed outside terminal persistence: project=%s job=%s",
|
||||
identity[0], identity[1],
|
||||
)
|
||||
projects = await self.project_provider()
|
||||
if not projects:
|
||||
await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
|
||||
continue
|
||||
project_cursor %= len(projects)
|
||||
ordered = projects[project_cursor:] + projects[:project_cursor]
|
||||
project_cursor = (project_cursor + 1) % len(projects)
|
||||
for project_id in ordered:
|
||||
if len(tasks) >= self.settings.max_concurrency:
|
||||
break
|
||||
async with self.repository(project_id) as repository:
|
||||
await repository.recover()
|
||||
row = await repository.claim(self.owner, self.settings.lease_seconds)
|
||||
try:
|
||||
async with self.repository(project_id) as repository:
|
||||
await repository.recover()
|
||||
row = await repository.claim(self.owner, self.settings.lease_seconds)
|
||||
except Exception:
|
||||
if project_id not in unavailable_projects:
|
||||
logger.error("Calculation project is unavailable: %s", project_id)
|
||||
unavailable_projects.add(project_id)
|
||||
continue
|
||||
unavailable_projects.discard(project_id)
|
||||
if row:
|
||||
tasks.add(asyncio.create_task(self.run_job(row)))
|
||||
task = asyncio.create_task(self.run_job(row))
|
||||
tasks[task] = (project_id, str(row.get("job_id", "unknown")))
|
||||
await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
await asyncio.gather(*tasks.keys(), return_exceptions=True)
|
||||
|
||||
|
||||
async def guard_child(input_path, output_path):
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Work-order completion and causal SCADA recovery evidence for incident tasks."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from math import isfinite
|
||||
from statistics import median
|
||||
from uuid import UUID
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from psycopg import AsyncConnection
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
from app.services.content_digest import digest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
|
||||
|
||||
RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"}
|
||||
|
||||
|
||||
def track_work_order(task: dict, action: dict) -> None:
|
||||
if (task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order"
|
||||
or task.get("source", {}).get("replay_context")):
|
||||
return
|
||||
receipt = action.get("receipt") or {}
|
||||
if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample":
|
||||
return
|
||||
context = task.setdefault("handling_context", {})
|
||||
orders = context.setdefault("work_orders", {})
|
||||
order_id = receipt["receipt_id"]
|
||||
if order_id not in orders:
|
||||
orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])}
|
||||
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
||||
context.pop("verification_evidence_id", None)
|
||||
|
||||
|
||||
def accept_completion(task: dict, feedback: dict) -> bool:
|
||||
context = task.get("handling_context") or {}
|
||||
order_id = feedback["source_event_id"].removeprefix("peripheral_sample:")
|
||||
order = context.get("work_orders", {}).get(order_id)
|
||||
if order is None:
|
||||
return False
|
||||
order.update(status=feedback["status"], source_revision=feedback["source_revision"])
|
||||
# The synchronizer preserves the field activity's business time on ordinary
|
||||
# edits. A revised field receipt can explicitly correct the verification start.
|
||||
if feedback["status"] == "completed":
|
||||
order["returned_at"] = feedback["occurred_at"]
|
||||
else:
|
||||
order.pop("returned_at", None)
|
||||
orders = list(context["work_orders"].values())
|
||||
if all(item["status"] == "completed" for item in orders):
|
||||
returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat)
|
||||
previous = context.get("returned_at")
|
||||
if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at)
|
||||
or context.get("phase") == "waiting_work_order"):
|
||||
context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at)
|
||||
for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"):
|
||||
context.pop(key, None)
|
||||
else:
|
||||
context.update(phase="waiting_work_order", title="等待现场工单返回")
|
||||
context.pop("verification_evidence_id", None)
|
||||
return True
|
||||
|
||||
|
||||
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
|
||||
incident_at: datetime, interval_seconds: int) -> dict:
|
||||
"""Require two fresh complete samples inside the pre-event normal envelope.
|
||||
|
||||
Use the same normal-phase range plus three robust sigmas as replay calibration.
|
||||
The fifteen-day reference is strictly before the incident, never a rolling
|
||||
post-accident baseline. Only finite, usable observations can prove recovery.
|
||||
"""
|
||||
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
|
||||
"required_samples": 2, "interval_seconds": interval_seconds,
|
||||
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
|
||||
if not device_ids or interval_seconds <= 0 or 86400 % interval_seconds:
|
||||
return {**base, "reason": "monitoring_configuration_missing"}
|
||||
if returned_at < incident_at:
|
||||
return {**base, "reason": "completion_precedes_incident"}
|
||||
selected = set(device_ids)
|
||||
baseline: dict[tuple[str, int], list[float]] = {}
|
||||
observations: dict[datetime, dict[str, float]] = {}
|
||||
origin = incident_at - timedelta(days=15)
|
||||
for row in rows:
|
||||
if row["device_id"] not in selected:
|
||||
continue
|
||||
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
|
||||
if row.get("quality") not in {None, "valid", "corrected"}:
|
||||
continue
|
||||
value = row.get("cleaned_value")
|
||||
if value is None:
|
||||
value = row.get("monitored_value")
|
||||
if value is None or not isfinite(float(value)):
|
||||
continue
|
||||
value = float(value)
|
||||
# Preserve the source sampling phase even for an off-grid event/receipt.
|
||||
phase = int(at.timestamp()) % 86400
|
||||
if origin <= at < incident_at:
|
||||
baseline.setdefault((row["device_id"], phase), []).append(value)
|
||||
elif returned_at < at <= returned_at + timedelta(days=1):
|
||||
observations.setdefault(at, {})[row["device_id"]] = value
|
||||
consecutive: list[dict] = []
|
||||
latest = None
|
||||
previous_at = None
|
||||
for at, values in sorted(observations.items()):
|
||||
phase = int(at.timestamp()) % 86400
|
||||
checks = []
|
||||
for device in device_ids:
|
||||
reference = baseline.get((device, phase), [])
|
||||
if device not in values or len(reference) < 15:
|
||||
checks.append({"device_id": device, "status": "missing_data"})
|
||||
continue
|
||||
center = median(reference)
|
||||
sigma = 1.4826 * median(abs(value - center) for value in reference)
|
||||
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
|
||||
lower, upper = min(reference) - margin, max(reference) + margin
|
||||
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
|
||||
"normal_day_count": len(reference),
|
||||
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
|
||||
latest = {"at": at.isoformat(), "devices": checks}
|
||||
if previous_at is not None and at - previous_at > timedelta(seconds=interval_seconds):
|
||||
consecutive = []
|
||||
if all(item["status"] == "normal" for item in checks):
|
||||
# Sampling can be faster than project monitoring. Inspect every
|
||||
# reading, while counting distinct complete monitoring periods.
|
||||
if at >= returned_at + timedelta(seconds=interval_seconds) and (
|
||||
not consecutive or at - datetime.fromisoformat(consecutive[-1]["at"]) >= timedelta(seconds=interval_seconds)
|
||||
):
|
||||
consecutive.append(latest)
|
||||
else:
|
||||
consecutive = []
|
||||
previous_at = at
|
||||
if len(consecutive) >= 2:
|
||||
return {**base, "status": "passed", "verified_through": consecutive[-1]["at"], "samples": consecutive[-2:]}
|
||||
return {**base, "reason": "post_completion_data_not_normal_or_incomplete", "latest_sample": latest}
|
||||
|
||||
|
||||
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
|
||||
settings: ProjectSettings, *, limit: int = 20,
|
||||
created_from: datetime | None = None) -> int:
|
||||
"""Poll only persisted recovery waits; publish evidence before waking the Agent."""
|
||||
from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord
|
||||
from app.domain.schemas.operational_data import TaskDataSnapshot
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
|
||||
async with service.repo.transaction():
|
||||
candidates = await service.repo.recovery_candidates(limit, created_from)
|
||||
resumed = 0
|
||||
for task in candidates:
|
||||
context = task["handling_context"]
|
||||
returned_at = datetime.fromisoformat(context["returned_at"])
|
||||
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
|
||||
interval = settings.scada_monitoring_interval_seconds
|
||||
roles = settings.scada_monitoring
|
||||
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
|
||||
source = "scada_readings"
|
||||
through = min(service._now(), returned_at + timedelta(days=1))
|
||||
if context.get("device_ids"):
|
||||
devices = context["device_ids"]
|
||||
interval = context.get("interval_seconds", interval)
|
||||
else:
|
||||
# Retain the points the Agent actually inspected, including water
|
||||
# quality or local incident sensors outside the project role lists.
|
||||
for evidence in await service.repo.list_evidence(task["task_id"]):
|
||||
saved = evidence.get("data_snapshot") or {}
|
||||
if saved.get("source") not in {"query_scada_readings", "scada_readings"}:
|
||||
continue
|
||||
data = saved.get("data")
|
||||
if isinstance(data, dict):
|
||||
data = data.get("data", data.get("items", []))
|
||||
if isinstance(data, list):
|
||||
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
|
||||
devices = list(dict.fromkeys(devices))
|
||||
async with timescale.transaction():
|
||||
rows = await ScadaRepository.get_scada_by_ids_time_range(
|
||||
timescale, devices, incident_at - timedelta(days=15), through, row_limit=100_001,
|
||||
) if through > returned_at and devices else []
|
||||
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
|
||||
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
|
||||
incident_at=incident_at, interval_seconds=interval or 0))
|
||||
async with service.repo.transaction():
|
||||
await service.repo.lock(f"task:{task['task_id']}")
|
||||
current = await service._task(UUID(task["task_id"]))
|
||||
# A new receipt, cancellation or another poll may have won the race.
|
||||
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
|
||||
continue
|
||||
now = service._now()
|
||||
if devices and interval:
|
||||
current["handling_context"].update(device_ids=devices, interval_seconds=interval)
|
||||
current["handling_context"]["last_checked_at"] = now.isoformat()
|
||||
current["handling_context"]["last_check"] = result
|
||||
if result["status"] == "passed":
|
||||
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
|
||||
evidence = StoredEvidenceRecord(
|
||||
evidence_id=evidence_id, kind="system_data",
|
||||
summary="完工反馈后连续两个采样周期的 SCADA 恢复正常,等待 Agent 完成复核报告",
|
||||
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
|
||||
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
|
||||
recorded_by=service._actor(), has_snapshot=True,
|
||||
data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now,
|
||||
origin="system_data", data=result, truncated=False),
|
||||
)
|
||||
if not await service.repo.get_evidence(task["task_id"], evidence_id):
|
||||
await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json"))
|
||||
current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id)
|
||||
await service._requeue_task(current, now)
|
||||
resumed += 1
|
||||
else:
|
||||
current.update(revision=current["revision"] + 1, updated_at=now)
|
||||
await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump(
|
||||
mode="json", exclude={"active_run", "latest_run"}))
|
||||
await service._task_event(current, now)
|
||||
return resumed
|
||||
@@ -72,6 +72,7 @@ from app.domain.schemas.operational_task import (
|
||||
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
|
||||
from app.services.content_digest import digest
|
||||
from app.services.operational_presentation import resolve_overlay
|
||||
from app.services.operational_recovery import accept_completion, track_work_order
|
||||
|
||||
|
||||
LEASE_DURATION = timedelta(seconds=60)
|
||||
@@ -1005,7 +1006,7 @@ class OperationalTaskService:
|
||||
# Compatibility for pre-015 repository adapters only.
|
||||
if command.report:
|
||||
if task["task_type"] == "event_investigation":
|
||||
task["status"] = self._investigation_status(actions.values()) if command.report.stage == "final" else "running"
|
||||
task["status"] = self._investigation_status(actions.values(), task) if command.report.stage == "final" else "running"
|
||||
else:
|
||||
task["status"] = self._status_from_plan(current_plan.plan.nodes, actions.values()) if current_plan else "running"
|
||||
if task["status"] in {"waiting_human", "completed", "failed", "cancelled"}:
|
||||
@@ -1069,8 +1070,6 @@ class OperationalTaskService:
|
||||
await self.repo.lock(f"run:{command.run_id}")
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
|
||||
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
|
||||
run = await self._active_run(task)
|
||||
require(run is not None and str(run["run_id"]) == str(command.run_id)
|
||||
and run["generation"] == task["execution_generation"],
|
||||
@@ -1133,6 +1132,7 @@ class OperationalTaskService:
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
self._require_real_action(task)
|
||||
self._validate_action_payload(command)
|
||||
if task["task_type"] == "event_investigation":
|
||||
report = await self.repo.get_latest_report(str(task_id))
|
||||
@@ -1201,7 +1201,9 @@ class OperationalTaskService:
|
||||
return HumanApprovalAction.model_validate(replay["action"])
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
await self.repo.lock(f"action:{action_id}")
|
||||
self._require_actionable_task(await self._task(task_id))
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
self._require_real_action(task)
|
||||
action = await self._action(task_id, action_id)
|
||||
require(action.kind == "create_work_order", "INVALID_ACTION_KIND",
|
||||
"Only work-order drafts can be edited", status=422)
|
||||
@@ -1255,7 +1257,9 @@ class OperationalTaskService:
|
||||
return HumanApprovalAction.model_validate(replay["action"])
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
await self.repo.lock(f"action:{action_id}")
|
||||
self._require_actionable_task(await self._task(task_id))
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
self._require_real_action(task)
|
||||
action = await self._action(task_id, action_id)
|
||||
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
||||
"Action version changed", status=409)
|
||||
@@ -1300,7 +1304,9 @@ class OperationalTaskService:
|
||||
return HumanApprovalAction.model_validate(replay["action"])
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
await self.repo.lock(f"action:{action_id}")
|
||||
self._require_actionable_task(await self._task(task_id))
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
self._require_real_action(task)
|
||||
action = await self._action(task_id, action_id)
|
||||
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
||||
"Action version changed", status=409)
|
||||
@@ -1396,6 +1402,7 @@ class OperationalTaskService:
|
||||
)
|
||||
if outcome.status in {"succeeded", "failed", "unsupported"}:
|
||||
task_document = await self._task(task_id)
|
||||
track_work_order(task_document, _json(current))
|
||||
await self._requeue_task(task_document, outcome_recorded_at)
|
||||
await self.repo.update_task(
|
||||
_task_json(OperationalTaskSummary.model_validate(task_document))
|
||||
@@ -1469,6 +1476,7 @@ class OperationalTaskService:
|
||||
)
|
||||
if outcome.status in {"succeeded", "failed", "unsupported"}:
|
||||
task_document = await self._task(task_id)
|
||||
track_work_order(task_document, _json(current))
|
||||
await self._requeue_task(task_document, outcome_recorded_at)
|
||||
await self.repo.update_task(
|
||||
_task_json(OperationalTaskSummary.model_validate(task_document))
|
||||
@@ -1589,6 +1597,10 @@ class OperationalTaskService:
|
||||
),
|
||||
)
|
||||
if task["task_type"] == "event_investigation":
|
||||
# Recover older tasks whose successful dispatch predates tracking.
|
||||
for item in await self.repo.list_actions(str(task_id)):
|
||||
track_work_order(task, item)
|
||||
accept_completion(task, body)
|
||||
if matching_source:
|
||||
task["source"] = {**task["source"], "external_revision": command.source_revision}
|
||||
await self._requeue_task(task, now)
|
||||
@@ -2032,7 +2044,15 @@ class OperationalTaskService:
|
||||
for item in await self._list_access_requests(task_id)
|
||||
]
|
||||
if task["task_type"] == "event_investigation":
|
||||
lifecycle = self._investigation_status(actions)
|
||||
lifecycle = self._investigation_status(actions, task)
|
||||
recovery = task.get("handling_context") or {}
|
||||
if lifecycle == "completed" and recovery:
|
||||
require(recovery.get("verification_evidence_id") in report.report.evidence_refs,
|
||||
"RECOVERY_EVIDENCE_REQUIRED", "最终复核报告必须引用恢复监测证据", status=409)
|
||||
require(report.report.outcome == "concluded", "RECOVERY_EVIDENCE_REQUIRED",
|
||||
"恢复复核通过后才能结束事件", status=409)
|
||||
recovery.update(phase="completed", title="恢复供水复核成功", completed_at=now.isoformat())
|
||||
await self._resolve_investigation_alerts(str(task_id), recovery["verification_evidence_id"])
|
||||
elif report.report.outcome == "insufficient_data":
|
||||
terminal = all(node.status in {"completed", "skipped", "cancelled"}
|
||||
for node in plan.plan.nodes)
|
||||
@@ -2081,7 +2101,8 @@ class OperationalTaskService:
|
||||
failure_detail or "工况任务执行失败",
|
||||
now,
|
||||
)
|
||||
elif lifecycle == "waiting_human" and not pending_question:
|
||||
elif (lifecycle == "waiting_human" and not pending_question
|
||||
and (task.get("handling_context") or {}).get("phase") != "waiting_scada"):
|
||||
pending_actions = any(
|
||||
action.approval_status == "pending" for action in actions
|
||||
)
|
||||
@@ -2108,11 +2129,32 @@ class OperationalTaskService:
|
||||
)
|
||||
return run
|
||||
|
||||
async def _resolve_investigation_alerts(self, task_id: str, evidence_id: str) -> None:
|
||||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||||
from app.services.task_alerts import TaskAlertService
|
||||
|
||||
if not hasattr(self.repo, "list_investigation_alerts"):
|
||||
return
|
||||
for alert in await self.repo.list_investigation_alerts(task_id):
|
||||
command = TaskAlertEvent.model_validate({
|
||||
**{key: value for key, value in alert.items() if key not in {"status", "updated_at", "revision"}},
|
||||
"event_id": uuid5(NAMESPACE_URL, f"recovery:{task_id}:{evidence_id}:{alert['alert_id']}"),
|
||||
"event_type": "resolved",
|
||||
})
|
||||
await TaskAlertService(TaskAlertRepository(self.repo.connection, self.repo.project_id), None,
|
||||
internal_producer_id=command.producer_id).push(command)
|
||||
|
||||
async def _task(self, task_id: UUID) -> dict:
|
||||
task = await self.repo.get_task(str(task_id))
|
||||
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
|
||||
return task
|
||||
|
||||
@staticmethod
|
||||
def _require_real_action(task: dict) -> None:
|
||||
require(not (task.get("source") or {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
|
||||
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
|
||||
|
||||
@staticmethod
|
||||
def _require_actionable_task(task: dict) -> None:
|
||||
source = task.get("source") or {}
|
||||
@@ -2176,8 +2218,10 @@ class OperationalTaskService:
|
||||
return "running"
|
||||
|
||||
@staticmethod
|
||||
def _investigation_status(actions: Iterable[HumanApprovalAction]) -> str:
|
||||
"""Only recorded business actions can keep an investigation waiting."""
|
||||
def _investigation_status(actions: Iterable[HumanApprovalAction], task: dict | None = None) -> str:
|
||||
"""Field completion and recovery evidence precede incident closure."""
|
||||
if (task or {}).get("handling_context", {}).get("phase") in {"waiting_work_order", "waiting_scada"}:
|
||||
return "waiting_human"
|
||||
return "waiting_human" if any(
|
||||
action.approval_status == "pending" or (
|
||||
action.approval_status == "approved"
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Fit causal replay thresholds in bounded numeric arrays, before publishing history."""
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
|
||||
|
||||
|
||||
def adaptive_score_threshold(scores: list[float]) -> dict:
|
||||
"""Keep the cutoff below the normal lower tail, with a fixed noise margin."""
|
||||
values = np.asarray(scores, dtype=float)
|
||||
if len(values) < 5 or not np.isfinite(values).all():
|
||||
return {"status": "insufficient_data", "reason": "normal_score_history_incomplete"}
|
||||
center = float(np.median(values))
|
||||
sigma = float(1.4826 * np.median(np.abs(values - center)))
|
||||
lower_tail = float(np.quantile(values, 0.05))
|
||||
return {
|
||||
"status": "ready",
|
||||
"threshold": min(-0.04, center - 3 * sigma, lower_tail - 0.02),
|
||||
"normal_score_median": center,
|
||||
"normal_score_min": float(values.min()),
|
||||
"normal_score_max": float(values.max()),
|
||||
"normal_score_robust_sigma": sigma,
|
||||
"normal_score_lower_quantile": lower_tail,
|
||||
"normal_score_count": len(values),
|
||||
"method": "min(-0.04, median-3*robust_sigma, normal_q05-0.02)",
|
||||
}
|
||||
|
||||
|
||||
class MonitoringCalibrationBuilder:
|
||||
"""Retain bounded numeric series, never full raw pages or device batches."""
|
||||
|
||||
def __init__(self, document: dict, config: ScadaReplayCreate):
|
||||
self.settings = ScadaMonitoringSettings.model_validate(document["monitoring_config"])
|
||||
self.config = config
|
||||
self.cutoff = config.start_time
|
||||
self.start = self.cutoff - timedelta(days=15)
|
||||
self.interval = int(document["monitoring_interval_seconds"])
|
||||
self.supported = self.interval % 60 == 0 and 86400 % self.interval == 0
|
||||
self.count = 15 * (86400 // self.interval) if self.supported else 0
|
||||
ids = dict.fromkeys(self.settings.pressure_device_ids + self.settings.outlet_flow_device_ids
|
||||
+ [device for rule in config.rules if rule.kind == "relative_change"
|
||||
for device in rule.device_ids])
|
||||
self.values = {device: np.full(self.count, np.nan) for device in ids}
|
||||
|
||||
def add(self, rows: list[dict]) -> None:
|
||||
for row in rows:
|
||||
device = row["device_id"]
|
||||
if device not in self.values:
|
||||
continue
|
||||
at = datetime.fromisoformat(row["time"])
|
||||
if at >= self.cutoff:
|
||||
raise ValueError("动态阈值拟合不能读取回放开始时刻及之后的观测")
|
||||
offset = (at - self.start).total_seconds()
|
||||
if (not self.supported or offset < 0 or offset % self.interval
|
||||
or row.get("quality") not in {"valid", "corrected"}
|
||||
or row.get("cleaned_value") is None):
|
||||
continue
|
||||
index = int(offset // self.interval)
|
||||
if np.isfinite(self.values[device][index]):
|
||||
raise ValueError("动态阈值拟合存在重复的设备时间点")
|
||||
self.values[device][index] = float(row["cleaned_value"])
|
||||
|
||||
def fit(self) -> dict:
|
||||
base = {
|
||||
"version": CALIBRATION_VERSION,
|
||||
"diagnostic_policy": DIAGNOSTIC_POLICY,
|
||||
"knowledge_cutoff": self.cutoff.isoformat(),
|
||||
"history_start": self.start.isoformat(),
|
||||
"required_samples_per_device": self.count,
|
||||
"pressure": {}, "outlet_flow": {}, "diagnostic": {},
|
||||
}
|
||||
if not self.supported:
|
||||
return {**base, "status": "insufficient_data", "reason": "unsupported_sampling_interval"}
|
||||
available = {device: int(np.isfinite(values).sum()) for device, values in self.values.items()}
|
||||
# A regular diurnal rise/reversal can be a huge percentage of a small
|
||||
# preceding flow. Judge its absolute value against causal normal readings
|
||||
# at the same sampling phase before treating that percentage as abnormal.
|
||||
points_per_day = 86400 // self.interval
|
||||
for device, values in self.values.items():
|
||||
if available[device] != self.count:
|
||||
continue
|
||||
days = values.reshape(15, points_per_day)
|
||||
center = np.median(days, axis=0)
|
||||
sigma = 1.4826 * np.median(np.abs(days - center), axis=0)
|
||||
margin = np.maximum(3 * sigma, np.maximum(np.abs(center) * 1e-6, 1e-9))
|
||||
base["diagnostic"][device] = {
|
||||
"phase_origin": self.start.isoformat(), "interval_seconds": self.interval,
|
||||
"lower": (days.min(axis=0) - margin).tolist(),
|
||||
"upper": (days.max(axis=0) + margin).tolist(),
|
||||
"method": "normal_same_phase_range_plus_3_robust_sigma", "normal_day_count": 15,
|
||||
}
|
||||
for kind, ids, direction, floor in (
|
||||
("pressure", self.settings.pressure_device_ids, -1, 3.0),
|
||||
("outlet_flow", self.settings.outlet_flow_device_ids, 1, 5.0),
|
||||
):
|
||||
for device in ids:
|
||||
if available[device] != self.count:
|
||||
continue
|
||||
rule = next((rule for rule in self.config.rules
|
||||
if rule.kind == "relative_change" and device in rule.device_ids), None)
|
||||
if rule is None:
|
||||
continue
|
||||
series = pd.Series(self.values[device])
|
||||
reference = series.rolling(rule.window_samples, min_periods=rule.minimum_samples).median().shift(1)
|
||||
changes = (direction * 100 * (series - reference) / reference.abs()).to_numpy()
|
||||
changes = changes[np.isfinite(changes)]
|
||||
if not len(changes):
|
||||
continue
|
||||
center = float(np.median(changes))
|
||||
sigma = float(1.4826 * np.median(np.abs(changes - center)))
|
||||
base[kind][device] = {
|
||||
"threshold_percent": max(floor, center + 3 * sigma),
|
||||
"normal_change_median": center,
|
||||
"normal_change_robust_sigma": sigma,
|
||||
"normal_change_count": int(len(changes)),
|
||||
"method": "normal_recent_window_change_median_plus_3_robust_sigma",
|
||||
"normal_profile": base["diagnostic"].get(device),
|
||||
}
|
||||
pressure_ids = self.settings.pressure_device_ids
|
||||
incomplete = {device: available[device] for device in pressure_ids if available[device] != self.count}
|
||||
if len(pressure_ids) < 5 or incomplete:
|
||||
return {**base, "status": "insufficient_data", "reason": "pressure_history_incomplete",
|
||||
"available_samples": available}
|
||||
from app.services import scada_replay_monitoring as monitoring
|
||||
|
||||
index = pd.date_range(start=self.start, periods=self.count, freq=pd.Timedelta(seconds=self.interval))
|
||||
frame = pd.DataFrame({device: self.values[device] for device in pressure_ids}, index=index)
|
||||
result = monitoring.run_burst_detection(
|
||||
network="scada-replay-baseline", username="scada-replay",
|
||||
observed_pressure_data=frame, points_per_day=86400 // self.interval,
|
||||
mu=monitoring.TARGET_MU,
|
||||
iforest_params={"n_estimators": monitoring.TARGET_N_ESTIMATORS,
|
||||
"random_state": monitoring.TARGET_RANDOM_STATE, "contamination": "auto"},
|
||||
sensor_nodes=pressure_ids,
|
||||
)
|
||||
score = adaptive_score_threshold([float(row["Score"]) for row in result["rows"]])
|
||||
return {**base, "status": score["status"], "burst_detection": score}
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Persist one accident across monitoring periods without moving its evidence cutoff."""
|
||||
from datetime import datetime
|
||||
from math import isfinite
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
|
||||
def _anchors(monitoring: dict, indicator: str = "pressure") -> dict:
|
||||
return {
|
||||
item["device_id"]: {"baseline": item["baseline"], "threshold_percent": item["threshold_percent"]}
|
||||
for item in monitoring.get(indicator, {}).get("devices", [])
|
||||
if item.get("hit") and item.get("baseline", 0) > 0
|
||||
and 0 < item.get("threshold_percent", 0) < 100
|
||||
}
|
||||
|
||||
|
||||
def update_incident(document: dict, monitoring: dict, sequence: int) -> tuple[dict | None, str | None]:
|
||||
incident = document.get("monitoring_incident")
|
||||
# Adopt the last task from an older replay document on its first new step.
|
||||
previous = document.get("last_monitoring") or {}
|
||||
if incident:
|
||||
incident.setdefault("burst_diagnosed", previous.get("status") == "suspected_burst")
|
||||
if not incident and previous.get("task_id"):
|
||||
incident = {
|
||||
"incident_id": str(uuid5(NAMESPACE_URL, f"scada_replay:{document['replay_id']}:monitor:"
|
||||
f"{previous['period_end']}:{previous['rule_version']}")),
|
||||
"started_at": previous["period_end"], "last_observed_at": previous["period_end"],
|
||||
"status": "active", "task_id": previous["task_id"], "recovery_count": 0,
|
||||
"pressure_anchors": _anchors(previous), "alert": None,
|
||||
"flow_anchors": _anchors(previous, "outlet_flow"),
|
||||
"burst_diagnosed": previous.get("status") == "suspected_burst",
|
||||
}
|
||||
document["monitoring_incident"] = incident
|
||||
if incident and incident.get("last_monitoring_id") == monitoring.get("monitoring_id") and monitoring.get("monitoring_id"):
|
||||
return incident, incident["last_action"]
|
||||
anomaly = (monitoring["status"] in {"attention", "suspected_burst"}
|
||||
or bool(monitoring.get("anomaly_signals"))
|
||||
or any(monitoring.get(key, {}).get("hit") is True
|
||||
for key in ("pressure", "outlet_flow", "burst_detection")))
|
||||
if anomaly and (not incident or incident["status"] == "resolved"):
|
||||
external = (f"scada_replay:{document['replay_id']}:monitor:"
|
||||
f"{monitoring['period_end']}:{monitoring['rule_version']}")
|
||||
incident = {
|
||||
"incident_id": str(uuid5(NAMESPACE_URL, external)), "external_id": external,
|
||||
"started_at": monitoring["period_end"], "status": "active", "task_id": None,
|
||||
"recovery_count": 0, "pressure_anchors": _anchors(monitoring), "alert": None,
|
||||
"flow_anchors": _anchors(monitoring, "outlet_flow"),
|
||||
"batch_sequence": sequence,
|
||||
}
|
||||
document["monitoring_incident"] = incident
|
||||
action = "raised"
|
||||
elif incident and incident["status"] == "active":
|
||||
# A later diagnostic stage may add pressure/flow evidence to the same task.
|
||||
# Freeze each newly abnormal device once; retain earlier references.
|
||||
for indicator, field in (("pressure", "pressure_anchors"), ("outlet_flow", "flow_anchors")):
|
||||
for device, anchor in _anchors(monitoring, indicator).items():
|
||||
incident.setdefault(field, {}).setdefault(device, anchor)
|
||||
current = {item["device_id"]: item for item in monitoring.get("pressure", {}).get("devices", [])}
|
||||
anchors = incident["pressure_anchors"]
|
||||
flow = {item["device_id"]: item for item in monitoring.get("outlet_flow", {}).get("devices", [])}
|
||||
# Rolling relative changes can become normal while absolute pressure stays low.
|
||||
# Confirm recovery against the frozen pre-accident references as well.
|
||||
recovered = monitoring["status"] == "normal" and not anomaly and all(
|
||||
monitoring.get(key, {}).get("status") == "ready"
|
||||
and monitoring[key].get("hit") is False for key in ("pressure", "burst_detection")
|
||||
) and all(
|
||||
device in current and isfinite(current[device]["value"])
|
||||
and current[device].get("observed_at") is not None
|
||||
and datetime.fromisoformat(current[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
|
||||
and current[device]["value"] >= anchor["baseline"] * (1 - anchor["threshold_percent"] / 100)
|
||||
for device, anchor in anchors.items()
|
||||
) and all(
|
||||
device in flow and isfinite(flow[device]["value"])
|
||||
and flow[device].get("observed_at") is not None
|
||||
and datetime.fromisoformat(flow[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
|
||||
and flow[device]["value"] <= anchor["baseline"] * (1 + anchor["threshold_percent"] / 100)
|
||||
for device, anchor in incident.get("flow_anchors", {}).items()
|
||||
)
|
||||
contiguous = bool(monitoring.get("period_start")) and (
|
||||
datetime.fromisoformat(monitoring["period_start"]) == datetime.fromisoformat(incident["last_observed_at"])
|
||||
)
|
||||
incident["recovery_count"] = (incident["recovery_count"] + 1 if contiguous else 1) if recovered else 0
|
||||
action = "resolved" if incident["recovery_count"] >= 2 else "ongoing"
|
||||
if action == "resolved":
|
||||
incident["status"] = "resolved"
|
||||
else:
|
||||
return None, None
|
||||
incident.update(last_observed_at=monitoring["period_end"],
|
||||
last_monitoring_id=monitoring.get("monitoring_id"), last_action=action)
|
||||
return incident, action
|
||||
|
||||
|
||||
def attach_incident(monitoring: dict, incident: dict, action: str) -> None:
|
||||
monitoring["task_id"] = incident["task_id"]
|
||||
monitoring["incident"] = {
|
||||
key: incident[key] for key in (
|
||||
"incident_id", "status", "started_at", "last_observed_at", "recovery_count",
|
||||
)
|
||||
} | {"action": action}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Clock-phase normal envelopes frozen before replay; never inspect future readings."""
|
||||
from datetime import datetime
|
||||
from math import isfinite
|
||||
|
||||
CALIBRATION_VERSION = "scada-monitor-calibration@2"
|
||||
DIAGNOSTIC_POLICY = "normal-phase-envelope@1"
|
||||
|
||||
|
||||
def weighted_diagnosis(document: dict) -> bool:
|
||||
return (document.get("monitoring_config") or {}).get("decision_policy") == "weighted"
|
||||
|
||||
|
||||
def current_calibration(document: dict, start: datetime) -> dict | None:
|
||||
calibration = document.get("monitoring_calibration") or {}
|
||||
if (calibration.get("version") == CALIBRATION_VERSION
|
||||
and calibration.get("diagnostic_policy") == DIAGNOSTIC_POLICY
|
||||
and calibration.get("knowledge_cutoff") == start.isoformat()):
|
||||
return calibration
|
||||
return None
|
||||
|
||||
|
||||
def diagnostic_signal(document: dict, event: dict) -> bool:
|
||||
"""Never revive a persisted fixed-percentage event as new diagnosis evidence."""
|
||||
if event.get("category") == "data_quality":
|
||||
return False
|
||||
if not weighted_diagnosis(document):
|
||||
return True
|
||||
rules = (document.get("config") or {}).get("rules", [])
|
||||
rule = next((item for item in rules if item["rule_id"] == event["rule_id"]), None)
|
||||
return bool(rule and (rule["kind"] == "bounds" or event.get("diagnostic_policy") == DIAGNOSTIC_POLICY))
|
||||
|
||||
|
||||
def normal_profile_bounds(profile: dict | None, at: datetime) -> tuple[float, float] | None:
|
||||
if not profile:
|
||||
return None
|
||||
interval = profile.get("interval_seconds", 0)
|
||||
if not profile.get("phase_origin"):
|
||||
return None
|
||||
origin = datetime.fromisoformat(profile["phase_origin"])
|
||||
lower, upper = profile.get("lower", []), profile.get("upper", [])
|
||||
offset = (at - origin).total_seconds()
|
||||
if interval <= 0 or offset < 0 or offset % interval or not lower or len(lower) != len(upper):
|
||||
return None
|
||||
slot = int(offset // interval) % len(lower)
|
||||
low, high = lower[slot], upper[slot]
|
||||
if low is None or high is None or not isfinite(low) or not isfinite(high):
|
||||
return None
|
||||
return float(low), float(high)
|
||||
+245
-42
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from fastapi import HTTPException
|
||||
@@ -14,6 +15,18 @@ from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.content_digest import digest
|
||||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||||
from app.services.scada_normal_profile import current_calibration, diagnostic_signal, weighted_diagnosis
|
||||
|
||||
|
||||
# 500 devices at 15-minute sampling need 720,000 rows for fifteen days.
|
||||
# Leave headroom without allowing dense sources to make preparation unbounded.
|
||||
BASELINE_PAGE_SIZE = 2000
|
||||
BASELINE_MAX_ROWS = 1_000_000
|
||||
BASELINE_MAX_BYTES = 512 * 1024 * 1024
|
||||
|
||||
|
||||
class BaselineBudgetExceeded(RuntimeError):
|
||||
"""The unpublished baseline exceeded its fixed preparation budget."""
|
||||
|
||||
|
||||
def parse_time(value: str) -> datetime:
|
||||
@@ -36,6 +49,10 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||||
step_pending=document.get("step_pending", False),
|
||||
monitoring_count=document.get("monitoring_count", 0),
|
||||
last_monitoring=document.get("last_monitoring"),
|
||||
# Old persisted documents without a policy still use three-signal detection.
|
||||
monitoring_config=({"decision_policy": "all_signals", **document["monitoring_config"]}
|
||||
if document.get("monitoring_config") else None),
|
||||
monitoring_calibration=document.get("monitoring_calibration"),
|
||||
)
|
||||
|
||||
|
||||
@@ -105,7 +122,7 @@ class ScadaReplayService:
|
||||
await self.ensure_storage()
|
||||
async with self.repo.control():
|
||||
document = await self.repo.get(replay_id)
|
||||
key, body = str(command.request_id), command.model_dump(mode="json")
|
||||
key, body = str(command.request_id), command.model_dump(mode="json", exclude_none=True)
|
||||
receipt = document.setdefault("receipts", {}).get(key)
|
||||
if receipt:
|
||||
if receipt != body:
|
||||
@@ -114,9 +131,13 @@ class ScadaReplayService:
|
||||
if command.expected_revision != document["revision"]:
|
||||
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
|
||||
action, status = command.action, document["status"]
|
||||
if document.get("step_pending") and action == "step":
|
||||
if (action in {"step", "monitor", "retry"} and document.get("warmed")
|
||||
and weighted_diagnosis(document)
|
||||
and current_calibration(document, parse_time(document["start_time"])) is None):
|
||||
raise HTTPException(409, "旧回放缺少新版历史诊断校准,请停止后新建演示")
|
||||
if document.get("step_pending") and action in {"step", "monitor"}:
|
||||
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
|
||||
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
|
||||
if action in {"step", "monitor", "retry"} and await self.repo.active_other(replay_id):
|
||||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||||
if action == "step" and status in {"ready", "paused"}:
|
||||
interval = document.get("monitoring_interval_seconds")
|
||||
@@ -126,8 +147,31 @@ class ScadaReplayService:
|
||||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||||
document["step_start"] = document["virtual_now"]
|
||||
document["step_anomaly_signals"] = []
|
||||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||||
document["status"], document["step_pending"] = "paused", True
|
||||
elif action == "monitor" and status in {"ready", "paused", "completed"}:
|
||||
target = command.at_time
|
||||
if target is None:
|
||||
raise HTTPException(422, "选定时刻监测需要提供 at_time")
|
||||
start = parse_time(document["start_time"])
|
||||
current = parse_time(document["virtual_now"])
|
||||
if not start <= target <= parse_time(document["end_time"]):
|
||||
raise HTTPException(422, "监测时刻必须在本轮演示区间内")
|
||||
if target < current:
|
||||
raise HTTPException(409, "监测历史时刻需要新建一轮回放,避免使用后续状态")
|
||||
interval = document.get("monitoring_interval_seconds")
|
||||
if interval is None:
|
||||
raise HTTPException(409, "本轮未固定监测周期,请停止后新建演示")
|
||||
previous = document.get("last_monitoring") or {}
|
||||
# Reusing an immutable result must not count as another normal
|
||||
# period, resolve an incident, or create another investigation.
|
||||
if not previous.get("period_end") or parse_time(previous["period_end"]) != target:
|
||||
document["step_start"] = (current if target > current
|
||||
else max(start, target - timedelta(seconds=interval))).isoformat()
|
||||
document["step_target"] = target.isoformat()
|
||||
document["step_anomaly_signals"] = []
|
||||
document["status"], document["step_pending"] = "paused", True
|
||||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||||
document["status"], document["step_pending"] = "stopped", False
|
||||
document.pop("step_target", None)
|
||||
@@ -161,11 +205,16 @@ class ScadaReplayService:
|
||||
# Failure never discards the prepared batch; retry resumes it.
|
||||
document = await self.repo.get(replay_id)
|
||||
document["status"], document["step_pending"] = "failed", False
|
||||
document["error"] = (
|
||||
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
|
||||
if "SOURCE_CHANGED" in str(exc)
|
||||
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
|
||||
)
|
||||
if isinstance(exc, HTTPException):
|
||||
document["error"] = str(exc.detail)
|
||||
elif isinstance(exc, BaselineBudgetExceeded):
|
||||
document["error"] = str(exc)
|
||||
else:
|
||||
document["error"] = (
|
||||
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
|
||||
if "SOURCE_CHANGED" in str(exc)
|
||||
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
|
||||
)
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
raise
|
||||
@@ -174,23 +223,9 @@ class ScadaReplayService:
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
replay_id = document["replay_id"]
|
||||
if not document.get("warmed"):
|
||||
import math
|
||||
window = max(rule.window_samples for rule in config.rules)
|
||||
async with self.timescale.transaction():
|
||||
warmup = await ScadaRepository.get_warmup_values(
|
||||
self.timescale, config.device_ids, config.start_time, window,
|
||||
)
|
||||
for row in warmup:
|
||||
value = row["monitored_value"]
|
||||
bounds = config.quality_ranges.get(row["device_id"])
|
||||
if value is not None and math.isfinite(value) and not (bounds and (
|
||||
bounds.lower is not None and value < bounds.lower
|
||||
or bounds.upper is not None and value > bounds.upper
|
||||
)):
|
||||
document["histories"].setdefault(row["device_id"], []).append(value)
|
||||
document["warmed"] = True
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
await self._freeze_baseline(document, config)
|
||||
if weighted_diagnosis(document) and current_calibration(document, config.start_time) is None:
|
||||
raise HTTPException(409, "旧回放缺少新版历史诊断校准,请停止后新建演示")
|
||||
batch = await self.repo.pending_batch(replay_id)
|
||||
if batch is None:
|
||||
step = document.get("step_pending", False)
|
||||
@@ -252,6 +287,15 @@ class ScadaReplayService:
|
||||
target = result.get("step_target")
|
||||
was_step = bool(target)
|
||||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||||
if was_step:
|
||||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||||
if diagnostic_signal(result, item)}
|
||||
for event in changes:
|
||||
if event["change"] == "raised" and diagnostic_signal(result, event):
|
||||
signals[event["event_id"]] = {key: event[key] for key in (
|
||||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||||
) if key in event}
|
||||
result["step_anomaly_signals"] = list(signals.values())
|
||||
monitoring = None
|
||||
if was_step and not result["step_pending"]:
|
||||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||||
@@ -261,7 +305,7 @@ class ScadaReplayService:
|
||||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||||
))
|
||||
previous = await self.repo.monitoring_outputs(
|
||||
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
|
||||
replay_id, role_ids, period_end - timedelta(days=15),
|
||||
period_end, batch["sequence"],
|
||||
) if role_ids else []
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
@@ -273,11 +317,25 @@ class ScadaReplayService:
|
||||
period_end=period_end,
|
||||
batch_sequence=batch["sequence"],
|
||||
)
|
||||
result["last_monitoring"] = monitoring
|
||||
# Any configured measurement rule can enter diagnosis, including
|
||||
# indicators outside the pressure/outlet roles used for burst scoring.
|
||||
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
|
||||
if diagnostic_signal(result, item)}
|
||||
for state in result.get("rule_states", {}).values():
|
||||
event = state.get("event")
|
||||
if event and diagnostic_signal(result, event):
|
||||
signals[event["event_id"]] = {key: event[key] for key in (
|
||||
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
|
||||
) if key in event}
|
||||
monitoring["anomaly_signals"] = list(signals.values())
|
||||
if signals and monitoring["status"] == "normal":
|
||||
monitoring["status"] = "attention"
|
||||
monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断"
|
||||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||||
if not result["step_pending"]:
|
||||
result.pop("step_target", None)
|
||||
result.pop("step_start", None)
|
||||
result.pop("step_anomaly_signals", None)
|
||||
if batch["refs"]:
|
||||
last = batch["refs"][-1]
|
||||
result["cursor"] = [last["time"], last["device_id"]]
|
||||
@@ -295,6 +353,80 @@ class ScadaReplayService:
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
await self.repo.save(result)
|
||||
|
||||
async def _freeze_baseline(self, document, config):
|
||||
"""Stage bounded pages, then atomically publish one causal source snapshot."""
|
||||
if document.get("warmed"):
|
||||
return
|
||||
start = config.start_time - timedelta(days=15)
|
||||
batch_count, row_count, byte_count, after = 0, 0, 0, None
|
||||
counts = {device_id: 0 for device_id in config.device_ids}
|
||||
valid_counts = dict(counts)
|
||||
histories = {device_id: [] for device_id in config.device_ids}
|
||||
window = max(rule.window_samples for rule in config.rules)
|
||||
calibration = None
|
||||
if (document.get("monitoring_config") or {}).get("decision_policy") == "weighted":
|
||||
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder
|
||||
calibration = MonitoringCalibrationBuilder(document, config)
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.clear_unpublished_baseline(document["replay_id"])
|
||||
# Keep only the current page in memory. Each BizDB write commits before
|
||||
# the next source read; clean_prepared baseline pages remain invisible.
|
||||
async with self.timescale.transaction():
|
||||
await ScadaRepository.begin_replay_baseline_snapshot(self.timescale)
|
||||
while True:
|
||||
rows = await ScadaRepository.get_raw_page(
|
||||
self.timescale, config.device_ids, start, config.start_time,
|
||||
config.start_time, min(BASELINE_PAGE_SIZE, BASELINE_MAX_ROWS - row_count + 1), after,
|
||||
)
|
||||
if not rows:
|
||||
break
|
||||
if row_count + len(rows) > BASELINE_MAX_ROWS:
|
||||
raise BaselineBudgetExceeded(
|
||||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_ROWS:,} 条准备预算,"
|
||||
"尚未发布;请减少回放设备或核查历史采样密度后重试"
|
||||
)
|
||||
outputs = prepare_cleaning(rows, config)
|
||||
if calibration is not None:
|
||||
calibration.add(outputs)
|
||||
batch = {
|
||||
"sequence": -batch_count - 1, "stage": "clean_prepared", "kind": "baseline",
|
||||
"at": outputs[-1]["time"], "outputs": outputs, "events": [],
|
||||
}
|
||||
batch_bytes = len(json.dumps(batch, ensure_ascii=False, allow_nan=False,
|
||||
separators=(",", ":")).encode("utf-8"))
|
||||
if byte_count + batch_bytes > BASELINE_MAX_BYTES:
|
||||
raise BaselineBudgetExceeded(
|
||||
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_BYTES // (1024 * 1024)} MiB 准备预算,"
|
||||
"尚未发布;请减少回放设备或核查历史采样密度后重试"
|
||||
)
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save_batch(document["replay_id"], batch)
|
||||
batch_count += 1
|
||||
row_count += len(outputs)
|
||||
byte_count += batch_bytes
|
||||
for row in outputs:
|
||||
device_id = row["device_id"]
|
||||
counts[device_id] += 1
|
||||
if row["quality"] == "valid":
|
||||
valid_counts[device_id] += 1
|
||||
histories[device_id].append(row["cleaned_value"])
|
||||
histories[device_id] = histories[device_id][-window:]
|
||||
last = rows[-1]
|
||||
after = (last["time"], last["device_id"])
|
||||
document["baseline"] = {
|
||||
"start_time": start.isoformat(), "end_time": config.start_time.isoformat(),
|
||||
"end_exclusive": True, "sample_counts": counts, "valid_sample_counts": valid_counts,
|
||||
"row_count": row_count, "batch_count": batch_count, "byte_count": byte_count,
|
||||
}
|
||||
document["histories"] = histories
|
||||
if calibration is not None:
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
document["monitoring_calibration"] = await bounded_calculation(calibration.fit)
|
||||
document["warmed"] = True
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.publish_baseline(document["replay_id"])
|
||||
await self.repo.save(document)
|
||||
|
||||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
@@ -309,9 +441,20 @@ class ScadaReplayService:
|
||||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context))
|
||||
for event in changes:
|
||||
# Period diagnosis owns measurement notifications during manual steps.
|
||||
# Retain rule events as evidence; quality alerts remain independent.
|
||||
if (not create_tasks and event.get("category") != "data_quality"
|
||||
and (event["change"] == "raised" or event.get("notification_suppressed"))):
|
||||
if event["change"] == "raised":
|
||||
event["batch_sequence"] = batch["sequence"]
|
||||
event["notification_suppressed"] = True
|
||||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"].update(
|
||||
batch_sequence=batch["sequence"], notification_suppressed=True,
|
||||
)
|
||||
continue
|
||||
if event["change"] == "raised":
|
||||
event["batch_sequence"] = batch["sequence"]
|
||||
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
|
||||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["batch_sequence"] = batch["sequence"]
|
||||
await alerts.push(TaskAlertEvent(
|
||||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||||
@@ -345,10 +488,30 @@ class ScadaReplayService:
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
event["task_id"] = str(task.task_id)
|
||||
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
|
||||
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["task_id"] = event["task_id"]
|
||||
|
||||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||||
if monitoring["status"] != "suspected_burst":
|
||||
from app.services.scada_monitoring_incidents import attach_incident, update_incident
|
||||
incident, action = update_incident(document, monitoring, batch["sequence"])
|
||||
if incident is None:
|
||||
return
|
||||
if incident.get("task_id"):
|
||||
if monitoring["status"] == "suspected_burst" and not incident.get("burst_diagnosed"):
|
||||
from uuid import UUID
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
tasks = OperationalTaskService(
|
||||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context),
|
||||
)
|
||||
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
|
||||
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||||
"batch_sequence": batch["sequence"],
|
||||
}, uuid5(NAMESPACE_URL, incident["incident_id"] + ":burst-evidence"))
|
||||
incident["burst_diagnosed"] = True
|
||||
if action == "resolved" and incident.get("alert"):
|
||||
await self._publish_monitoring_alert(document, incident, "resolved")
|
||||
attach_incident(monitoring, incident, action)
|
||||
return
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
@@ -358,29 +521,44 @@ class ScadaReplayService:
|
||||
self.context,
|
||||
permissions_for_context(self.context),
|
||||
)
|
||||
external = (
|
||||
f"scada_replay:{document['replay_id']}:monitor:"
|
||||
f"{monitoring['period_end']}:{monitoring['rule_version']}"
|
||||
)
|
||||
external = incident["external_id"]
|
||||
pressure = monitoring["pressure"]
|
||||
flow = monitoring["outlet_flow"]
|
||||
detector = monitoring["burst_detection"]
|
||||
score = detector.get("target_score")
|
||||
score_text = f"{score:.4f}" if score is not None else "数据不足"
|
||||
summary = (
|
||||
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
|
||||
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
|
||||
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
|
||||
f"SCADA 数据诊断发现指标异常,进入异常调查与爆管诊断。"
|
||||
f"爆管侦测得分 {score_text};"
|
||||
f"压力下降测点 {len(pressure.get('hit_device_ids', []))} 个;"
|
||||
f"出厂边界流量增加测点 {len(flow.get('hit_device_ids', []))} 个。"
|
||||
"任一指标异常即启动调查,加权分仅用于评估爆管嫌疑,不能据此提前确诊。"
|
||||
)
|
||||
if monitoring.get("anomaly_signals"):
|
||||
signals = monitoring["anomaly_signals"]
|
||||
devices = {device for item in signals for device in item.get("device_ids", [])}
|
||||
summary += f" 单项异常 {len(signals)} 项,涉及 {len(devices)} 个测点;明细通过冻结 SCADA 查询核验。"
|
||||
if monitoring.get("weighted"):
|
||||
weighted = monitoring["weighted"]
|
||||
weights = weighted["weights"]
|
||||
summary += (
|
||||
f" 综合评分 {weighted['score']:.2f},爆管嫌疑门槛 {weighted['threshold']:.2f}"
|
||||
f"(侦测 {weights['burst_detection']:.0%}、压力 {weights['pressure']:.0%}、"
|
||||
f"流量 {weights['outlet_flow']:.0%})。"
|
||||
)
|
||||
if detector.get("score_threshold") is not None:
|
||||
summary += f" 侦测动态阈值 {detector['score_threshold']:.4f}。"
|
||||
command = OperationalTaskCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, external),
|
||||
task_type="event_investigation",
|
||||
workflow="burst_report",
|
||||
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
|
||||
workflow="scada_anomaly",
|
||||
title=f"SCADA 异常调查 · {monitoring['period_end']}",
|
||||
summary=summary,
|
||||
source=TaskSource(
|
||||
kind="external_system",
|
||||
external_id=external,
|
||||
external_revision=1,
|
||||
event_kind="scada_burst_suspected",
|
||||
event_kind="scada_anomaly",
|
||||
replay_context={
|
||||
"replay_id": document["replay_id"],
|
||||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||||
@@ -390,8 +568,33 @@ class ScadaReplayService:
|
||||
trigger=TaskTrigger(
|
||||
kind="event",
|
||||
occurred_at=monitoring["period_end"],
|
||||
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
|
||||
detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管",
|
||||
),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
monitoring["task_id"] = str(task.task_id)
|
||||
incident["task_id"] = str(task.task_id)
|
||||
incident["burst_diagnosed"] = monitoring["status"] == "suspected_burst"
|
||||
incident["alert"] = {
|
||||
"alert_id": incident["incident_id"],
|
||||
"producer_id": "scada-replay:" + document["replay_id"],
|
||||
"occurred_at": incident["started_at"],
|
||||
"condition_id": f"operational-task:{task.task_id}",
|
||||
"source": {"kind": "scada_replay", "replay_id": document["replay_id"],
|
||||
"rule_id": "scada_diagnosis", "batch_sequence": batch["sequence"]},
|
||||
"title": "SCADA 指标异常", "description": "已启动调查,详细诊断见调查报告。", "severity": "warning",
|
||||
}
|
||||
await self._publish_monitoring_alert(document, incident, "raised")
|
||||
attach_incident(monitoring, incident, action)
|
||||
|
||||
async def _publish_monitoring_alert(self, document, incident, change):
|
||||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||||
from app.services.task_alerts import TaskAlertService
|
||||
alerts = TaskAlertService(
|
||||
TaskAlertRepository(self.repo.connection, self.context.project_id), self.context,
|
||||
internal_producer_id="scada-replay:" + document["replay_id"],
|
||||
)
|
||||
await alerts.push(TaskAlertEvent(
|
||||
**incident["alert"], event_type=change,
|
||||
event_id=uuid5(NAMESPACE_URL, incident["incident_id"] + ":" + change),
|
||||
))
|
||||
|
||||
@@ -9,6 +9,8 @@ from uuid import NAMESPACE_URL, uuid5
|
||||
import pandas as pd
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.services.scada_normal_profile import current_calibration, normal_profile_bounds
|
||||
from app.services.burst_detection import (
|
||||
TARGET_MU,
|
||||
TARGET_N_ESTIMATORS,
|
||||
@@ -42,6 +44,7 @@ def _relative_indicator(
|
||||
period_end: datetime,
|
||||
*,
|
||||
direction: str,
|
||||
calibration: dict | None = None,
|
||||
) -> dict[str, Any]:
|
||||
details: list[dict[str, Any]] = []
|
||||
gaps: list[dict[str, str]] = []
|
||||
@@ -54,30 +57,51 @@ def _relative_indicator(
|
||||
if rule is None:
|
||||
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
|
||||
continue
|
||||
valid = _valid_rows(rows, device_id)
|
||||
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
|
||||
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
|
||||
if len(baseline) < rule.minimum_samples:
|
||||
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
|
||||
profile = calibration.get(device_id) if calibration is not None else None
|
||||
if calibration is not None and profile is None:
|
||||
gaps.append({"device_id": device_id, "reason": "adaptive_history_not_ready"})
|
||||
continue
|
||||
threshold = float(profile["threshold_percent"]) if profile else rule.threshold
|
||||
valid = _valid_rows(rows, device_id)
|
||||
current = [row for row in valid if period_start < _time(row["time"]) <= period_end]
|
||||
latest = current[-1] if current else None
|
||||
if not current:
|
||||
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
|
||||
continue
|
||||
baseline = ([row for row in valid if _time(row["time"]) < _time(latest["time"])]
|
||||
[-rule.window_samples:]) if latest else []
|
||||
if len(baseline) < rule.minimum_samples:
|
||||
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
|
||||
continue
|
||||
reference = median(float(row["cleaned_value"]) for row in baseline)
|
||||
if reference == 0:
|
||||
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
|
||||
continue
|
||||
latest = current[-1]
|
||||
value = float(latest["cleaned_value"])
|
||||
change = (value - reference) / abs(reference) * 100
|
||||
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
|
||||
hit = change <= -threshold if direction == "decrease" else change >= threshold
|
||||
bounds = normal_profile_bounds(profile.get("normal_profile"), _time(latest["time"])) if profile else None
|
||||
if profile and bounds is None:
|
||||
gaps.append({"device_id": device_id, "reason": "normal_phase_history_not_ready"})
|
||||
continue
|
||||
if bounds is not None:
|
||||
hit = hit and not bounds[0] <= value <= bounds[1]
|
||||
details.append({
|
||||
"device_id": device_id,
|
||||
"observed_at": _time(latest["time"]).isoformat(),
|
||||
"baseline": reference,
|
||||
"baseline_method": "recent_samples_median",
|
||||
"baseline_start": _time(baseline[0]["time"]).isoformat(),
|
||||
"baseline_end": _time(baseline[-1]["time"]).isoformat(),
|
||||
"baseline_sample_count": len(baseline),
|
||||
"value": value,
|
||||
"value_change": value - reference,
|
||||
"unit": rule.unit,
|
||||
"change_percent": change,
|
||||
"threshold_percent": rule.threshold,
|
||||
"threshold_percent": threshold,
|
||||
**({"normal_phase_lower": bounds[0], "normal_phase_upper": bounds[1]} if bounds else {}),
|
||||
**({"threshold_method": profile["method"],
|
||||
"configured_threshold_percent": rule.threshold} if profile else {}),
|
||||
"hit": hit,
|
||||
})
|
||||
return {
|
||||
@@ -94,6 +118,8 @@ def _burst_detection(
|
||||
pressure_device_ids: list[str],
|
||||
interval_seconds: int,
|
||||
period_end: datetime,
|
||||
*,
|
||||
score_threshold: float = TARGET_SCORE_THRESHOLD,
|
||||
) -> dict[str, Any]:
|
||||
if len(pressure_device_ids) < 5:
|
||||
return {"status": "sensor_coverage_insufficient", "hit": False,
|
||||
@@ -112,7 +138,10 @@ def _burst_detection(
|
||||
common_times = set(values) if common_times is None else common_times & set(values)
|
||||
if not common_times:
|
||||
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
|
||||
target = max(common_times)
|
||||
target = period_end
|
||||
if target not in common_times:
|
||||
return {"status": "current_reading_missing", "hit": False,
|
||||
"target_time": target.isoformat(), "latest_common_time": max(common_times).isoformat()}
|
||||
expected = [target - timedelta(seconds=interval_seconds * offset)
|
||||
for offset in range(count - 1, -1, -1)]
|
||||
incomplete = [device_id for device_id, values in series_by_device.items()
|
||||
@@ -142,10 +171,11 @@ def _burst_detection(
|
||||
score = float(latest["Score"])
|
||||
return {
|
||||
"status": "ready",
|
||||
"hit": score <= TARGET_SCORE_THRESHOLD,
|
||||
"hit": score <= score_threshold,
|
||||
"target_time": target.isoformat(),
|
||||
"target_score": score,
|
||||
"score_threshold": TARGET_SCORE_THRESHOLD,
|
||||
"score_interpretation": "lower_is_more_anomalous_not_probability",
|
||||
"score_threshold": score_threshold,
|
||||
"sensor_count": len(pressure_device_ids),
|
||||
"reference_day_count": 14,
|
||||
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
|
||||
@@ -185,17 +215,64 @@ def evaluate_monitoring_period(
|
||||
flow_ids = monitoring.get("outlet_flow_device_ids", [])
|
||||
selected = set(config.device_ids)
|
||||
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
|
||||
if not pressure_ids or not flow_ids or role_gaps:
|
||||
weighted_policy = monitoring.get("decision_policy") == "weighted"
|
||||
if not weighted_policy and (not pressure_ids or not flow_ids or role_gaps):
|
||||
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
|
||||
"role_gaps": role_gaps,
|
||||
"pressure": {"status": "unconfigured", "hit": False},
|
||||
"outlet_flow": {"status": "unconfigured", "hit": False},
|
||||
"burst_detection": {"status": "not_run", "hit": False}}
|
||||
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
|
||||
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
|
||||
detector = _burst_detection(
|
||||
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
|
||||
calibration = current_calibration(document, config.start_time) if weighted_policy else None
|
||||
if config.start_time > period_end:
|
||||
calibration = None
|
||||
base["role_gaps"] = role_gaps
|
||||
pressure = _relative_indicator(
|
||||
rows, config, pressure_ids, period_start, period_end, direction="decrease",
|
||||
calibration=(calibration or {}).get("pressure", {}) if weighted_policy else None,
|
||||
)
|
||||
outlet_flow = _relative_indicator(
|
||||
rows, config, flow_ids, period_start, period_end, direction="increase",
|
||||
calibration=(calibration or {}).get("outlet_flow", {}) if weighted_policy else None,
|
||||
) if flow_ids else {"status": "unconfigured", "hit": False, "hit_device_ids": [], "devices": [], "gaps": []}
|
||||
detector = {"status": "baseline_not_ready", "hit": False} if weighted_policy and (
|
||||
not calibration or calibration.get("status") != "ready"
|
||||
) else _burst_detection(
|
||||
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
|
||||
score_threshold=calibration["burst_detection"]["threshold"] if calibration else TARGET_SCORE_THRESHOLD,
|
||||
)
|
||||
if weighted_policy:
|
||||
settings = ScadaMonitoringSettings.model_validate(monitoring)
|
||||
weights = settings.weights.model_dump()
|
||||
indicators = {"burst_detection": detector, "pressure": pressure, "outlet_flow": outlet_flow}
|
||||
contributions = {key: weights[key] if indicator["status"] == "ready" and indicator["hit"] else 0.0
|
||||
for key, indicator in indicators.items()}
|
||||
score = round(sum(contributions.values()), 10)
|
||||
complete = pressure["status"] == detector["status"] == "ready"
|
||||
diagnostic_missing = sorted({device for rule in config.rules if rule.kind == "relative_change"
|
||||
for device in rule.device_ids}
|
||||
- set((calibration or {}).get("diagnostic", {})))
|
||||
if not complete:
|
||||
status, detail = "insufficient_data", (
|
||||
"旧回放缺少新版历史诊断校准,请停止后新建演示" if calibration is None
|
||||
else "压力数据或爆管侦测基线不足,尚不能作出组合判定"
|
||||
)
|
||||
elif score >= settings.alert_threshold:
|
||||
status, detail = "suspected_burst", "爆管相关指标加权达到疑似爆管门槛"
|
||||
elif any(indicator["hit"] for indicator in indicators.values()):
|
||||
status, detail = "attention", "部分指标异常,进入调查;综合结果尚未达到爆管嫌疑门槛"
|
||||
elif diagnostic_missing:
|
||||
status, detail = "insufficient_data", "部分测点缺少完整历史同期基线,不能判定全部指标正常"
|
||||
else:
|
||||
status, detail = "normal", "本周期爆管相关指标均未命中"
|
||||
detail += f"(综合 {score:.2f} / 门槛 {settings.alert_threshold:.2f},流量权重 {weights['outlet_flow']:.0%})"
|
||||
return {**base, "status": status, "detail": detail, "pressure": pressure,
|
||||
"outlet_flow": outlet_flow, "burst_detection": detector,
|
||||
"weighted": {"score": score, "threshold": settings.alert_threshold,
|
||||
"weights": weights, "contributions": contributions,
|
||||
"unavailable_indicators": [key for key, value in indicators.items() if value["status"] != "ready"],
|
||||
"calibration_cutoff": calibration["knowledge_cutoff"] if calibration else None,
|
||||
"diagnostic_unavailable_device_ids": diagnostic_missing,
|
||||
"missing_weight_policy": "zero_contribution_without_renormalization"}}
|
||||
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
|
||||
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
|
||||
if not complete:
|
||||
|
||||
@@ -6,6 +6,9 @@ from statistics import median
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
import re
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_normal_profile import (
|
||||
DIAGNOSTIC_POLICY, current_calibration, normal_profile_bounds, weighted_diagnosis,
|
||||
)
|
||||
|
||||
|
||||
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
|
||||
@@ -43,6 +46,8 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
|
||||
by_id = {row["device_id"]: row for row in rows}
|
||||
changes = []
|
||||
devices = {device["device_id"]: device for device in result.get("devices", [])}
|
||||
weighted = weighted_diagnosis(result)
|
||||
calibration = current_calibration(result, config.start_time) if weighted else None
|
||||
for device_id in config.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
invalid = row is not None and row["quality"] != "valid"
|
||||
@@ -75,7 +80,8 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
|
||||
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
|
||||
state["event"] = None
|
||||
for rule in config.rules:
|
||||
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
|
||||
state_key = f"{rule.rule_id}:{DIAGNOSTIC_POLICY}" if weighted and rule.kind == "relative_change" else rule.rule_id
|
||||
state = states.setdefault(state_key, {"hits": 0, "recoveries": 0, "event": None})
|
||||
hits, usable, reasons = [], [], []
|
||||
for device_id in rule.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
@@ -86,6 +92,17 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
|
||||
triggered = ((rule.lower is not None and value < rule.lower)
|
||||
or (rule.upper is not None and value > rule.upper))
|
||||
reason = f"{device_id}={value} {rule.unit},超出配置范围"
|
||||
elif weighted:
|
||||
# The percentage in the compatibility rule is not a diagnosis
|
||||
# threshold. Learn each device's normal same-phase range first.
|
||||
profile = (calibration or {}).get("diagnostic", {}).get(device_id)
|
||||
bounds = normal_profile_bounds(profile, at)
|
||||
if bounds is None:
|
||||
continue
|
||||
low, high = bounds
|
||||
triggered = (value < low or value > high if rule.direction == "both"
|
||||
else value > high if rule.direction == "increase" else value < low)
|
||||
reason = f"{device_id}={value} {rule.unit},超出冻结正常同时段范围 [{low:.4f}, {high:.4f}] {rule.unit}"
|
||||
else:
|
||||
previous = histories.get(device_id, [])[-rule.window_samples:]
|
||||
if len(previous) < rule.minimum_samples:
|
||||
@@ -106,10 +123,12 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
|
||||
state["hits"] += 1
|
||||
state["recoveries"] = 0
|
||||
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
|
||||
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
|
||||
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{state_key}:{at.isoformat()}"))
|
||||
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
|
||||
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
|
||||
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
|
||||
if weighted and rule.kind == "relative_change":
|
||||
event.update(diagnostic_policy=DIAGNOSTIC_POLICY, state_key=state_key)
|
||||
state["event"] = event
|
||||
changes.append({**event, "change": "raised"})
|
||||
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
|
||||
@@ -128,7 +147,9 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
|
||||
# Incomplete frames cannot count as a consecutive hit/recovery.
|
||||
elif not usable and any(device in by_id for device in rule.device_ids):
|
||||
state["hits"] = state["recoveries"] = 0
|
||||
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
|
||||
frozen = {device for rule in config.rules if states[
|
||||
f"{rule.rule_id}:{DIAGNOSTIC_POLICY}" if weighted and rule.kind == "relative_change" else rule.rule_id
|
||||
]["event"]
|
||||
for device in rule.device_ids}
|
||||
for row in rows:
|
||||
if row["quality"] == "valid" and row["device_id"] not in frozen:
|
||||
|
||||
@@ -12,14 +12,23 @@ def check_replay_window(context, start, end):
|
||||
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
|
||||
|
||||
|
||||
def check_snapshot_start(run, start):
|
||||
replay_start = datetime.fromisoformat(run["start_time"])
|
||||
if start < replay_start:
|
||||
baseline = run.get("baseline")
|
||||
if not baseline:
|
||||
raise HTTPException(422, "本轮回放未冻结历史对照,请新建回放后查询历史基线")
|
||||
if start < datetime.fromisoformat(baseline["start_time"]):
|
||||
raise HTTPException(422, "查询起点早于本轮已冻结的历史基线")
|
||||
|
||||
|
||||
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
|
||||
check_replay_window(context, start, end)
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if not set(device_ids) <= set(run["config"]["device_ids"]):
|
||||
raise HTTPException(403, "设备或回放不属于当前调查")
|
||||
if start < datetime.fromisoformat(run["start_time"]):
|
||||
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
|
||||
check_snapshot_start(run, start)
|
||||
rows = await repository.detected_outputs(
|
||||
context["replay_id"], device_ids, start, end, context["batch_sequence"],
|
||||
limit=limit, offset=offset,
|
||||
@@ -40,9 +49,7 @@ async def replay_burst_location_inputs(connection, project_id, context, args):
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if run:
|
||||
replay_start = datetime.fromisoformat(run["start_time"])
|
||||
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
|
||||
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
|
||||
check_snapshot_start(run, min(args.scada_burst_start, args.scada_normal_start))
|
||||
devices = {item["device_id"]: item for item in run.get("devices", [])}
|
||||
monitoring = run.get("monitoring_config") or {}
|
||||
pressure_devices = monitoring.get("pressure_device_ids", [])
|
||||
|
||||
Reference in New Issue
Block a user