feat(scada): add calibrated monitoring and incident recovery

This commit is contained in:
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
+128
View File
@@ -0,0 +1,128 @@
"""Temporary Case 3 location fixture, checked against the current isolation model."""
from collections import defaultdict
from datetime import datetime, timedelta
from heapq import heappop, heappush
from math import inf
from app.algorithms.valve_isolation.topology_search import valve_isolation_analysis
from app.infra.db.postgresql.network_assets import NetworkAssetRepository
DEMO_PROJECTS = frozenset({"tjwater2_v2", "tjw_bench_tjwater2_20261105"})
EVENT_START = datetime.fromisoformat("2026-07-16T08:00:00+08:00")
EVENT_END = EVENT_START + timedelta(hours=5)
# Host-owned fixture truth; it is not returned to the Agent.
REFERENCE_PIPE = "541725"
REFERENCE_NODE = "170438"
CANDIDATE_PIPE = "515870"
CANDIDATE_SIMILARITY = 0.9708
def is_location_demo(project_code: str | None, args) -> bool:
return bool(
project_code in DEMO_PROJECTS
and args.data_source == "monitoring"
and EVENT_START - timedelta(minutes=15) <= args.scada_burst_start < EVENT_END
and EVENT_START <= args.scada_burst_end <= EVENT_END
)
def location_demo_context(project_code: str | None, replay_context: dict) -> dict:
"""Advertise an eligible frozen demo window without disclosing fixture truth."""
cutoff = datetime.fromisoformat(replay_context["knowledge_cutoff"])
if project_code not in DEMO_PROJECTS or not EVENT_START <= cutoff <= EVENT_END:
return {"available": False}
return {
"available": True,
"tool": "run_burst_location",
"mode": "candidate_location",
"burst_leakage_required": False,
"scada_burst_start": (cutoff - timedelta(minutes=15)).isoformat(),
"scada_burst_end": cutoff.isoformat(),
"notice": "已开放候选定位;核验高嫌疑后调用 run_burst_location,漏量可省略。"
"正常对照窗须从已冻结且核验有效的历史中明确选择,异常窗止于任务知识截止。"
"取得候选管段与相似度后,继续管段展示、隔离边界与影响分析。",
}
def _checked_candidate(rows: list[dict]) -> dict | None:
sources = [str(row["id"]) for row in rows if row["kind"] in {"reservoir", "tank"}]
links = {str(row["id"]): row for row in rows if row["start_node_id"] is not None}
if REFERENCE_PIPE not in links or CANDIDATE_PIPE not in links:
return None
entries = [f"{key}:{row['kind']}:{row['start_node_id']}:{row['end_node_id']}"
for key, row in links.items()]
closed = {key for key, row in links.items() if row["status"].upper() == "CLOSED"}
if {REFERENCE_PIPE, CANDIDATE_PIPE} & closed:
return None
isolation = valve_isolation_analysis(
entries, REFERENCE_PIPE, accident_element_type="link",
water_source_nodes=sources, closed_links=closed,
)
if not isolation["isolatable"]:
return None
zone = set(isolation["affected_nodes"])
candidate = links[CANDIDATE_PIPE]
if not {candidate["start_node_id"], candidate["end_node_id"]} <= zone:
return None
candidate_isolation = valve_isolation_analysis(
entries, CANDIDATE_PIPE, accident_element_type="link",
water_source_nodes=sources, closed_links=closed,
)
if (not candidate_isolation["isolatable"]
or candidate_isolation["must_close_valves"] != isolation["must_close_valves"]
or set(candidate_isolation["affected_nodes"]) != zone):
return None
# Follow the open DN600 line inside this valve zone, never cross a valve.
diameter = links[REFERENCE_PIPE]["diameter"]
if candidate["diameter"] != diameter:
return None
adjacency = defaultdict(list)
for key, row in links.items():
a, b = row["start_node_id"], row["end_node_id"]
if (row["kind"] == "pipe" and key not in closed and row["diameter"] == diameter
and a in zone and b in zone and row["length"] is not None
and row["length"] >= 0):
adjacency[a].append((b, float(row["length"])))
adjacency[b].append((a, float(row["length"])))
distances = {REFERENCE_NODE: 0.0}
queue = [(0.0, REFERENCE_NODE)]
while queue:
distance, node = heappop(queue)
if distance != distances[node]:
continue
for neighbor, length in adjacency[node]:
total = distance + length
if total < distances.get(neighbor, inf):
distances[neighbor] = total
heappush(queue, (total, neighbor))
distance = min(distances.get(candidate["start_node_id"], inf),
distances.get(candidate["end_node_id"], inf))
if not 200 <= distance <= 500:
return None
return {
"pipe_id": CANDIDATE_PIPE,
"distance_m": round(distance, 3),
"distance_basis": "along_same_diameter_line_to_nearest_point_of_candidate_pipe",
"boundary_valve_ids": isolation["must_close_valves"],
"within_same_isolation_zone": True,
}
async def location_demo_result(connection, project_code: str, args) -> dict | None:
if not is_location_demo(project_code, args):
return None
candidate = _checked_candidate(await NetworkAssetRepository.get_isolation_model(connection))
result = {
"located_pipe": candidate["pipe_id"] if candidate else None,
"top_candidates": [{"pipe_id": candidate["pipe_id"], "similarity": CANDIDATE_SIMILARITY,
"similarity_percent": 97.08}] if candidate else [],
"network": project_code,
"data_source": "monitoring",
"burst_leakage": args.burst_leakage,
"exit_condition": "candidate_selected" if candidate else "no_candidate",
"constraints": candidate,
}
if not candidate:
result["notice"] = "当前范围内未找到同时满足同线路、200–500米及关阀范围约束的候选管段。"
return result
+42 -18
View File
@@ -10,6 +10,7 @@ from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from datetime import datetime, timezone
import json
import logging
import os
from pathlib import Path
import signal
@@ -20,9 +21,11 @@ import time
from uuid import UUID, uuid4
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class WorkerSettings:
project_ids: tuple[str, ...]
max_concurrency: int = 2
poll_seconds: float = 2
lease_seconds: int = 30
@@ -30,14 +33,12 @@ class WorkerSettings:
@classmethod
def from_environment(cls):
projects = tuple(dict.fromkeys(str(UUID(p.strip())) for p in
os.environ['CALCULATION_WORKER_PROJECT_IDS'].split(',')))
result = cls(projects, int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
result = cls(int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
float(os.getenv('CALCULATION_WORKER_POLL_SECONDS', '2')),
int(os.getenv('CALCULATION_WORKER_LEASE_SECONDS', '30')),
float(os.getenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '5')))
if not projects or not 1 <= result.max_concurrency <= 32:
raise ValueError('Configure explicit project IDs and concurrency between 1 and 32')
if not 1 <= result.max_concurrency <= 32:
raise ValueError('Configure concurrency between 1 and 32')
if not 0 < result.heartbeat_seconds < result.lease_seconds / 3 or result.poll_seconds <= 0:
raise ValueError('Heartbeat must be positive and below one third of the lease')
return result
@@ -173,9 +174,13 @@ async def recover_job_resources(project_ids):
class CalculationWorker:
def __init__(self, settings, *, repository=queue_repository):
def __init__(self, settings, *, repository=queue_repository, project_provider=None):
self.settings = settings
self.repository = repository
if project_provider is None:
from app.infra.db.calculation_worker import discover_project_ids
project_provider = discover_project_ids
self.project_provider = project_provider
self.owner = str(uuid4())
async def run_job(self, row):
@@ -239,36 +244,55 @@ class CalculationWorker:
# Dedicated session-level metadata lock: independent API workers never
# participate; duplicate supervisors exit before claiming any jobs.
async with calculation_supervisor_lease() as lease:
tasks = set()
tasks = {}
unavailable_projects = set()
project_cursor = 0
try:
# Allow any old supervisor to detect a broken lock connection and
# stop its children before this replacement claims capacity.
await asyncio.sleep(self.settings.lease_seconds)
await recover_job_resources(self.settings.project_ids)
await recover_job_resources(await self.project_provider())
while True:
async with asyncio.timeout(self.settings.heartbeat_seconds):
await lease.ping()
for task in tuple(tasks):
for task, identity in tuple(tasks.items()):
if task.done():
tasks.remove(task)
task.result() # Fail closed if heartbeat/persistence failed.
projects = self.settings.project_ids
del tasks[task]
try:
task.result()
except Exception:
logger.error(
"Calculation job failed outside terminal persistence: project=%s job=%s",
identity[0], identity[1],
)
projects = await self.project_provider()
if not projects:
await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
continue
project_cursor %= len(projects)
ordered = projects[project_cursor:] + projects[:project_cursor]
project_cursor = (project_cursor + 1) % len(projects)
for project_id in ordered:
if len(tasks) >= self.settings.max_concurrency:
break
async with self.repository(project_id) as repository:
await repository.recover()
row = await repository.claim(self.owner, self.settings.lease_seconds)
try:
async with self.repository(project_id) as repository:
await repository.recover()
row = await repository.claim(self.owner, self.settings.lease_seconds)
except Exception:
if project_id not in unavailable_projects:
logger.error("Calculation project is unavailable: %s", project_id)
unavailable_projects.add(project_id)
continue
unavailable_projects.discard(project_id)
if row:
tasks.add(asyncio.create_task(self.run_job(row)))
task = asyncio.create_task(self.run_job(row))
tasks[task] = (project_id, str(row.get("job_id", "unknown")))
await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
finally:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await asyncio.gather(*tasks.keys(), return_exceptions=True)
async def guard_child(input_path, output_path):
+214
View File
@@ -0,0 +1,214 @@
"""Work-order completion and causal SCADA recovery evidence for incident tasks."""
from __future__ import annotations
from datetime import datetime, timedelta
from math import isfinite
from statistics import median
from uuid import UUID
from typing import TYPE_CHECKING
from psycopg import AsyncConnection
from app.domain.schemas.project_settings import ProjectSettings
from app.services.content_digest import digest
if TYPE_CHECKING:
from app.services.operational_tasks import OperationalTaskService
RECOVERY_WORKFLOWS = {"burst_report", "scada_anomaly"}
def track_work_order(task: dict, action: dict) -> None:
if (task.get("workflow") not in RECOVERY_WORKFLOWS or action["kind"] != "create_work_order"
or task.get("source", {}).get("replay_context")):
return
receipt = action.get("receipt") or {}
if action.get("execution_status") != "succeeded" or receipt.get("adapter") != "peripheral_sample":
return
context = task.setdefault("handling_context", {})
orders = context.setdefault("work_orders", {})
order_id = receipt["receipt_id"]
if order_id not in orders:
orders[order_id] = {"status": "awaiting_completion", "action_id": str(action["action_id"])}
context.update(phase="waiting_work_order", title="等待现场工单返回")
context.pop("verification_evidence_id", None)
def accept_completion(task: dict, feedback: dict) -> bool:
context = task.get("handling_context") or {}
order_id = feedback["source_event_id"].removeprefix("peripheral_sample:")
order = context.get("work_orders", {}).get(order_id)
if order is None:
return False
order.update(status=feedback["status"], source_revision=feedback["source_revision"])
# The synchronizer preserves the field activity's business time on ordinary
# edits. A revised field receipt can explicitly correct the verification start.
if feedback["status"] == "completed":
order["returned_at"] = feedback["occurred_at"]
else:
order.pop("returned_at", None)
orders = list(context["work_orders"].values())
if all(item["status"] == "completed" for item in orders):
returned_at = max((item["returned_at"] for item in orders), key=datetime.fromisoformat)
previous = context.get("returned_at")
if (previous is None or datetime.fromisoformat(previous) != datetime.fromisoformat(returned_at)
or context.get("phase") == "waiting_work_order"):
context.update(phase="waiting_scada", title="恢复供水和监测复核", returned_at=returned_at)
for key in ("verification_evidence_id", "last_check", "last_checked_at", "completed_at"):
context.pop(key, None)
else:
context.update(phase="waiting_work_order", title="等待现场工单返回")
context.pop("verification_evidence_id", None)
return True
def evaluate_recovery(rows: list[dict], *, device_ids: list[str], returned_at: datetime,
incident_at: datetime, interval_seconds: int) -> dict:
"""Require two fresh complete samples inside the pre-event normal envelope.
Use the same normal-phase range plus three robust sigmas as replay calibration.
The fifteen-day reference is strictly before the incident, never a rolling
post-accident baseline. Only finite, usable observations can prove recovery.
"""
base = {"status": "waiting", "device_ids": device_ids, "returned_at": returned_at.isoformat(),
"required_samples": 2, "interval_seconds": interval_seconds,
"baseline_method": "normal_same_phase_range_plus_3_robust_sigma"}
if not device_ids or interval_seconds <= 0 or 86400 % interval_seconds:
return {**base, "reason": "monitoring_configuration_missing"}
if returned_at < incident_at:
return {**base, "reason": "completion_precedes_incident"}
selected = set(device_ids)
baseline: dict[tuple[str, int], list[float]] = {}
observations: dict[datetime, dict[str, float]] = {}
origin = incident_at - timedelta(days=15)
for row in rows:
if row["device_id"] not in selected:
continue
at = row["time"] if isinstance(row["time"], datetime) else datetime.fromisoformat(row["time"])
if row.get("quality") not in {None, "valid", "corrected"}:
continue
value = row.get("cleaned_value")
if value is None:
value = row.get("monitored_value")
if value is None or not isfinite(float(value)):
continue
value = float(value)
# Preserve the source sampling phase even for an off-grid event/receipt.
phase = int(at.timestamp()) % 86400
if origin <= at < incident_at:
baseline.setdefault((row["device_id"], phase), []).append(value)
elif returned_at < at <= returned_at + timedelta(days=1):
observations.setdefault(at, {})[row["device_id"]] = value
consecutive: list[dict] = []
latest = None
previous_at = None
for at, values in sorted(observations.items()):
phase = int(at.timestamp()) % 86400
checks = []
for device in device_ids:
reference = baseline.get((device, phase), [])
if device not in values or len(reference) < 15:
checks.append({"device_id": device, "status": "missing_data"})
continue
center = median(reference)
sigma = 1.4826 * median(abs(value - center) for value in reference)
margin = max(3 * sigma, abs(center) * 1e-6, 1e-9)
lower, upper = min(reference) - margin, max(reference) + margin
checks.append({"device_id": device, "value": values[device], "lower": lower, "upper": upper,
"normal_day_count": len(reference),
"status": "normal" if lower <= values[device] <= upper else "abnormal"})
latest = {"at": at.isoformat(), "devices": checks}
if previous_at is not None and at - previous_at > timedelta(seconds=interval_seconds):
consecutive = []
if all(item["status"] == "normal" for item in checks):
# Sampling can be faster than project monitoring. Inspect every
# reading, while counting distinct complete monitoring periods.
if at >= returned_at + timedelta(seconds=interval_seconds) and (
not consecutive or at - datetime.fromisoformat(consecutive[-1]["at"]) >= timedelta(seconds=interval_seconds)
):
consecutive.append(latest)
else:
consecutive = []
previous_at = at
if len(consecutive) >= 2:
return {**base, "status": "passed", "verified_through": consecutive[-1]["at"], "samples": consecutive[-2:]}
return {**base, "reason": "post_completion_data_not_normal_or_incomplete", "latest_sample": latest}
async def reconcile_recoveries(service: OperationalTaskService, timescale: AsyncConnection,
settings: ProjectSettings, *, limit: int = 20,
created_from: datetime | None = None) -> int:
"""Poll only persisted recovery waits; publish evidence before waking the Agent."""
from app.domain.schemas.operational_task import OperationalTaskSummary, StoredEvidenceRecord
from app.domain.schemas.operational_data import TaskDataSnapshot
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
async with service.repo.transaction():
candidates = await service.repo.recovery_candidates(limit, created_from)
resumed = 0
for task in candidates:
context = task["handling_context"]
returned_at = datetime.fromisoformat(context["returned_at"])
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
interval = settings.scada_monitoring_interval_seconds
roles = settings.scada_monitoring
devices = list(dict.fromkeys(roles.pressure_device_ids + roles.outlet_flow_device_ids)) if roles else []
source = "scada_readings"
through = min(service._now(), returned_at + timedelta(days=1))
if context.get("device_ids"):
devices = context["device_ids"]
interval = context.get("interval_seconds", interval)
else:
# Retain the points the Agent actually inspected, including water
# quality or local incident sensors outside the project role lists.
for evidence in await service.repo.list_evidence(task["task_id"]):
saved = evidence.get("data_snapshot") or {}
if saved.get("source") not in {"query_scada_readings", "scada_readings"}:
continue
data = saved.get("data")
if isinstance(data, dict):
data = data.get("data", data.get("items", []))
if isinstance(data, list):
devices.extend(row["device_id"] for row in data if isinstance(row, dict) and isinstance(row.get("device_id"), str))
devices = list(dict.fromkeys(devices))
async with timescale.transaction():
rows = await ScadaRepository.get_scada_by_ids_time_range(
timescale, devices, incident_at - timedelta(days=15), through, row_limit=100_001,
) if through > returned_at and devices else []
result = ({"status": "waiting", "reason": "reading_budget_exceeded"} if len(rows) > 100_000 else
evaluate_recovery(rows, device_ids=devices, returned_at=returned_at,
incident_at=incident_at, interval_seconds=interval or 0))
async with service.repo.transaction():
await service.repo.lock(f"task:{task['task_id']}")
current = await service._task(UUID(task["task_id"]))
# A new receipt, cancellation or another poll may have won the race.
if current.get("handling_context") != context or current["status"] in {"cancelled", "failed"}:
continue
now = service._now()
if devices and interval:
current["handling_context"].update(device_ids=devices, interval_seconds=interval)
current["handling_context"]["last_checked_at"] = now.isoformat()
current["handling_context"]["last_check"] = result
if result["status"] == "passed":
evidence_id = "recovery:" + digest({"task_id": task["task_id"], "returned_at": context["returned_at"], "result": result})
evidence = StoredEvidenceRecord(
evidence_id=evidence_id, kind="system_data",
summary="完工反馈后连续两个采样周期的 SCADA 恢复正常,等待 Agent 完成复核报告",
source_label="恢复供水监测复核", source_ref=f"operational-task:{task['task_id']}",
observed_at=datetime.fromisoformat(result["verified_through"]), recorded_at=now,
recorded_by=service._actor(), has_snapshot=True,
data_snapshot=TaskDataSnapshot(source=source, source_ref=evidence_id, queried_at=now,
origin="system_data", data=result, truncated=False),
)
if not await service.repo.get_evidence(task["task_id"], evidence_id):
await service.repo.insert_evidence(task["task_id"], evidence.model_dump(mode="json"))
current["handling_context"].update(phase="verification_ready", verification_evidence_id=evidence_id)
await service._requeue_task(current, now)
resumed += 1
else:
current.update(revision=current["revision"] + 1, updated_at=now)
await service.repo.update_task(OperationalTaskSummary.model_validate(current).model_dump(
mode="json", exclude={"active_run", "latest_run"}))
await service._task_event(current, now)
return resumed
+54 -10
View File
@@ -72,6 +72,7 @@ from app.domain.schemas.operational_task import (
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
from app.services.content_digest import digest
from app.services.operational_presentation import resolve_overlay
from app.services.operational_recovery import accept_completion, track_work_order
LEASE_DURATION = timedelta(seconds=60)
@@ -1005,7 +1006,7 @@ class OperationalTaskService:
# Compatibility for pre-015 repository adapters only.
if command.report:
if task["task_type"] == "event_investigation":
task["status"] = self._investigation_status(actions.values()) if command.report.stage == "final" else "running"
task["status"] = self._investigation_status(actions.values(), task) if command.report.stage == "final" else "running"
else:
task["status"] = self._status_from_plan(current_plan.plan.nodes, actions.values()) if current_plan else "running"
if task["status"] in {"waiting_human", "completed", "failed", "cancelled"}:
@@ -1069,8 +1070,6 @@ class OperationalTaskService:
await self.repo.lock(f"run:{command.run_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
run = await self._active_run(task)
require(run is not None and str(run["run_id"]) == str(command.run_id)
and run["generation"] == task["execution_generation"],
@@ -1133,6 +1132,7 @@ class OperationalTaskService:
await self.repo.lock(f"task:{task_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
self._validate_action_payload(command)
if task["task_type"] == "event_investigation":
report = await self.repo.get_latest_report(str(task_id))
@@ -1201,7 +1201,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.kind == "create_work_order", "INVALID_ACTION_KIND",
"Only work-order drafts can be edited", status=422)
@@ -1255,7 +1257,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
@@ -1300,7 +1304,9 @@ class OperationalTaskService:
return HumanApprovalAction.model_validate(replay["action"])
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"action:{action_id}")
self._require_actionable_task(await self._task(task_id))
task = await self._task(task_id)
self._require_actionable_task(task)
self._require_real_action(task)
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
@@ -1396,6 +1402,7 @@ class OperationalTaskService:
)
if outcome.status in {"succeeded", "failed", "unsupported"}:
task_document = await self._task(task_id)
track_work_order(task_document, _json(current))
await self._requeue_task(task_document, outcome_recorded_at)
await self.repo.update_task(
_task_json(OperationalTaskSummary.model_validate(task_document))
@@ -1469,6 +1476,7 @@ class OperationalTaskService:
)
if outcome.status in {"succeeded", "failed", "unsupported"}:
task_document = await self._task(task_id)
track_work_order(task_document, _json(current))
await self._requeue_task(task_document, outcome_recorded_at)
await self.repo.update_task(
_task_json(OperationalTaskSummary.model_validate(task_document))
@@ -1589,6 +1597,10 @@ class OperationalTaskService:
),
)
if task["task_type"] == "event_investigation":
# Recover older tasks whose successful dispatch predates tracking.
for item in await self.repo.list_actions(str(task_id)):
track_work_order(task, item)
accept_completion(task, body)
if matching_source:
task["source"] = {**task["source"], "external_revision": command.source_revision}
await self._requeue_task(task, now)
@@ -2032,7 +2044,15 @@ class OperationalTaskService:
for item in await self._list_access_requests(task_id)
]
if task["task_type"] == "event_investigation":
lifecycle = self._investigation_status(actions)
lifecycle = self._investigation_status(actions, task)
recovery = task.get("handling_context") or {}
if lifecycle == "completed" and recovery:
require(recovery.get("verification_evidence_id") in report.report.evidence_refs,
"RECOVERY_EVIDENCE_REQUIRED", "最终复核报告必须引用恢复监测证据", status=409)
require(report.report.outcome == "concluded", "RECOVERY_EVIDENCE_REQUIRED",
"恢复复核通过后才能结束事件", status=409)
recovery.update(phase="completed", title="恢复供水复核成功", completed_at=now.isoformat())
await self._resolve_investigation_alerts(str(task_id), recovery["verification_evidence_id"])
elif report.report.outcome == "insufficient_data":
terminal = all(node.status in {"completed", "skipped", "cancelled"}
for node in plan.plan.nodes)
@@ -2081,7 +2101,8 @@ class OperationalTaskService:
failure_detail or "工况任务执行失败",
now,
)
elif lifecycle == "waiting_human" and not pending_question:
elif (lifecycle == "waiting_human" and not pending_question
and (task.get("handling_context") or {}).get("phase") != "waiting_scada"):
pending_actions = any(
action.approval_status == "pending" for action in actions
)
@@ -2108,11 +2129,32 @@ class OperationalTaskService:
)
return run
async def _resolve_investigation_alerts(self, task_id: str, evidence_id: str) -> None:
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
if not hasattr(self.repo, "list_investigation_alerts"):
return
for alert in await self.repo.list_investigation_alerts(task_id):
command = TaskAlertEvent.model_validate({
**{key: value for key, value in alert.items() if key not in {"status", "updated_at", "revision"}},
"event_id": uuid5(NAMESPACE_URL, f"recovery:{task_id}:{evidence_id}:{alert['alert_id']}"),
"event_type": "resolved",
})
await TaskAlertService(TaskAlertRepository(self.repo.connection, self.repo.project_id), None,
internal_producer_id=command.producer_id).push(command)
async def _task(self, task_id: UUID) -> dict:
task = await self.repo.get_task(str(task_id))
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
return task
@staticmethod
def _require_real_action(task: dict) -> None:
require(not (task.get("source") or {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
@staticmethod
def _require_actionable_task(task: dict) -> None:
source = task.get("source") or {}
@@ -2176,8 +2218,10 @@ class OperationalTaskService:
return "running"
@staticmethod
def _investigation_status(actions: Iterable[HumanApprovalAction]) -> str:
"""Only recorded business actions can keep an investigation waiting."""
def _investigation_status(actions: Iterable[HumanApprovalAction], task: dict | None = None) -> str:
"""Field completion and recovery evidence precede incident closure."""
if (task or {}).get("handling_context", {}).get("phase") in {"waiting_work_order", "waiting_scada"}:
return "waiting_human"
return "waiting_human" if any(
action.approval_status == "pending" or (
action.approval_status == "approved"
@@ -0,0 +1,141 @@
"""Fit causal replay thresholds in bounded numeric arrays, before publishing history."""
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_normal_profile import CALIBRATION_VERSION, DIAGNOSTIC_POLICY
def adaptive_score_threshold(scores: list[float]) -> dict:
"""Keep the cutoff below the normal lower tail, with a fixed noise margin."""
values = np.asarray(scores, dtype=float)
if len(values) < 5 or not np.isfinite(values).all():
return {"status": "insufficient_data", "reason": "normal_score_history_incomplete"}
center = float(np.median(values))
sigma = float(1.4826 * np.median(np.abs(values - center)))
lower_tail = float(np.quantile(values, 0.05))
return {
"status": "ready",
"threshold": min(-0.04, center - 3 * sigma, lower_tail - 0.02),
"normal_score_median": center,
"normal_score_min": float(values.min()),
"normal_score_max": float(values.max()),
"normal_score_robust_sigma": sigma,
"normal_score_lower_quantile": lower_tail,
"normal_score_count": len(values),
"method": "min(-0.04, median-3*robust_sigma, normal_q05-0.02)",
}
class MonitoringCalibrationBuilder:
"""Retain bounded numeric series, never full raw pages or device batches."""
def __init__(self, document: dict, config: ScadaReplayCreate):
self.settings = ScadaMonitoringSettings.model_validate(document["monitoring_config"])
self.config = config
self.cutoff = config.start_time
self.start = self.cutoff - timedelta(days=15)
self.interval = int(document["monitoring_interval_seconds"])
self.supported = self.interval % 60 == 0 and 86400 % self.interval == 0
self.count = 15 * (86400 // self.interval) if self.supported else 0
ids = dict.fromkeys(self.settings.pressure_device_ids + self.settings.outlet_flow_device_ids
+ [device for rule in config.rules if rule.kind == "relative_change"
for device in rule.device_ids])
self.values = {device: np.full(self.count, np.nan) for device in ids}
def add(self, rows: list[dict]) -> None:
for row in rows:
device = row["device_id"]
if device not in self.values:
continue
at = datetime.fromisoformat(row["time"])
if at >= self.cutoff:
raise ValueError("动态阈值拟合不能读取回放开始时刻及之后的观测")
offset = (at - self.start).total_seconds()
if (not self.supported or offset < 0 or offset % self.interval
or row.get("quality") not in {"valid", "corrected"}
or row.get("cleaned_value") is None):
continue
index = int(offset // self.interval)
if np.isfinite(self.values[device][index]):
raise ValueError("动态阈值拟合存在重复的设备时间点")
self.values[device][index] = float(row["cleaned_value"])
def fit(self) -> dict:
base = {
"version": CALIBRATION_VERSION,
"diagnostic_policy": DIAGNOSTIC_POLICY,
"knowledge_cutoff": self.cutoff.isoformat(),
"history_start": self.start.isoformat(),
"required_samples_per_device": self.count,
"pressure": {}, "outlet_flow": {}, "diagnostic": {},
}
if not self.supported:
return {**base, "status": "insufficient_data", "reason": "unsupported_sampling_interval"}
available = {device: int(np.isfinite(values).sum()) for device, values in self.values.items()}
# A regular diurnal rise/reversal can be a huge percentage of a small
# preceding flow. Judge its absolute value against causal normal readings
# at the same sampling phase before treating that percentage as abnormal.
points_per_day = 86400 // self.interval
for device, values in self.values.items():
if available[device] != self.count:
continue
days = values.reshape(15, points_per_day)
center = np.median(days, axis=0)
sigma = 1.4826 * np.median(np.abs(days - center), axis=0)
margin = np.maximum(3 * sigma, np.maximum(np.abs(center) * 1e-6, 1e-9))
base["diagnostic"][device] = {
"phase_origin": self.start.isoformat(), "interval_seconds": self.interval,
"lower": (days.min(axis=0) - margin).tolist(),
"upper": (days.max(axis=0) + margin).tolist(),
"method": "normal_same_phase_range_plus_3_robust_sigma", "normal_day_count": 15,
}
for kind, ids, direction, floor in (
("pressure", self.settings.pressure_device_ids, -1, 3.0),
("outlet_flow", self.settings.outlet_flow_device_ids, 1, 5.0),
):
for device in ids:
if available[device] != self.count:
continue
rule = next((rule for rule in self.config.rules
if rule.kind == "relative_change" and device in rule.device_ids), None)
if rule is None:
continue
series = pd.Series(self.values[device])
reference = series.rolling(rule.window_samples, min_periods=rule.minimum_samples).median().shift(1)
changes = (direction * 100 * (series - reference) / reference.abs()).to_numpy()
changes = changes[np.isfinite(changes)]
if not len(changes):
continue
center = float(np.median(changes))
sigma = float(1.4826 * np.median(np.abs(changes - center)))
base[kind][device] = {
"threshold_percent": max(floor, center + 3 * sigma),
"normal_change_median": center,
"normal_change_robust_sigma": sigma,
"normal_change_count": int(len(changes)),
"method": "normal_recent_window_change_median_plus_3_robust_sigma",
"normal_profile": base["diagnostic"].get(device),
}
pressure_ids = self.settings.pressure_device_ids
incomplete = {device: available[device] for device in pressure_ids if available[device] != self.count}
if len(pressure_ids) < 5 or incomplete:
return {**base, "status": "insufficient_data", "reason": "pressure_history_incomplete",
"available_samples": available}
from app.services import scada_replay_monitoring as monitoring
index = pd.date_range(start=self.start, periods=self.count, freq=pd.Timedelta(seconds=self.interval))
frame = pd.DataFrame({device: self.values[device] for device in pressure_ids}, index=index)
result = monitoring.run_burst_detection(
network="scada-replay-baseline", username="scada-replay",
observed_pressure_data=frame, points_per_day=86400 // self.interval,
mu=monitoring.TARGET_MU,
iforest_params={"n_estimators": monitoring.TARGET_N_ESTIMATORS,
"random_state": monitoring.TARGET_RANDOM_STATE, "contamination": "auto"},
sensor_nodes=pressure_ids,
)
score = adaptive_score_threshold([float(row["Score"]) for row in result["rows"]])
return {**base, "status": score["status"], "burst_detection": score}
@@ -0,0 +1,98 @@
"""Persist one accident across monitoring periods without moving its evidence cutoff."""
from datetime import datetime
from math import isfinite
from uuid import NAMESPACE_URL, uuid5
def _anchors(monitoring: dict, indicator: str = "pressure") -> dict:
return {
item["device_id"]: {"baseline": item["baseline"], "threshold_percent": item["threshold_percent"]}
for item in monitoring.get(indicator, {}).get("devices", [])
if item.get("hit") and item.get("baseline", 0) > 0
and 0 < item.get("threshold_percent", 0) < 100
}
def update_incident(document: dict, monitoring: dict, sequence: int) -> tuple[dict | None, str | None]:
incident = document.get("monitoring_incident")
# Adopt the last task from an older replay document on its first new step.
previous = document.get("last_monitoring") or {}
if incident:
incident.setdefault("burst_diagnosed", previous.get("status") == "suspected_burst")
if not incident and previous.get("task_id"):
incident = {
"incident_id": str(uuid5(NAMESPACE_URL, f"scada_replay:{document['replay_id']}:monitor:"
f"{previous['period_end']}:{previous['rule_version']}")),
"started_at": previous["period_end"], "last_observed_at": previous["period_end"],
"status": "active", "task_id": previous["task_id"], "recovery_count": 0,
"pressure_anchors": _anchors(previous), "alert": None,
"flow_anchors": _anchors(previous, "outlet_flow"),
"burst_diagnosed": previous.get("status") == "suspected_burst",
}
document["monitoring_incident"] = incident
if incident and incident.get("last_monitoring_id") == monitoring.get("monitoring_id") and monitoring.get("monitoring_id"):
return incident, incident["last_action"]
anomaly = (monitoring["status"] in {"attention", "suspected_burst"}
or bool(monitoring.get("anomaly_signals"))
or any(monitoring.get(key, {}).get("hit") is True
for key in ("pressure", "outlet_flow", "burst_detection")))
if anomaly and (not incident or incident["status"] == "resolved"):
external = (f"scada_replay:{document['replay_id']}:monitor:"
f"{monitoring['period_end']}:{monitoring['rule_version']}")
incident = {
"incident_id": str(uuid5(NAMESPACE_URL, external)), "external_id": external,
"started_at": monitoring["period_end"], "status": "active", "task_id": None,
"recovery_count": 0, "pressure_anchors": _anchors(monitoring), "alert": None,
"flow_anchors": _anchors(monitoring, "outlet_flow"),
"batch_sequence": sequence,
}
document["monitoring_incident"] = incident
action = "raised"
elif incident and incident["status"] == "active":
# A later diagnostic stage may add pressure/flow evidence to the same task.
# Freeze each newly abnormal device once; retain earlier references.
for indicator, field in (("pressure", "pressure_anchors"), ("outlet_flow", "flow_anchors")):
for device, anchor in _anchors(monitoring, indicator).items():
incident.setdefault(field, {}).setdefault(device, anchor)
current = {item["device_id"]: item for item in monitoring.get("pressure", {}).get("devices", [])}
anchors = incident["pressure_anchors"]
flow = {item["device_id"]: item for item in monitoring.get("outlet_flow", {}).get("devices", [])}
# Rolling relative changes can become normal while absolute pressure stays low.
# Confirm recovery against the frozen pre-accident references as well.
recovered = monitoring["status"] == "normal" and not anomaly and all(
monitoring.get(key, {}).get("status") == "ready"
and monitoring[key].get("hit") is False for key in ("pressure", "burst_detection")
) and all(
device in current and isfinite(current[device]["value"])
and current[device].get("observed_at") is not None
and datetime.fromisoformat(current[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
and current[device]["value"] >= anchor["baseline"] * (1 - anchor["threshold_percent"] / 100)
for device, anchor in anchors.items()
) and all(
device in flow and isfinite(flow[device]["value"])
and flow[device].get("observed_at") is not None
and datetime.fromisoformat(flow[device]["observed_at"]) > datetime.fromisoformat(incident["last_observed_at"])
and flow[device]["value"] <= anchor["baseline"] * (1 + anchor["threshold_percent"] / 100)
for device, anchor in incident.get("flow_anchors", {}).items()
)
contiguous = bool(monitoring.get("period_start")) and (
datetime.fromisoformat(monitoring["period_start"]) == datetime.fromisoformat(incident["last_observed_at"])
)
incident["recovery_count"] = (incident["recovery_count"] + 1 if contiguous else 1) if recovered else 0
action = "resolved" if incident["recovery_count"] >= 2 else "ongoing"
if action == "resolved":
incident["status"] = "resolved"
else:
return None, None
incident.update(last_observed_at=monitoring["period_end"],
last_monitoring_id=monitoring.get("monitoring_id"), last_action=action)
return incident, action
def attach_incident(monitoring: dict, incident: dict, action: str) -> None:
monitoring["task_id"] = incident["task_id"]
monitoring["incident"] = {
key: incident[key] for key in (
"incident_id", "status", "started_at", "last_observed_at", "recovery_count",
)
} | {"action": action}
+48
View File
@@ -0,0 +1,48 @@
"""Clock-phase normal envelopes frozen before replay; never inspect future readings."""
from datetime import datetime
from math import isfinite
CALIBRATION_VERSION = "scada-monitor-calibration@2"
DIAGNOSTIC_POLICY = "normal-phase-envelope@1"
def weighted_diagnosis(document: dict) -> bool:
return (document.get("monitoring_config") or {}).get("decision_policy") == "weighted"
def current_calibration(document: dict, start: datetime) -> dict | None:
calibration = document.get("monitoring_calibration") or {}
if (calibration.get("version") == CALIBRATION_VERSION
and calibration.get("diagnostic_policy") == DIAGNOSTIC_POLICY
and calibration.get("knowledge_cutoff") == start.isoformat()):
return calibration
return None
def diagnostic_signal(document: dict, event: dict) -> bool:
"""Never revive a persisted fixed-percentage event as new diagnosis evidence."""
if event.get("category") == "data_quality":
return False
if not weighted_diagnosis(document):
return True
rules = (document.get("config") or {}).get("rules", [])
rule = next((item for item in rules if item["rule_id"] == event["rule_id"]), None)
return bool(rule and (rule["kind"] == "bounds" or event.get("diagnostic_policy") == DIAGNOSTIC_POLICY))
def normal_profile_bounds(profile: dict | None, at: datetime) -> tuple[float, float] | None:
if not profile:
return None
interval = profile.get("interval_seconds", 0)
if not profile.get("phase_origin"):
return None
origin = datetime.fromisoformat(profile["phase_origin"])
lower, upper = profile.get("lower", []), profile.get("upper", [])
offset = (at - origin).total_seconds()
if interval <= 0 or offset < 0 or offset % interval or not lower or len(lower) != len(upper):
return None
slot = int(offset // interval) % len(lower)
low, high = lower[slot], upper[slot]
if low is None or high is None or not isfinite(low) or not isfinite(high):
return None
return float(low), float(high)
+245 -42
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import json
from uuid import NAMESPACE_URL, uuid4, uuid5
from fastapi import HTTPException
@@ -14,6 +15,18 @@ from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.content_digest import digest
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
from app.services.scada_replay_monitoring import evaluate_monitoring_period
from app.services.scada_normal_profile import current_calibration, diagnostic_signal, weighted_diagnosis
# 500 devices at 15-minute sampling need 720,000 rows for fifteen days.
# Leave headroom without allowing dense sources to make preparation unbounded.
BASELINE_PAGE_SIZE = 2000
BASELINE_MAX_ROWS = 1_000_000
BASELINE_MAX_BYTES = 512 * 1024 * 1024
class BaselineBudgetExceeded(RuntimeError):
"""The unpublished baseline exceeded its fixed preparation budget."""
def parse_time(value: str) -> datetime:
@@ -36,6 +49,10 @@ def snapshot(document: dict) -> ScadaReplaySnapshot:
step_pending=document.get("step_pending", False),
monitoring_count=document.get("monitoring_count", 0),
last_monitoring=document.get("last_monitoring"),
# Old persisted documents without a policy still use three-signal detection.
monitoring_config=({"decision_policy": "all_signals", **document["monitoring_config"]}
if document.get("monitoring_config") else None),
monitoring_calibration=document.get("monitoring_calibration"),
)
@@ -105,7 +122,7 @@ class ScadaReplayService:
await self.ensure_storage()
async with self.repo.control():
document = await self.repo.get(replay_id)
key, body = str(command.request_id), command.model_dump(mode="json")
key, body = str(command.request_id), command.model_dump(mode="json", exclude_none=True)
receipt = document.setdefault("receipts", {}).get(key)
if receipt:
if receipt != body:
@@ -114,9 +131,13 @@ class ScadaReplayService:
if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
action, status = command.action, document["status"]
if document.get("step_pending") and action == "step":
if (action in {"step", "monitor", "retry"} and document.get("warmed")
and weighted_diagnosis(document)
and current_calibration(document, parse_time(document["start_time"])) is None):
raise HTTPException(409, "旧回放缺少新版历史诊断校准,请停止后新建演示")
if document.get("step_pending") and action in {"step", "monitor"}:
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
if action in {"step", "monitor", "retry"} and await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有另一条活动回放")
if action == "step" and status in {"ready", "paused"}:
interval = document.get("monitoring_interval_seconds")
@@ -126,8 +147,31 @@ class ScadaReplayService:
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
document["step_start"] = document["virtual_now"]
document["step_anomaly_signals"] = []
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
document["status"], document["step_pending"] = "paused", True
elif action == "monitor" and status in {"ready", "paused", "completed"}:
target = command.at_time
if target is None:
raise HTTPException(422, "选定时刻监测需要提供 at_time")
start = parse_time(document["start_time"])
current = parse_time(document["virtual_now"])
if not start <= target <= parse_time(document["end_time"]):
raise HTTPException(422, "监测时刻必须在本轮演示区间内")
if target < current:
raise HTTPException(409, "监测历史时刻需要新建一轮回放,避免使用后续状态")
interval = document.get("monitoring_interval_seconds")
if interval is None:
raise HTTPException(409, "本轮未固定监测周期,请停止后新建演示")
previous = document.get("last_monitoring") or {}
# Reusing an immutable result must not count as another normal
# period, resolve an incident, or create another investigation.
if not previous.get("period_end") or parse_time(previous["period_end"]) != target:
document["step_start"] = (current if target > current
else max(start, target - timedelta(seconds=interval))).isoformat()
document["step_target"] = target.isoformat()
document["step_anomaly_signals"] = []
document["status"], document["step_pending"] = "paused", True
elif action == "stop" and status not in {"completed", "stopped"}:
document["status"], document["step_pending"] = "stopped", False
document.pop("step_target", None)
@@ -161,11 +205,16 @@ class ScadaReplayService:
# Failure never discards the prepared batch; retry resumes it.
document = await self.repo.get(replay_id)
document["status"], document["step_pending"] = "failed", False
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
)
if isinstance(exc, HTTPException):
document["error"] = str(exc.detail)
elif isinstance(exc, BaselineBudgetExceeded):
document["error"] = str(exc)
else:
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
)
async with self.repo.connection.transaction():
await self.repo.save(document)
raise
@@ -174,23 +223,9 @@ class ScadaReplayService:
config = ScadaReplayCreate.model_validate(document["config"])
replay_id = document["replay_id"]
if not document.get("warmed"):
import math
window = max(rule.window_samples for rule in config.rules)
async with self.timescale.transaction():
warmup = await ScadaRepository.get_warmup_values(
self.timescale, config.device_ids, config.start_time, window,
)
for row in warmup:
value = row["monitored_value"]
bounds = config.quality_ranges.get(row["device_id"])
if value is not None and math.isfinite(value) and not (bounds and (
bounds.lower is not None and value < bounds.lower
or bounds.upper is not None and value > bounds.upper
)):
document["histories"].setdefault(row["device_id"], []).append(value)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.save(document)
await self._freeze_baseline(document, config)
if weighted_diagnosis(document) and current_calibration(document, config.start_time) is None:
raise HTTPException(409, "旧回放缺少新版历史诊断校准,请停止后新建演示")
batch = await self.repo.pending_batch(replay_id)
if batch is None:
step = document.get("step_pending", False)
@@ -252,6 +287,15 @@ class ScadaReplayService:
target = result.get("step_target")
was_step = bool(target)
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
if was_step:
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
if diagnostic_signal(result, item)}
for event in changes:
if event["change"] == "raised" and diagnostic_signal(result, event):
signals[event["event_id"]] = {key: event[key] for key in (
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
) if key in event}
result["step_anomaly_signals"] = list(signals.values())
monitoring = None
if was_step and not result["step_pending"]:
period_start = parse_time(result.get("step_start", result["start_time"]))
@@ -261,7 +305,7 @@ class ScadaReplayService:
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
))
previous = await self.repo.monitoring_outputs(
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
replay_id, role_ids, period_end - timedelta(days=15),
period_end, batch["sequence"],
) if role_ids else []
from app.mcp.analysis_service import bounded_calculation
@@ -273,11 +317,25 @@ class ScadaReplayService:
period_end=period_end,
batch_sequence=batch["sequence"],
)
result["last_monitoring"] = monitoring
# Any configured measurement rule can enter diagnosis, including
# indicators outside the pressure/outlet roles used for burst scoring.
signals = {item["event_id"]: item for item in result.get("step_anomaly_signals", [])
if diagnostic_signal(result, item)}
for state in result.get("rule_states", {}).values():
event = state.get("event")
if event and diagnostic_signal(result, event):
signals[event["event_id"]] = {key: event[key] for key in (
"event_id", "rule_id", "device_ids", "detail", "diagnostic_policy"
) if key in event}
monitoring["anomaly_signals"] = list(signals.values())
if signals and monitoring["status"] == "normal":
monitoring["status"] = "attention"
monitoring["detail"] = "本周期曾检出测点异常,进入异常调查与爆管诊断"
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]:
result.pop("step_target", None)
result.pop("step_start", None)
result.pop("step_anomaly_signals", None)
if batch["refs"]:
last = batch["refs"][-1]
result["cursor"] = [last["time"], last["device_id"]]
@@ -295,6 +353,80 @@ class ScadaReplayService:
await self.repo.save_batch(replay_id, batch)
await self.repo.save(result)
async def _freeze_baseline(self, document, config):
"""Stage bounded pages, then atomically publish one causal source snapshot."""
if document.get("warmed"):
return
start = config.start_time - timedelta(days=15)
batch_count, row_count, byte_count, after = 0, 0, 0, None
counts = {device_id: 0 for device_id in config.device_ids}
valid_counts = dict(counts)
histories = {device_id: [] for device_id in config.device_ids}
window = max(rule.window_samples for rule in config.rules)
calibration = None
if (document.get("monitoring_config") or {}).get("decision_policy") == "weighted":
from app.services.scada_monitoring_calibration import MonitoringCalibrationBuilder
calibration = MonitoringCalibrationBuilder(document, config)
async with self.repo.connection.transaction():
await self.repo.clear_unpublished_baseline(document["replay_id"])
# Keep only the current page in memory. Each BizDB write commits before
# the next source read; clean_prepared baseline pages remain invisible.
async with self.timescale.transaction():
await ScadaRepository.begin_replay_baseline_snapshot(self.timescale)
while True:
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, start, config.start_time,
config.start_time, min(BASELINE_PAGE_SIZE, BASELINE_MAX_ROWS - row_count + 1), after,
)
if not rows:
break
if row_count + len(rows) > BASELINE_MAX_ROWS:
raise BaselineBudgetExceeded(
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_ROWS:,} 条准备预算,"
"尚未发布;请减少回放设备或核查历史采样密度后重试"
)
outputs = prepare_cleaning(rows, config)
if calibration is not None:
calibration.add(outputs)
batch = {
"sequence": -batch_count - 1, "stage": "clean_prepared", "kind": "baseline",
"at": outputs[-1]["time"], "outputs": outputs, "events": [],
}
batch_bytes = len(json.dumps(batch, ensure_ascii=False, allow_nan=False,
separators=(",", ":")).encode("utf-8"))
if byte_count + batch_bytes > BASELINE_MAX_BYTES:
raise BaselineBudgetExceeded(
f"BASELINE_BUDGET_EXCEEDED: 十五天历史基线超过 {BASELINE_MAX_BYTES // (1024 * 1024)} MiB 准备预算,"
"尚未发布;请减少回放设备或核查历史采样密度后重试"
)
async with self.repo.connection.transaction():
await self.repo.save_batch(document["replay_id"], batch)
batch_count += 1
row_count += len(outputs)
byte_count += batch_bytes
for row in outputs:
device_id = row["device_id"]
counts[device_id] += 1
if row["quality"] == "valid":
valid_counts[device_id] += 1
histories[device_id].append(row["cleaned_value"])
histories[device_id] = histories[device_id][-window:]
last = rows[-1]
after = (last["time"], last["device_id"])
document["baseline"] = {
"start_time": start.isoformat(), "end_time": config.start_time.isoformat(),
"end_exclusive": True, "sample_counts": counts, "valid_sample_counts": valid_counts,
"row_count": row_count, "batch_count": batch_count, "byte_count": byte_count,
}
document["histories"] = histories
if calibration is not None:
from app.mcp.analysis_service import bounded_calculation
document["monitoring_calibration"] = await bounded_calculation(calibration.fit)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.publish_baseline(document["replay_id"])
await self.repo.save(document)
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
@@ -309,9 +441,20 @@ class ScadaReplayService:
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context))
for event in changes:
# Period diagnosis owns measurement notifications during manual steps.
# Retain rule events as evidence; quality alerts remain independent.
if (not create_tasks and event.get("category") != "data_quality"
and (event["change"] == "raised" or event.get("notification_suppressed"))):
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
event["notification_suppressed"] = True
document["rule_states"][event.get("state_key", event["rule_id"])]["event"].update(
batch_sequence=batch["sequence"], notification_suppressed=True,
)
continue
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["batch_sequence"] = batch["sequence"]
await alerts.push(TaskAlertEvent(
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
@@ -345,10 +488,30 @@ class ScadaReplayService:
)
task = await tasks.create(command)
event["task_id"] = str(task.task_id)
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
document["rule_states"][event.get("state_key", event["rule_id"])]["event"]["task_id"] = event["task_id"]
async def _create_monitoring_task(self, document, monitoring, batch):
if monitoring["status"] != "suspected_burst":
from app.services.scada_monitoring_incidents import attach_incident, update_incident
incident, action = update_incident(document, monitoring, batch["sequence"])
if incident is None:
return
if incident.get("task_id"):
if monitoring["status"] == "suspected_burst" and not incident.get("burst_diagnosed"):
from uuid import UUID
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
tasks = OperationalTaskService(
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context),
)
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
"batch_sequence": batch["sequence"],
}, uuid5(NAMESPACE_URL, incident["incident_id"] + ":burst-evidence"))
incident["burst_diagnosed"] = True
if action == "resolved" and incident.get("alert"):
await self._publish_monitoring_alert(document, incident, "resolved")
attach_incident(monitoring, incident, action)
return
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
@@ -358,29 +521,44 @@ class ScadaReplayService:
self.context,
permissions_for_context(self.context),
)
external = (
f"scada_replay:{document['replay_id']}:monitor:"
f"{monitoring['period_end']}:{monitoring['rule_version']}"
)
external = incident["external_id"]
pressure = monitoring["pressure"]
flow = monitoring["outlet_flow"]
detector = monitoring["burst_detection"]
score = detector.get("target_score")
score_text = f"{score:.4f}" if score is not None else "数据不足"
summary = (
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
f"SCADA 数据诊断发现指标异常,进入异常调查与爆管诊断。"
f"爆管侦测得分 {score_text};"
f"压力下降测点 {len(pressure.get('hit_device_ids', []))} 个;"
f"出厂边界流量增加测点 {len(flow.get('hit_device_ids', []))} 个。"
"任一指标异常即启动调查,加权分仅用于评估爆管嫌疑,不能据此提前确诊。"
)
if monitoring.get("anomaly_signals"):
signals = monitoring["anomaly_signals"]
devices = {device for item in signals for device in item.get("device_ids", [])}
summary += f" 单项异常 {len(signals)} 项,涉及 {len(devices)} 个测点;明细通过冻结 SCADA 查询核验。"
if monitoring.get("weighted"):
weighted = monitoring["weighted"]
weights = weighted["weights"]
summary += (
f" 综合评分 {weighted['score']:.2f},爆管嫌疑门槛 {weighted['threshold']:.2f}"
f"(侦测 {weights['burst_detection']:.0%}、压力 {weights['pressure']:.0%}、"
f"流量 {weights['outlet_flow']:.0%})。"
)
if detector.get("score_threshold") is not None:
summary += f" 侦测动态阈值 {detector['score_threshold']:.4f}。"
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external),
task_type="event_investigation",
workflow="burst_report",
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
workflow="scada_anomaly",
title=f"SCADA 异常调查 · {monitoring['period_end']}",
summary=summary,
source=TaskSource(
kind="external_system",
external_id=external,
external_revision=1,
event_kind="scada_burst_suspected",
event_kind="scada_anomaly",
replay_context={
"replay_id": document["replay_id"],
"knowledge_cutoff": monitoring["knowledge_cutoff"],
@@ -390,8 +568,33 @@ class ScadaReplayService:
trigger=TaskTrigger(
kind="event",
occurred_at=monitoring["period_end"],
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
detail="SCADA 任一指标异常启动调查,核验数据后进行爆管诊断;异常不等于已确诊爆管",
),
)
task = await tasks.create(command)
monitoring["task_id"] = str(task.task_id)
incident["task_id"] = str(task.task_id)
incident["burst_diagnosed"] = monitoring["status"] == "suspected_burst"
incident["alert"] = {
"alert_id": incident["incident_id"],
"producer_id": "scada-replay:" + document["replay_id"],
"occurred_at": incident["started_at"],
"condition_id": f"operational-task:{task.task_id}",
"source": {"kind": "scada_replay", "replay_id": document["replay_id"],
"rule_id": "scada_diagnosis", "batch_sequence": batch["sequence"]},
"title": "SCADA 指标异常", "description": "已启动调查,详细诊断见调查报告。", "severity": "warning",
}
await self._publish_monitoring_alert(document, incident, "raised")
attach_incident(monitoring, incident, action)
async def _publish_monitoring_alert(self, document, incident, change):
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
alerts = TaskAlertService(
TaskAlertRepository(self.repo.connection, self.context.project_id), self.context,
internal_producer_id="scada-replay:" + document["replay_id"],
)
await alerts.push(TaskAlertEvent(
**incident["alert"], event_type=change,
event_id=uuid5(NAMESPACE_URL, incident["incident_id"] + ":" + change),
))
+93 -16
View File
@@ -9,6 +9,8 @@ from uuid import NAMESPACE_URL, uuid5
import pandas as pd
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.services.scada_normal_profile import current_calibration, normal_profile_bounds
from app.services.burst_detection import (
TARGET_MU,
TARGET_N_ESTIMATORS,
@@ -42,6 +44,7 @@ def _relative_indicator(
period_end: datetime,
*,
direction: str,
calibration: dict | None = None,
) -> dict[str, Any]:
details: list[dict[str, Any]] = []
gaps: list[dict[str, str]] = []
@@ -54,30 +57,51 @@ def _relative_indicator(
if rule is None:
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
continue
valid = _valid_rows(rows, device_id)
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
if len(baseline) < rule.minimum_samples:
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
profile = calibration.get(device_id) if calibration is not None else None
if calibration is not None and profile is None:
gaps.append({"device_id": device_id, "reason": "adaptive_history_not_ready"})
continue
threshold = float(profile["threshold_percent"]) if profile else rule.threshold
valid = _valid_rows(rows, device_id)
current = [row for row in valid if period_start < _time(row["time"]) <= period_end]
latest = current[-1] if current else None
if not current:
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
continue
baseline = ([row for row in valid if _time(row["time"]) < _time(latest["time"])]
[-rule.window_samples:]) if latest else []
if len(baseline) < rule.minimum_samples:
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
continue
reference = median(float(row["cleaned_value"]) for row in baseline)
if reference == 0:
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
continue
latest = current[-1]
value = float(latest["cleaned_value"])
change = (value - reference) / abs(reference) * 100
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
hit = change <= -threshold if direction == "decrease" else change >= threshold
bounds = normal_profile_bounds(profile.get("normal_profile"), _time(latest["time"])) if profile else None
if profile and bounds is None:
gaps.append({"device_id": device_id, "reason": "normal_phase_history_not_ready"})
continue
if bounds is not None:
hit = hit and not bounds[0] <= value <= bounds[1]
details.append({
"device_id": device_id,
"observed_at": _time(latest["time"]).isoformat(),
"baseline": reference,
"baseline_method": "recent_samples_median",
"baseline_start": _time(baseline[0]["time"]).isoformat(),
"baseline_end": _time(baseline[-1]["time"]).isoformat(),
"baseline_sample_count": len(baseline),
"value": value,
"value_change": value - reference,
"unit": rule.unit,
"change_percent": change,
"threshold_percent": rule.threshold,
"threshold_percent": threshold,
**({"normal_phase_lower": bounds[0], "normal_phase_upper": bounds[1]} if bounds else {}),
**({"threshold_method": profile["method"],
"configured_threshold_percent": rule.threshold} if profile else {}),
"hit": hit,
})
return {
@@ -94,6 +118,8 @@ def _burst_detection(
pressure_device_ids: list[str],
interval_seconds: int,
period_end: datetime,
*,
score_threshold: float = TARGET_SCORE_THRESHOLD,
) -> dict[str, Any]:
if len(pressure_device_ids) < 5:
return {"status": "sensor_coverage_insufficient", "hit": False,
@@ -112,7 +138,10 @@ def _burst_detection(
common_times = set(values) if common_times is None else common_times & set(values)
if not common_times:
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
target = max(common_times)
target = period_end
if target not in common_times:
return {"status": "current_reading_missing", "hit": False,
"target_time": target.isoformat(), "latest_common_time": max(common_times).isoformat()}
expected = [target - timedelta(seconds=interval_seconds * offset)
for offset in range(count - 1, -1, -1)]
incomplete = [device_id for device_id, values in series_by_device.items()
@@ -142,10 +171,11 @@ def _burst_detection(
score = float(latest["Score"])
return {
"status": "ready",
"hit": score <= TARGET_SCORE_THRESHOLD,
"hit": score <= score_threshold,
"target_time": target.isoformat(),
"target_score": score,
"score_threshold": TARGET_SCORE_THRESHOLD,
"score_interpretation": "lower_is_more_anomalous_not_probability",
"score_threshold": score_threshold,
"sensor_count": len(pressure_device_ids),
"reference_day_count": 14,
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
@@ -185,17 +215,64 @@ def evaluate_monitoring_period(
flow_ids = monitoring.get("outlet_flow_device_ids", [])
selected = set(config.device_ids)
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
if not pressure_ids or not flow_ids or role_gaps:
weighted_policy = monitoring.get("decision_policy") == "weighted"
if not weighted_policy and (not pressure_ids or not flow_ids or role_gaps):
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
"role_gaps": role_gaps,
"pressure": {"status": "unconfigured", "hit": False},
"outlet_flow": {"status": "unconfigured", "hit": False},
"burst_detection": {"status": "not_run", "hit": False}}
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
detector = _burst_detection(
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
calibration = current_calibration(document, config.start_time) if weighted_policy else None
if config.start_time > period_end:
calibration = None
base["role_gaps"] = role_gaps
pressure = _relative_indicator(
rows, config, pressure_ids, period_start, period_end, direction="decrease",
calibration=(calibration or {}).get("pressure", {}) if weighted_policy else None,
)
outlet_flow = _relative_indicator(
rows, config, flow_ids, period_start, period_end, direction="increase",
calibration=(calibration or {}).get("outlet_flow", {}) if weighted_policy else None,
) if flow_ids else {"status": "unconfigured", "hit": False, "hit_device_ids": [], "devices": [], "gaps": []}
detector = {"status": "baseline_not_ready", "hit": False} if weighted_policy and (
not calibration or calibration.get("status") != "ready"
) else _burst_detection(
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
score_threshold=calibration["burst_detection"]["threshold"] if calibration else TARGET_SCORE_THRESHOLD,
)
if weighted_policy:
settings = ScadaMonitoringSettings.model_validate(monitoring)
weights = settings.weights.model_dump()
indicators = {"burst_detection": detector, "pressure": pressure, "outlet_flow": outlet_flow}
contributions = {key: weights[key] if indicator["status"] == "ready" and indicator["hit"] else 0.0
for key, indicator in indicators.items()}
score = round(sum(contributions.values()), 10)
complete = pressure["status"] == detector["status"] == "ready"
diagnostic_missing = sorted({device for rule in config.rules if rule.kind == "relative_change"
for device in rule.device_ids}
- set((calibration or {}).get("diagnostic", {})))
if not complete:
status, detail = "insufficient_data", (
"旧回放缺少新版历史诊断校准,请停止后新建演示" if calibration is None
else "压力数据或爆管侦测基线不足,尚不能作出组合判定"
)
elif score >= settings.alert_threshold:
status, detail = "suspected_burst", "爆管相关指标加权达到疑似爆管门槛"
elif any(indicator["hit"] for indicator in indicators.values()):
status, detail = "attention", "部分指标异常,进入调查;综合结果尚未达到爆管嫌疑门槛"
elif diagnostic_missing:
status, detail = "insufficient_data", "部分测点缺少完整历史同期基线,不能判定全部指标正常"
else:
status, detail = "normal", "本周期爆管相关指标均未命中"
detail += f"(综合 {score:.2f} / 门槛 {settings.alert_threshold:.2f},流量权重 {weights['outlet_flow']:.0%})"
return {**base, "status": status, "detail": detail, "pressure": pressure,
"outlet_flow": outlet_flow, "burst_detection": detector,
"weighted": {"score": score, "threshold": settings.alert_threshold,
"weights": weights, "contributions": contributions,
"unavailable_indicators": [key for key, value in indicators.items() if value["status"] != "ready"],
"calibration_cutoff": calibration["knowledge_cutoff"] if calibration else None,
"diagnostic_unavailable_device_ids": diagnostic_missing,
"missing_weight_policy": "zero_contribution_without_renormalization"}}
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
if not complete:
+24 -3
View File
@@ -6,6 +6,9 @@ from statistics import median
from uuid import NAMESPACE_URL, uuid5
import re
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.scada_normal_profile import (
DIAGNOSTIC_POLICY, current_calibration, normal_profile_bounds, weighted_diagnosis,
)
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
@@ -43,6 +46,8 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
by_id = {row["device_id"]: row for row in rows}
changes = []
devices = {device["device_id"]: device for device in result.get("devices", [])}
weighted = weighted_diagnosis(result)
calibration = current_calibration(result, config.start_time) if weighted else None
for device_id in config.device_ids:
row = by_id.get(device_id)
invalid = row is not None and row["quality"] != "valid"
@@ -75,7 +80,8 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
state["event"] = None
for rule in config.rules:
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
state_key = f"{rule.rule_id}:{DIAGNOSTIC_POLICY}" if weighted and rule.kind == "relative_change" else rule.rule_id
state = states.setdefault(state_key, {"hits": 0, "recoveries": 0, "event": None})
hits, usable, reasons = [], [], []
for device_id in rule.device_ids:
row = by_id.get(device_id)
@@ -86,6 +92,17 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
triggered = ((rule.lower is not None and value < rule.lower)
or (rule.upper is not None and value > rule.upper))
reason = f"{device_id}={value} {rule.unit},超出配置范围"
elif weighted:
# The percentage in the compatibility rule is not a diagnosis
# threshold. Learn each device's normal same-phase range first.
profile = (calibration or {}).get("diagnostic", {}).get(device_id)
bounds = normal_profile_bounds(profile, at)
if bounds is None:
continue
low, high = bounds
triggered = (value < low or value > high if rule.direction == "both"
else value > high if rule.direction == "increase" else value < low)
reason = f"{device_id}={value} {rule.unit},超出冻结正常同时段范围 [{low:.4f}, {high:.4f}] {rule.unit}"
else:
previous = histories.get(device_id, [])[-rule.window_samples:]
if len(previous) < rule.minimum_samples:
@@ -106,10 +123,12 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
state["hits"] += 1
state["recoveries"] = 0
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{state_key}:{at.isoformat()}"))
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
if weighted and rule.kind == "relative_change":
event.update(diagnostic_policy=DIAGNOSTIC_POLICY, state_key=state_key)
state["event"] = event
changes.append({**event, "change": "raised"})
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
@@ -128,7 +147,9 @@ def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at
# Incomplete frames cannot count as a consecutive hit/recovery.
elif not usable and any(device in by_id for device in rule.device_ids):
state["hits"] = state["recoveries"] = 0
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
frozen = {device for rule in config.rules if states[
f"{rule.rule_id}:{DIAGNOSTIC_POLICY}" if weighted and rule.kind == "relative_change" else rule.rule_id
]["event"]
for device in rule.device_ids}
for row in rows:
if row["quality"] == "valid" and row["device_id"] not in frozen:
+12 -5
View File
@@ -12,14 +12,23 @@ def check_replay_window(context, start, end):
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
def check_snapshot_start(run, start):
replay_start = datetime.fromisoformat(run["start_time"])
if start < replay_start:
baseline = run.get("baseline")
if not baseline:
raise HTTPException(422, "本轮回放未冻结历史对照,请新建回放后查询历史基线")
if start < datetime.fromisoformat(baseline["start_time"]):
raise HTTPException(422, "查询起点早于本轮已冻结的历史基线")
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
check_replay_window(context, start, end)
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if not set(device_ids) <= set(run["config"]["device_ids"]):
raise HTTPException(403, "设备或回放不属于当前调查")
if start < datetime.fromisoformat(run["start_time"]):
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
check_snapshot_start(run, start)
rows = await repository.detected_outputs(
context["replay_id"], device_ids, start, end, context["batch_sequence"],
limit=limit, offset=offset,
@@ -40,9 +49,7 @@ async def replay_burst_location_inputs(connection, project_id, context, args):
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if run:
replay_start = datetime.fromisoformat(run["start_time"])
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
check_snapshot_start(run, min(args.scada_burst_start, args.scada_normal_start))
devices = {item["device_id"]: item for item in run.get("devices", [])}
monitoring = run.get("monitoring_config") or {}
pressure_devices = monitoring.get("pressure_device_ids", [])