feat(scada): add project monitoring replay
Generic Container CI/CD / test-build-publish (push) Successful in 3m39s
Server CI/CD v2 / build-test-publish-and-deploy (push) Successful in 3m39s

This commit is contained in:
2026-11-04 14:27:21 +08:00
parent f25423dec4
commit 26fa5d1909
53 changed files with 5153 additions and 20 deletions
+34
View File
@@ -161,6 +161,14 @@ class OperationalTaskService:
)
async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail:
if command.source.replay_context:
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
replay_context = command.source.replay_context
replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id)
require(replay_doc.get("delivered_until") is not None
and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"])
and replay_context.batch_sequence <= replay_doc["batch_sequence"],
"REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422)
body = _json(command)
request_id = str(command.request_id)
scope = "task:create"
@@ -219,6 +227,27 @@ class OperationalTaskService:
await self._task_event(task_document, now)
return await self.detail(task_id)
async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None:
"""A recovery is new evidence, coalesced through the existing run lifecycle."""
scope = f"task:{task_id}:replay-evidence"
async with self.repo.transaction():
if await self._request_replay(scope, request_id, replay_context):
return
await self.repo.lock(f"task:{task_id}")
task = await self._task(task_id)
prior = task["source"].get("replay_context")
require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID",
"Replay evidence belongs to another investigation", status=409)
require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]),
"REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409)
task["source"]["replay_context"] = replay_context
task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1
now = self._now()
await self._requeue_task(task, now)
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
await self._task_event(task, now)
await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)})
async def list(
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
peripheral_work_order_id: UUID | None = None,
@@ -336,6 +365,9 @@ class OperationalTaskService:
selected_report=await self.report(task_id, report_revision) if report_revision is not None else None,
)
async def replay_context(self, task_id: UUID) -> dict | None:
return (await self._task(task_id)).get("source", {}).get("replay_context")
async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool:
task = await self._task(task_id)
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
@@ -1037,6 +1069,8 @@ class OperationalTaskService:
await self.repo.lock(f"run:{command.run_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
run = await self._active_run(task)
require(run is not None and str(run["run_id"]) == str(command.run_id)
and run["generation"] == task["execution_generation"],
+66
View File
@@ -0,0 +1,66 @@
"""Read-only chronological SCADA paging shared by REST and replay processing."""
from __future__ import annotations
import base64
import hashlib
import json
import math
from datetime import datetime
from app.domain.schemas.scada import ScadaRawPage, ScadaRawReading
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
def validate_window(start: datetime, end: datetime) -> None:
if start.tzinfo is None or end.tzinfo is None or start >= end:
raise ValueError("需要带时区、结束晚于开始的 SCADA 时间区间")
def raw_reading(row: dict) -> ScadaRawReading:
value = row["monitored_value"]
invalid = value is not None and not math.isfinite(float(value))
return ScadaRawReading(
time=row["time"], device_id=row["device_id"],
monitored_value=None if invalid else value,
quality="invalid" if invalid else "missing" if value is None else "unvalidated",
)
async def read_raw_page(
conn, *, project_id: str, device_ids: list[str], start: datetime,
end: datetime, limit: int = 500, cursor: str | None = None,
through: datetime | None = None,
) -> ScadaRawPage:
validate_window(start, end)
ids = sorted(set(device_ids))
if not ids or len(ids) > 500 or not 1 <= limit <= 500:
raise ValueError("请选择 1 至 500 个设备,每页最多 500 条")
cap = min(through or end, end)
if cap.tzinfo is None or cap < start:
raise ValueError("模拟时间不在可读取区间内")
signature = hashlib.sha256(json.dumps(
[project_id, ids, start.isoformat(), end.isoformat()], separators=(",", ":")
).encode()).hexdigest()
after = None
if cursor:
try:
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
frozen = datetime.fromisoformat(data["through"])
last = datetime.fromisoformat(data["time"])
if (data["query"] != signature or frozen.tzinfo is None or last.tzinfo is None
or frozen > cap or not start <= last <= frozen or data["device"] not in ids):
raise ValueError()
cap, after = frozen, (last, data["device"])
except (ValueError, KeyError, TypeError, UnicodeError) as exc:
raise ValueError("SCADA 分页游标无效或查询范围已改变") from exc
rows = await ScadaRepository.get_raw_page(conn, ids, start, end, cap, limit + 1, after)
selected = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = selected[-1]
next_cursor = base64.urlsafe_b64encode(json.dumps({
"query": signature, "through": cap.isoformat(),
"time": last["time"].isoformat(), "device": last["device_id"],
}, separators=(",", ":")).encode()).decode()
return ScadaRawPage(items=[raw_reading(row) for row in selected],
next_cursor=next_cursor, scanned_through=cap)
+397
View File
@@ -0,0 +1,397 @@
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from uuid import NAMESPACE_URL, uuid4, uuid5
from fastapi import HTTPException
from app.auth.permissions import permissions_for_context
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
from app.infra.db.postgresql.scada import ScadaInfoRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.content_digest import digest
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
from app.services.scada_replay_monitoring import evaluate_monitoring_period
def parse_time(value: str) -> datetime:
return datetime.fromisoformat(value)
def snapshot(document: dict) -> ScadaReplaySnapshot:
return ScadaReplaySnapshot(
**{key: document[key] for key in (
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
"virtual_now", "created_at", "updated_at",
)},
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
devices=document["devices"], rules=document["config"]["rules"],
latest=list(document.get("latest", {}).values()),
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
error=document.get("error"),
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
step_pending=document.get("step_pending", False),
monitoring_count=document.get("monitoring_count", 0),
last_monitoring=document.get("last_monitoring"),
)
class ScadaReplayService:
def __init__(self, repository, context, timescale=None, *, clock=None):
self.repo, self.context, self.timescale = repository, context, timescale
self.clock = clock or (lambda: datetime.now(timezone.utc))
async def ensure_storage(self):
if not await self.repo.available():
raise HTTPException(503, "SCADA 回放存储未配置,需要应用 031_scada_replays.sql")
async def create(
self,
command: ScadaReplayCreate,
*,
monitoring_interval_seconds: int | None,
monitoring_config: ScadaMonitoringSettings | None = None,
):
await self.ensure_storage()
async with self.repo.control():
previous = await self.repo.by_request(command.request_id)
config = command.model_dump(mode="json")
if previous:
if previous["config"] != config:
raise HTTPException(409, "创建请求号已用于不同回放配置")
return snapshot(previous)
if monitoring_interval_seconds is None:
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds,再创建演示")
replay_id = str(uuid4())
if await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有回放,请先停止原回放再新建")
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
if not set(command.device_ids) <= metadata.keys():
raise HTTPException(422, "设备不属于当前项目")
for rule in command.rules:
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
raise HTTPException(422, "规则单位与设备单位不一致,请分组配置")
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
raise HTTPException(422, "水质规则必须明确指标名称,不能仅凭单位解释读数")
if monitoring_config:
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
if not role_ids <= metadata.keys():
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
for device in monitoring_config.outlet_flow_device_ids):
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
now = self.clock().isoformat()
document = {
"replay_id": replay_id, "project_id": str(self.context.project_id),
"created_by": str(self.context.user_id), "config": config,
"revision": 1, "status": "ready", "start_time": config["start_time"],
"end_time": config["end_time"], "virtual_now": config["start_time"],
"created_at": now, "updated_at": now,
"monitoring_interval_seconds": monitoring_interval_seconds,
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
"devices": [metadata[device] for device in command.device_ids],
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
}
await self.repo.insert(document)
return snapshot(document)
async def command(self, replay_id, command: ScadaReplayCommand):
await self.ensure_storage()
async with self.repo.control():
document = await self.repo.get(replay_id)
key, body = str(command.request_id), command.model_dump(mode="json")
receipt = document.setdefault("receipts", {}).get(key)
if receipt:
if receipt != body:
raise HTTPException(409, "命令请求号已用于不同内容")
return snapshot(document)
if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
action, status = command.action, document["status"]
if document.get("step_pending") and action == "step":
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有另一条活动回放")
if action == "step" and status in {"ready", "paused"}:
interval = document.get("monitoring_interval_seconds")
if interval is None:
raise HTTPException(409, "旧回放未固定监测周期,请停止后新建演示")
start = parse_time(document["start_time"])
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
document["step_start"] = document["virtual_now"]
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
document["status"], document["step_pending"] = "paused", True
elif action == "stop" and status not in {"completed", "stopped"}:
document["status"], document["step_pending"] = "stopped", False
document.pop("step_target", None)
document.pop("step_start", None)
elif action == "retry" and status == "failed":
if "SOURCE_CHANGED" in (document.get("error") or ""):
raise HTTPException(409, "原始数据发生变化,需要停止并新建回放")
document["status"], document["error"] = "paused", None
document["step_pending"] = bool(document.get("step_target"))
else:
raise HTTPException(409, "当前回放状态不支持此操作")
document["revision"] += 1
document["receipts"][key] = body
if len(document["receipts"]) > 256:
del document["receipts"][next(iter(document["receipts"]))]
await self.repo.save(document)
return snapshot(document)
async def process_pending_step(self, replay_id):
await self.ensure_storage()
async with self.repo.processing() as owned:
if not owned:
raise HTTPException(409, "当前时间步正在由另一请求处理")
while True:
document = await self.repo.get(replay_id)
if not document.get("step_pending"):
return snapshot(document)
try:
await self._advance(document)
except Exception as exc:
# Failure never discards the prepared batch; retry resumes it.
document = await self.repo.get(replay_id)
document["status"], document["step_pending"] = "failed", False
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
)
async with self.repo.connection.transaction():
await self.repo.save(document)
raise
async def _advance(self, document):
config = ScadaReplayCreate.model_validate(document["config"])
replay_id = document["replay_id"]
if not document.get("warmed"):
import math
window = max(rule.window_samples for rule in config.rules)
async with self.timescale.transaction():
warmup = await ScadaRepository.get_warmup_values(
self.timescale, config.device_ids, config.start_time, window,
)
for row in warmup:
value = row["monitored_value"]
bounds = config.quality_ranges.get(row["device_id"])
if value is not None and math.isfinite(value) and not (bounds and (
bounds.lower is not None and value < bounds.lower
or bounds.upper is not None and value > bounds.upper
)):
document["histories"].setdefault(row["device_id"], []).append(value)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.save(document)
batch = await self.repo.pending_batch(replay_id)
if batch is None:
step = document.get("step_pending", False)
virtual = parse_time(document["virtual_now"])
through = parse_time(document["step_target"]) if step else virtual
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
if document.get("cursor") else None)
async with self.timescale.transaction():
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, config.start_time, config.end_time,
through, 501, after, frame=True,
)
if len(rows) > 500:
raise ValueError("SCADA frame exceeds selected device bound")
at = rows[0]["time"] if rows else through
if step:
virtual = through
document["virtual_now"] = virtual.isoformat()
if not rows and document.get("processed_until") == at.isoformat() and not step:
async with self.repo.connection.transaction():
await self.repo.save(document)
return
sequence = document["batch_sequence"] + 1
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
"source_hash": row["source_hash"]} for row in rows]
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
"refs": refs, "input_digest": digest(refs), "events": []}
async with self.repo.connection.transaction():
document["batch_sequence"] = sequence
document["delivered_until"] = at.isoformat()
await self.repo.save_batch(replay_id, batch)
await self.repo.save(document)
else:
rows = None
if batch["stage"] == "selected":
if rows is None:
# Only crash recovery reloads the selected raw frame.
at = parse_time(batch["at"])
async with self.timescale.transaction():
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, at, config.end_time, at, 501,
) if batch["refs"] else []
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
"source_hash": row["source_hash"]} for row in rows]
if digest(actual) != batch["input_digest"]:
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
batch["outputs"] = prepare_cleaning(rows, config)
batch["stage"] = "clean_prepared"
async with self.repo.connection.transaction():
await self.repo.save_batch(replay_id, batch)
if batch["stage"] == "clean_prepared":
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
batch["stage"] = "cleaned"
async with self.repo.connection.transaction():
await self.repo.save_batch(replay_id, batch)
if batch["stage"] == "cleaned":
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
result["processed_until"] = batch["at"]
target = result.get("step_target")
was_step = bool(target)
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
monitoring = None
if was_step and not result["step_pending"]:
period_start = parse_time(result.get("step_start", result["start_time"]))
period_end = parse_time(target)
role = result.get("monitoring_config") or {}
role_ids = list(dict.fromkeys(
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
))
previous = await self.repo.monitoring_outputs(
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
period_end, batch["sequence"],
) if role_ids else []
from app.mcp.analysis_service import bounded_calculation
monitoring = await bounded_calculation(
evaluate_monitoring_period,
result,
previous + batch["outputs"],
period_start=period_start,
period_end=period_end,
batch_sequence=batch["sequence"],
)
result["last_monitoring"] = monitoring
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]:
result.pop("step_target", None)
result.pop("step_start", None)
if batch["refs"]:
last = batch["refs"][-1]
result["cursor"] = [last["time"], last["device_id"]]
if parse_time(batch["at"]) >= config.end_time:
result["status"] = "completed"
batch["stage"], batch["events"] = "detected", changes
if monitoring is not None:
batch["monitoring"] = monitoring
async with self.repo.connection.transaction():
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
if monitoring is not None:
await self._create_monitoring_task(result, monitoring, batch)
batch["monitoring"] = monitoring
result["last_monitoring"] = monitoring
await self.repo.save_batch(replay_id, batch)
await self.repo.save(result)
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
from uuid import UUID
producer = "scada-replay:" + document["replay_id"]
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
self.context, internal_producer_id=producer)
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context))
for event in changes:
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
await alerts.push(TaskAlertEvent(
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
occurred_at=event["occurred_at"], condition_id=None,
source={"kind": "scada_replay", "replay_id": document["replay_id"],
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
))
if event["change"] != "raised":
if event.get("task_id"):
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
"batch_sequence": batch["sequence"],
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
continue
if event.get("category") == "data_quality":
continue
if not create_tasks:
continue
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
summary=event["detail"], source=TaskSource(
kind="external_system", external_id=external, external_revision=1,
event_kind="scada_anomaly",
replay_context={"replay_id": document["replay_id"],
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
),
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号,原因尚待调查"),
)
task = await tasks.create(command)
event["task_id"] = str(task.task_id)
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
async def _create_monitoring_task(self, document, monitoring, batch):
if monitoring["status"] != "suspected_burst":
return
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
tasks = OperationalTaskService(
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context,
permissions_for_context(self.context),
)
external = (
f"scada_replay:{document['replay_id']}:monitor:"
f"{monitoring['period_end']}:{monitoring['rule_version']}"
)
pressure = monitoring["pressure"]
flow = monitoring["outlet_flow"]
detector = monitoring["burst_detection"]
summary = (
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
)
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external),
task_type="event_investigation",
workflow="burst_report",
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
summary=summary,
source=TaskSource(
kind="external_system",
external_id=external,
external_revision=1,
event_kind="scada_burst_suspected",
replay_context={
"replay_id": document["replay_id"],
"knowledge_cutoff": monitoring["knowledge_cutoff"],
"batch_sequence": batch["sequence"],
},
),
trigger=TaskTrigger(
kind="event",
occurred_at=monitoring["period_end"],
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
),
)
task = await tasks.create(command)
monitoring["task_id"] = str(task.task_id)
+210
View File
@@ -0,0 +1,210 @@
"""Deterministic monitoring summaries built only from frozen replay outputs."""
from __future__ import annotations
from datetime import datetime, timedelta
from statistics import median
from typing import Any
from uuid import NAMESPACE_URL, uuid5
import pandas as pd
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.burst_detection import (
TARGET_MU,
TARGET_N_ESTIMATORS,
TARGET_RANDOM_STATE,
TARGET_SCORE_THRESHOLD,
run_burst_detection,
)
def _time(value: str | datetime) -> datetime:
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
def _valid_rows(rows: list[dict], device_id: str) -> list[dict]:
return sorted(
(
row for row in rows
if row.get("device_id") == device_id
and row.get("quality") in {"valid", "corrected"}
and row.get("cleaned_value") is not None
),
key=lambda row: _time(row["time"]),
)
def _relative_indicator(
rows: list[dict],
config: ScadaReplayCreate,
device_ids: list[str],
period_start: datetime,
period_end: datetime,
*,
direction: str,
) -> dict[str, Any]:
details: list[dict[str, Any]] = []
gaps: list[dict[str, str]] = []
for device_id in device_ids:
rule = next((
item for item in config.rules
if item.kind == "relative_change" and device_id in item.device_ids
and item.threshold is not None
), None)
if rule is None:
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
continue
valid = _valid_rows(rows, device_id)
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
if len(baseline) < rule.minimum_samples:
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
continue
if not current:
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
continue
reference = median(float(row["cleaned_value"]) for row in baseline)
if reference == 0:
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
continue
latest = current[-1]
value = float(latest["cleaned_value"])
change = (value - reference) / abs(reference) * 100
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
details.append({
"device_id": device_id,
"observed_at": _time(latest["time"]).isoformat(),
"baseline": reference,
"value": value,
"change_percent": change,
"threshold_percent": rule.threshold,
"hit": hit,
})
return {
"status": "ready" if not gaps else "insufficient_data",
"hit": any(item["hit"] for item in details),
"hit_device_ids": [item["device_id"] for item in details if item["hit"]],
"devices": details,
"gaps": gaps,
}
def _burst_detection(
rows: list[dict],
pressure_device_ids: list[str],
interval_seconds: int,
period_end: datetime,
) -> dict[str, Any]:
if len(pressure_device_ids) < 5:
return {"status": "sensor_coverage_insufficient", "hit": False,
"required_sensors": 5, "available_sensors": len(pressure_device_ids)}
if interval_seconds % 60 or 86400 % interval_seconds:
return {"status": "unsupported_sampling_interval", "hit": False,
"interval_seconds": interval_seconds}
points_per_day = 86400 // interval_seconds
count = 15 * points_per_day
series_by_device: dict[str, dict[datetime, float]] = {}
common_times: set[datetime] | None = None
for device_id in pressure_device_ids:
values = {_time(row["time"]): float(row["cleaned_value"])
for row in _valid_rows(rows, device_id) if _time(row["time"]) <= period_end}
series_by_device[device_id] = values
common_times = set(values) if common_times is None else common_times & set(values)
if not common_times:
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
target = max(common_times)
expected = [target - timedelta(seconds=interval_seconds * offset)
for offset in range(count - 1, -1, -1)]
incomplete = [device_id for device_id, values in series_by_device.items()
if any(at not in values for at in expected)]
if incomplete:
return {"status": "baseline_not_ready", "hit": False, "required_samples": count,
"incomplete_device_ids": incomplete}
frame = pd.DataFrame({device_id: [series_by_device[device_id][at] for at in expected]
for device_id in pressure_device_ids}, index=expected)
try:
result = run_burst_detection(
network="scada-replay",
username="scada-replay",
observed_pressure_data=frame,
points_per_day=points_per_day,
mu=TARGET_MU,
iforest_params={
"n_estimators": TARGET_N_ESTIMATORS,
"random_state": TARGET_RANDOM_STATE,
"contamination": "auto",
},
sensor_nodes=pressure_device_ids,
)
except (TypeError, ValueError) as exc:
return {"status": "analysis_failed", "hit": False, "reason": str(exc)[:300]}
latest = result["rows"][-1]
score = float(latest["Score"])
return {
"status": "ready",
"hit": score <= TARGET_SCORE_THRESHOLD,
"target_time": target.isoformat(),
"target_score": score,
"score_threshold": TARGET_SCORE_THRESHOLD,
"sensor_count": len(pressure_device_ids),
"reference_day_count": 14,
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
}
def evaluate_monitoring_period(
document: dict,
rows: list[dict],
*,
period_start: datetime,
period_end: datetime,
batch_sequence: int,
) -> dict[str, Any]:
config = ScadaReplayCreate.model_validate(document["config"])
monitoring = document.get("monitoring_config")
rule_version = (monitoring or {}).get("rule_version", "unconfigured")
monitoring_id = str(uuid5(
NAMESPACE_URL,
f"{document['replay_id']}:{period_end.isoformat()}:{rule_version}",
))
base = {
"monitoring_id": monitoring_id,
"period_start": period_start.isoformat(),
"period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(),
"batch_sequence": batch_sequence,
"rule_version": rule_version,
"task_id": None,
}
if not monitoring:
return {**base, "status": "insufficient_data", "detail": "项目未配置 SCADA 监测设备角色",
"pressure": {"status": "unconfigured", "hit": False},
"outlet_flow": {"status": "unconfigured", "hit": False},
"burst_detection": {"status": "not_run", "hit": False}}
pressure_ids = monitoring.get("pressure_device_ids", [])
flow_ids = monitoring.get("outlet_flow_device_ids", [])
selected = set(config.device_ids)
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
if not pressure_ids or not flow_ids or role_gaps:
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
"role_gaps": role_gaps,
"pressure": {"status": "unconfigured", "hit": False},
"outlet_flow": {"status": "unconfigured", "hit": False},
"burst_detection": {"status": "not_run", "hit": False}}
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
detector = _burst_detection(
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
)
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
if not complete:
status, detail = "insufficient_data", "监测数据、设备角色或爆管侦测基线不足"
elif all(hits):
status, detail = "suspected_burst", "爆管侦测、压力下降与出厂边界流量增加相互支持"
elif any(hits):
status, detail = "attention", "仅部分爆管相关指标命中,需要继续观察或人工核查"
else:
status, detail = "normal", "本周期爆管相关指标均未命中"
return {**base, "status": status, "detail": detail, "pressure": pressure,
"outlet_flow": outlet_flow, "burst_detection": detector}
+139
View File
@@ -0,0 +1,139 @@
"""Causal batch quality checks and grouped anomaly episodes, no database IO."""
from copy import deepcopy
from datetime import datetime
from math import isfinite
from statistics import median
from uuid import NAMESPACE_URL, uuid5
import re
from app.domain.schemas.scada_replay import ScadaReplayCreate
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
result = []
for row in rows:
value = row["monitored_value"]
quality, reason = "valid", "原始值通过数值与已配置量程校验"
if value is None:
quality, reason = "missing", "原始读数缺失"
elif not isfinite(float(value)):
quality, reason = "invalid", "原始读数不是有限数值"
else:
bounds = config.quality_ranges.get(row["device_id"])
if bounds and ((bounds.lower is not None and value < bounds.lower)
or (bounds.upper is not None and value > bounds.upper)):
quality, reason = "invalid", "超出本次明确配置的仪表量程"
result.append({
"time": row["time"].isoformat() if isinstance(row["time"], datetime) else row["time"],
"device_id": row["device_id"], "source_hash": row["source_hash"],
"monitored_value": value if value is not None and isfinite(float(value)) else None,
"cleaned_value": value if quality == "valid" else None,
"quality": quality, "reason": reason,
})
return result
def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at: datetime) -> tuple[dict, list[dict]]:
"""Consume a complete timestamp; never treat missing data as recovery."""
result = deepcopy(document)
histories = result.setdefault("histories", {})
states = result.setdefault("rule_states", {})
latest = result.setdefault("latest", {})
for row in rows:
latest[row["device_id"]] = row
by_id = {row["device_id"]: row for row in rows}
changes = []
devices = {device["device_id"]: device for device in result.get("devices", [])}
for device_id in config.device_ids:
row = by_id.get(device_id)
invalid = row is not None and row["quality"] != "valid"
previous = latest.get(device_id)
frequency = str(devices.get(device_id, {}).get("transmission_frequency", ""))
match = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
if match:
seconds = int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3])
else:
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
seconds = int(minutes[1]) * 60 if minutes else 0
prior_time = datetime.fromisoformat(previous["time"]) if previous else config.start_time
missing = not row and seconds > 0 and (at - prior_time).total_seconds() >= seconds * 2
key = "quality-" + device_id
state = states.setdefault(key, {"event": None, "recoveries": 0})
if invalid or missing:
state["recoveries"] = 0
if state["event"] is None:
event = {"event_id": str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{key}:{at.isoformat()}")),
"rule_id": key, "device_ids": [device_id], "occurred_at": at.isoformat(),
"last_observed_at": at.isoformat(), "status": "active", "category": "data_quality",
"detail": row["reason"] if invalid else "连续两个设备周期未读取到观测",
"task_id": None}
state["event"] = event
changes.append({**event, "change": "raised"})
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
elif row and row["quality"] == "valid" and state["event"]:
state["recoveries"] += 1
if state["recoveries"] >= 2:
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
state["event"] = None
for rule in config.rules:
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
hits, usable, reasons = [], [], []
for device_id in rule.device_ids:
row = by_id.get(device_id)
if not row or row["quality"] != "valid":
continue
value = row["cleaned_value"]
if rule.kind == "bounds":
triggered = ((rule.lower is not None and value < rule.lower)
or (rule.upper is not None and value > rule.upper))
reason = f"{device_id}={value} {rule.unit},超出配置范围"
else:
previous = histories.get(device_id, [])[-rule.window_samples:]
if len(previous) < rule.minimum_samples:
continue
baseline = median(previous)
if baseline == 0:
continue
change = (value - baseline) / abs(baseline) * 100
triggered = (abs(change) >= rule.threshold if rule.direction == "both"
else change >= rule.threshold if rule.direction == "increase"
else change <= -rule.threshold)
reason = f"{device_id} 较历史中位值变化 {change:.1f}%,阈值 {rule.threshold}%"
usable.append(device_id)
if triggered:
hits.append(device_id)
reasons.append(reason)
if len(hits) >= rule.minimum_devices:
state["hits"] += 1
state["recoveries"] = 0
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
state["event"] = event
changes.append({**event, "change": "raised"})
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
elif state["event"]:
state["event"]["last_observed_at"] = at.isoformat()
elif len(usable) == len(rule.device_ids):
state["hits"] = 0
if state["event"]:
state["recoveries"] += 1
if state["recoveries"] >= rule.recovery_samples:
event = {**state["event"], "status": "resolved", "resolved_at": at.isoformat(),
"change": "resolved"}
changes.append(event)
state["event"] = None
state["recoveries"] = 0
# Incomplete frames cannot count as a consecutive hit/recovery.
elif not usable and any(device in by_id for device in rule.device_ids):
state["hits"] = state["recoveries"] = 0
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
for device in rule.device_ids}
for row in rows:
if row["quality"] == "valid" and row["device_id"] not in frozen:
history = histories.setdefault(row["device_id"], [])
history.append(row["cleaned_value"])
histories[row["device_id"]] = history[-96:]
result["processed_count"] = result.get("processed_count", 0) + len(rows)
return result, changes
+95
View File
@@ -0,0 +1,95 @@
"""Server-enforced replay knowledge bounds and immutable corrected snapshots."""
from datetime import datetime
from fastapi import HTTPException
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
def check_replay_window(context, start, end):
cap = datetime.fromisoformat(context["knowledge_cutoff"])
if start is None or end is None or start.tzinfo is None or end.tzinfo is None or start > end or end > cap:
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
check_replay_window(context, start, end)
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if not set(device_ids) <= set(run["config"]["device_ids"]):
raise HTTPException(403, "设备或回放不属于当前调查")
if start < datetime.fromisoformat(run["start_time"]):
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
rows = await repository.detected_outputs(
context["replay_id"], device_ids, start, end, context["batch_sequence"],
limit=limit, offset=offset,
)
return [{key: row[key] for key in (
"time", "device_id", "monitored_value", "cleaned_value", "quality", "reason"
)} for row in rows]
async def replay_burst_location_inputs(connection, project_id, context, args):
"""Build location series from immutable replay outputs, never live TSDB rows."""
if args.data_source != "monitoring":
raise HTTPException(422, "回放定位只支持 monitoring 数据源")
if args.scada_normal_start is None or args.scada_normal_end is None:
raise HTTPException(422, "回放定位必须显式提供已处理范围内的正常对照窗口")
check_replay_window(context, args.scada_burst_start, args.scada_burst_end)
check_replay_window(context, args.scada_normal_start, args.scada_normal_end)
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if run:
replay_start = datetime.fromisoformat(run["start_time"])
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
devices = {item["device_id"]: item for item in run.get("devices", [])}
monitoring = run.get("monitoring_config") or {}
pressure_devices = monitoring.get("pressure_device_ids", [])
pressure_map = {str(devices[item]["node_id"]): item for item in pressure_devices
if item in devices and devices[item].get("node_id")}
requested_pressure = args.pressure_scada_ids or list(pressure_map)
if not requested_pressure or not set(requested_pressure) <= set(pressure_map):
raise HTTPException(422, "压力节点必须来自本轮已配置的回放监测角色")
flow_devices = monitoring.get("outlet_flow_device_ids", [])
flow_map = {str(devices[item]["link_id"]): item for item in flow_devices
if item in devices and devices[item].get("link_id")}
requested_flow = args.flow_scada_ids or []
if not set(requested_flow) <= set(flow_map):
raise HTTPException(422, "流量要素必须来自本轮已配置的出厂边界流量角色")
selected = [pressure_map[item] for item in requested_pressure] + [flow_map[item] for item in requested_flow]
rows = await repository.detected_outputs(
context["replay_id"], selected,
min(args.scada_burst_start, args.scada_normal_start),
max(args.scada_burst_end, args.scada_normal_end),
context["batch_sequence"],
)
def averages(mapping, start, end):
result = {}
for element_id, device_id in mapping.items():
values = [float(row["cleaned_value"]) for row in rows
if row.get("device_id") == device_id
and row.get("quality") in {"valid", "corrected"}
and row.get("cleaned_value") is not None
and start <= datetime.fromisoformat(row["time"]) <= end]
if not values:
raise HTTPException(422, f"回放窗口缺少 {device_id} 的有效清洗观测")
result[element_id] = sum(values) / len(values)
return result
pressure_mapping = {item: pressure_map[item] for item in requested_pressure}
flow_mapping = {item: flow_map[item] for item in requested_flow}
return {
"data_source": "monitoring",
"burst_leakage": args.burst_leakage,
"pressure_scada_ids": requested_pressure,
"burst_pressure": averages(pressure_mapping, args.scada_burst_start, args.scada_burst_end),
"normal_pressure": averages(pressure_mapping, args.scada_normal_start, args.scada_normal_end),
**({
"flow_scada_ids": requested_flow,
"burst_flow": averages(flow_mapping, args.scada_burst_start, args.scada_burst_end),
"normal_flow": averages(flow_mapping, args.scada_normal_start, args.scada_normal_end),
"use_scada_flow": True,
} if requested_flow else {}),
}