feat(scada): add project monitoring replay
Generic Container CI/CD / test-build-publish (push) Successful in 3m39s
Server CI/CD v2 / build-test-publish-and-deploy (push) Successful in 3m39s

This commit is contained in:
2026-11-04 14:27:21 +08:00
parent f25423dec4
commit 26fa5d1909
53 changed files with 5153 additions and 20 deletions
+2
View File
@@ -49,6 +49,7 @@ def _project_response(project: models.Project) -> AdminProjectResponse:
gs_workspace=project.gs_workspace,
map_extent=project.map_extent,
map_config=getattr(project, "map_config", {}) or {},
settings=getattr(project, "settings", {}) or {},
status=project.status,
created_at=project.created_at,
updated_at=project.updated_at,
@@ -276,6 +277,7 @@ async def create_admin_project(
gs_workspace=payload.gs_workspace,
map_extent=payload.map_extent,
map_config=payload.map_config.model_dump(exclude_unset=True),
settings=payload.settings.model_dump(exclude_unset=True),
status=payload.status,
creator_user_id=current_user.id,
)
+1
View File
@@ -45,6 +45,7 @@ async def get_project_metadata(
gs_workspace=project.gs_workspace,
map_extent=project.map_extent,
map_config=getattr(project, "map_config", {}) or {},
settings=getattr(project, "settings", {}) or {},
status=project.status,
project_role=ctx.project_role,
)
+1
View File
@@ -320,6 +320,7 @@ async def provision_project(
gs_workspace=project.gs_workspace,
map_extent=project.map_extent,
map_config=getattr(project, "map_config", {}) or {},
settings=getattr(project, "settings", {}) or {},
status=project.status,
created_at=project.created_at,
updated_at=project.updated_at,
+18 -4
View File
@@ -567,10 +567,24 @@ async def query_operational_task_data(
if not await service.can_read_source(task_id, query.source):
raise DomainError("PERMISSION_DENIED", "Task has no approved access to this data source", status=403)
source_permission = SCADA_VIEW if query.source.startswith("scada_") else SIMULATION_VIEW
snapshot = await read_task_data(
context=ctx, query=query, credential=authorization,
delegated=source_permission not in _task_permissions(ctx),
)
from app.services.scada_replay_queries import replay_snapshot_rows
replay_context = await service.replay_context(task_id)
if replay_context:
if query.source != "scada_readings":
raise HTTPException(403, "回放调查数据接口仅开放本轮 SCADA 快照,请通过 MCP 查询设备目录")
rows = await replay_snapshot_rows(conn, ctx.project_id, replay_context, query.device_ids,
query.start, query.end, 1001)
from app.services.content_digest import digest
from datetime import datetime, timezone
snapshot = TaskDataSnapshot(
source="scada_readings", source_ref=f"scada_replay:{replay_context['replay_id']}:{digest(rows)}",
queried_at=datetime.now(timezone.utc), origin="synthetic", data=rows[:1000], truncated=len(rows) > 1000,
)
else:
snapshot = await read_task_data(
context=ctx, query=query, credential=authorization,
delegated=source_permission not in _task_permissions(ctx),
)
evidence, stored_snapshot = await service.record_data_snapshot(task_id, snapshot)
return TaskDataQueryResult(evidence=evidence, snapshot=stored_snapshot)
except TaskDataError as exc:
+1
View File
@@ -35,6 +35,7 @@ async def get_project_info_endpoint(
gs_workspace=project_detail.gs_workspace,
map_extent=project_detail.map_extent,
map_config=getattr(project_detail, "map_config", {}) or {},
settings=getattr(project_detail, "settings", {}) or {},
status=project_detail.status,
project_role="viewer", # Default role for public access
)
+114
View File
@@ -0,0 +1,114 @@
"""SCADA replay controls. Manual steps are processed by the command request."""
from uuid import UUID
from datetime import datetime
from fastapi import APIRouter, Depends, Query, HTTPException
from app.auth.metadata_dependencies import get_metadata_repository
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, require_permission, permissions_for_context
from app.auth.project_dependencies import ProjectContext, get_project_pg_connection, get_project_timescale_connection
from app.domain.schemas.scada_replay import (
ScadaReplayCapabilities, ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot,
ScadaReplayEventsPage, ScadaReplayReadings,
)
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.domain.schemas.project_settings import ProjectSettings
from app.services.scada_replay import ScadaReplayService, snapshot
router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"])
async def repository(
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
connection=Depends(get_project_pg_connection),
):
return ScadaReplayRepository(connection, ctx.project_id)
async def project_settings(
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
metadata_repo=Depends(get_metadata_repository),
) -> ProjectSettings:
project = await metadata_repo.get_project_by_id(ctx.project_id)
if project is None:
raise HTTPException(404, "当前项目不存在")
return ProjectSettings.model_validate(project.settings or {})
@router.get("/capabilities", response_model=ScadaReplayCapabilities)
async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_permission(SCADA_VIEW)),
settings=Depends(project_settings)):
available = await repo.available()
current = await repo.current() if available else None
monitoring = settings.scada_monitoring
monitoring_ready = bool(
monitoring
and len(monitoring.pressure_device_ids) >= 5
and monitoring.outlet_flow_device_ids
)
if monitoring_ready:
monitoring_reason = None
elif monitoring is None:
monitoring_reason = "项目尚未配置压力与出厂边界流量监测角色"
elif len(monitoring.pressure_device_ids) < 5:
monitoring_reason = "爆管侦测至少需要 5 个压力监测设备"
else:
monitoring_reason = "项目尚未配置出厂边界流量设备"
return ScadaReplayCapabilities(
available=available, can_write=SCADA_CLEAN in permissions_for_context(ctx),
reason=None if available else "需先部署 SCADA 回放存储结构",
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
burst_monitoring_ready=monitoring_ready,
burst_monitoring_reason=monitoring_reason,
current=snapshot(current) if current else None,
)
@router.post("", response_model=ScadaReplaySnapshot, status_code=201)
async def create_replay(command: ScadaReplayCreate, repo=Depends(repository),
ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)):
return await ScadaReplayService(repo, ctx).create(
command,
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
monitoring_config=settings.scada_monitoring,
)
@router.get("/{replay_id}", response_model=ScadaReplaySnapshot)
async def get_replay(replay_id: UUID, repo=Depends(repository)):
await ScadaReplayService(repo, None).ensure_storage()
return snapshot(await repo.get(replay_id))
@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot)
async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depends(repository),
ctx=Depends(require_permission(SCADA_CLEAN)),
timescale=Depends(get_project_timescale_connection)):
service = ScadaReplayService(repo, ctx, timescale)
result = await service.command(replay_id, command)
if command.action in {"step", "retry"} and result.step_pending:
return await service.process_pending_step(replay_id)
return result
@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage)
async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200),
repo=Depends(repository)):
await ScadaReplayService(repo, None).ensure_storage()
await repo.get(replay_id)
items = await repo.events(replay_id, offset, limit + 1)
return ScadaReplayEventsPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None)
@router.get("/{replay_id}/observations", response_model=ScadaReplayReadings)
async def replay_observations(replay_id: UUID, through: datetime, repo=Depends(repository)):
await ScadaReplayService(repo, None).ensure_storage()
document = await repo.get(replay_id)
if (through.tzinfo is None or not document.get("delivered_until")
or through > datetime.fromisoformat(document["delivered_until"])
or through < datetime.fromisoformat(document["start_time"])):
raise HTTPException(422, "回看仅允许本轮已读取的时间区间")
return ScadaReplayReadings(
items=await repo.latest_outputs(replay_id, through),
through=through,
)
+44 -1
View File
@@ -1,10 +1,13 @@
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
from typing import List
from datetime import datetime
from uuid import UUID
from psycopg import AsyncConnection
from pydantic import BaseModel, Field, field_validator
from app.domain.schemas.scada import ScadaLatestReadingsResponse
from app.domain.schemas.scada import ScadaLatestReadingsResponse, ScadaRawPage
from app.auth.project_dependencies import get_project_context, ProjectContext
from app.services.scada_raw import read_raw_page, validate_window
from app.infra.db.postgresql.scada import ScadaInfoRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from .dependencies import get_postgres_connection, get_timescale_connection
@@ -13,6 +16,46 @@ router = APIRouter()
SCADA_BATCH_MAX_ITEMS = 10_000
@router.get("/timeseries/scada-readings/raw", response_model=ScadaRawPage,
summary="按时间顺序分页读取 SCADA 原始值")
async def get_scada_raw(
start_time: datetime, end_time: datetime, device_ids: str,
limit: int = Query(500, ge=1, le=500),
cursor: str | None = Query(None, max_length=2048),
replay_id: UUID | None = None,
conn: AsyncConnection = Depends(get_timescale_connection),
postgres_conn: AsyncConnection = Depends(get_postgres_connection),
ctx: ProjectContext = Depends(get_project_context),
) -> ScadaRawPage:
try:
validate_window(start_time, end_time)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
ids = list(dict.fromkeys(item.strip() for item in device_ids.split(",") if item.strip()))
if not ids or len(ids) > 500:
raise HTTPException(422, "请选择 1 至 500 个 SCADA 设备")
existing = await ScadaInfoRepository.get_existing_device_ids(postgres_conn, ids)
if set(ids) != existing:
raise HTTPException(422, "查询包含当前项目未配置的 SCADA 设备")
try:
through = None
if replay_id is not None:
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
from app.services.scada_replay import ScadaReplayService
repo = ScadaReplayRepository(postgres_conn, ctx.project_id)
await ScadaReplayService(repo, ctx).ensure_storage()
replay = await repo.get(replay_id)
if not set(ids) <= set(replay["config"]["device_ids"]):
raise HTTPException(403, "设备超出回放范围")
if start_time < datetime.fromisoformat(replay["start_time"]) or end_time > datetime.fromisoformat(replay["end_time"]):
raise HTTPException(422, "查询超出选定回放区间")
through = datetime.fromisoformat(replay["virtual_now"])
return await read_raw_page(conn, project_id=str(ctx.project_id), device_ids=ids,
start=start_time, end=end_time, limit=limit, cursor=cursor, through=through)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
class ScadaReadingBatchItem(BaseModel):
time: datetime
device_id: str = Field(min_length=1)
+2
View File
@@ -61,8 +61,10 @@ from app.auth.permissions import (
require_method_permission,
require_permission,
)
from app.api.v1.endpoints import scada_replay
api_router = APIRouter()
api_router.include_router(scada_replay.router)
webgis_access = Depends(
require_method_permission(
+15 -1
View File
@@ -2,9 +2,10 @@ from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from app.domain.schemas.project_map_config import ProjectMapConfig, ProjectMapConfigUpdate
from app.domain.schemas.project_settings import ProjectSettings
BusinessRole = Literal["admin", "service", "user"]
@@ -72,6 +73,7 @@ class ProjectMemberResponse(BaseModel):
class AdminProjectCreateRequest(BaseModel):
settings: ProjectSettings = Field(default_factory=ProjectSettings)
name: str = Field(..., min_length=1, max_length=100)
code: str = Field(..., min_length=1, max_length=50)
description: str | None = None
@@ -82,6 +84,10 @@ class AdminProjectCreateRequest(BaseModel):
class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
settings: ProjectSettings | None = Field(
default=None,
description="按顶层键合并项目配置;省略的键保持原值,嵌套对象整体替换",
)
name: str | None = Field(default=None, min_length=1, max_length=100)
code: str | None = Field(default=None, min_length=1, max_length=50)
description: str | None = None
@@ -89,8 +95,16 @@ class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
map_extent: dict | None = None
status: ProjectStatus | None = None
@field_validator("settings")
@classmethod
def reject_null_settings(cls, value: ProjectSettings | None) -> ProjectSettings:
if value is None:
raise ValueError("settings must be an object")
return value
class AdminProjectResponse(BaseModel):
settings: ProjectSettings = Field(default_factory=ProjectSettings)
project_id: UUID
name: str
code: str
+2
View File
@@ -4,9 +4,11 @@ from uuid import UUID
from pydantic import BaseModel, Field
from app.domain.schemas.project_map_config import ProjectMapConfig
from app.domain.schemas.project_settings import ProjectSettings
class ProjectMetaResponse(BaseModel):
settings: ProjectSettings = Field(default_factory=ProjectSettings)
project_id: UUID
name: str
code: str
+7
View File
@@ -41,12 +41,19 @@ ExecutionStatus = Literal[
]
class ScadaReplayTaskContext(StrictModel):
replay_id: UUID
knowledge_cutoff: WorkbenchDateTime
batch_sequence: PositiveInt
class TaskSource(StrictModel):
kind: TaskSourceKind
external_id: NonEmptyStr | None = None
external_revision: PositiveInt | None = None
event_kind: NonEmptyStr | None = None
deleted_at: WorkbenchDateTime | None = None
replay_context: ScadaReplayTaskContext | None = None
class TaskTrigger(StrictModel):
+44
View File
@@ -0,0 +1,44 @@
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
class ScadaMonitoringSettings(BaseModel):
"""Project-owned device roles used by the manual SCADA monitoring demo."""
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
rule_version: str = Field(
default="burst-monitor-v1",
min_length=1,
max_length=80,
pattern=r"^[a-zA-Z0-9_.-]+$",
)
pressure_device_ids: list[str] = Field(default_factory=list, max_length=20)
outlet_flow_device_ids: list[str] = Field(default_factory=list, max_length=20)
@model_validator(mode="after")
def distinct_roles(self):
for values in (self.pressure_device_ids, self.outlet_flow_device_ids):
if len(set(values)) != len(values) or any(not value.strip() for value in values):
raise ValueError("SCADA 监测角色中的设备标识不能为空或重复")
if set(self.pressure_device_ids) & set(self.outlet_flow_device_ids):
raise ValueError("同一设备不能同时作为压力与出厂边界流量")
return self
class ProjectSettings(BaseModel):
"""Extensible, non-secret project parameters stored in MetaDB as JSON."""
model_config = ConfigDict(extra="allow", allow_inf_nan=False)
__pydantic_extra__: dict[str, JsonValue] = Field(init=False)
scada_monitoring_interval_seconds: int | None = Field(
default=None,
strict=True,
ge=60,
le=31_536_000,
description="SCADA 主动监测演示步长,单位秒;null 表示未配置",
)
scada_monitoring: ScadaMonitoringSettings | None = Field(
default=None,
description="主动监测使用的项目设备角色;不创建定时任务",
)
+18 -1
View File
@@ -1,8 +1,8 @@
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
class ScadaDeviceResponse(BaseModel):
"""Project SCADA metadata keyed by the canonical device identifier."""
@@ -47,3 +47,20 @@ class ScadaLatestReadingsResponse(BaseModel):
offset: int = Field(ge=0)
observed_count: int = Field(ge=0)
latest_observed_at: datetime | None = None
class ScadaRawReading(BaseModel):
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
time: datetime
device_id: str
monitored_value: float | None
quality: Literal["unvalidated", "missing", "invalid"] = "unvalidated"
class ScadaRawPage(BaseModel):
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
items: list[ScadaRawReading]
next_cursor: str | None
scanned_through: datetime
+142
View File
@@ -0,0 +1,142 @@
"""Persisted replay controls and explicit, unit-aware demonstration rules."""
from datetime import datetime, timedelta
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, model_validator
class ReplayModel(BaseModel):
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
class ScadaReplayRule(ReplayModel):
rule_id: str = Field(min_length=1, max_length=80, pattern=r"^[a-zA-Z0-9_-]+$")
device_ids: list[str] = Field(min_length=1, max_length=500)
kind: Literal["relative_change", "bounds"]
direction: Literal["both", "increase", "decrease"] = "both"
threshold: float | None = Field(None, gt=0)
lower: float | None = None
upper: float | None = None
unit: str = Field(min_length=1, max_length=40)
metric_name: str = Field(default="", max_length=80)
minimum_samples: int = Field(3, ge=1, le=96)
window_samples: int = Field(4, ge=1, le=96)
consecutive_samples: int = Field(1, ge=1, le=20)
recovery_samples: int = Field(2, ge=1, le=20)
minimum_devices: int = Field(1, ge=1, le=500)
@model_validator(mode="after")
def parameters(self):
if len(set(self.device_ids)) != len(self.device_ids):
raise ValueError("规则内设备不能重复")
if self.minimum_devices > len(self.device_ids):
raise ValueError("同时异常设备数超过规则设备数")
if self.kind == "relative_change" and (self.threshold is None or self.minimum_samples > self.window_samples):
raise ValueError("变化率规则需要阈值和足够的历史窗口")
if self.kind == "bounds" and self.lower is None and self.upper is None:
raise ValueError("上下限规则至少配置一个阈值")
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
raise ValueError("规则下限必须小于上限")
return self
class ScadaQualityRange(ReplayModel):
lower: float | None = None
upper: float | None = None
@model_validator(mode="after")
def bounds(self):
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
raise ValueError("仪表量程下限必须小于上限")
return self
class ScadaReplayCreate(ReplayModel):
request_id: UUID
start_time: datetime
end_time: datetime
device_ids: list[str] = Field(min_length=1, max_length=500)
rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30)
quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict)
@model_validator(mode="after")
def configuration(self):
if (self.start_time.tzinfo is None or self.end_time.tzinfo is None
or not timedelta(0) < self.end_time - self.start_time <= timedelta(days=31)):
raise ValueError("请选择带时区、最长 31 天的有效时间区间")
ids = set(self.device_ids)
if len(ids) != len(self.device_ids) or any(not value.strip() for value in ids):
raise ValueError("设备标识不能为空或重复")
if len({rule.rule_id for rule in self.rules}) != len(self.rules):
raise ValueError("规则标识不能重复")
if any(not set(rule.device_ids) <= ids for rule in self.rules) or not set(self.quality_ranges) <= ids:
raise ValueError("规则及量程设备必须属于本次回放")
return self
class ScadaReplayCommand(ReplayModel):
request_id: UUID
expected_revision: int = Field(ge=1)
action: Literal["step", "stop", "retry"]
class ScadaMonitoringResult(ReplayModel):
monitoring_id: UUID
period_start: datetime
period_end: datetime
knowledge_cutoff: datetime
batch_sequence: int = Field(ge=1)
rule_version: str = Field(min_length=1, max_length=80)
status: Literal["normal", "attention", "insufficient_data", "suspected_burst"]
detail: str = Field(min_length=1, max_length=1000)
task_id: UUID | None = None
pressure: dict
outlet_flow: dict
burst_detection: dict
role_gaps: list[str] = Field(default_factory=list)
class ScadaReplaySnapshot(ReplayModel):
replay_id: UUID
project_id: UUID
revision: int
status: Literal["ready", "paused", "completed", "stopped", "failed"]
start_time: datetime
end_time: datetime
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
step_pending: bool = False
monitoring_count: int = 0
last_monitoring: ScadaMonitoringResult | None = None
virtual_now: datetime
delivered_until: datetime | None = None
processed_until: datetime | None = None
processed_count: int = 0
anomaly_count: int = 0
devices: list[dict]
rules: list[ScadaReplayRule]
latest: list[dict]
active_events: list[dict]
error: str | None = None
created_at: datetime
updated_at: datetime
class ScadaReplayCapabilities(ReplayModel):
available: bool
can_write: bool
reason: str | None = None
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
burst_monitoring_ready: bool = False
burst_monitoring_reason: str | None = None
current: ScadaReplaySnapshot | None = None
class ScadaReplayEventsPage(ReplayModel):
items: list[dict]
next_offset: int | None
class ScadaReplayReadings(ReplayModel):
items: list[dict]
through: datetime
+3 -3
View File
@@ -5,7 +5,7 @@ from typing import Literal
from uuid import UUID
from app.domain.schemas.shared import NonEmptyStr, PositiveInt, StrictModel
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchDateTime
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchScadaReplaySource, WorkbenchDateTime
class TaskAlertCommand(StrictModel):
@@ -14,7 +14,7 @@ class TaskAlertCommand(StrictModel):
event_type: Literal["raised", "acknowledged", "resolved"]
occurred_at: WorkbenchDateTime
condition_id: NonEmptyStr | None
source: WorkbenchAlertSource
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
title: NonEmptyStr
description: NonEmptyStr
severity: Literal["warning", "critical"]
@@ -29,7 +29,7 @@ class TaskAlertState(StrictModel):
revision: PositiveInt
producer_id: NonEmptyStr
condition_id: NonEmptyStr | None
source: WorkbenchAlertSource
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
title: NonEmptyStr
description: NonEmptyStr
occurred_at: WorkbenchDateTime
+9 -2
View File
@@ -49,10 +49,17 @@ class WorkbenchAlertSource(StrictModel):
run_id: NonEmptyStr
class WorkbenchScadaReplaySource(StrictModel):
kind: Literal["scada_replay"]
replay_id: NonEmptyStr
rule_id: NonEmptyStr
batch_sequence: int = Field(ge=1)
class WorkbenchRuntimeAlert(StrictModel):
id: NonEmptyStr
condition_id: NonEmptyStr | None
source: WorkbenchAlertSource
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
title: NonEmptyStr
description: NonEmptyStr
occurred_at: WorkbenchDateTime
@@ -62,7 +69,7 @@ class WorkbenchRuntimeAlert(StrictModel):
class WorkbenchPersistedAlert(StrictModel):
alert_id: NonEmptyStr
condition_id: NonEmptyStr | None
source: WorkbenchAlertSource
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
title: NonEmptyStr
description: NonEmptyStr
occurred_at: WorkbenchDateTime
+1
View File
@@ -44,6 +44,7 @@ class Project(Base):
gs_workspace: Mapped[str] = mapped_column(String(100), unique=True)
map_extent: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
map_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
settings: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
status: Mapped[str] = mapped_column(String(20), default="active")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.utcnow
@@ -67,6 +67,7 @@ class ProjectDetail:
map_extent: Optional[dict]
status: str
map_config: dict = field(default_factory=dict)
settings: dict = field(default_factory=dict)
@dataclass(frozen=True)
@@ -223,6 +224,7 @@ class MetadataRepository:
status: str,
creator_user_id: UUID | None = None,
map_config: dict | None = None,
settings: dict | None = None,
) -> models.Project:
project = models.Project(
id=uuid4(),
@@ -233,6 +235,7 @@ class MetadataRepository:
map_extent=map_extent,
status=status,
map_config=map_config or {},
settings=settings or {},
created_at=_utcnow(),
updated_at=_utcnow(),
)
@@ -263,6 +266,7 @@ class MetadataRepository:
timescale_dsn: str,
pool_min_size: int = 1,
pool_max_size: int = 4,
settings: dict | None = None,
) -> models.Project:
"""Atomically expose a fully provisioned project and both DB routes."""
business_secret = _encrypt_database_secret(business_dsn)
@@ -276,6 +280,7 @@ class MetadataRepository:
map_extent=map_extent,
status="active",
map_config={},
settings=settings or {},
created_at=_utcnow(),
updated_at=_utcnow(),
)
@@ -333,6 +338,8 @@ class MetadataRepository:
setattr(project, field, updates[field])
if "map_config" in updates:
project.map_config = {**(project.map_config or {}), **updates["map_config"]}
if "settings" in updates:
project.settings = {**(project.settings or {}), **updates["settings"]}
project.updated_at = _utcnow()
await self.session.commit()
await self.session.refresh(project)
@@ -350,6 +357,7 @@ class MetadataRepository:
gs_workspace=project.gs_workspace,
map_extent=project.map_extent,
map_config=project.map_config or {},
settings=project.settings or {},
status=project.status,
)
+188
View File
@@ -0,0 +1,188 @@
"""Short-transaction storage for replay control and resumable batches."""
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from fastapi import HTTPException
from psycopg.types.json import Jsonb
class ScadaReplayRepository:
def __init__(self, connection, project_id):
self.connection, self.project_id = connection, str(project_id)
self.lock_key = f"scada-replay:{self.project_id}"
async def available(self) -> bool:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT to_regclass('peripheral_sample.scada_replays') IS NOT NULL "
"AND to_regclass('peripheral_sample.scada_replay_batches') IS NOT NULL AS ready"
)
return (await cur.fetchone())["ready"]
@asynccontextmanager
async def control(self):
async with self.connection.transaction():
await self.connection.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (self.lock_key,))
yield
@asynccontextmanager
async def processing(self):
# Session lock outlives short transactions, never an open SQL transaction
# across Timescale IO. A dead connection automatically releases ownership.
async with self.connection.transaction():
cur = await self.connection.execute("SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS owned", (self.lock_key,))
owned = (await cur.fetchone())["owned"]
try:
yield owned
finally:
if owned:
async with self.connection.transaction():
await self.connection.execute("SELECT pg_advisory_unlock(hashtextextended(%s,0))", (self.lock_key,))
async def get(self, replay_id) -> dict:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id=%s",
(self.project_id, str(replay_id)),
)
row = await cur.fetchone()
if not row:
raise HTTPException(404, "当前项目不存在此 SCADA 回放")
return row["document"]
async def current(self) -> dict | None:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s"
" ORDER BY created_at DESC LIMIT 1", (self.project_id,),
)
row = await cur.fetchone()
return row["document"] if row else None
async def by_request(self, request_id) -> dict | None:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND request_id=%s",
(self.project_id, str(request_id)),
)
row = await cur.fetchone()
return row["document"] if row else None
async def active_other(self, replay_id) -> bool:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT 1 FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id<>%s "
"AND status IN ('ready','paused') LIMIT 1",
(self.project_id, str(replay_id)),
)
return await cur.fetchone() is not None
async def insert(self, document: dict):
await self.connection.execute(
"INSERT INTO peripheral_sample.scada_replays(project_id,replay_id,request_id,status,document) "
"VALUES(%s,%s,%s,%s,%s)",
(self.project_id, document["replay_id"], document["config"]["request_id"], document["status"], Jsonb(document)),
)
async def save(self, document: dict):
document["updated_at"] = datetime.now(timezone.utc).isoformat()
await self.connection.execute(
"UPDATE peripheral_sample.scada_replays SET status=%s,document=%s,updated_at=now() "
"WHERE project_id=%s AND replay_id=%s",
(document["status"], Jsonb(document), self.project_id, document["replay_id"]),
)
async def pending_batch(self, replay_id) -> dict | None:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replay_batches "
"WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1",
(self.project_id, str(replay_id)),
)
row = await cur.fetchone()
return row["document"] if row else None
async def save_batch(self, replay_id, batch: dict):
await self.connection.execute(
"INSERT INTO peripheral_sample.scada_replay_batches(project_id,replay_id,sequence,stage,document) "
"VALUES(%s,%s,%s,%s,%s) ON CONFLICT(project_id,replay_id,sequence) "
"DO UPDATE SET stage=excluded.stage,document=excluded.document",
(self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)),
)
async def events(self, replay_id, offset=0, limit=100) -> list[dict]:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
"WHERE b.project_id=%s AND b.replay_id=%s ORDER BY b.sequence DESC LIMIT %s OFFSET %s",
(self.project_id, str(replay_id), limit, offset),
)
return [r["event"] for r in await cur.fetchall()]
async def monitoring_outputs(
self,
replay_id,
device_ids: list[str],
start: datetime,
end: datetime,
before_sequence: int,
) -> list[dict]:
"""Read only committed replay outputs available before the current batch."""
if not device_ids:
return []
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' AND b.sequence<%s "
"AND r.value->>'device_id'=ANY(%s) "
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'",
(self.project_id, str(replay_id), before_sequence, device_ids, start, end),
)
return [row["reading"] for row in await cur.fetchall()]
async def detected_outputs(
self,
replay_id,
device_ids: list[str],
start: datetime,
end: datetime,
before_sequence: int,
*,
limit: int | None = None,
offset: int = 0,
) -> list[dict]:
"""Read immutable detector outputs available to a bounded replay investigation."""
statement = (
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence<=%s AND b.stage='detected' "
"AND r.value->>'device_id'=ANY(%s) "
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'"
)
params: tuple = (
self.project_id, str(replay_id), before_sequence, device_ids, start, end,
)
if limit is not None:
statement += " LIMIT %s OFFSET %s"
params += (limit, offset)
async with self.connection.transaction():
cur = await self.connection.execute(statement, params)
return [row["reading"] for row in await cur.fetchall()]
async def latest_outputs(self, replay_id, through: datetime, limit: int = 500) -> list[dict]:
"""Return the latest committed cleaned output for each replay device."""
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading "
"FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') "
"AND (r.value->>'time')::timestamptz<=%s "
"ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s",
(self.project_id, str(replay_id), through, limit),
)
return [row["reading"] for row in await cur.fetchall()]
@@ -9,6 +9,86 @@ class ScadaRepository:
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
@staticmethod
async def get_warmup_values(
conn: AsyncConnection,
device_ids: list[str],
before: datetime,
per_device_limit: int,
) -> list[dict]:
"""Return the latest raw values before a replay starts for detector warmup."""
async with conn.cursor() as cur:
await cur.execute(
"SELECT d.device_id,h.monitored_value FROM unnest(%s::text[]) d(device_id) "
"CROSS JOIN LATERAL (SELECT monitored_value,time FROM scada.measurements "
"WHERE device_id=d.device_id AND time<%s ORDER BY time DESC LIMIT %s) h "
"ORDER BY d.device_id,h.time",
(device_ids, before, per_device_limit),
)
return [dict(row) for row in await cur.fetchall()]
@staticmethod
async def get_raw_page(
conn: AsyncConnection, device_ids: list[str], start: datetime,
end: datetime, through: datetime, limit: int,
after: tuple[datetime, str] | None = None, *, frame: bool = False,
) -> list[dict]:
"""Read raw only, with a stable composite key and a bounded frame/page."""
predicate = "device_id=ANY(%s) AND time>=%s AND time<%s AND time<=%s"
params: tuple = (device_ids, start, end, through)
if after is not None:
predicate += " AND (time,device_id)>(%s,%s)"
params += after
if frame:
statement = (
"WITH next_time AS (SELECT min(time) AS time FROM scada.measurements WHERE "
+ predicate + ") SELECT m.time,m.device_id,m.monitored_value,"
"md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
"FROM scada.measurements m WHERE m.device_id=ANY(%s) "
"AND m.time=(SELECT time FROM next_time) ORDER BY m.time,m.device_id LIMIT %s"
)
params += (device_ids, limit)
else:
statement = (
"SELECT time,device_id,monitored_value,"
"md5(COALESCE(monitored_value::text,'NULL')) AS source_hash "
"FROM scada.measurements WHERE " + predicate
+ " ORDER BY time,device_id LIMIT %s"
)
params += (limit,)
async with conn.cursor() as cur:
await cur.execute(statement, params)
return [dict(row) for row in await cur.fetchall()]
@staticmethod
async def apply_replay_cleaning(conn: AsyncConnection, rows: list[dict]) -> None:
"""Guard every raw version; update corrected values without inserting raw."""
if not rows:
return
async with conn.transaction():
async with conn.cursor() as cur:
await cur.execute(
"SELECT m.time,m.device_id,md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
"FROM scada.measurements m JOIN unnest(%s::timestamptz[],%s::text[]) "
"b(time,device_id) USING(time,device_id) FOR UPDATE OF m",
([r["time"] for r in rows], [r["device_id"] for r in rows]),
)
existing = {(r["time"].isoformat(), r["device_id"]): r["source_hash"]
for r in await cur.fetchall()}
if len(existing) != len(rows) or any(
existing.get((datetime.fromisoformat(r["time"]).isoformat(), r["device_id"]))
!= r["source_hash"] for r in rows
):
raise ValueError("SOURCE_CHANGED: SCADA 原始值在处理期间发生变化")
await cur.execute(
"UPDATE scada.measurements m SET cleaned_value=b.value "
"FROM unnest(%s::timestamptz[],%s::text[],%s::double precision[]) "
"b(time,device_id,value) WHERE m.time=b.time AND m.device_id=b.device_id "
"AND m.cleaned_value IS DISTINCT FROM b.value",
([r["time"] for r in rows], [r["device_id"] for r in rows],
[r["cleaned_value"] for r in rows]),
)
@staticmethod
async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
if not data:
+16
View File
@@ -98,3 +98,19 @@ async def bounded_calculation(function, *args, **kwargs):
finally:
_slots.release()
return await asyncio.to_thread(run)
async def calculate_replay_burst_location(
parameters: dict[str, Any], *, network: str, username: str,
) -> dict[str, Any]:
"""Run location with Server-built frozen replay series and no live SCADA query."""
from app.services.burst_location import run_burst_location_by_network
return await bounded_calculation(
run_burst_location_by_network,
network=network,
username=username,
scheme_name=None,
final_candidates_csv_path=None,
**parameters,
)
+1
View File
@@ -30,6 +30,7 @@ async def verify_task(scope: QueryScope, binding: BoundContext) -> None:
task = await repo.get_task(str(binding.task_id))
if not task:
raise HTTPException(404, "Task is not in this project")
binding.replay_context = task.get("source", {}).get("replay_context")
if binding.purpose == "task_collaboration":
return
if OPERATIONAL_TASK_EXECUTE not in permissions:
+1
View File
@@ -291,5 +291,6 @@ class BoundContext(BaseModel):
task_id: UUID | None = None
run_id: UUID | None = None
execution_request_id: UUID | None = None
replay_context: dict | None = None
expires_at: datetime
revoked_at: datetime | None = None
+46 -1
View File
@@ -66,8 +66,23 @@ def page_result(rows, args):
async def authorize_query(
scope: QueryScope, binding: BoundContext, tool: QueryTool, args: BaseModel
) -> None:
replay_context = getattr(binding, "replay_context", None)
if tool.name not in binding.tool_cap:
raise HTTPException(403, "Query is outside the captured tool catalog")
if replay_context:
from app.services.scada_replay_queries import check_replay_window
allowed = {"get_task_context", "get_task_evidence", "list_scada_devices", "get_scada_device",
"query_scada_readings", "get_network_schema", "run_burst_location",
"get_calculation_job", "get_calculation_result", "list_calculation_jobs"}
if tool.name not in allowed:
raise HTTPException(403, "回放调查当前仅开放冻结 SCADA 证据与模型目录查询;请使用 query_scada_readings")
if tool.name == "query_scada_readings":
check_replay_window(replay_context, args.start, args.end)
if tool.name == "run_burst_location":
if args.data_source != "monitoring" or args.scada_normal_start is None or args.scada_normal_end is None:
raise HTTPException(422, "回放定位只支持显式异常窗和正常对照窗的 monitoring 输入")
check_replay_window(replay_context, args.scada_burst_start, args.scada_burst_end)
check_replay_window(replay_context, args.scada_normal_start, args.scada_normal_end)
permissions = permissions_for_context(scope.project)
if tool.mutating:
if binding.purpose != "ordinary_chat" or binding.task_id is not None or scope.project.project_role != "member":
@@ -149,7 +164,37 @@ async def execute_query(
gaps = []
source_ref = "mcp:" + str(uuid4())
evidence_id = None
if tool.computation:
replay_context = getattr(binding, "replay_context", None)
if name == "query_scada_readings" and replay_context:
from app.services.scada_replay_queries import replay_snapshot_rows
rows = await replay_snapshot_rows(conn, scope.project.project_id, replay_context,
args.device_ids, args.start, args.end, args.limit + 1, args.offset)
data, page, truncated = page_result(rows, args)
origin = "synthetic"
gaps.append("历史数据回放证据,仅覆盖该轮调查已提交的批次,不表示现场实时观测。")
elif name == "run_burst_location" and replay_context:
from app.mcp.analysis_service import calculate_replay_burst_location
from app.services.scada_replay_queries import replay_burst_location_inputs
parameters = await replay_burst_location_inputs(
conn, scope.project.project_id, replay_context, args,
)
await conn.commit()
async with scope.simulation_routing():
result = await calculate_replay_burst_location(
parameters,
network=scope.project.project_code,
username=scope.user.username,
)
data = {
"parameters": args.model_dump(mode="json", exclude={"context_handle"}),
"result": result,
}
origin = "synthetic"
gaps = [
"定位只使用任务知识截止前已固化的回放清洗输出;候选排序依赖漏量假设、模型和测点覆盖,仍需现场核实。"
]
elif tool.computation:
from app.mcp.analysis_service import TIMESCALE_CALCULATIONS, calculate
# Context/receipt reads may have started an implicit transaction. Release
+1
View File
@@ -9,6 +9,7 @@ import yaml
SKILL_ROOT = Path(__file__).resolve().parents[2] / "resources" / "work-order-skills"
WORKFLOW_SKILLS = {
"scada_anomaly": "scada-anomaly",
"burst_report": "burst-report",
"water_quality_complaint": "water-quality-complaint",
"general": "appointment-follow-up",
+34
View File
@@ -161,6 +161,14 @@ class OperationalTaskService:
)
async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail:
if command.source.replay_context:
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
replay_context = command.source.replay_context
replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id)
require(replay_doc.get("delivered_until") is not None
and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"])
and replay_context.batch_sequence <= replay_doc["batch_sequence"],
"REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422)
body = _json(command)
request_id = str(command.request_id)
scope = "task:create"
@@ -219,6 +227,27 @@ class OperationalTaskService:
await self._task_event(task_document, now)
return await self.detail(task_id)
async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None:
"""A recovery is new evidence, coalesced through the existing run lifecycle."""
scope = f"task:{task_id}:replay-evidence"
async with self.repo.transaction():
if await self._request_replay(scope, request_id, replay_context):
return
await self.repo.lock(f"task:{task_id}")
task = await self._task(task_id)
prior = task["source"].get("replay_context")
require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID",
"Replay evidence belongs to another investigation", status=409)
require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]),
"REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409)
task["source"]["replay_context"] = replay_context
task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1
now = self._now()
await self._requeue_task(task, now)
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
await self._task_event(task, now)
await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)})
async def list(
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
peripheral_work_order_id: UUID | None = None,
@@ -336,6 +365,9 @@ class OperationalTaskService:
selected_report=await self.report(task_id, report_revision) if report_revision is not None else None,
)
async def replay_context(self, task_id: UUID) -> dict | None:
return (await self._task(task_id)).get("source", {}).get("replay_context")
async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool:
task = await self._task(task_id)
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
@@ -1037,6 +1069,8 @@ class OperationalTaskService:
await self.repo.lock(f"run:{command.run_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
run = await self._active_run(task)
require(run is not None and str(run["run_id"]) == str(command.run_id)
and run["generation"] == task["execution_generation"],
+66
View File
@@ -0,0 +1,66 @@
"""Read-only chronological SCADA paging shared by REST and replay processing."""
from __future__ import annotations
import base64
import hashlib
import json
import math
from datetime import datetime
from app.domain.schemas.scada import ScadaRawPage, ScadaRawReading
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
def validate_window(start: datetime, end: datetime) -> None:
if start.tzinfo is None or end.tzinfo is None or start >= end:
raise ValueError("需要带时区、结束晚于开始的 SCADA 时间区间")
def raw_reading(row: dict) -> ScadaRawReading:
value = row["monitored_value"]
invalid = value is not None and not math.isfinite(float(value))
return ScadaRawReading(
time=row["time"], device_id=row["device_id"],
monitored_value=None if invalid else value,
quality="invalid" if invalid else "missing" if value is None else "unvalidated",
)
async def read_raw_page(
conn, *, project_id: str, device_ids: list[str], start: datetime,
end: datetime, limit: int = 500, cursor: str | None = None,
through: datetime | None = None,
) -> ScadaRawPage:
validate_window(start, end)
ids = sorted(set(device_ids))
if not ids or len(ids) > 500 or not 1 <= limit <= 500:
raise ValueError("请选择 1 至 500 个设备,每页最多 500 条")
cap = min(through or end, end)
if cap.tzinfo is None or cap < start:
raise ValueError("模拟时间不在可读取区间内")
signature = hashlib.sha256(json.dumps(
[project_id, ids, start.isoformat(), end.isoformat()], separators=(",", ":")
).encode()).hexdigest()
after = None
if cursor:
try:
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
frozen = datetime.fromisoformat(data["through"])
last = datetime.fromisoformat(data["time"])
if (data["query"] != signature or frozen.tzinfo is None or last.tzinfo is None
or frozen > cap or not start <= last <= frozen or data["device"] not in ids):
raise ValueError()
cap, after = frozen, (last, data["device"])
except (ValueError, KeyError, TypeError, UnicodeError) as exc:
raise ValueError("SCADA 分页游标无效或查询范围已改变") from exc
rows = await ScadaRepository.get_raw_page(conn, ids, start, end, cap, limit + 1, after)
selected = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = selected[-1]
next_cursor = base64.urlsafe_b64encode(json.dumps({
"query": signature, "through": cap.isoformat(),
"time": last["time"].isoformat(), "device": last["device_id"],
}, separators=(",", ":")).encode()).decode()
return ScadaRawPage(items=[raw_reading(row) for row in selected],
next_cursor=next_cursor, scanned_through=cap)
+397
View File
@@ -0,0 +1,397 @@
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from uuid import NAMESPACE_URL, uuid4, uuid5
from fastapi import HTTPException
from app.auth.permissions import permissions_for_context
from app.domain.schemas.project_settings import ScadaMonitoringSettings
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
from app.infra.db.postgresql.scada import ScadaInfoRepository
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
from app.services.content_digest import digest
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
from app.services.scada_replay_monitoring import evaluate_monitoring_period
def parse_time(value: str) -> datetime:
return datetime.fromisoformat(value)
def snapshot(document: dict) -> ScadaReplaySnapshot:
return ScadaReplaySnapshot(
**{key: document[key] for key in (
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
"virtual_now", "created_at", "updated_at",
)},
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
devices=document["devices"], rules=document["config"]["rules"],
latest=list(document.get("latest", {}).values()),
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
error=document.get("error"),
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
step_pending=document.get("step_pending", False),
monitoring_count=document.get("monitoring_count", 0),
last_monitoring=document.get("last_monitoring"),
)
class ScadaReplayService:
def __init__(self, repository, context, timescale=None, *, clock=None):
self.repo, self.context, self.timescale = repository, context, timescale
self.clock = clock or (lambda: datetime.now(timezone.utc))
async def ensure_storage(self):
if not await self.repo.available():
raise HTTPException(503, "SCADA 回放存储未配置,需要应用 031_scada_replays.sql")
async def create(
self,
command: ScadaReplayCreate,
*,
monitoring_interval_seconds: int | None,
monitoring_config: ScadaMonitoringSettings | None = None,
):
await self.ensure_storage()
async with self.repo.control():
previous = await self.repo.by_request(command.request_id)
config = command.model_dump(mode="json")
if previous:
if previous["config"] != config:
raise HTTPException(409, "创建请求号已用于不同回放配置")
return snapshot(previous)
if monitoring_interval_seconds is None:
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds,再创建演示")
replay_id = str(uuid4())
if await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有回放,请先停止原回放再新建")
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
if not set(command.device_ids) <= metadata.keys():
raise HTTPException(422, "设备不属于当前项目")
for rule in command.rules:
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
raise HTTPException(422, "规则单位与设备单位不一致,请分组配置")
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
raise HTTPException(422, "水质规则必须明确指标名称,不能仅凭单位解释读数")
if monitoring_config:
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
if not role_ids <= metadata.keys():
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
for device in monitoring_config.outlet_flow_device_ids):
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
now = self.clock().isoformat()
document = {
"replay_id": replay_id, "project_id": str(self.context.project_id),
"created_by": str(self.context.user_id), "config": config,
"revision": 1, "status": "ready", "start_time": config["start_time"],
"end_time": config["end_time"], "virtual_now": config["start_time"],
"created_at": now, "updated_at": now,
"monitoring_interval_seconds": monitoring_interval_seconds,
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
"devices": [metadata[device] for device in command.device_ids],
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
}
await self.repo.insert(document)
return snapshot(document)
async def command(self, replay_id, command: ScadaReplayCommand):
await self.ensure_storage()
async with self.repo.control():
document = await self.repo.get(replay_id)
key, body = str(command.request_id), command.model_dump(mode="json")
receipt = document.setdefault("receipts", {}).get(key)
if receipt:
if receipt != body:
raise HTTPException(409, "命令请求号已用于不同内容")
return snapshot(document)
if command.expected_revision != document["revision"]:
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
action, status = command.action, document["status"]
if document.get("step_pending") and action == "step":
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
raise HTTPException(409, "当前项目已有另一条活动回放")
if action == "step" and status in {"ready", "paused"}:
interval = document.get("monitoring_interval_seconds")
if interval is None:
raise HTTPException(409, "旧回放未固定监测周期,请停止后新建演示")
start = parse_time(document["start_time"])
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
document["step_start"] = document["virtual_now"]
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
document["status"], document["step_pending"] = "paused", True
elif action == "stop" and status not in {"completed", "stopped"}:
document["status"], document["step_pending"] = "stopped", False
document.pop("step_target", None)
document.pop("step_start", None)
elif action == "retry" and status == "failed":
if "SOURCE_CHANGED" in (document.get("error") or ""):
raise HTTPException(409, "原始数据发生变化,需要停止并新建回放")
document["status"], document["error"] = "paused", None
document["step_pending"] = bool(document.get("step_target"))
else:
raise HTTPException(409, "当前回放状态不支持此操作")
document["revision"] += 1
document["receipts"][key] = body
if len(document["receipts"]) > 256:
del document["receipts"][next(iter(document["receipts"]))]
await self.repo.save(document)
return snapshot(document)
async def process_pending_step(self, replay_id):
await self.ensure_storage()
async with self.repo.processing() as owned:
if not owned:
raise HTTPException(409, "当前时间步正在由另一请求处理")
while True:
document = await self.repo.get(replay_id)
if not document.get("step_pending"):
return snapshot(document)
try:
await self._advance(document)
except Exception as exc:
# Failure never discards the prepared batch; retry resumes it.
document = await self.repo.get(replay_id)
document["status"], document["step_pending"] = "failed", False
document["error"] = (
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
if "SOURCE_CHANGED" in str(exc)
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
)
async with self.repo.connection.transaction():
await self.repo.save(document)
raise
async def _advance(self, document):
config = ScadaReplayCreate.model_validate(document["config"])
replay_id = document["replay_id"]
if not document.get("warmed"):
import math
window = max(rule.window_samples for rule in config.rules)
async with self.timescale.transaction():
warmup = await ScadaRepository.get_warmup_values(
self.timescale, config.device_ids, config.start_time, window,
)
for row in warmup:
value = row["monitored_value"]
bounds = config.quality_ranges.get(row["device_id"])
if value is not None and math.isfinite(value) and not (bounds and (
bounds.lower is not None and value < bounds.lower
or bounds.upper is not None and value > bounds.upper
)):
document["histories"].setdefault(row["device_id"], []).append(value)
document["warmed"] = True
async with self.repo.connection.transaction():
await self.repo.save(document)
batch = await self.repo.pending_batch(replay_id)
if batch is None:
step = document.get("step_pending", False)
virtual = parse_time(document["virtual_now"])
through = parse_time(document["step_target"]) if step else virtual
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
if document.get("cursor") else None)
async with self.timescale.transaction():
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, config.start_time, config.end_time,
through, 501, after, frame=True,
)
if len(rows) > 500:
raise ValueError("SCADA frame exceeds selected device bound")
at = rows[0]["time"] if rows else through
if step:
virtual = through
document["virtual_now"] = virtual.isoformat()
if not rows and document.get("processed_until") == at.isoformat() and not step:
async with self.repo.connection.transaction():
await self.repo.save(document)
return
sequence = document["batch_sequence"] + 1
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
"source_hash": row["source_hash"]} for row in rows]
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
"refs": refs, "input_digest": digest(refs), "events": []}
async with self.repo.connection.transaction():
document["batch_sequence"] = sequence
document["delivered_until"] = at.isoformat()
await self.repo.save_batch(replay_id, batch)
await self.repo.save(document)
else:
rows = None
if batch["stage"] == "selected":
if rows is None:
# Only crash recovery reloads the selected raw frame.
at = parse_time(batch["at"])
async with self.timescale.transaction():
rows = await ScadaRepository.get_raw_page(
self.timescale, config.device_ids, at, config.end_time, at, 501,
) if batch["refs"] else []
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
"source_hash": row["source_hash"]} for row in rows]
if digest(actual) != batch["input_digest"]:
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
batch["outputs"] = prepare_cleaning(rows, config)
batch["stage"] = "clean_prepared"
async with self.repo.connection.transaction():
await self.repo.save_batch(replay_id, batch)
if batch["stage"] == "clean_prepared":
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
batch["stage"] = "cleaned"
async with self.repo.connection.transaction():
await self.repo.save_batch(replay_id, batch)
if batch["stage"] == "cleaned":
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
result["processed_until"] = batch["at"]
target = result.get("step_target")
was_step = bool(target)
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
monitoring = None
if was_step and not result["step_pending"]:
period_start = parse_time(result.get("step_start", result["start_time"]))
period_end = parse_time(target)
role = result.get("monitoring_config") or {}
role_ids = list(dict.fromkeys(
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
))
previous = await self.repo.monitoring_outputs(
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
period_end, batch["sequence"],
) if role_ids else []
from app.mcp.analysis_service import bounded_calculation
monitoring = await bounded_calculation(
evaluate_monitoring_period,
result,
previous + batch["outputs"],
period_start=period_start,
period_end=period_end,
batch_sequence=batch["sequence"],
)
result["last_monitoring"] = monitoring
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
if not result["step_pending"]:
result.pop("step_target", None)
result.pop("step_start", None)
if batch["refs"]:
last = batch["refs"][-1]
result["cursor"] = [last["time"], last["device_id"]]
if parse_time(batch["at"]) >= config.end_time:
result["status"] = "completed"
batch["stage"], batch["events"] = "detected", changes
if monitoring is not None:
batch["monitoring"] = monitoring
async with self.repo.connection.transaction():
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
if monitoring is not None:
await self._create_monitoring_task(result, monitoring, batch)
batch["monitoring"] = monitoring
result["last_monitoring"] = monitoring
await self.repo.save_batch(replay_id, batch)
await self.repo.save(result)
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
from app.domain.schemas.task_alert import TaskAlertEvent
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
from app.services.task_alerts import TaskAlertService
from uuid import UUID
producer = "scada-replay:" + document["replay_id"]
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
self.context, internal_producer_id=producer)
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context))
for event in changes:
if event["change"] == "raised":
event["batch_sequence"] = batch["sequence"]
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
await alerts.push(TaskAlertEvent(
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
occurred_at=event["occurred_at"], condition_id=None,
source={"kind": "scada_replay", "replay_id": document["replay_id"],
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
))
if event["change"] != "raised":
if event.get("task_id"):
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
"batch_sequence": batch["sequence"],
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
continue
if event.get("category") == "data_quality":
continue
if not create_tasks:
continue
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
summary=event["detail"], source=TaskSource(
kind="external_system", external_id=external, external_revision=1,
event_kind="scada_anomaly",
replay_context={"replay_id": document["replay_id"],
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
),
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号,原因尚待调查"),
)
task = await tasks.create(command)
event["task_id"] = str(task.task_id)
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
async def _create_monitoring_task(self, document, monitoring, batch):
if monitoring["status"] != "suspected_burst":
return
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
from app.services.operational_tasks import OperationalTaskService
tasks = OperationalTaskService(
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context,
permissions_for_context(self.context),
)
external = (
f"scada_replay:{document['replay_id']}:monitor:"
f"{monitoring['period_end']}:{monitoring['rule_version']}"
)
pressure = monitoring["pressure"]
flow = monitoring["outlet_flow"]
detector = monitoring["burst_detection"]
summary = (
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
)
command = OperationalTaskCreate(
request_id=uuid5(NAMESPACE_URL, external),
task_type="event_investigation",
workflow="burst_report",
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
summary=summary,
source=TaskSource(
kind="external_system",
external_id=external,
external_revision=1,
event_kind="scada_burst_suspected",
replay_context={
"replay_id": document["replay_id"],
"knowledge_cutoff": monitoring["knowledge_cutoff"],
"batch_sequence": batch["sequence"],
},
),
trigger=TaskTrigger(
kind="event",
occurred_at=monitoring["period_end"],
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
),
)
task = await tasks.create(command)
monitoring["task_id"] = str(task.task_id)
+210
View File
@@ -0,0 +1,210 @@
"""Deterministic monitoring summaries built only from frozen replay outputs."""
from __future__ import annotations
from datetime import datetime, timedelta
from statistics import median
from typing import Any
from uuid import NAMESPACE_URL, uuid5
import pandas as pd
from app.domain.schemas.scada_replay import ScadaReplayCreate
from app.services.burst_detection import (
TARGET_MU,
TARGET_N_ESTIMATORS,
TARGET_RANDOM_STATE,
TARGET_SCORE_THRESHOLD,
run_burst_detection,
)
def _time(value: str | datetime) -> datetime:
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
def _valid_rows(rows: list[dict], device_id: str) -> list[dict]:
return sorted(
(
row for row in rows
if row.get("device_id") == device_id
and row.get("quality") in {"valid", "corrected"}
and row.get("cleaned_value") is not None
),
key=lambda row: _time(row["time"]),
)
def _relative_indicator(
rows: list[dict],
config: ScadaReplayCreate,
device_ids: list[str],
period_start: datetime,
period_end: datetime,
*,
direction: str,
) -> dict[str, Any]:
details: list[dict[str, Any]] = []
gaps: list[dict[str, str]] = []
for device_id in device_ids:
rule = next((
item for item in config.rules
if item.kind == "relative_change" and device_id in item.device_ids
and item.threshold is not None
), None)
if rule is None:
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
continue
valid = _valid_rows(rows, device_id)
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
if len(baseline) < rule.minimum_samples:
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
continue
if not current:
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
continue
reference = median(float(row["cleaned_value"]) for row in baseline)
if reference == 0:
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
continue
latest = current[-1]
value = float(latest["cleaned_value"])
change = (value - reference) / abs(reference) * 100
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
details.append({
"device_id": device_id,
"observed_at": _time(latest["time"]).isoformat(),
"baseline": reference,
"value": value,
"change_percent": change,
"threshold_percent": rule.threshold,
"hit": hit,
})
return {
"status": "ready" if not gaps else "insufficient_data",
"hit": any(item["hit"] for item in details),
"hit_device_ids": [item["device_id"] for item in details if item["hit"]],
"devices": details,
"gaps": gaps,
}
def _burst_detection(
rows: list[dict],
pressure_device_ids: list[str],
interval_seconds: int,
period_end: datetime,
) -> dict[str, Any]:
if len(pressure_device_ids) < 5:
return {"status": "sensor_coverage_insufficient", "hit": False,
"required_sensors": 5, "available_sensors": len(pressure_device_ids)}
if interval_seconds % 60 or 86400 % interval_seconds:
return {"status": "unsupported_sampling_interval", "hit": False,
"interval_seconds": interval_seconds}
points_per_day = 86400 // interval_seconds
count = 15 * points_per_day
series_by_device: dict[str, dict[datetime, float]] = {}
common_times: set[datetime] | None = None
for device_id in pressure_device_ids:
values = {_time(row["time"]): float(row["cleaned_value"])
for row in _valid_rows(rows, device_id) if _time(row["time"]) <= period_end}
series_by_device[device_id] = values
common_times = set(values) if common_times is None else common_times & set(values)
if not common_times:
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
target = max(common_times)
expected = [target - timedelta(seconds=interval_seconds * offset)
for offset in range(count - 1, -1, -1)]
incomplete = [device_id for device_id, values in series_by_device.items()
if any(at not in values for at in expected)]
if incomplete:
return {"status": "baseline_not_ready", "hit": False, "required_samples": count,
"incomplete_device_ids": incomplete}
frame = pd.DataFrame({device_id: [series_by_device[device_id][at] for at in expected]
for device_id in pressure_device_ids}, index=expected)
try:
result = run_burst_detection(
network="scada-replay",
username="scada-replay",
observed_pressure_data=frame,
points_per_day=points_per_day,
mu=TARGET_MU,
iforest_params={
"n_estimators": TARGET_N_ESTIMATORS,
"random_state": TARGET_RANDOM_STATE,
"contamination": "auto",
},
sensor_nodes=pressure_device_ids,
)
except (TypeError, ValueError) as exc:
return {"status": "analysis_failed", "hit": False, "reason": str(exc)[:300]}
latest = result["rows"][-1]
score = float(latest["Score"])
return {
"status": "ready",
"hit": score <= TARGET_SCORE_THRESHOLD,
"target_time": target.isoformat(),
"target_score": score,
"score_threshold": TARGET_SCORE_THRESHOLD,
"sensor_count": len(pressure_device_ids),
"reference_day_count": 14,
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
}
def evaluate_monitoring_period(
document: dict,
rows: list[dict],
*,
period_start: datetime,
period_end: datetime,
batch_sequence: int,
) -> dict[str, Any]:
config = ScadaReplayCreate.model_validate(document["config"])
monitoring = document.get("monitoring_config")
rule_version = (monitoring or {}).get("rule_version", "unconfigured")
monitoring_id = str(uuid5(
NAMESPACE_URL,
f"{document['replay_id']}:{period_end.isoformat()}:{rule_version}",
))
base = {
"monitoring_id": monitoring_id,
"period_start": period_start.isoformat(),
"period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(),
"batch_sequence": batch_sequence,
"rule_version": rule_version,
"task_id": None,
}
if not monitoring:
return {**base, "status": "insufficient_data", "detail": "项目未配置 SCADA 监测设备角色",
"pressure": {"status": "unconfigured", "hit": False},
"outlet_flow": {"status": "unconfigured", "hit": False},
"burst_detection": {"status": "not_run", "hit": False}}
pressure_ids = monitoring.get("pressure_device_ids", [])
flow_ids = monitoring.get("outlet_flow_device_ids", [])
selected = set(config.device_ids)
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
if not pressure_ids or not flow_ids or role_gaps:
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
"role_gaps": role_gaps,
"pressure": {"status": "unconfigured", "hit": False},
"outlet_flow": {"status": "unconfigured", "hit": False},
"burst_detection": {"status": "not_run", "hit": False}}
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
detector = _burst_detection(
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
)
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
if not complete:
status, detail = "insufficient_data", "监测数据、设备角色或爆管侦测基线不足"
elif all(hits):
status, detail = "suspected_burst", "爆管侦测、压力下降与出厂边界流量增加相互支持"
elif any(hits):
status, detail = "attention", "仅部分爆管相关指标命中,需要继续观察或人工核查"
else:
status, detail = "normal", "本周期爆管相关指标均未命中"
return {**base, "status": status, "detail": detail, "pressure": pressure,
"outlet_flow": outlet_flow, "burst_detection": detector}
+139
View File
@@ -0,0 +1,139 @@
"""Causal batch quality checks and grouped anomaly episodes, no database IO."""
from copy import deepcopy
from datetime import datetime
from math import isfinite
from statistics import median
from uuid import NAMESPACE_URL, uuid5
import re
from app.domain.schemas.scada_replay import ScadaReplayCreate
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
result = []
for row in rows:
value = row["monitored_value"]
quality, reason = "valid", "原始值通过数值与已配置量程校验"
if value is None:
quality, reason = "missing", "原始读数缺失"
elif not isfinite(float(value)):
quality, reason = "invalid", "原始读数不是有限数值"
else:
bounds = config.quality_ranges.get(row["device_id"])
if bounds and ((bounds.lower is not None and value < bounds.lower)
or (bounds.upper is not None and value > bounds.upper)):
quality, reason = "invalid", "超出本次明确配置的仪表量程"
result.append({
"time": row["time"].isoformat() if isinstance(row["time"], datetime) else row["time"],
"device_id": row["device_id"], "source_hash": row["source_hash"],
"monitored_value": value if value is not None and isfinite(float(value)) else None,
"cleaned_value": value if quality == "valid" else None,
"quality": quality, "reason": reason,
})
return result
def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at: datetime) -> tuple[dict, list[dict]]:
"""Consume a complete timestamp; never treat missing data as recovery."""
result = deepcopy(document)
histories = result.setdefault("histories", {})
states = result.setdefault("rule_states", {})
latest = result.setdefault("latest", {})
for row in rows:
latest[row["device_id"]] = row
by_id = {row["device_id"]: row for row in rows}
changes = []
devices = {device["device_id"]: device for device in result.get("devices", [])}
for device_id in config.device_ids:
row = by_id.get(device_id)
invalid = row is not None and row["quality"] != "valid"
previous = latest.get(device_id)
frequency = str(devices.get(device_id, {}).get("transmission_frequency", ""))
match = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
if match:
seconds = int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3])
else:
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
seconds = int(minutes[1]) * 60 if minutes else 0
prior_time = datetime.fromisoformat(previous["time"]) if previous else config.start_time
missing = not row and seconds > 0 and (at - prior_time).total_seconds() >= seconds * 2
key = "quality-" + device_id
state = states.setdefault(key, {"event": None, "recoveries": 0})
if invalid or missing:
state["recoveries"] = 0
if state["event"] is None:
event = {"event_id": str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{key}:{at.isoformat()}")),
"rule_id": key, "device_ids": [device_id], "occurred_at": at.isoformat(),
"last_observed_at": at.isoformat(), "status": "active", "category": "data_quality",
"detail": row["reason"] if invalid else "连续两个设备周期未读取到观测",
"task_id": None}
state["event"] = event
changes.append({**event, "change": "raised"})
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
elif row and row["quality"] == "valid" and state["event"]:
state["recoveries"] += 1
if state["recoveries"] >= 2:
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
state["event"] = None
for rule in config.rules:
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
hits, usable, reasons = [], [], []
for device_id in rule.device_ids:
row = by_id.get(device_id)
if not row or row["quality"] != "valid":
continue
value = row["cleaned_value"]
if rule.kind == "bounds":
triggered = ((rule.lower is not None and value < rule.lower)
or (rule.upper is not None and value > rule.upper))
reason = f"{device_id}={value} {rule.unit},超出配置范围"
else:
previous = histories.get(device_id, [])[-rule.window_samples:]
if len(previous) < rule.minimum_samples:
continue
baseline = median(previous)
if baseline == 0:
continue
change = (value - baseline) / abs(baseline) * 100
triggered = (abs(change) >= rule.threshold if rule.direction == "both"
else change >= rule.threshold if rule.direction == "increase"
else change <= -rule.threshold)
reason = f"{device_id} 较历史中位值变化 {change:.1f}%,阈值 {rule.threshold}%"
usable.append(device_id)
if triggered:
hits.append(device_id)
reasons.append(reason)
if len(hits) >= rule.minimum_devices:
state["hits"] += 1
state["recoveries"] = 0
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
state["event"] = event
changes.append({**event, "change": "raised"})
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
elif state["event"]:
state["event"]["last_observed_at"] = at.isoformat()
elif len(usable) == len(rule.device_ids):
state["hits"] = 0
if state["event"]:
state["recoveries"] += 1
if state["recoveries"] >= rule.recovery_samples:
event = {**state["event"], "status": "resolved", "resolved_at": at.isoformat(),
"change": "resolved"}
changes.append(event)
state["event"] = None
state["recoveries"] = 0
# Incomplete frames cannot count as a consecutive hit/recovery.
elif not usable and any(device in by_id for device in rule.device_ids):
state["hits"] = state["recoveries"] = 0
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
for device in rule.device_ids}
for row in rows:
if row["quality"] == "valid" and row["device_id"] not in frozen:
history = histories.setdefault(row["device_id"], [])
history.append(row["cleaned_value"])
histories[row["device_id"]] = history[-96:]
result["processed_count"] = result.get("processed_count", 0) + len(rows)
return result, changes
+95
View File
@@ -0,0 +1,95 @@
"""Server-enforced replay knowledge bounds and immutable corrected snapshots."""
from datetime import datetime
from fastapi import HTTPException
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
def check_replay_window(context, start, end):
cap = datetime.fromisoformat(context["knowledge_cutoff"])
if start is None or end is None or start.tzinfo is None or end.tzinfo is None or start > end or end > cap:
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
check_replay_window(context, start, end)
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if not set(device_ids) <= set(run["config"]["device_ids"]):
raise HTTPException(403, "设备或回放不属于当前调查")
if start < datetime.fromisoformat(run["start_time"]):
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
rows = await repository.detected_outputs(
context["replay_id"], device_ids, start, end, context["batch_sequence"],
limit=limit, offset=offset,
)
return [{key: row[key] for key in (
"time", "device_id", "monitored_value", "cleaned_value", "quality", "reason"
)} for row in rows]
async def replay_burst_location_inputs(connection, project_id, context, args):
"""Build location series from immutable replay outputs, never live TSDB rows."""
if args.data_source != "monitoring":
raise HTTPException(422, "回放定位只支持 monitoring 数据源")
if args.scada_normal_start is None or args.scada_normal_end is None:
raise HTTPException(422, "回放定位必须显式提供已处理范围内的正常对照窗口")
check_replay_window(context, args.scada_burst_start, args.scada_burst_end)
check_replay_window(context, args.scada_normal_start, args.scada_normal_end)
repository = ScadaReplayRepository(connection, project_id)
run = await repository.get(context["replay_id"])
if run:
replay_start = datetime.fromisoformat(run["start_time"])
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
devices = {item["device_id"]: item for item in run.get("devices", [])}
monitoring = run.get("monitoring_config") or {}
pressure_devices = monitoring.get("pressure_device_ids", [])
pressure_map = {str(devices[item]["node_id"]): item for item in pressure_devices
if item in devices and devices[item].get("node_id")}
requested_pressure = args.pressure_scada_ids or list(pressure_map)
if not requested_pressure or not set(requested_pressure) <= set(pressure_map):
raise HTTPException(422, "压力节点必须来自本轮已配置的回放监测角色")
flow_devices = monitoring.get("outlet_flow_device_ids", [])
flow_map = {str(devices[item]["link_id"]): item for item in flow_devices
if item in devices and devices[item].get("link_id")}
requested_flow = args.flow_scada_ids or []
if not set(requested_flow) <= set(flow_map):
raise HTTPException(422, "流量要素必须来自本轮已配置的出厂边界流量角色")
selected = [pressure_map[item] for item in requested_pressure] + [flow_map[item] for item in requested_flow]
rows = await repository.detected_outputs(
context["replay_id"], selected,
min(args.scada_burst_start, args.scada_normal_start),
max(args.scada_burst_end, args.scada_normal_end),
context["batch_sequence"],
)
def averages(mapping, start, end):
result = {}
for element_id, device_id in mapping.items():
values = [float(row["cleaned_value"]) for row in rows
if row.get("device_id") == device_id
and row.get("quality") in {"valid", "corrected"}
and row.get("cleaned_value") is not None
and start <= datetime.fromisoformat(row["time"]) <= end]
if not values:
raise HTTPException(422, f"回放窗口缺少 {device_id} 的有效清洗观测")
result[element_id] = sum(values) / len(values)
return result
pressure_mapping = {item: pressure_map[item] for item in requested_pressure}
flow_mapping = {item: flow_map[item] for item in requested_flow}
return {
"data_source": "monitoring",
"burst_leakage": args.burst_leakage,
"pressure_scada_ids": requested_pressure,
"burst_pressure": averages(pressure_mapping, args.scada_burst_start, args.scada_burst_end),
"normal_pressure": averages(pressure_mapping, args.scada_normal_start, args.scada_normal_end),
**({
"flow_scada_ids": requested_flow,
"burst_flow": averages(flow_mapping, args.scada_burst_start, args.scada_burst_end),
"normal_flow": averages(flow_mapping, args.scada_normal_start, args.scada_normal_end),
"use_scada_flow": True,
} if requested_flow else {}),
}