feat(scada): add project monitoring replay
This commit is contained in:
@@ -49,6 +49,7 @@ def _project_response(project: models.Project) -> AdminProjectResponse:
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
created_at=project.created_at,
|
||||
updated_at=project.updated_at,
|
||||
@@ -276,6 +277,7 @@ async def create_admin_project(
|
||||
gs_workspace=payload.gs_workspace,
|
||||
map_extent=payload.map_extent,
|
||||
map_config=payload.map_config.model_dump(exclude_unset=True),
|
||||
settings=payload.settings.model_dump(exclude_unset=True),
|
||||
status=payload.status,
|
||||
creator_user_id=current_user.id,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ async def get_project_metadata(
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
project_role=ctx.project_role,
|
||||
)
|
||||
|
||||
@@ -320,6 +320,7 @@ async def provision_project(
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
created_at=project.created_at,
|
||||
updated_at=project.updated_at,
|
||||
|
||||
@@ -567,10 +567,24 @@ async def query_operational_task_data(
|
||||
if not await service.can_read_source(task_id, query.source):
|
||||
raise DomainError("PERMISSION_DENIED", "Task has no approved access to this data source", status=403)
|
||||
source_permission = SCADA_VIEW if query.source.startswith("scada_") else SIMULATION_VIEW
|
||||
snapshot = await read_task_data(
|
||||
context=ctx, query=query, credential=authorization,
|
||||
delegated=source_permission not in _task_permissions(ctx),
|
||||
)
|
||||
from app.services.scada_replay_queries import replay_snapshot_rows
|
||||
replay_context = await service.replay_context(task_id)
|
||||
if replay_context:
|
||||
if query.source != "scada_readings":
|
||||
raise HTTPException(403, "回放调查数据接口仅开放本轮 SCADA 快照,请通过 MCP 查询设备目录")
|
||||
rows = await replay_snapshot_rows(conn, ctx.project_id, replay_context, query.device_ids,
|
||||
query.start, query.end, 1001)
|
||||
from app.services.content_digest import digest
|
||||
from datetime import datetime, timezone
|
||||
snapshot = TaskDataSnapshot(
|
||||
source="scada_readings", source_ref=f"scada_replay:{replay_context['replay_id']}:{digest(rows)}",
|
||||
queried_at=datetime.now(timezone.utc), origin="synthetic", data=rows[:1000], truncated=len(rows) > 1000,
|
||||
)
|
||||
else:
|
||||
snapshot = await read_task_data(
|
||||
context=ctx, query=query, credential=authorization,
|
||||
delegated=source_permission not in _task_permissions(ctx),
|
||||
)
|
||||
evidence, stored_snapshot = await service.record_data_snapshot(task_id, snapshot)
|
||||
return TaskDataQueryResult(evidence=evidence, snapshot=stored_snapshot)
|
||||
except TaskDataError as exc:
|
||||
|
||||
@@ -35,6 +35,7 @@ async def get_project_info_endpoint(
|
||||
gs_workspace=project_detail.gs_workspace,
|
||||
map_extent=project_detail.map_extent,
|
||||
map_config=getattr(project_detail, "map_config", {}) or {},
|
||||
settings=getattr(project_detail, "settings", {}) or {},
|
||||
status=project_detail.status,
|
||||
project_role="viewer", # Default role for public access
|
||||
)
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""SCADA replay controls. Manual steps are processed by the command request."""
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
|
||||
from app.auth.metadata_dependencies import get_metadata_repository
|
||||
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, require_permission, permissions_for_context
|
||||
from app.auth.project_dependencies import ProjectContext, get_project_pg_connection, get_project_timescale_connection
|
||||
from app.domain.schemas.scada_replay import (
|
||||
ScadaReplayCapabilities, ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot,
|
||||
ScadaReplayEventsPage, ScadaReplayReadings,
|
||||
)
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
from app.services.scada_replay import ScadaReplayService, snapshot
|
||||
|
||||
router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"])
|
||||
|
||||
|
||||
async def repository(
|
||||
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
|
||||
connection=Depends(get_project_pg_connection),
|
||||
):
|
||||
return ScadaReplayRepository(connection, ctx.project_id)
|
||||
|
||||
|
||||
async def project_settings(
|
||||
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
|
||||
metadata_repo=Depends(get_metadata_repository),
|
||||
) -> ProjectSettings:
|
||||
project = await metadata_repo.get_project_by_id(ctx.project_id)
|
||||
if project is None:
|
||||
raise HTTPException(404, "当前项目不存在")
|
||||
return ProjectSettings.model_validate(project.settings or {})
|
||||
|
||||
|
||||
@router.get("/capabilities", response_model=ScadaReplayCapabilities)
|
||||
async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_permission(SCADA_VIEW)),
|
||||
settings=Depends(project_settings)):
|
||||
available = await repo.available()
|
||||
current = await repo.current() if available else None
|
||||
monitoring = settings.scada_monitoring
|
||||
monitoring_ready = bool(
|
||||
monitoring
|
||||
and len(monitoring.pressure_device_ids) >= 5
|
||||
and monitoring.outlet_flow_device_ids
|
||||
)
|
||||
if monitoring_ready:
|
||||
monitoring_reason = None
|
||||
elif monitoring is None:
|
||||
monitoring_reason = "项目尚未配置压力与出厂边界流量监测角色"
|
||||
elif len(monitoring.pressure_device_ids) < 5:
|
||||
monitoring_reason = "爆管侦测至少需要 5 个压力监测设备"
|
||||
else:
|
||||
monitoring_reason = "项目尚未配置出厂边界流量设备"
|
||||
return ScadaReplayCapabilities(
|
||||
available=available, can_write=SCADA_CLEAN in permissions_for_context(ctx),
|
||||
reason=None if available else "需先部署 SCADA 回放存储结构",
|
||||
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
|
||||
burst_monitoring_ready=monitoring_ready,
|
||||
burst_monitoring_reason=monitoring_reason,
|
||||
current=snapshot(current) if current else None,
|
||||
)
|
||||
|
||||
|
||||
@router.post("", response_model=ScadaReplaySnapshot, status_code=201)
|
||||
async def create_replay(command: ScadaReplayCreate, repo=Depends(repository),
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)):
|
||||
return await ScadaReplayService(repo, ctx).create(
|
||||
command,
|
||||
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
|
||||
monitoring_config=settings.scada_monitoring,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{replay_id}", response_model=ScadaReplaySnapshot)
|
||||
async def get_replay(replay_id: UUID, repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
return snapshot(await repo.get(replay_id))
|
||||
|
||||
|
||||
@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot)
|
||||
async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depends(repository),
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)),
|
||||
timescale=Depends(get_project_timescale_connection)):
|
||||
service = ScadaReplayService(repo, ctx, timescale)
|
||||
result = await service.command(replay_id, command)
|
||||
if command.action in {"step", "retry"} and result.step_pending:
|
||||
return await service.process_pending_step(replay_id)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage)
|
||||
async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200),
|
||||
repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
await repo.get(replay_id)
|
||||
items = await repo.events(replay_id, offset, limit + 1)
|
||||
return ScadaReplayEventsPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None)
|
||||
|
||||
|
||||
@router.get("/{replay_id}/observations", response_model=ScadaReplayReadings)
|
||||
async def replay_observations(replay_id: UUID, through: datetime, repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
document = await repo.get(replay_id)
|
||||
if (through.tzinfo is None or not document.get("delivered_until")
|
||||
or through > datetime.fromisoformat(document["delivered_until"])
|
||||
or through < datetime.fromisoformat(document["start_time"])):
|
||||
raise HTTPException(422, "回看仅允许本轮已读取的时间区间")
|
||||
return ScadaReplayReadings(
|
||||
items=await repo.latest_outputs(replay_id, through),
|
||||
through=through,
|
||||
)
|
||||
@@ -1,10 +1,13 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
|
||||
from typing import List
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from psycopg import AsyncConnection
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.domain.schemas.scada import ScadaLatestReadingsResponse
|
||||
from app.domain.schemas.scada import ScadaLatestReadingsResponse, ScadaRawPage
|
||||
from app.auth.project_dependencies import get_project_context, ProjectContext
|
||||
from app.services.scada_raw import read_raw_page, validate_window
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from .dependencies import get_postgres_connection, get_timescale_connection
|
||||
@@ -13,6 +16,46 @@ router = APIRouter()
|
||||
SCADA_BATCH_MAX_ITEMS = 10_000
|
||||
|
||||
|
||||
@router.get("/timeseries/scada-readings/raw", response_model=ScadaRawPage,
|
||||
summary="按时间顺序分页读取 SCADA 原始值")
|
||||
async def get_scada_raw(
|
||||
start_time: datetime, end_time: datetime, device_ids: str,
|
||||
limit: int = Query(500, ge=1, le=500),
|
||||
cursor: str | None = Query(None, max_length=2048),
|
||||
replay_id: UUID | None = None,
|
||||
conn: AsyncConnection = Depends(get_timescale_connection),
|
||||
postgres_conn: AsyncConnection = Depends(get_postgres_connection),
|
||||
ctx: ProjectContext = Depends(get_project_context),
|
||||
) -> ScadaRawPage:
|
||||
try:
|
||||
validate_window(start_time, end_time)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
ids = list(dict.fromkeys(item.strip() for item in device_ids.split(",") if item.strip()))
|
||||
if not ids or len(ids) > 500:
|
||||
raise HTTPException(422, "请选择 1 至 500 个 SCADA 设备")
|
||||
existing = await ScadaInfoRepository.get_existing_device_ids(postgres_conn, ids)
|
||||
if set(ids) != existing:
|
||||
raise HTTPException(422, "查询包含当前项目未配置的 SCADA 设备")
|
||||
try:
|
||||
through = None
|
||||
if replay_id is not None:
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.services.scada_replay import ScadaReplayService
|
||||
repo = ScadaReplayRepository(postgres_conn, ctx.project_id)
|
||||
await ScadaReplayService(repo, ctx).ensure_storage()
|
||||
replay = await repo.get(replay_id)
|
||||
if not set(ids) <= set(replay["config"]["device_ids"]):
|
||||
raise HTTPException(403, "设备超出回放范围")
|
||||
if start_time < datetime.fromisoformat(replay["start_time"]) or end_time > datetime.fromisoformat(replay["end_time"]):
|
||||
raise HTTPException(422, "查询超出选定回放区间")
|
||||
through = datetime.fromisoformat(replay["virtual_now"])
|
||||
return await read_raw_page(conn, project_id=str(ctx.project_id), device_ids=ids,
|
||||
start=start_time, end=end_time, limit=limit, cursor=cursor, through=through)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
|
||||
|
||||
class ScadaReadingBatchItem(BaseModel):
|
||||
time: datetime
|
||||
device_id: str = Field(min_length=1)
|
||||
|
||||
@@ -61,8 +61,10 @@ from app.auth.permissions import (
|
||||
require_method_permission,
|
||||
require_permission,
|
||||
)
|
||||
from app.api.v1.endpoints import scada_replay
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(scada_replay.router)
|
||||
|
||||
webgis_access = Depends(
|
||||
require_method_permission(
|
||||
|
||||
@@ -2,9 +2,10 @@ from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from app.domain.schemas.project_map_config import ProjectMapConfig, ProjectMapConfigUpdate
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
|
||||
BusinessRole = Literal["admin", "service", "user"]
|
||||
@@ -72,6 +73,7 @@ class ProjectMemberResponse(BaseModel):
|
||||
|
||||
|
||||
class AdminProjectCreateRequest(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
code: str = Field(..., min_length=1, max_length=50)
|
||||
description: str | None = None
|
||||
@@ -82,6 +84,10 @@ class AdminProjectCreateRequest(BaseModel):
|
||||
|
||||
|
||||
class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
|
||||
settings: ProjectSettings | None = Field(
|
||||
default=None,
|
||||
description="按顶层键合并项目配置;省略的键保持原值,嵌套对象整体替换",
|
||||
)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=100)
|
||||
code: str | None = Field(default=None, min_length=1, max_length=50)
|
||||
description: str | None = None
|
||||
@@ -89,8 +95,16 @@ class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
|
||||
map_extent: dict | None = None
|
||||
status: ProjectStatus | None = None
|
||||
|
||||
@field_validator("settings")
|
||||
@classmethod
|
||||
def reject_null_settings(cls, value: ProjectSettings | None) -> ProjectSettings:
|
||||
if value is None:
|
||||
raise ValueError("settings must be an object")
|
||||
return value
|
||||
|
||||
|
||||
class AdminProjectResponse(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
project_id: UUID
|
||||
name: str
|
||||
code: str
|
||||
|
||||
@@ -4,9 +4,11 @@ from uuid import UUID
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.domain.schemas.project_map_config import ProjectMapConfig
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
|
||||
class ProjectMetaResponse(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
project_id: UUID
|
||||
name: str
|
||||
code: str
|
||||
|
||||
@@ -41,12 +41,19 @@ ExecutionStatus = Literal[
|
||||
]
|
||||
|
||||
|
||||
class ScadaReplayTaskContext(StrictModel):
|
||||
replay_id: UUID
|
||||
knowledge_cutoff: WorkbenchDateTime
|
||||
batch_sequence: PositiveInt
|
||||
|
||||
|
||||
class TaskSource(StrictModel):
|
||||
kind: TaskSourceKind
|
||||
external_id: NonEmptyStr | None = None
|
||||
external_revision: PositiveInt | None = None
|
||||
event_kind: NonEmptyStr | None = None
|
||||
deleted_at: WorkbenchDateTime | None = None
|
||||
replay_context: ScadaReplayTaskContext | None = None
|
||||
|
||||
|
||||
class TaskTrigger(StrictModel):
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
|
||||
|
||||
|
||||
class ScadaMonitoringSettings(BaseModel):
|
||||
"""Project-owned device roles used by the manual SCADA monitoring demo."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
rule_version: str = Field(
|
||||
default="burst-monitor-v1",
|
||||
min_length=1,
|
||||
max_length=80,
|
||||
pattern=r"^[a-zA-Z0-9_.-]+$",
|
||||
)
|
||||
pressure_device_ids: list[str] = Field(default_factory=list, max_length=20)
|
||||
outlet_flow_device_ids: list[str] = Field(default_factory=list, max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def distinct_roles(self):
|
||||
for values in (self.pressure_device_ids, self.outlet_flow_device_ids):
|
||||
if len(set(values)) != len(values) or any(not value.strip() for value in values):
|
||||
raise ValueError("SCADA 监测角色中的设备标识不能为空或重复")
|
||||
if set(self.pressure_device_ids) & set(self.outlet_flow_device_ids):
|
||||
raise ValueError("同一设备不能同时作为压力与出厂边界流量")
|
||||
return self
|
||||
|
||||
|
||||
class ProjectSettings(BaseModel):
|
||||
"""Extensible, non-secret project parameters stored in MetaDB as JSON."""
|
||||
|
||||
model_config = ConfigDict(extra="allow", allow_inf_nan=False)
|
||||
|
||||
__pydantic_extra__: dict[str, JsonValue] = Field(init=False)
|
||||
scada_monitoring_interval_seconds: int | None = Field(
|
||||
default=None,
|
||||
strict=True,
|
||||
ge=60,
|
||||
le=31_536_000,
|
||||
description="SCADA 主动监测演示步长,单位秒;null 表示未配置",
|
||||
)
|
||||
scada_monitoring: ScadaMonitoringSettings | None = Field(
|
||||
default=None,
|
||||
description="主动监测使用的项目设备角色;不创建定时任务",
|
||||
)
|
||||
@@ -1,8 +1,8 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ScadaDeviceResponse(BaseModel):
|
||||
"""Project SCADA metadata keyed by the canonical device identifier."""
|
||||
|
||||
@@ -47,3 +47,20 @@ class ScadaLatestReadingsResponse(BaseModel):
|
||||
offset: int = Field(ge=0)
|
||||
observed_count: int = Field(ge=0)
|
||||
latest_observed_at: datetime | None = None
|
||||
|
||||
|
||||
class ScadaRawReading(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
time: datetime
|
||||
device_id: str
|
||||
monitored_value: float | None
|
||||
quality: Literal["unvalidated", "missing", "invalid"] = "unvalidated"
|
||||
|
||||
|
||||
class ScadaRawPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
items: list[ScadaRawReading]
|
||||
next_cursor: str | None
|
||||
scanned_through: datetime
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Persisted replay controls and explicit, unit-aware demonstration rules."""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class ReplayModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
|
||||
class ScadaReplayRule(ReplayModel):
|
||||
rule_id: str = Field(min_length=1, max_length=80, pattern=r"^[a-zA-Z0-9_-]+$")
|
||||
device_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
kind: Literal["relative_change", "bounds"]
|
||||
direction: Literal["both", "increase", "decrease"] = "both"
|
||||
threshold: float | None = Field(None, gt=0)
|
||||
lower: float | None = None
|
||||
upper: float | None = None
|
||||
unit: str = Field(min_length=1, max_length=40)
|
||||
metric_name: str = Field(default="", max_length=80)
|
||||
minimum_samples: int = Field(3, ge=1, le=96)
|
||||
window_samples: int = Field(4, ge=1, le=96)
|
||||
consecutive_samples: int = Field(1, ge=1, le=20)
|
||||
recovery_samples: int = Field(2, ge=1, le=20)
|
||||
minimum_devices: int = Field(1, ge=1, le=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def parameters(self):
|
||||
if len(set(self.device_ids)) != len(self.device_ids):
|
||||
raise ValueError("规则内设备不能重复")
|
||||
if self.minimum_devices > len(self.device_ids):
|
||||
raise ValueError("同时异常设备数超过规则设备数")
|
||||
if self.kind == "relative_change" and (self.threshold is None or self.minimum_samples > self.window_samples):
|
||||
raise ValueError("变化率规则需要阈值和足够的历史窗口")
|
||||
if self.kind == "bounds" and self.lower is None and self.upper is None:
|
||||
raise ValueError("上下限规则至少配置一个阈值")
|
||||
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
|
||||
raise ValueError("规则下限必须小于上限")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaQualityRange(ReplayModel):
|
||||
lower: float | None = None
|
||||
upper: float | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounds(self):
|
||||
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
|
||||
raise ValueError("仪表量程下限必须小于上限")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaReplayCreate(ReplayModel):
|
||||
request_id: UUID
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
device_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30)
|
||||
quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def configuration(self):
|
||||
if (self.start_time.tzinfo is None or self.end_time.tzinfo is None
|
||||
or not timedelta(0) < self.end_time - self.start_time <= timedelta(days=31)):
|
||||
raise ValueError("请选择带时区、最长 31 天的有效时间区间")
|
||||
ids = set(self.device_ids)
|
||||
if len(ids) != len(self.device_ids) or any(not value.strip() for value in ids):
|
||||
raise ValueError("设备标识不能为空或重复")
|
||||
if len({rule.rule_id for rule in self.rules}) != len(self.rules):
|
||||
raise ValueError("规则标识不能重复")
|
||||
if any(not set(rule.device_ids) <= ids for rule in self.rules) or not set(self.quality_ranges) <= ids:
|
||||
raise ValueError("规则及量程设备必须属于本次回放")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaReplayCommand(ReplayModel):
|
||||
request_id: UUID
|
||||
expected_revision: int = Field(ge=1)
|
||||
action: Literal["step", "stop", "retry"]
|
||||
|
||||
|
||||
class ScadaMonitoringResult(ReplayModel):
|
||||
monitoring_id: UUID
|
||||
period_start: datetime
|
||||
period_end: datetime
|
||||
knowledge_cutoff: datetime
|
||||
batch_sequence: int = Field(ge=1)
|
||||
rule_version: str = Field(min_length=1, max_length=80)
|
||||
status: Literal["normal", "attention", "insufficient_data", "suspected_burst"]
|
||||
detail: str = Field(min_length=1, max_length=1000)
|
||||
task_id: UUID | None = None
|
||||
pressure: dict
|
||||
outlet_flow: dict
|
||||
burst_detection: dict
|
||||
role_gaps: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ScadaReplaySnapshot(ReplayModel):
|
||||
replay_id: UUID
|
||||
project_id: UUID
|
||||
revision: int
|
||||
status: Literal["ready", "paused", "completed", "stopped", "failed"]
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
|
||||
step_pending: bool = False
|
||||
monitoring_count: int = 0
|
||||
last_monitoring: ScadaMonitoringResult | None = None
|
||||
virtual_now: datetime
|
||||
delivered_until: datetime | None = None
|
||||
processed_until: datetime | None = None
|
||||
processed_count: int = 0
|
||||
anomaly_count: int = 0
|
||||
devices: list[dict]
|
||||
rules: list[ScadaReplayRule]
|
||||
latest: list[dict]
|
||||
active_events: list[dict]
|
||||
error: str | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ScadaReplayCapabilities(ReplayModel):
|
||||
available: bool
|
||||
can_write: bool
|
||||
reason: str | None = None
|
||||
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
|
||||
burst_monitoring_ready: bool = False
|
||||
burst_monitoring_reason: str | None = None
|
||||
current: ScadaReplaySnapshot | None = None
|
||||
|
||||
|
||||
class ScadaReplayEventsPage(ReplayModel):
|
||||
items: list[dict]
|
||||
next_offset: int | None
|
||||
|
||||
|
||||
class ScadaReplayReadings(ReplayModel):
|
||||
items: list[dict]
|
||||
through: datetime
|
||||
@@ -5,7 +5,7 @@ from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.schemas.shared import NonEmptyStr, PositiveInt, StrictModel
|
||||
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchDateTime
|
||||
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchScadaReplaySource, WorkbenchDateTime
|
||||
|
||||
|
||||
class TaskAlertCommand(StrictModel):
|
||||
@@ -14,7 +14,7 @@ class TaskAlertCommand(StrictModel):
|
||||
event_type: Literal["raised", "acknowledged", "resolved"]
|
||||
occurred_at: WorkbenchDateTime
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
severity: Literal["warning", "critical"]
|
||||
@@ -29,7 +29,7 @@ class TaskAlertState(StrictModel):
|
||||
revision: PositiveInt
|
||||
producer_id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
|
||||
@@ -49,10 +49,17 @@ class WorkbenchAlertSource(StrictModel):
|
||||
run_id: NonEmptyStr
|
||||
|
||||
|
||||
class WorkbenchScadaReplaySource(StrictModel):
|
||||
kind: Literal["scada_replay"]
|
||||
replay_id: NonEmptyStr
|
||||
rule_id: NonEmptyStr
|
||||
batch_sequence: int = Field(ge=1)
|
||||
|
||||
|
||||
class WorkbenchRuntimeAlert(StrictModel):
|
||||
id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
@@ -62,7 +69,7 @@ class WorkbenchRuntimeAlert(StrictModel):
|
||||
class WorkbenchPersistedAlert(StrictModel):
|
||||
alert_id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
|
||||
@@ -44,6 +44,7 @@ class Project(Base):
|
||||
gs_workspace: Mapped[str] = mapped_column(String(100), unique=True)
|
||||
map_extent: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
|
||||
map_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
|
||||
settings: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
|
||||
status: Mapped[str] = mapped_column(String(20), default="active")
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow
|
||||
|
||||
@@ -67,6 +67,7 @@ class ProjectDetail:
|
||||
map_extent: Optional[dict]
|
||||
status: str
|
||||
map_config: dict = field(default_factory=dict)
|
||||
settings: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -223,6 +224,7 @@ class MetadataRepository:
|
||||
status: str,
|
||||
creator_user_id: UUID | None = None,
|
||||
map_config: dict | None = None,
|
||||
settings: dict | None = None,
|
||||
) -> models.Project:
|
||||
project = models.Project(
|
||||
id=uuid4(),
|
||||
@@ -233,6 +235,7 @@ class MetadataRepository:
|
||||
map_extent=map_extent,
|
||||
status=status,
|
||||
map_config=map_config or {},
|
||||
settings=settings or {},
|
||||
created_at=_utcnow(),
|
||||
updated_at=_utcnow(),
|
||||
)
|
||||
@@ -263,6 +266,7 @@ class MetadataRepository:
|
||||
timescale_dsn: str,
|
||||
pool_min_size: int = 1,
|
||||
pool_max_size: int = 4,
|
||||
settings: dict | None = None,
|
||||
) -> models.Project:
|
||||
"""Atomically expose a fully provisioned project and both DB routes."""
|
||||
business_secret = _encrypt_database_secret(business_dsn)
|
||||
@@ -276,6 +280,7 @@ class MetadataRepository:
|
||||
map_extent=map_extent,
|
||||
status="active",
|
||||
map_config={},
|
||||
settings=settings or {},
|
||||
created_at=_utcnow(),
|
||||
updated_at=_utcnow(),
|
||||
)
|
||||
@@ -333,6 +338,8 @@ class MetadataRepository:
|
||||
setattr(project, field, updates[field])
|
||||
if "map_config" in updates:
|
||||
project.map_config = {**(project.map_config or {}), **updates["map_config"]}
|
||||
if "settings" in updates:
|
||||
project.settings = {**(project.settings or {}), **updates["settings"]}
|
||||
project.updated_at = _utcnow()
|
||||
await self.session.commit()
|
||||
await self.session.refresh(project)
|
||||
@@ -350,6 +357,7 @@ class MetadataRepository:
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=project.map_config or {},
|
||||
settings=project.settings or {},
|
||||
status=project.status,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Short-transaction storage for replay control and resumable batches."""
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import HTTPException
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
|
||||
class ScadaReplayRepository:
|
||||
def __init__(self, connection, project_id):
|
||||
self.connection, self.project_id = connection, str(project_id)
|
||||
self.lock_key = f"scada-replay:{self.project_id}"
|
||||
|
||||
async def available(self) -> bool:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT to_regclass('peripheral_sample.scada_replays') IS NOT NULL "
|
||||
"AND to_regclass('peripheral_sample.scada_replay_batches') IS NOT NULL AS ready"
|
||||
)
|
||||
return (await cur.fetchone())["ready"]
|
||||
|
||||
@asynccontextmanager
|
||||
async def control(self):
|
||||
async with self.connection.transaction():
|
||||
await self.connection.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (self.lock_key,))
|
||||
yield
|
||||
|
||||
@asynccontextmanager
|
||||
async def processing(self):
|
||||
# Session lock outlives short transactions, never an open SQL transaction
|
||||
# across Timescale IO. A dead connection automatically releases ownership.
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute("SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS owned", (self.lock_key,))
|
||||
owned = (await cur.fetchone())["owned"]
|
||||
try:
|
||||
yield owned
|
||||
finally:
|
||||
if owned:
|
||||
async with self.connection.transaction():
|
||||
await self.connection.execute("SELECT pg_advisory_unlock(hashtextextended(%s,0))", (self.lock_key,))
|
||||
|
||||
async def get(self, replay_id) -> dict:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id=%s",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if not row:
|
||||
raise HTTPException(404, "当前项目不存在此 SCADA 回放")
|
||||
return row["document"]
|
||||
|
||||
async def current(self) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s"
|
||||
" ORDER BY created_at DESC LIMIT 1", (self.project_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def by_request(self, request_id) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND request_id=%s",
|
||||
(self.project_id, str(request_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def active_other(self, replay_id) -> bool:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT 1 FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id<>%s "
|
||||
"AND status IN ('ready','paused') LIMIT 1",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
return await cur.fetchone() is not None
|
||||
|
||||
async def insert(self, document: dict):
|
||||
await self.connection.execute(
|
||||
"INSERT INTO peripheral_sample.scada_replays(project_id,replay_id,request_id,status,document) "
|
||||
"VALUES(%s,%s,%s,%s,%s)",
|
||||
(self.project_id, document["replay_id"], document["config"]["request_id"], document["status"], Jsonb(document)),
|
||||
)
|
||||
|
||||
async def save(self, document: dict):
|
||||
document["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||||
await self.connection.execute(
|
||||
"UPDATE peripheral_sample.scada_replays SET status=%s,document=%s,updated_at=now() "
|
||||
"WHERE project_id=%s AND replay_id=%s",
|
||||
(document["status"], Jsonb(document), self.project_id, document["replay_id"]),
|
||||
)
|
||||
|
||||
async def pending_batch(self, replay_id) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replay_batches "
|
||||
"WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def save_batch(self, replay_id, batch: dict):
|
||||
await self.connection.execute(
|
||||
"INSERT INTO peripheral_sample.scada_replay_batches(project_id,replay_id,sequence,stage,document) "
|
||||
"VALUES(%s,%s,%s,%s,%s) ON CONFLICT(project_id,replay_id,sequence) "
|
||||
"DO UPDATE SET stage=excluded.stage,document=excluded.document",
|
||||
(self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)),
|
||||
)
|
||||
|
||||
async def events(self, replay_id, offset=0, limit=100) -> list[dict]:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s ORDER BY b.sequence DESC LIMIT %s OFFSET %s",
|
||||
(self.project_id, str(replay_id), limit, offset),
|
||||
)
|
||||
return [r["event"] for r in await cur.fetchall()]
|
||||
|
||||
async def monitoring_outputs(
|
||||
self,
|
||||
replay_id,
|
||||
device_ids: list[str],
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
before_sequence: int,
|
||||
) -> list[dict]:
|
||||
"""Read only committed replay outputs available before the current batch."""
|
||||
if not device_ids:
|
||||
return []
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' AND b.sequence<%s "
|
||||
"AND r.value->>'device_id'=ANY(%s) "
|
||||
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'",
|
||||
(self.project_id, str(replay_id), before_sequence, device_ids, start, end),
|
||||
)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
|
||||
async def detected_outputs(
|
||||
self,
|
||||
replay_id,
|
||||
device_ids: list[str],
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
before_sequence: int,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Read immutable detector outputs available to a bounded replay investigation."""
|
||||
statement = (
|
||||
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence<=%s AND b.stage='detected' "
|
||||
"AND r.value->>'device_id'=ANY(%s) "
|
||||
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'"
|
||||
)
|
||||
params: tuple = (
|
||||
self.project_id, str(replay_id), before_sequence, device_ids, start, end,
|
||||
)
|
||||
if limit is not None:
|
||||
statement += " LIMIT %s OFFSET %s"
|
||||
params += (limit, offset)
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(statement, params)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
|
||||
async def latest_outputs(self, replay_id, through: datetime, limit: int = 500) -> list[dict]:
|
||||
"""Return the latest committed cleaned output for each replay device."""
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading "
|
||||
"FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') "
|
||||
"AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s",
|
||||
(self.project_id, str(replay_id), through, limit),
|
||||
)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
@@ -9,6 +9,86 @@ class ScadaRepository:
|
||||
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
|
||||
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
|
||||
|
||||
@staticmethod
|
||||
async def get_warmup_values(
|
||||
conn: AsyncConnection,
|
||||
device_ids: list[str],
|
||||
before: datetime,
|
||||
per_device_limit: int,
|
||||
) -> list[dict]:
|
||||
"""Return the latest raw values before a replay starts for detector warmup."""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"SELECT d.device_id,h.monitored_value FROM unnest(%s::text[]) d(device_id) "
|
||||
"CROSS JOIN LATERAL (SELECT monitored_value,time FROM scada.measurements "
|
||||
"WHERE device_id=d.device_id AND time<%s ORDER BY time DESC LIMIT %s) h "
|
||||
"ORDER BY d.device_id,h.time",
|
||||
(device_ids, before, per_device_limit),
|
||||
)
|
||||
return [dict(row) for row in await cur.fetchall()]
|
||||
|
||||
@staticmethod
|
||||
async def get_raw_page(
|
||||
conn: AsyncConnection, device_ids: list[str], start: datetime,
|
||||
end: datetime, through: datetime, limit: int,
|
||||
after: tuple[datetime, str] | None = None, *, frame: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Read raw only, with a stable composite key and a bounded frame/page."""
|
||||
predicate = "device_id=ANY(%s) AND time>=%s AND time<%s AND time<=%s"
|
||||
params: tuple = (device_ids, start, end, through)
|
||||
if after is not None:
|
||||
predicate += " AND (time,device_id)>(%s,%s)"
|
||||
params += after
|
||||
if frame:
|
||||
statement = (
|
||||
"WITH next_time AS (SELECT min(time) AS time FROM scada.measurements WHERE "
|
||||
+ predicate + ") SELECT m.time,m.device_id,m.monitored_value,"
|
||||
"md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements m WHERE m.device_id=ANY(%s) "
|
||||
"AND m.time=(SELECT time FROM next_time) ORDER BY m.time,m.device_id LIMIT %s"
|
||||
)
|
||||
params += (device_ids, limit)
|
||||
else:
|
||||
statement = (
|
||||
"SELECT time,device_id,monitored_value,"
|
||||
"md5(COALESCE(monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements WHERE " + predicate
|
||||
+ " ORDER BY time,device_id LIMIT %s"
|
||||
)
|
||||
params += (limit,)
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(statement, params)
|
||||
return [dict(row) for row in await cur.fetchall()]
|
||||
|
||||
@staticmethod
|
||||
async def apply_replay_cleaning(conn: AsyncConnection, rows: list[dict]) -> None:
|
||||
"""Guard every raw version; update corrected values without inserting raw."""
|
||||
if not rows:
|
||||
return
|
||||
async with conn.transaction():
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"SELECT m.time,m.device_id,md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements m JOIN unnest(%s::timestamptz[],%s::text[]) "
|
||||
"b(time,device_id) USING(time,device_id) FOR UPDATE OF m",
|
||||
([r["time"] for r in rows], [r["device_id"] for r in rows]),
|
||||
)
|
||||
existing = {(r["time"].isoformat(), r["device_id"]): r["source_hash"]
|
||||
for r in await cur.fetchall()}
|
||||
if len(existing) != len(rows) or any(
|
||||
existing.get((datetime.fromisoformat(r["time"]).isoformat(), r["device_id"]))
|
||||
!= r["source_hash"] for r in rows
|
||||
):
|
||||
raise ValueError("SOURCE_CHANGED: SCADA 原始值在处理期间发生变化")
|
||||
await cur.execute(
|
||||
"UPDATE scada.measurements m SET cleaned_value=b.value "
|
||||
"FROM unnest(%s::timestamptz[],%s::text[],%s::double precision[]) "
|
||||
"b(time,device_id,value) WHERE m.time=b.time AND m.device_id=b.device_id "
|
||||
"AND m.cleaned_value IS DISTINCT FROM b.value",
|
||||
([r["time"] for r in rows], [r["device_id"] for r in rows],
|
||||
[r["cleaned_value"] for r in rows]),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
|
||||
if not data:
|
||||
|
||||
@@ -98,3 +98,19 @@ async def bounded_calculation(function, *args, **kwargs):
|
||||
finally:
|
||||
_slots.release()
|
||||
return await asyncio.to_thread(run)
|
||||
|
||||
|
||||
async def calculate_replay_burst_location(
|
||||
parameters: dict[str, Any], *, network: str, username: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Run location with Server-built frozen replay series and no live SCADA query."""
|
||||
from app.services.burst_location import run_burst_location_by_network
|
||||
|
||||
return await bounded_calculation(
|
||||
run_burst_location_by_network,
|
||||
network=network,
|
||||
username=username,
|
||||
scheme_name=None,
|
||||
final_candidates_csv_path=None,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ async def verify_task(scope: QueryScope, binding: BoundContext) -> None:
|
||||
task = await repo.get_task(str(binding.task_id))
|
||||
if not task:
|
||||
raise HTTPException(404, "Task is not in this project")
|
||||
binding.replay_context = task.get("source", {}).get("replay_context")
|
||||
if binding.purpose == "task_collaboration":
|
||||
return
|
||||
if OPERATIONAL_TASK_EXECUTE not in permissions:
|
||||
|
||||
@@ -291,5 +291,6 @@ class BoundContext(BaseModel):
|
||||
task_id: UUID | None = None
|
||||
run_id: UUID | None = None
|
||||
execution_request_id: UUID | None = None
|
||||
replay_context: dict | None = None
|
||||
expires_at: datetime
|
||||
revoked_at: datetime | None = None
|
||||
|
||||
@@ -66,8 +66,23 @@ def page_result(rows, args):
|
||||
async def authorize_query(
|
||||
scope: QueryScope, binding: BoundContext, tool: QueryTool, args: BaseModel
|
||||
) -> None:
|
||||
replay_context = getattr(binding, "replay_context", None)
|
||||
if tool.name not in binding.tool_cap:
|
||||
raise HTTPException(403, "Query is outside the captured tool catalog")
|
||||
if replay_context:
|
||||
from app.services.scada_replay_queries import check_replay_window
|
||||
allowed = {"get_task_context", "get_task_evidence", "list_scada_devices", "get_scada_device",
|
||||
"query_scada_readings", "get_network_schema", "run_burst_location",
|
||||
"get_calculation_job", "get_calculation_result", "list_calculation_jobs"}
|
||||
if tool.name not in allowed:
|
||||
raise HTTPException(403, "回放调查当前仅开放冻结 SCADA 证据与模型目录查询;请使用 query_scada_readings")
|
||||
if tool.name == "query_scada_readings":
|
||||
check_replay_window(replay_context, args.start, args.end)
|
||||
if tool.name == "run_burst_location":
|
||||
if args.data_source != "monitoring" or args.scada_normal_start is None or args.scada_normal_end is None:
|
||||
raise HTTPException(422, "回放定位只支持显式异常窗和正常对照窗的 monitoring 输入")
|
||||
check_replay_window(replay_context, args.scada_burst_start, args.scada_burst_end)
|
||||
check_replay_window(replay_context, args.scada_normal_start, args.scada_normal_end)
|
||||
permissions = permissions_for_context(scope.project)
|
||||
if tool.mutating:
|
||||
if binding.purpose != "ordinary_chat" or binding.task_id is not None or scope.project.project_role != "member":
|
||||
@@ -149,7 +164,37 @@ async def execute_query(
|
||||
gaps = []
|
||||
source_ref = "mcp:" + str(uuid4())
|
||||
evidence_id = None
|
||||
if tool.computation:
|
||||
replay_context = getattr(binding, "replay_context", None)
|
||||
if name == "query_scada_readings" and replay_context:
|
||||
from app.services.scada_replay_queries import replay_snapshot_rows
|
||||
rows = await replay_snapshot_rows(conn, scope.project.project_id, replay_context,
|
||||
args.device_ids, args.start, args.end, args.limit + 1, args.offset)
|
||||
data, page, truncated = page_result(rows, args)
|
||||
origin = "synthetic"
|
||||
gaps.append("历史数据回放证据,仅覆盖该轮调查已提交的批次,不表示现场实时观测。")
|
||||
elif name == "run_burst_location" and replay_context:
|
||||
from app.mcp.analysis_service import calculate_replay_burst_location
|
||||
from app.services.scada_replay_queries import replay_burst_location_inputs
|
||||
|
||||
parameters = await replay_burst_location_inputs(
|
||||
conn, scope.project.project_id, replay_context, args,
|
||||
)
|
||||
await conn.commit()
|
||||
async with scope.simulation_routing():
|
||||
result = await calculate_replay_burst_location(
|
||||
parameters,
|
||||
network=scope.project.project_code,
|
||||
username=scope.user.username,
|
||||
)
|
||||
data = {
|
||||
"parameters": args.model_dump(mode="json", exclude={"context_handle"}),
|
||||
"result": result,
|
||||
}
|
||||
origin = "synthetic"
|
||||
gaps = [
|
||||
"定位只使用任务知识截止前已固化的回放清洗输出;候选排序依赖漏量假设、模型和测点覆盖,仍需现场核实。"
|
||||
]
|
||||
elif tool.computation:
|
||||
from app.mcp.analysis_service import TIMESCALE_CALCULATIONS, calculate
|
||||
|
||||
# Context/receipt reads may have started an implicit transaction. Release
|
||||
|
||||
@@ -9,6 +9,7 @@ import yaml
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[2] / "resources" / "work-order-skills"
|
||||
WORKFLOW_SKILLS = {
|
||||
"scada_anomaly": "scada-anomaly",
|
||||
"burst_report": "burst-report",
|
||||
"water_quality_complaint": "water-quality-complaint",
|
||||
"general": "appointment-follow-up",
|
||||
|
||||
@@ -161,6 +161,14 @@ class OperationalTaskService:
|
||||
)
|
||||
|
||||
async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail:
|
||||
if command.source.replay_context:
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
replay_context = command.source.replay_context
|
||||
replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id)
|
||||
require(replay_doc.get("delivered_until") is not None
|
||||
and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"])
|
||||
and replay_context.batch_sequence <= replay_doc["batch_sequence"],
|
||||
"REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422)
|
||||
body = _json(command)
|
||||
request_id = str(command.request_id)
|
||||
scope = "task:create"
|
||||
@@ -219,6 +227,27 @@ class OperationalTaskService:
|
||||
await self._task_event(task_document, now)
|
||||
return await self.detail(task_id)
|
||||
|
||||
async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None:
|
||||
"""A recovery is new evidence, coalesced through the existing run lifecycle."""
|
||||
scope = f"task:{task_id}:replay-evidence"
|
||||
async with self.repo.transaction():
|
||||
if await self._request_replay(scope, request_id, replay_context):
|
||||
return
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
task = await self._task(task_id)
|
||||
prior = task["source"].get("replay_context")
|
||||
require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID",
|
||||
"Replay evidence belongs to another investigation", status=409)
|
||||
require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]),
|
||||
"REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409)
|
||||
task["source"]["replay_context"] = replay_context
|
||||
task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1
|
||||
now = self._now()
|
||||
await self._requeue_task(task, now)
|
||||
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
||||
await self._task_event(task, now)
|
||||
await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)})
|
||||
|
||||
async def list(
|
||||
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
|
||||
peripheral_work_order_id: UUID | None = None,
|
||||
@@ -336,6 +365,9 @@ class OperationalTaskService:
|
||||
selected_report=await self.report(task_id, report_revision) if report_revision is not None else None,
|
||||
)
|
||||
|
||||
async def replay_context(self, task_id: UUID) -> dict | None:
|
||||
return (await self._task(task_id)).get("source", {}).get("replay_context")
|
||||
|
||||
async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool:
|
||||
task = await self._task(task_id)
|
||||
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
|
||||
@@ -1037,6 +1069,8 @@ class OperationalTaskService:
|
||||
await self.repo.lock(f"run:{command.run_id}")
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
|
||||
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
|
||||
run = await self._active_run(task)
|
||||
require(run is not None and str(run["run_id"]) == str(command.run_id)
|
||||
and run["generation"] == task["execution_generation"],
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Read-only chronological SCADA paging shared by REST and replay processing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime
|
||||
|
||||
from app.domain.schemas.scada import ScadaRawPage, ScadaRawReading
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
|
||||
|
||||
def validate_window(start: datetime, end: datetime) -> None:
|
||||
if start.tzinfo is None or end.tzinfo is None or start >= end:
|
||||
raise ValueError("需要带时区、结束晚于开始的 SCADA 时间区间")
|
||||
|
||||
|
||||
def raw_reading(row: dict) -> ScadaRawReading:
|
||||
value = row["monitored_value"]
|
||||
invalid = value is not None and not math.isfinite(float(value))
|
||||
return ScadaRawReading(
|
||||
time=row["time"], device_id=row["device_id"],
|
||||
monitored_value=None if invalid else value,
|
||||
quality="invalid" if invalid else "missing" if value is None else "unvalidated",
|
||||
)
|
||||
|
||||
|
||||
async def read_raw_page(
|
||||
conn, *, project_id: str, device_ids: list[str], start: datetime,
|
||||
end: datetime, limit: int = 500, cursor: str | None = None,
|
||||
through: datetime | None = None,
|
||||
) -> ScadaRawPage:
|
||||
validate_window(start, end)
|
||||
ids = sorted(set(device_ids))
|
||||
if not ids or len(ids) > 500 or not 1 <= limit <= 500:
|
||||
raise ValueError("请选择 1 至 500 个设备,每页最多 500 条")
|
||||
cap = min(through or end, end)
|
||||
if cap.tzinfo is None or cap < start:
|
||||
raise ValueError("模拟时间不在可读取区间内")
|
||||
signature = hashlib.sha256(json.dumps(
|
||||
[project_id, ids, start.isoformat(), end.isoformat()], separators=(",", ":")
|
||||
).encode()).hexdigest()
|
||||
after = None
|
||||
if cursor:
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
frozen = datetime.fromisoformat(data["through"])
|
||||
last = datetime.fromisoformat(data["time"])
|
||||
if (data["query"] != signature or frozen.tzinfo is None or last.tzinfo is None
|
||||
or frozen > cap or not start <= last <= frozen or data["device"] not in ids):
|
||||
raise ValueError()
|
||||
cap, after = frozen, (last, data["device"])
|
||||
except (ValueError, KeyError, TypeError, UnicodeError) as exc:
|
||||
raise ValueError("SCADA 分页游标无效或查询范围已改变") from exc
|
||||
rows = await ScadaRepository.get_raw_page(conn, ids, start, end, cap, limit + 1, after)
|
||||
selected = rows[:limit]
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = selected[-1]
|
||||
next_cursor = base64.urlsafe_b64encode(json.dumps({
|
||||
"query": signature, "through": cap.isoformat(),
|
||||
"time": last["time"].isoformat(), "device": last["device_id"],
|
||||
}, separators=(",", ":")).encode()).decode()
|
||||
return ScadaRawPage(items=[raw_reading(row) for row in selected],
|
||||
next_cursor=next_cursor, scanned_through=cap)
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from fastapi import HTTPException
|
||||
from app.auth.permissions import permissions_for_context
|
||||
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.content_digest import digest
|
||||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||||
|
||||
|
||||
def parse_time(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||||
return ScadaReplaySnapshot(
|
||||
**{key: document[key] for key in (
|
||||
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
|
||||
"virtual_now", "created_at", "updated_at",
|
||||
)},
|
||||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||||
devices=document["devices"], rules=document["config"]["rules"],
|
||||
latest=list(document.get("latest", {}).values()),
|
||||
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
|
||||
error=document.get("error"),
|
||||
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
|
||||
step_pending=document.get("step_pending", False),
|
||||
monitoring_count=document.get("monitoring_count", 0),
|
||||
last_monitoring=document.get("last_monitoring"),
|
||||
)
|
||||
|
||||
|
||||
class ScadaReplayService:
|
||||
def __init__(self, repository, context, timescale=None, *, clock=None):
|
||||
self.repo, self.context, self.timescale = repository, context, timescale
|
||||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||||
|
||||
async def ensure_storage(self):
|
||||
if not await self.repo.available():
|
||||
raise HTTPException(503, "SCADA 回放存储未配置,需要应用 031_scada_replays.sql")
|
||||
|
||||
async def create(
|
||||
self,
|
||||
command: ScadaReplayCreate,
|
||||
*,
|
||||
monitoring_interval_seconds: int | None,
|
||||
monitoring_config: ScadaMonitoringSettings | None = None,
|
||||
):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.control():
|
||||
previous = await self.repo.by_request(command.request_id)
|
||||
config = command.model_dump(mode="json")
|
||||
if previous:
|
||||
if previous["config"] != config:
|
||||
raise HTTPException(409, "创建请求号已用于不同回放配置")
|
||||
return snapshot(previous)
|
||||
if monitoring_interval_seconds is None:
|
||||
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds,再创建演示")
|
||||
replay_id = str(uuid4())
|
||||
if await self.repo.active_other(replay_id):
|
||||
raise HTTPException(409, "当前项目已有回放,请先停止原回放再新建")
|
||||
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
|
||||
if not set(command.device_ids) <= metadata.keys():
|
||||
raise HTTPException(422, "设备不属于当前项目")
|
||||
for rule in command.rules:
|
||||
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
|
||||
raise HTTPException(422, "规则单位与设备单位不一致,请分组配置")
|
||||
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
|
||||
raise HTTPException(422, "水质规则必须明确指标名称,不能仅凭单位解释读数")
|
||||
if monitoring_config:
|
||||
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
|
||||
if not role_ids <= metadata.keys():
|
||||
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
|
||||
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
|
||||
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
|
||||
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
|
||||
for device in monitoring_config.outlet_flow_device_ids):
|
||||
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
|
||||
now = self.clock().isoformat()
|
||||
document = {
|
||||
"replay_id": replay_id, "project_id": str(self.context.project_id),
|
||||
"created_by": str(self.context.user_id), "config": config,
|
||||
"revision": 1, "status": "ready", "start_time": config["start_time"],
|
||||
"end_time": config["end_time"], "virtual_now": config["start_time"],
|
||||
"created_at": now, "updated_at": now,
|
||||
"monitoring_interval_seconds": monitoring_interval_seconds,
|
||||
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
|
||||
"devices": [metadata[device] for device in command.device_ids],
|
||||
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
|
||||
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
|
||||
}
|
||||
await self.repo.insert(document)
|
||||
return snapshot(document)
|
||||
|
||||
async def command(self, replay_id, command: ScadaReplayCommand):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.control():
|
||||
document = await self.repo.get(replay_id)
|
||||
key, body = str(command.request_id), command.model_dump(mode="json")
|
||||
receipt = document.setdefault("receipts", {}).get(key)
|
||||
if receipt:
|
||||
if receipt != body:
|
||||
raise HTTPException(409, "命令请求号已用于不同内容")
|
||||
return snapshot(document)
|
||||
if command.expected_revision != document["revision"]:
|
||||
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
|
||||
action, status = command.action, document["status"]
|
||||
if document.get("step_pending") and action == "step":
|
||||
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
|
||||
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
|
||||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||||
if action == "step" and status in {"ready", "paused"}:
|
||||
interval = document.get("monitoring_interval_seconds")
|
||||
if interval is None:
|
||||
raise HTTPException(409, "旧回放未固定监测周期,请停止后新建演示")
|
||||
start = parse_time(document["start_time"])
|
||||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||||
document["step_start"] = document["virtual_now"]
|
||||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||||
document["status"], document["step_pending"] = "paused", True
|
||||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||||
document["status"], document["step_pending"] = "stopped", False
|
||||
document.pop("step_target", None)
|
||||
document.pop("step_start", None)
|
||||
elif action == "retry" and status == "failed":
|
||||
if "SOURCE_CHANGED" in (document.get("error") or ""):
|
||||
raise HTTPException(409, "原始数据发生变化,需要停止并新建回放")
|
||||
document["status"], document["error"] = "paused", None
|
||||
document["step_pending"] = bool(document.get("step_target"))
|
||||
else:
|
||||
raise HTTPException(409, "当前回放状态不支持此操作")
|
||||
document["revision"] += 1
|
||||
document["receipts"][key] = body
|
||||
if len(document["receipts"]) > 256:
|
||||
del document["receipts"][next(iter(document["receipts"]))]
|
||||
await self.repo.save(document)
|
||||
return snapshot(document)
|
||||
|
||||
async def process_pending_step(self, replay_id):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.processing() as owned:
|
||||
if not owned:
|
||||
raise HTTPException(409, "当前时间步正在由另一请求处理")
|
||||
while True:
|
||||
document = await self.repo.get(replay_id)
|
||||
if not document.get("step_pending"):
|
||||
return snapshot(document)
|
||||
try:
|
||||
await self._advance(document)
|
||||
except Exception as exc:
|
||||
# Failure never discards the prepared batch; retry resumes it.
|
||||
document = await self.repo.get(replay_id)
|
||||
document["status"], document["step_pending"] = "failed", False
|
||||
document["error"] = (
|
||||
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
|
||||
if "SOURCE_CHANGED" in str(exc)
|
||||
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
|
||||
)
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
raise
|
||||
|
||||
async def _advance(self, document):
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
replay_id = document["replay_id"]
|
||||
if not document.get("warmed"):
|
||||
import math
|
||||
window = max(rule.window_samples for rule in config.rules)
|
||||
async with self.timescale.transaction():
|
||||
warmup = await ScadaRepository.get_warmup_values(
|
||||
self.timescale, config.device_ids, config.start_time, window,
|
||||
)
|
||||
for row in warmup:
|
||||
value = row["monitored_value"]
|
||||
bounds = config.quality_ranges.get(row["device_id"])
|
||||
if value is not None and math.isfinite(value) and not (bounds and (
|
||||
bounds.lower is not None and value < bounds.lower
|
||||
or bounds.upper is not None and value > bounds.upper
|
||||
)):
|
||||
document["histories"].setdefault(row["device_id"], []).append(value)
|
||||
document["warmed"] = True
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
batch = await self.repo.pending_batch(replay_id)
|
||||
if batch is None:
|
||||
step = document.get("step_pending", False)
|
||||
virtual = parse_time(document["virtual_now"])
|
||||
through = parse_time(document["step_target"]) if step else virtual
|
||||
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
|
||||
if document.get("cursor") else None)
|
||||
async with self.timescale.transaction():
|
||||
rows = await ScadaRepository.get_raw_page(
|
||||
self.timescale, config.device_ids, config.start_time, config.end_time,
|
||||
through, 501, after, frame=True,
|
||||
)
|
||||
if len(rows) > 500:
|
||||
raise ValueError("SCADA frame exceeds selected device bound")
|
||||
at = rows[0]["time"] if rows else through
|
||||
if step:
|
||||
virtual = through
|
||||
document["virtual_now"] = virtual.isoformat()
|
||||
if not rows and document.get("processed_until") == at.isoformat() and not step:
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
return
|
||||
sequence = document["batch_sequence"] + 1
|
||||
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||||
"source_hash": row["source_hash"]} for row in rows]
|
||||
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
|
||||
"refs": refs, "input_digest": digest(refs), "events": []}
|
||||
async with self.repo.connection.transaction():
|
||||
document["batch_sequence"] = sequence
|
||||
document["delivered_until"] = at.isoformat()
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
await self.repo.save(document)
|
||||
else:
|
||||
rows = None
|
||||
if batch["stage"] == "selected":
|
||||
if rows is None:
|
||||
# Only crash recovery reloads the selected raw frame.
|
||||
at = parse_time(batch["at"])
|
||||
async with self.timescale.transaction():
|
||||
rows = await ScadaRepository.get_raw_page(
|
||||
self.timescale, config.device_ids, at, config.end_time, at, 501,
|
||||
) if batch["refs"] else []
|
||||
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||||
"source_hash": row["source_hash"]} for row in rows]
|
||||
if digest(actual) != batch["input_digest"]:
|
||||
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
|
||||
batch["outputs"] = prepare_cleaning(rows, config)
|
||||
batch["stage"] = "clean_prepared"
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
if batch["stage"] == "clean_prepared":
|
||||
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
|
||||
batch["stage"] = "cleaned"
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
if batch["stage"] == "cleaned":
|
||||
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
|
||||
result["processed_until"] = batch["at"]
|
||||
target = result.get("step_target")
|
||||
was_step = bool(target)
|
||||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||||
monitoring = None
|
||||
if was_step and not result["step_pending"]:
|
||||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||||
period_end = parse_time(target)
|
||||
role = result.get("monitoring_config") or {}
|
||||
role_ids = list(dict.fromkeys(
|
||||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||||
))
|
||||
previous = await self.repo.monitoring_outputs(
|
||||
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
|
||||
period_end, batch["sequence"],
|
||||
) if role_ids else []
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
monitoring = await bounded_calculation(
|
||||
evaluate_monitoring_period,
|
||||
result,
|
||||
previous + batch["outputs"],
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
batch_sequence=batch["sequence"],
|
||||
)
|
||||
result["last_monitoring"] = monitoring
|
||||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||||
if not result["step_pending"]:
|
||||
result.pop("step_target", None)
|
||||
result.pop("step_start", None)
|
||||
if batch["refs"]:
|
||||
last = batch["refs"][-1]
|
||||
result["cursor"] = [last["time"], last["device_id"]]
|
||||
if parse_time(batch["at"]) >= config.end_time:
|
||||
result["status"] = "completed"
|
||||
batch["stage"], batch["events"] = "detected", changes
|
||||
if monitoring is not None:
|
||||
batch["monitoring"] = monitoring
|
||||
async with self.repo.connection.transaction():
|
||||
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
|
||||
if monitoring is not None:
|
||||
await self._create_monitoring_task(result, monitoring, batch)
|
||||
batch["monitoring"] = monitoring
|
||||
result["last_monitoring"] = monitoring
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
await self.repo.save(result)
|
||||
|
||||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||||
from app.services.task_alerts import TaskAlertService
|
||||
from uuid import UUID
|
||||
producer = "scada-replay:" + document["replay_id"]
|
||||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, internal_producer_id=producer)
|
||||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context))
|
||||
for event in changes:
|
||||
if event["change"] == "raised":
|
||||
event["batch_sequence"] = batch["sequence"]
|
||||
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
|
||||
await alerts.push(TaskAlertEvent(
|
||||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||||
occurred_at=event["occurred_at"], condition_id=None,
|
||||
source={"kind": "scada_replay", "replay_id": document["replay_id"],
|
||||
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
|
||||
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
|
||||
))
|
||||
if event["change"] != "raised":
|
||||
if event.get("task_id"):
|
||||
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
|
||||
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
|
||||
"batch_sequence": batch["sequence"],
|
||||
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
|
||||
continue
|
||||
if event.get("category") == "data_quality":
|
||||
continue
|
||||
if not create_tasks:
|
||||
continue
|
||||
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
|
||||
command = OperationalTaskCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
|
||||
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
|
||||
summary=event["detail"], source=TaskSource(
|
||||
kind="external_system", external_id=external, external_revision=1,
|
||||
event_kind="scada_anomaly",
|
||||
replay_context={"replay_id": document["replay_id"],
|
||||
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
|
||||
),
|
||||
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号,原因尚待调查"),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
event["task_id"] = str(task.task_id)
|
||||
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
|
||||
|
||||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||||
if monitoring["status"] != "suspected_burst":
|
||||
return
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
tasks = OperationalTaskService(
|
||||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context,
|
||||
permissions_for_context(self.context),
|
||||
)
|
||||
external = (
|
||||
f"scada_replay:{document['replay_id']}:monitor:"
|
||||
f"{monitoring['period_end']}:{monitoring['rule_version']}"
|
||||
)
|
||||
pressure = monitoring["pressure"]
|
||||
flow = monitoring["outlet_flow"]
|
||||
detector = monitoring["burst_detection"]
|
||||
summary = (
|
||||
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
|
||||
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
|
||||
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
|
||||
)
|
||||
command = OperationalTaskCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, external),
|
||||
task_type="event_investigation",
|
||||
workflow="burst_report",
|
||||
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
|
||||
summary=summary,
|
||||
source=TaskSource(
|
||||
kind="external_system",
|
||||
external_id=external,
|
||||
external_revision=1,
|
||||
event_kind="scada_burst_suspected",
|
||||
replay_context={
|
||||
"replay_id": document["replay_id"],
|
||||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||||
"batch_sequence": batch["sequence"],
|
||||
},
|
||||
),
|
||||
trigger=TaskTrigger(
|
||||
kind="event",
|
||||
occurred_at=monitoring["period_end"],
|
||||
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
|
||||
),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
monitoring["task_id"] = str(task.task_id)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Deterministic monitoring summaries built only from frozen replay outputs."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from statistics import median
|
||||
from typing import Any
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.burst_detection import (
|
||||
TARGET_MU,
|
||||
TARGET_N_ESTIMATORS,
|
||||
TARGET_RANDOM_STATE,
|
||||
TARGET_SCORE_THRESHOLD,
|
||||
run_burst_detection,
|
||||
)
|
||||
|
||||
|
||||
def _time(value: str | datetime) -> datetime:
|
||||
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def _valid_rows(rows: list[dict], device_id: str) -> list[dict]:
|
||||
return sorted(
|
||||
(
|
||||
row for row in rows
|
||||
if row.get("device_id") == device_id
|
||||
and row.get("quality") in {"valid", "corrected"}
|
||||
and row.get("cleaned_value") is not None
|
||||
),
|
||||
key=lambda row: _time(row["time"]),
|
||||
)
|
||||
|
||||
|
||||
def _relative_indicator(
|
||||
rows: list[dict],
|
||||
config: ScadaReplayCreate,
|
||||
device_ids: list[str],
|
||||
period_start: datetime,
|
||||
period_end: datetime,
|
||||
*,
|
||||
direction: str,
|
||||
) -> dict[str, Any]:
|
||||
details: list[dict[str, Any]] = []
|
||||
gaps: list[dict[str, str]] = []
|
||||
for device_id in device_ids:
|
||||
rule = next((
|
||||
item for item in config.rules
|
||||
if item.kind == "relative_change" and device_id in item.device_ids
|
||||
and item.threshold is not None
|
||||
), None)
|
||||
if rule is None:
|
||||
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
|
||||
continue
|
||||
valid = _valid_rows(rows, device_id)
|
||||
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
|
||||
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
|
||||
if len(baseline) < rule.minimum_samples:
|
||||
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
|
||||
continue
|
||||
if not current:
|
||||
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
|
||||
continue
|
||||
reference = median(float(row["cleaned_value"]) for row in baseline)
|
||||
if reference == 0:
|
||||
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
|
||||
continue
|
||||
latest = current[-1]
|
||||
value = float(latest["cleaned_value"])
|
||||
change = (value - reference) / abs(reference) * 100
|
||||
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
|
||||
details.append({
|
||||
"device_id": device_id,
|
||||
"observed_at": _time(latest["time"]).isoformat(),
|
||||
"baseline": reference,
|
||||
"value": value,
|
||||
"change_percent": change,
|
||||
"threshold_percent": rule.threshold,
|
||||
"hit": hit,
|
||||
})
|
||||
return {
|
||||
"status": "ready" if not gaps else "insufficient_data",
|
||||
"hit": any(item["hit"] for item in details),
|
||||
"hit_device_ids": [item["device_id"] for item in details if item["hit"]],
|
||||
"devices": details,
|
||||
"gaps": gaps,
|
||||
}
|
||||
|
||||
|
||||
def _burst_detection(
|
||||
rows: list[dict],
|
||||
pressure_device_ids: list[str],
|
||||
interval_seconds: int,
|
||||
period_end: datetime,
|
||||
) -> dict[str, Any]:
|
||||
if len(pressure_device_ids) < 5:
|
||||
return {"status": "sensor_coverage_insufficient", "hit": False,
|
||||
"required_sensors": 5, "available_sensors": len(pressure_device_ids)}
|
||||
if interval_seconds % 60 or 86400 % interval_seconds:
|
||||
return {"status": "unsupported_sampling_interval", "hit": False,
|
||||
"interval_seconds": interval_seconds}
|
||||
points_per_day = 86400 // interval_seconds
|
||||
count = 15 * points_per_day
|
||||
series_by_device: dict[str, dict[datetime, float]] = {}
|
||||
common_times: set[datetime] | None = None
|
||||
for device_id in pressure_device_ids:
|
||||
values = {_time(row["time"]): float(row["cleaned_value"])
|
||||
for row in _valid_rows(rows, device_id) if _time(row["time"]) <= period_end}
|
||||
series_by_device[device_id] = values
|
||||
common_times = set(values) if common_times is None else common_times & set(values)
|
||||
if not common_times:
|
||||
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
|
||||
target = max(common_times)
|
||||
expected = [target - timedelta(seconds=interval_seconds * offset)
|
||||
for offset in range(count - 1, -1, -1)]
|
||||
incomplete = [device_id for device_id, values in series_by_device.items()
|
||||
if any(at not in values for at in expected)]
|
||||
if incomplete:
|
||||
return {"status": "baseline_not_ready", "hit": False, "required_samples": count,
|
||||
"incomplete_device_ids": incomplete}
|
||||
frame = pd.DataFrame({device_id: [series_by_device[device_id][at] for at in expected]
|
||||
for device_id in pressure_device_ids}, index=expected)
|
||||
try:
|
||||
result = run_burst_detection(
|
||||
network="scada-replay",
|
||||
username="scada-replay",
|
||||
observed_pressure_data=frame,
|
||||
points_per_day=points_per_day,
|
||||
mu=TARGET_MU,
|
||||
iforest_params={
|
||||
"n_estimators": TARGET_N_ESTIMATORS,
|
||||
"random_state": TARGET_RANDOM_STATE,
|
||||
"contamination": "auto",
|
||||
},
|
||||
sensor_nodes=pressure_device_ids,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return {"status": "analysis_failed", "hit": False, "reason": str(exc)[:300]}
|
||||
latest = result["rows"][-1]
|
||||
score = float(latest["Score"])
|
||||
return {
|
||||
"status": "ready",
|
||||
"hit": score <= TARGET_SCORE_THRESHOLD,
|
||||
"target_time": target.isoformat(),
|
||||
"target_score": score,
|
||||
"score_threshold": TARGET_SCORE_THRESHOLD,
|
||||
"sensor_count": len(pressure_device_ids),
|
||||
"reference_day_count": 14,
|
||||
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_monitoring_period(
|
||||
document: dict,
|
||||
rows: list[dict],
|
||||
*,
|
||||
period_start: datetime,
|
||||
period_end: datetime,
|
||||
batch_sequence: int,
|
||||
) -> dict[str, Any]:
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
monitoring = document.get("monitoring_config")
|
||||
rule_version = (monitoring or {}).get("rule_version", "unconfigured")
|
||||
monitoring_id = str(uuid5(
|
||||
NAMESPACE_URL,
|
||||
f"{document['replay_id']}:{period_end.isoformat()}:{rule_version}",
|
||||
))
|
||||
base = {
|
||||
"monitoring_id": monitoring_id,
|
||||
"period_start": period_start.isoformat(),
|
||||
"period_end": period_end.isoformat(),
|
||||
"knowledge_cutoff": period_end.isoformat(),
|
||||
"batch_sequence": batch_sequence,
|
||||
"rule_version": rule_version,
|
||||
"task_id": None,
|
||||
}
|
||||
if not monitoring:
|
||||
return {**base, "status": "insufficient_data", "detail": "项目未配置 SCADA 监测设备角色",
|
||||
"pressure": {"status": "unconfigured", "hit": False},
|
||||
"outlet_flow": {"status": "unconfigured", "hit": False},
|
||||
"burst_detection": {"status": "not_run", "hit": False}}
|
||||
pressure_ids = monitoring.get("pressure_device_ids", [])
|
||||
flow_ids = monitoring.get("outlet_flow_device_ids", [])
|
||||
selected = set(config.device_ids)
|
||||
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
|
||||
if not pressure_ids or not flow_ids or role_gaps:
|
||||
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
|
||||
"role_gaps": role_gaps,
|
||||
"pressure": {"status": "unconfigured", "hit": False},
|
||||
"outlet_flow": {"status": "unconfigured", "hit": False},
|
||||
"burst_detection": {"status": "not_run", "hit": False}}
|
||||
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
|
||||
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
|
||||
detector = _burst_detection(
|
||||
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
|
||||
)
|
||||
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
|
||||
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
|
||||
if not complete:
|
||||
status, detail = "insufficient_data", "监测数据、设备角色或爆管侦测基线不足"
|
||||
elif all(hits):
|
||||
status, detail = "suspected_burst", "爆管侦测、压力下降与出厂边界流量增加相互支持"
|
||||
elif any(hits):
|
||||
status, detail = "attention", "仅部分爆管相关指标命中,需要继续观察或人工核查"
|
||||
else:
|
||||
status, detail = "normal", "本周期爆管相关指标均未命中"
|
||||
return {**base, "status": status, "detail": detail, "pressure": pressure,
|
||||
"outlet_flow": outlet_flow, "burst_detection": detector}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Causal batch quality checks and grouped anomaly episodes, no database IO."""
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from math import isfinite
|
||||
from statistics import median
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
import re
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
|
||||
|
||||
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
|
||||
result = []
|
||||
for row in rows:
|
||||
value = row["monitored_value"]
|
||||
quality, reason = "valid", "原始值通过数值与已配置量程校验"
|
||||
if value is None:
|
||||
quality, reason = "missing", "原始读数缺失"
|
||||
elif not isfinite(float(value)):
|
||||
quality, reason = "invalid", "原始读数不是有限数值"
|
||||
else:
|
||||
bounds = config.quality_ranges.get(row["device_id"])
|
||||
if bounds and ((bounds.lower is not None and value < bounds.lower)
|
||||
or (bounds.upper is not None and value > bounds.upper)):
|
||||
quality, reason = "invalid", "超出本次明确配置的仪表量程"
|
||||
result.append({
|
||||
"time": row["time"].isoformat() if isinstance(row["time"], datetime) else row["time"],
|
||||
"device_id": row["device_id"], "source_hash": row["source_hash"],
|
||||
"monitored_value": value if value is not None and isfinite(float(value)) else None,
|
||||
"cleaned_value": value if quality == "valid" else None,
|
||||
"quality": quality, "reason": reason,
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at: datetime) -> tuple[dict, list[dict]]:
|
||||
"""Consume a complete timestamp; never treat missing data as recovery."""
|
||||
result = deepcopy(document)
|
||||
histories = result.setdefault("histories", {})
|
||||
states = result.setdefault("rule_states", {})
|
||||
latest = result.setdefault("latest", {})
|
||||
for row in rows:
|
||||
latest[row["device_id"]] = row
|
||||
by_id = {row["device_id"]: row for row in rows}
|
||||
changes = []
|
||||
devices = {device["device_id"]: device for device in result.get("devices", [])}
|
||||
for device_id in config.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
invalid = row is not None and row["quality"] != "valid"
|
||||
previous = latest.get(device_id)
|
||||
frequency = str(devices.get(device_id, {}).get("transmission_frequency", ""))
|
||||
match = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
|
||||
if match:
|
||||
seconds = int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3])
|
||||
else:
|
||||
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
|
||||
seconds = int(minutes[1]) * 60 if minutes else 0
|
||||
prior_time = datetime.fromisoformat(previous["time"]) if previous else config.start_time
|
||||
missing = not row and seconds > 0 and (at - prior_time).total_seconds() >= seconds * 2
|
||||
key = "quality-" + device_id
|
||||
state = states.setdefault(key, {"event": None, "recoveries": 0})
|
||||
if invalid or missing:
|
||||
state["recoveries"] = 0
|
||||
if state["event"] is None:
|
||||
event = {"event_id": str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{key}:{at.isoformat()}")),
|
||||
"rule_id": key, "device_ids": [device_id], "occurred_at": at.isoformat(),
|
||||
"last_observed_at": at.isoformat(), "status": "active", "category": "data_quality",
|
||||
"detail": row["reason"] if invalid else "连续两个设备周期未读取到观测",
|
||||
"task_id": None}
|
||||
state["event"] = event
|
||||
changes.append({**event, "change": "raised"})
|
||||
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
|
||||
elif row and row["quality"] == "valid" and state["event"]:
|
||||
state["recoveries"] += 1
|
||||
if state["recoveries"] >= 2:
|
||||
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
|
||||
state["event"] = None
|
||||
for rule in config.rules:
|
||||
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
|
||||
hits, usable, reasons = [], [], []
|
||||
for device_id in rule.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
if not row or row["quality"] != "valid":
|
||||
continue
|
||||
value = row["cleaned_value"]
|
||||
if rule.kind == "bounds":
|
||||
triggered = ((rule.lower is not None and value < rule.lower)
|
||||
or (rule.upper is not None and value > rule.upper))
|
||||
reason = f"{device_id}={value} {rule.unit},超出配置范围"
|
||||
else:
|
||||
previous = histories.get(device_id, [])[-rule.window_samples:]
|
||||
if len(previous) < rule.minimum_samples:
|
||||
continue
|
||||
baseline = median(previous)
|
||||
if baseline == 0:
|
||||
continue
|
||||
change = (value - baseline) / abs(baseline) * 100
|
||||
triggered = (abs(change) >= rule.threshold if rule.direction == "both"
|
||||
else change >= rule.threshold if rule.direction == "increase"
|
||||
else change <= -rule.threshold)
|
||||
reason = f"{device_id} 较历史中位值变化 {change:.1f}%,阈值 {rule.threshold}%"
|
||||
usable.append(device_id)
|
||||
if triggered:
|
||||
hits.append(device_id)
|
||||
reasons.append(reason)
|
||||
if len(hits) >= rule.minimum_devices:
|
||||
state["hits"] += 1
|
||||
state["recoveries"] = 0
|
||||
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
|
||||
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
|
||||
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
|
||||
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
|
||||
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
|
||||
state["event"] = event
|
||||
changes.append({**event, "change": "raised"})
|
||||
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
|
||||
elif state["event"]:
|
||||
state["event"]["last_observed_at"] = at.isoformat()
|
||||
elif len(usable) == len(rule.device_ids):
|
||||
state["hits"] = 0
|
||||
if state["event"]:
|
||||
state["recoveries"] += 1
|
||||
if state["recoveries"] >= rule.recovery_samples:
|
||||
event = {**state["event"], "status": "resolved", "resolved_at": at.isoformat(),
|
||||
"change": "resolved"}
|
||||
changes.append(event)
|
||||
state["event"] = None
|
||||
state["recoveries"] = 0
|
||||
# Incomplete frames cannot count as a consecutive hit/recovery.
|
||||
elif not usable and any(device in by_id for device in rule.device_ids):
|
||||
state["hits"] = state["recoveries"] = 0
|
||||
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
|
||||
for device in rule.device_ids}
|
||||
for row in rows:
|
||||
if row["quality"] == "valid" and row["device_id"] not in frozen:
|
||||
history = histories.setdefault(row["device_id"], [])
|
||||
history.append(row["cleaned_value"])
|
||||
histories[row["device_id"]] = history[-96:]
|
||||
result["processed_count"] = result.get("processed_count", 0) + len(rows)
|
||||
return result, changes
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Server-enforced replay knowledge bounds and immutable corrected snapshots."""
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
|
||||
|
||||
def check_replay_window(context, start, end):
|
||||
cap = datetime.fromisoformat(context["knowledge_cutoff"])
|
||||
if start is None or end is None or start.tzinfo is None or end.tzinfo is None or start > end or end > cap:
|
||||
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
|
||||
|
||||
|
||||
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
|
||||
check_replay_window(context, start, end)
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if not set(device_ids) <= set(run["config"]["device_ids"]):
|
||||
raise HTTPException(403, "设备或回放不属于当前调查")
|
||||
if start < datetime.fromisoformat(run["start_time"]):
|
||||
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
|
||||
rows = await repository.detected_outputs(
|
||||
context["replay_id"], device_ids, start, end, context["batch_sequence"],
|
||||
limit=limit, offset=offset,
|
||||
)
|
||||
return [{key: row[key] for key in (
|
||||
"time", "device_id", "monitored_value", "cleaned_value", "quality", "reason"
|
||||
)} for row in rows]
|
||||
|
||||
|
||||
async def replay_burst_location_inputs(connection, project_id, context, args):
|
||||
"""Build location series from immutable replay outputs, never live TSDB rows."""
|
||||
if args.data_source != "monitoring":
|
||||
raise HTTPException(422, "回放定位只支持 monitoring 数据源")
|
||||
if args.scada_normal_start is None or args.scada_normal_end is None:
|
||||
raise HTTPException(422, "回放定位必须显式提供已处理范围内的正常对照窗口")
|
||||
check_replay_window(context, args.scada_burst_start, args.scada_burst_end)
|
||||
check_replay_window(context, args.scada_normal_start, args.scada_normal_end)
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if run:
|
||||
replay_start = datetime.fromisoformat(run["start_time"])
|
||||
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
|
||||
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
|
||||
devices = {item["device_id"]: item for item in run.get("devices", [])}
|
||||
monitoring = run.get("monitoring_config") or {}
|
||||
pressure_devices = monitoring.get("pressure_device_ids", [])
|
||||
pressure_map = {str(devices[item]["node_id"]): item for item in pressure_devices
|
||||
if item in devices and devices[item].get("node_id")}
|
||||
requested_pressure = args.pressure_scada_ids or list(pressure_map)
|
||||
if not requested_pressure or not set(requested_pressure) <= set(pressure_map):
|
||||
raise HTTPException(422, "压力节点必须来自本轮已配置的回放监测角色")
|
||||
flow_devices = monitoring.get("outlet_flow_device_ids", [])
|
||||
flow_map = {str(devices[item]["link_id"]): item for item in flow_devices
|
||||
if item in devices and devices[item].get("link_id")}
|
||||
requested_flow = args.flow_scada_ids or []
|
||||
if not set(requested_flow) <= set(flow_map):
|
||||
raise HTTPException(422, "流量要素必须来自本轮已配置的出厂边界流量角色")
|
||||
selected = [pressure_map[item] for item in requested_pressure] + [flow_map[item] for item in requested_flow]
|
||||
rows = await repository.detected_outputs(
|
||||
context["replay_id"], selected,
|
||||
min(args.scada_burst_start, args.scada_normal_start),
|
||||
max(args.scada_burst_end, args.scada_normal_end),
|
||||
context["batch_sequence"],
|
||||
)
|
||||
|
||||
def averages(mapping, start, end):
|
||||
result = {}
|
||||
for element_id, device_id in mapping.items():
|
||||
values = [float(row["cleaned_value"]) for row in rows
|
||||
if row.get("device_id") == device_id
|
||||
and row.get("quality") in {"valid", "corrected"}
|
||||
and row.get("cleaned_value") is not None
|
||||
and start <= datetime.fromisoformat(row["time"]) <= end]
|
||||
if not values:
|
||||
raise HTTPException(422, f"回放窗口缺少 {device_id} 的有效清洗观测")
|
||||
result[element_id] = sum(values) / len(values)
|
||||
return result
|
||||
|
||||
pressure_mapping = {item: pressure_map[item] for item in requested_pressure}
|
||||
flow_mapping = {item: flow_map[item] for item in requested_flow}
|
||||
return {
|
||||
"data_source": "monitoring",
|
||||
"burst_leakage": args.burst_leakage,
|
||||
"pressure_scada_ids": requested_pressure,
|
||||
"burst_pressure": averages(pressure_mapping, args.scada_burst_start, args.scada_burst_end),
|
||||
"normal_pressure": averages(pressure_mapping, args.scada_normal_start, args.scada_normal_end),
|
||||
**({
|
||||
"flow_scada_ids": requested_flow,
|
||||
"burst_flow": averages(flow_mapping, args.scada_burst_start, args.scada_burst_end),
|
||||
"normal_flow": averages(flow_mapping, args.scada_normal_start, args.scada_normal_end),
|
||||
"use_scada_flow": True,
|
||||
} if requested_flow else {}),
|
||||
}
|
||||
Reference in New Issue
Block a user