Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
26fa5d1909 | ||
|
|
f25423dec4 |
@@ -49,6 +49,7 @@ def _project_response(project: models.Project) -> AdminProjectResponse:
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
created_at=project.created_at,
|
||||
updated_at=project.updated_at,
|
||||
@@ -276,6 +277,7 @@ async def create_admin_project(
|
||||
gs_workspace=payload.gs_workspace,
|
||||
map_extent=payload.map_extent,
|
||||
map_config=payload.map_config.model_dump(exclude_unset=True),
|
||||
settings=payload.settings.model_dump(exclude_unset=True),
|
||||
status=payload.status,
|
||||
creator_user_id=current_user.id,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ async def get_project_metadata(
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
project_role=ctx.project_role,
|
||||
)
|
||||
|
||||
@@ -320,6 +320,7 @@ async def provision_project(
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=getattr(project, "map_config", {}) or {},
|
||||
settings=getattr(project, "settings", {}) or {},
|
||||
status=project.status,
|
||||
created_at=project.created_at,
|
||||
updated_at=project.updated_at,
|
||||
|
||||
@@ -567,10 +567,24 @@ async def query_operational_task_data(
|
||||
if not await service.can_read_source(task_id, query.source):
|
||||
raise DomainError("PERMISSION_DENIED", "Task has no approved access to this data source", status=403)
|
||||
source_permission = SCADA_VIEW if query.source.startswith("scada_") else SIMULATION_VIEW
|
||||
snapshot = await read_task_data(
|
||||
context=ctx, query=query, credential=authorization,
|
||||
delegated=source_permission not in _task_permissions(ctx),
|
||||
)
|
||||
from app.services.scada_replay_queries import replay_snapshot_rows
|
||||
replay_context = await service.replay_context(task_id)
|
||||
if replay_context:
|
||||
if query.source != "scada_readings":
|
||||
raise HTTPException(403, "回放调查数据接口仅开放本轮 SCADA 快照,请通过 MCP 查询设备目录")
|
||||
rows = await replay_snapshot_rows(conn, ctx.project_id, replay_context, query.device_ids,
|
||||
query.start, query.end, 1001)
|
||||
from app.services.content_digest import digest
|
||||
from datetime import datetime, timezone
|
||||
snapshot = TaskDataSnapshot(
|
||||
source="scada_readings", source_ref=f"scada_replay:{replay_context['replay_id']}:{digest(rows)}",
|
||||
queried_at=datetime.now(timezone.utc), origin="synthetic", data=rows[:1000], truncated=len(rows) > 1000,
|
||||
)
|
||||
else:
|
||||
snapshot = await read_task_data(
|
||||
context=ctx, query=query, credential=authorization,
|
||||
delegated=source_permission not in _task_permissions(ctx),
|
||||
)
|
||||
evidence, stored_snapshot = await service.record_data_snapshot(task_id, snapshot)
|
||||
return TaskDataQueryResult(evidence=evidence, snapshot=stored_snapshot)
|
||||
except TaskDataError as exc:
|
||||
|
||||
@@ -35,6 +35,7 @@ async def get_project_info_endpoint(
|
||||
gs_workspace=project_detail.gs_workspace,
|
||||
map_extent=project_detail.map_extent,
|
||||
map_config=getattr(project_detail, "map_config", {}) or {},
|
||||
settings=getattr(project_detail, "settings", {}) or {},
|
||||
status=project_detail.status,
|
||||
project_role="viewer", # Default role for public access
|
||||
)
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""SCADA replay controls. Manual steps are processed by the command request."""
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
|
||||
from app.auth.metadata_dependencies import get_metadata_repository
|
||||
from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, require_permission, permissions_for_context
|
||||
from app.auth.project_dependencies import ProjectContext, get_project_pg_connection, get_project_timescale_connection
|
||||
from app.domain.schemas.scada_replay import (
|
||||
ScadaReplayCapabilities, ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot,
|
||||
ScadaReplayEventsPage, ScadaReplayReadings,
|
||||
)
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
from app.services.scada_replay import ScadaReplayService, snapshot
|
||||
|
||||
router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"])
|
||||
|
||||
|
||||
async def repository(
|
||||
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
|
||||
connection=Depends(get_project_pg_connection),
|
||||
):
|
||||
return ScadaReplayRepository(connection, ctx.project_id)
|
||||
|
||||
|
||||
async def project_settings(
|
||||
ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)),
|
||||
metadata_repo=Depends(get_metadata_repository),
|
||||
) -> ProjectSettings:
|
||||
project = await metadata_repo.get_project_by_id(ctx.project_id)
|
||||
if project is None:
|
||||
raise HTTPException(404, "当前项目不存在")
|
||||
return ProjectSettings.model_validate(project.settings or {})
|
||||
|
||||
|
||||
@router.get("/capabilities", response_model=ScadaReplayCapabilities)
|
||||
async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_permission(SCADA_VIEW)),
|
||||
settings=Depends(project_settings)):
|
||||
available = await repo.available()
|
||||
current = await repo.current() if available else None
|
||||
monitoring = settings.scada_monitoring
|
||||
monitoring_ready = bool(
|
||||
monitoring
|
||||
and len(monitoring.pressure_device_ids) >= 5
|
||||
and monitoring.outlet_flow_device_ids
|
||||
)
|
||||
if monitoring_ready:
|
||||
monitoring_reason = None
|
||||
elif monitoring is None:
|
||||
monitoring_reason = "项目尚未配置压力与出厂边界流量监测角色"
|
||||
elif len(monitoring.pressure_device_ids) < 5:
|
||||
monitoring_reason = "爆管侦测至少需要 5 个压力监测设备"
|
||||
else:
|
||||
monitoring_reason = "项目尚未配置出厂边界流量设备"
|
||||
return ScadaReplayCapabilities(
|
||||
available=available, can_write=SCADA_CLEAN in permissions_for_context(ctx),
|
||||
reason=None if available else "需先部署 SCADA 回放存储结构",
|
||||
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
|
||||
burst_monitoring_ready=monitoring_ready,
|
||||
burst_monitoring_reason=monitoring_reason,
|
||||
current=snapshot(current) if current else None,
|
||||
)
|
||||
|
||||
|
||||
@router.post("", response_model=ScadaReplaySnapshot, status_code=201)
|
||||
async def create_replay(command: ScadaReplayCreate, repo=Depends(repository),
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)):
|
||||
return await ScadaReplayService(repo, ctx).create(
|
||||
command,
|
||||
monitoring_interval_seconds=settings.scada_monitoring_interval_seconds,
|
||||
monitoring_config=settings.scada_monitoring,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{replay_id}", response_model=ScadaReplaySnapshot)
|
||||
async def get_replay(replay_id: UUID, repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
return snapshot(await repo.get(replay_id))
|
||||
|
||||
|
||||
@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot)
|
||||
async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depends(repository),
|
||||
ctx=Depends(require_permission(SCADA_CLEAN)),
|
||||
timescale=Depends(get_project_timescale_connection)):
|
||||
service = ScadaReplayService(repo, ctx, timescale)
|
||||
result = await service.command(replay_id, command)
|
||||
if command.action in {"step", "retry"} and result.step_pending:
|
||||
return await service.process_pending_step(replay_id)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage)
|
||||
async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200),
|
||||
repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
await repo.get(replay_id)
|
||||
items = await repo.events(replay_id, offset, limit + 1)
|
||||
return ScadaReplayEventsPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None)
|
||||
|
||||
|
||||
@router.get("/{replay_id}/observations", response_model=ScadaReplayReadings)
|
||||
async def replay_observations(replay_id: UUID, through: datetime, repo=Depends(repository)):
|
||||
await ScadaReplayService(repo, None).ensure_storage()
|
||||
document = await repo.get(replay_id)
|
||||
if (through.tzinfo is None or not document.get("delivered_until")
|
||||
or through > datetime.fromisoformat(document["delivered_until"])
|
||||
or through < datetime.fromisoformat(document["start_time"])):
|
||||
raise HTTPException(422, "回看仅允许本轮已读取的时间区间")
|
||||
return ScadaReplayReadings(
|
||||
items=await repo.latest_outputs(replay_id, through),
|
||||
through=through,
|
||||
)
|
||||
@@ -1,10 +1,13 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
|
||||
from typing import List
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from psycopg import AsyncConnection
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.domain.schemas.scada import ScadaLatestReadingsResponse
|
||||
from app.domain.schemas.scada import ScadaLatestReadingsResponse, ScadaRawPage
|
||||
from app.auth.project_dependencies import get_project_context, ProjectContext
|
||||
from app.services.scada_raw import read_raw_page, validate_window
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from .dependencies import get_postgres_connection, get_timescale_connection
|
||||
@@ -13,6 +16,46 @@ router = APIRouter()
|
||||
SCADA_BATCH_MAX_ITEMS = 10_000
|
||||
|
||||
|
||||
@router.get("/timeseries/scada-readings/raw", response_model=ScadaRawPage,
|
||||
summary="按时间顺序分页读取 SCADA 原始值")
|
||||
async def get_scada_raw(
|
||||
start_time: datetime, end_time: datetime, device_ids: str,
|
||||
limit: int = Query(500, ge=1, le=500),
|
||||
cursor: str | None = Query(None, max_length=2048),
|
||||
replay_id: UUID | None = None,
|
||||
conn: AsyncConnection = Depends(get_timescale_connection),
|
||||
postgres_conn: AsyncConnection = Depends(get_postgres_connection),
|
||||
ctx: ProjectContext = Depends(get_project_context),
|
||||
) -> ScadaRawPage:
|
||||
try:
|
||||
validate_window(start_time, end_time)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
ids = list(dict.fromkeys(item.strip() for item in device_ids.split(",") if item.strip()))
|
||||
if not ids or len(ids) > 500:
|
||||
raise HTTPException(422, "请选择 1 至 500 个 SCADA 设备")
|
||||
existing = await ScadaInfoRepository.get_existing_device_ids(postgres_conn, ids)
|
||||
if set(ids) != existing:
|
||||
raise HTTPException(422, "查询包含当前项目未配置的 SCADA 设备")
|
||||
try:
|
||||
through = None
|
||||
if replay_id is not None:
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.services.scada_replay import ScadaReplayService
|
||||
repo = ScadaReplayRepository(postgres_conn, ctx.project_id)
|
||||
await ScadaReplayService(repo, ctx).ensure_storage()
|
||||
replay = await repo.get(replay_id)
|
||||
if not set(ids) <= set(replay["config"]["device_ids"]):
|
||||
raise HTTPException(403, "设备超出回放范围")
|
||||
if start_time < datetime.fromisoformat(replay["start_time"]) or end_time > datetime.fromisoformat(replay["end_time"]):
|
||||
raise HTTPException(422, "查询超出选定回放区间")
|
||||
through = datetime.fromisoformat(replay["virtual_now"])
|
||||
return await read_raw_page(conn, project_id=str(ctx.project_id), device_ids=ids,
|
||||
start=start_time, end=end_time, limit=limit, cursor=cursor, through=through)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
|
||||
|
||||
class ScadaReadingBatchItem(BaseModel):
|
||||
time: datetime
|
||||
device_id: str = Field(min_length=1)
|
||||
|
||||
@@ -61,8 +61,10 @@ from app.auth.permissions import (
|
||||
require_method_permission,
|
||||
require_permission,
|
||||
)
|
||||
from app.api.v1.endpoints import scada_replay
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(scada_replay.router)
|
||||
|
||||
webgis_access = Depends(
|
||||
require_method_permission(
|
||||
|
||||
@@ -2,9 +2,10 @@ from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from app.domain.schemas.project_map_config import ProjectMapConfig, ProjectMapConfigUpdate
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
|
||||
BusinessRole = Literal["admin", "service", "user"]
|
||||
@@ -72,6 +73,7 @@ class ProjectMemberResponse(BaseModel):
|
||||
|
||||
|
||||
class AdminProjectCreateRequest(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
code: str = Field(..., min_length=1, max_length=50)
|
||||
description: str | None = None
|
||||
@@ -82,6 +84,10 @@ class AdminProjectCreateRequest(BaseModel):
|
||||
|
||||
|
||||
class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
|
||||
settings: ProjectSettings | None = Field(
|
||||
default=None,
|
||||
description="按顶层键合并项目配置;省略的键保持原值,嵌套对象整体替换",
|
||||
)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=100)
|
||||
code: str | None = Field(default=None, min_length=1, max_length=50)
|
||||
description: str | None = None
|
||||
@@ -89,8 +95,16 @@ class AdminProjectUpdateRequest(ProjectMapConfigUpdate):
|
||||
map_extent: dict | None = None
|
||||
status: ProjectStatus | None = None
|
||||
|
||||
@field_validator("settings")
|
||||
@classmethod
|
||||
def reject_null_settings(cls, value: ProjectSettings | None) -> ProjectSettings:
|
||||
if value is None:
|
||||
raise ValueError("settings must be an object")
|
||||
return value
|
||||
|
||||
|
||||
class AdminProjectResponse(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
project_id: UUID
|
||||
name: str
|
||||
code: str
|
||||
|
||||
@@ -4,9 +4,11 @@ from uuid import UUID
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.domain.schemas.project_map_config import ProjectMapConfig
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
|
||||
class ProjectMetaResponse(BaseModel):
|
||||
settings: ProjectSettings = Field(default_factory=ProjectSettings)
|
||||
project_id: UUID
|
||||
name: str
|
||||
code: str
|
||||
|
||||
@@ -41,12 +41,19 @@ ExecutionStatus = Literal[
|
||||
]
|
||||
|
||||
|
||||
class ScadaReplayTaskContext(StrictModel):
|
||||
replay_id: UUID
|
||||
knowledge_cutoff: WorkbenchDateTime
|
||||
batch_sequence: PositiveInt
|
||||
|
||||
|
||||
class TaskSource(StrictModel):
|
||||
kind: TaskSourceKind
|
||||
external_id: NonEmptyStr | None = None
|
||||
external_revision: PositiveInt | None = None
|
||||
event_kind: NonEmptyStr | None = None
|
||||
deleted_at: WorkbenchDateTime | None = None
|
||||
replay_context: ScadaReplayTaskContext | None = None
|
||||
|
||||
|
||||
class TaskTrigger(StrictModel):
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
|
||||
|
||||
|
||||
class ScadaMonitoringSettings(BaseModel):
|
||||
"""Project-owned device roles used by the manual SCADA monitoring demo."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
rule_version: str = Field(
|
||||
default="burst-monitor-v1",
|
||||
min_length=1,
|
||||
max_length=80,
|
||||
pattern=r"^[a-zA-Z0-9_.-]+$",
|
||||
)
|
||||
pressure_device_ids: list[str] = Field(default_factory=list, max_length=20)
|
||||
outlet_flow_device_ids: list[str] = Field(default_factory=list, max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def distinct_roles(self):
|
||||
for values in (self.pressure_device_ids, self.outlet_flow_device_ids):
|
||||
if len(set(values)) != len(values) or any(not value.strip() for value in values):
|
||||
raise ValueError("SCADA 监测角色中的设备标识不能为空或重复")
|
||||
if set(self.pressure_device_ids) & set(self.outlet_flow_device_ids):
|
||||
raise ValueError("同一设备不能同时作为压力与出厂边界流量")
|
||||
return self
|
||||
|
||||
|
||||
class ProjectSettings(BaseModel):
|
||||
"""Extensible, non-secret project parameters stored in MetaDB as JSON."""
|
||||
|
||||
model_config = ConfigDict(extra="allow", allow_inf_nan=False)
|
||||
|
||||
__pydantic_extra__: dict[str, JsonValue] = Field(init=False)
|
||||
scada_monitoring_interval_seconds: int | None = Field(
|
||||
default=None,
|
||||
strict=True,
|
||||
ge=60,
|
||||
le=31_536_000,
|
||||
description="SCADA 主动监测演示步长,单位秒;null 表示未配置",
|
||||
)
|
||||
scada_monitoring: ScadaMonitoringSettings | None = Field(
|
||||
default=None,
|
||||
description="主动监测使用的项目设备角色;不创建定时任务",
|
||||
)
|
||||
@@ -1,8 +1,8 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ScadaDeviceResponse(BaseModel):
|
||||
"""Project SCADA metadata keyed by the canonical device identifier."""
|
||||
|
||||
@@ -47,3 +47,20 @@ class ScadaLatestReadingsResponse(BaseModel):
|
||||
offset: int = Field(ge=0)
|
||||
observed_count: int = Field(ge=0)
|
||||
latest_observed_at: datetime | None = None
|
||||
|
||||
|
||||
class ScadaRawReading(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
time: datetime
|
||||
device_id: str
|
||||
monitored_value: float | None
|
||||
quality: Literal["unvalidated", "missing", "invalid"] = "unvalidated"
|
||||
|
||||
|
||||
class ScadaRawPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
items: list[ScadaRawReading]
|
||||
next_cursor: str | None
|
||||
scanned_through: datetime
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Persisted replay controls and explicit, unit-aware demonstration rules."""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class ReplayModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
|
||||
class ScadaReplayRule(ReplayModel):
|
||||
rule_id: str = Field(min_length=1, max_length=80, pattern=r"^[a-zA-Z0-9_-]+$")
|
||||
device_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
kind: Literal["relative_change", "bounds"]
|
||||
direction: Literal["both", "increase", "decrease"] = "both"
|
||||
threshold: float | None = Field(None, gt=0)
|
||||
lower: float | None = None
|
||||
upper: float | None = None
|
||||
unit: str = Field(min_length=1, max_length=40)
|
||||
metric_name: str = Field(default="", max_length=80)
|
||||
minimum_samples: int = Field(3, ge=1, le=96)
|
||||
window_samples: int = Field(4, ge=1, le=96)
|
||||
consecutive_samples: int = Field(1, ge=1, le=20)
|
||||
recovery_samples: int = Field(2, ge=1, le=20)
|
||||
minimum_devices: int = Field(1, ge=1, le=500)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def parameters(self):
|
||||
if len(set(self.device_ids)) != len(self.device_ids):
|
||||
raise ValueError("规则内设备不能重复")
|
||||
if self.minimum_devices > len(self.device_ids):
|
||||
raise ValueError("同时异常设备数超过规则设备数")
|
||||
if self.kind == "relative_change" and (self.threshold is None or self.minimum_samples > self.window_samples):
|
||||
raise ValueError("变化率规则需要阈值和足够的历史窗口")
|
||||
if self.kind == "bounds" and self.lower is None and self.upper is None:
|
||||
raise ValueError("上下限规则至少配置一个阈值")
|
||||
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
|
||||
raise ValueError("规则下限必须小于上限")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaQualityRange(ReplayModel):
|
||||
lower: float | None = None
|
||||
upper: float | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounds(self):
|
||||
if self.lower is not None and self.upper is not None and self.lower >= self.upper:
|
||||
raise ValueError("仪表量程下限必须小于上限")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaReplayCreate(ReplayModel):
|
||||
request_id: UUID
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
device_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30)
|
||||
quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def configuration(self):
|
||||
if (self.start_time.tzinfo is None or self.end_time.tzinfo is None
|
||||
or not timedelta(0) < self.end_time - self.start_time <= timedelta(days=31)):
|
||||
raise ValueError("请选择带时区、最长 31 天的有效时间区间")
|
||||
ids = set(self.device_ids)
|
||||
if len(ids) != len(self.device_ids) or any(not value.strip() for value in ids):
|
||||
raise ValueError("设备标识不能为空或重复")
|
||||
if len({rule.rule_id for rule in self.rules}) != len(self.rules):
|
||||
raise ValueError("规则标识不能重复")
|
||||
if any(not set(rule.device_ids) <= ids for rule in self.rules) or not set(self.quality_ranges) <= ids:
|
||||
raise ValueError("规则及量程设备必须属于本次回放")
|
||||
return self
|
||||
|
||||
|
||||
class ScadaReplayCommand(ReplayModel):
|
||||
request_id: UUID
|
||||
expected_revision: int = Field(ge=1)
|
||||
action: Literal["step", "stop", "retry"]
|
||||
|
||||
|
||||
class ScadaMonitoringResult(ReplayModel):
|
||||
monitoring_id: UUID
|
||||
period_start: datetime
|
||||
period_end: datetime
|
||||
knowledge_cutoff: datetime
|
||||
batch_sequence: int = Field(ge=1)
|
||||
rule_version: str = Field(min_length=1, max_length=80)
|
||||
status: Literal["normal", "attention", "insufficient_data", "suspected_burst"]
|
||||
detail: str = Field(min_length=1, max_length=1000)
|
||||
task_id: UUID | None = None
|
||||
pressure: dict
|
||||
outlet_flow: dict
|
||||
burst_detection: dict
|
||||
role_gaps: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ScadaReplaySnapshot(ReplayModel):
|
||||
replay_id: UUID
|
||||
project_id: UUID
|
||||
revision: int
|
||||
status: Literal["ready", "paused", "completed", "stopped", "failed"]
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
|
||||
step_pending: bool = False
|
||||
monitoring_count: int = 0
|
||||
last_monitoring: ScadaMonitoringResult | None = None
|
||||
virtual_now: datetime
|
||||
delivered_until: datetime | None = None
|
||||
processed_until: datetime | None = None
|
||||
processed_count: int = 0
|
||||
anomaly_count: int = 0
|
||||
devices: list[dict]
|
||||
rules: list[ScadaReplayRule]
|
||||
latest: list[dict]
|
||||
active_events: list[dict]
|
||||
error: str | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ScadaReplayCapabilities(ReplayModel):
|
||||
available: bool
|
||||
can_write: bool
|
||||
reason: str | None = None
|
||||
monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000)
|
||||
burst_monitoring_ready: bool = False
|
||||
burst_monitoring_reason: str | None = None
|
||||
current: ScadaReplaySnapshot | None = None
|
||||
|
||||
|
||||
class ScadaReplayEventsPage(ReplayModel):
|
||||
items: list[dict]
|
||||
next_offset: int | None
|
||||
|
||||
|
||||
class ScadaReplayReadings(ReplayModel):
|
||||
items: list[dict]
|
||||
through: datetime
|
||||
@@ -5,7 +5,7 @@ from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.schemas.shared import NonEmptyStr, PositiveInt, StrictModel
|
||||
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchDateTime
|
||||
from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchScadaReplaySource, WorkbenchDateTime
|
||||
|
||||
|
||||
class TaskAlertCommand(StrictModel):
|
||||
@@ -14,7 +14,7 @@ class TaskAlertCommand(StrictModel):
|
||||
event_type: Literal["raised", "acknowledged", "resolved"]
|
||||
occurred_at: WorkbenchDateTime
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
severity: Literal["warning", "critical"]
|
||||
@@ -29,7 +29,7 @@ class TaskAlertState(StrictModel):
|
||||
revision: PositiveInt
|
||||
producer_id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
|
||||
@@ -49,10 +49,17 @@ class WorkbenchAlertSource(StrictModel):
|
||||
run_id: NonEmptyStr
|
||||
|
||||
|
||||
class WorkbenchScadaReplaySource(StrictModel):
|
||||
kind: Literal["scada_replay"]
|
||||
replay_id: NonEmptyStr
|
||||
rule_id: NonEmptyStr
|
||||
batch_sequence: int = Field(ge=1)
|
||||
|
||||
|
||||
class WorkbenchRuntimeAlert(StrictModel):
|
||||
id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
@@ -62,7 +69,7 @@ class WorkbenchRuntimeAlert(StrictModel):
|
||||
class WorkbenchPersistedAlert(StrictModel):
|
||||
alert_id: NonEmptyStr
|
||||
condition_id: NonEmptyStr | None
|
||||
source: WorkbenchAlertSource
|
||||
source: WorkbenchAlertSource | WorkbenchScadaReplaySource
|
||||
title: NonEmptyStr
|
||||
description: NonEmptyStr
|
||||
occurred_at: WorkbenchDateTime
|
||||
|
||||
@@ -44,6 +44,7 @@ class Project(Base):
|
||||
gs_workspace: Mapped[str] = mapped_column(String(100), unique=True)
|
||||
map_extent: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
|
||||
map_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
|
||||
settings: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
|
||||
status: Mapped[str] = mapped_column(String(20), default="active")
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=datetime.utcnow
|
||||
|
||||
@@ -67,6 +67,7 @@ class ProjectDetail:
|
||||
map_extent: Optional[dict]
|
||||
status: str
|
||||
map_config: dict = field(default_factory=dict)
|
||||
settings: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -223,6 +224,7 @@ class MetadataRepository:
|
||||
status: str,
|
||||
creator_user_id: UUID | None = None,
|
||||
map_config: dict | None = None,
|
||||
settings: dict | None = None,
|
||||
) -> models.Project:
|
||||
project = models.Project(
|
||||
id=uuid4(),
|
||||
@@ -233,6 +235,7 @@ class MetadataRepository:
|
||||
map_extent=map_extent,
|
||||
status=status,
|
||||
map_config=map_config or {},
|
||||
settings=settings or {},
|
||||
created_at=_utcnow(),
|
||||
updated_at=_utcnow(),
|
||||
)
|
||||
@@ -263,6 +266,7 @@ class MetadataRepository:
|
||||
timescale_dsn: str,
|
||||
pool_min_size: int = 1,
|
||||
pool_max_size: int = 4,
|
||||
settings: dict | None = None,
|
||||
) -> models.Project:
|
||||
"""Atomically expose a fully provisioned project and both DB routes."""
|
||||
business_secret = _encrypt_database_secret(business_dsn)
|
||||
@@ -276,6 +280,7 @@ class MetadataRepository:
|
||||
map_extent=map_extent,
|
||||
status="active",
|
||||
map_config={},
|
||||
settings=settings or {},
|
||||
created_at=_utcnow(),
|
||||
updated_at=_utcnow(),
|
||||
)
|
||||
@@ -333,6 +338,8 @@ class MetadataRepository:
|
||||
setattr(project, field, updates[field])
|
||||
if "map_config" in updates:
|
||||
project.map_config = {**(project.map_config or {}), **updates["map_config"]}
|
||||
if "settings" in updates:
|
||||
project.settings = {**(project.settings or {}), **updates["settings"]}
|
||||
project.updated_at = _utcnow()
|
||||
await self.session.commit()
|
||||
await self.session.refresh(project)
|
||||
@@ -350,6 +357,7 @@ class MetadataRepository:
|
||||
gs_workspace=project.gs_workspace,
|
||||
map_extent=project.map_extent,
|
||||
map_config=project.map_config or {},
|
||||
settings=project.settings or {},
|
||||
status=project.status,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Short-transaction storage for replay control and resumable batches."""
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import HTTPException
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
|
||||
class ScadaReplayRepository:
|
||||
def __init__(self, connection, project_id):
|
||||
self.connection, self.project_id = connection, str(project_id)
|
||||
self.lock_key = f"scada-replay:{self.project_id}"
|
||||
|
||||
async def available(self) -> bool:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT to_regclass('peripheral_sample.scada_replays') IS NOT NULL "
|
||||
"AND to_regclass('peripheral_sample.scada_replay_batches') IS NOT NULL AS ready"
|
||||
)
|
||||
return (await cur.fetchone())["ready"]
|
||||
|
||||
@asynccontextmanager
|
||||
async def control(self):
|
||||
async with self.connection.transaction():
|
||||
await self.connection.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (self.lock_key,))
|
||||
yield
|
||||
|
||||
@asynccontextmanager
|
||||
async def processing(self):
|
||||
# Session lock outlives short transactions, never an open SQL transaction
|
||||
# across Timescale IO. A dead connection automatically releases ownership.
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute("SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS owned", (self.lock_key,))
|
||||
owned = (await cur.fetchone())["owned"]
|
||||
try:
|
||||
yield owned
|
||||
finally:
|
||||
if owned:
|
||||
async with self.connection.transaction():
|
||||
await self.connection.execute("SELECT pg_advisory_unlock(hashtextextended(%s,0))", (self.lock_key,))
|
||||
|
||||
async def get(self, replay_id) -> dict:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id=%s",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if not row:
|
||||
raise HTTPException(404, "当前项目不存在此 SCADA 回放")
|
||||
return row["document"]
|
||||
|
||||
async def current(self) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s"
|
||||
" ORDER BY created_at DESC LIMIT 1", (self.project_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def by_request(self, request_id) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND request_id=%s",
|
||||
(self.project_id, str(request_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def active_other(self, replay_id) -> bool:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT 1 FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id<>%s "
|
||||
"AND status IN ('ready','paused') LIMIT 1",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
return await cur.fetchone() is not None
|
||||
|
||||
async def insert(self, document: dict):
|
||||
await self.connection.execute(
|
||||
"INSERT INTO peripheral_sample.scada_replays(project_id,replay_id,request_id,status,document) "
|
||||
"VALUES(%s,%s,%s,%s,%s)",
|
||||
(self.project_id, document["replay_id"], document["config"]["request_id"], document["status"], Jsonb(document)),
|
||||
)
|
||||
|
||||
async def save(self, document: dict):
|
||||
document["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||||
await self.connection.execute(
|
||||
"UPDATE peripheral_sample.scada_replays SET status=%s,document=%s,updated_at=now() "
|
||||
"WHERE project_id=%s AND replay_id=%s",
|
||||
(document["status"], Jsonb(document), self.project_id, document["replay_id"]),
|
||||
)
|
||||
|
||||
async def pending_batch(self, replay_id) -> dict | None:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT document FROM peripheral_sample.scada_replay_batches "
|
||||
"WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1",
|
||||
(self.project_id, str(replay_id)),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def save_batch(self, replay_id, batch: dict):
|
||||
await self.connection.execute(
|
||||
"INSERT INTO peripheral_sample.scada_replay_batches(project_id,replay_id,sequence,stage,document) "
|
||||
"VALUES(%s,%s,%s,%s,%s) ON CONFLICT(project_id,replay_id,sequence) "
|
||||
"DO UPDATE SET stage=excluded.stage,document=excluded.document",
|
||||
(self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)),
|
||||
)
|
||||
|
||||
async def events(self, replay_id, offset=0, limit=100) -> list[dict]:
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s ORDER BY b.sequence DESC LIMIT %s OFFSET %s",
|
||||
(self.project_id, str(replay_id), limit, offset),
|
||||
)
|
||||
return [r["event"] for r in await cur.fetchall()]
|
||||
|
||||
async def monitoring_outputs(
|
||||
self,
|
||||
replay_id,
|
||||
device_ids: list[str],
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
before_sequence: int,
|
||||
) -> list[dict]:
|
||||
"""Read only committed replay outputs available before the current batch."""
|
||||
if not device_ids:
|
||||
return []
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' AND b.sequence<%s "
|
||||
"AND r.value->>'device_id'=ANY(%s) "
|
||||
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'",
|
||||
(self.project_id, str(replay_id), before_sequence, device_ids, start, end),
|
||||
)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
|
||||
async def detected_outputs(
|
||||
self,
|
||||
replay_id,
|
||||
device_ids: list[str],
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
before_sequence: int,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Read immutable detector outputs available to a bounded replay investigation."""
|
||||
statement = (
|
||||
"SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence<=%s AND b.stage='detected' "
|
||||
"AND r.value->>'device_id'=ANY(%s) "
|
||||
"AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'"
|
||||
)
|
||||
params: tuple = (
|
||||
self.project_id, str(replay_id), before_sequence, device_ids, start, end,
|
||||
)
|
||||
if limit is not None:
|
||||
statement += " LIMIT %s OFFSET %s"
|
||||
params += (limit, offset)
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(statement, params)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
|
||||
async def latest_outputs(self, replay_id, through: datetime, limit: int = 500) -> list[dict]:
|
||||
"""Return the latest committed cleaned output for each replay device."""
|
||||
async with self.connection.transaction():
|
||||
cur = await self.connection.execute(
|
||||
"SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading "
|
||||
"FROM peripheral_sample.scada_replay_batches b "
|
||||
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
|
||||
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') "
|
||||
"AND (r.value->>'time')::timestamptz<=%s "
|
||||
"ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s",
|
||||
(self.project_id, str(replay_id), through, limit),
|
||||
)
|
||||
return [row["reading"] for row in await cur.fetchall()]
|
||||
@@ -9,6 +9,86 @@ class ScadaRepository:
|
||||
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
|
||||
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
|
||||
|
||||
@staticmethod
|
||||
async def get_warmup_values(
|
||||
conn: AsyncConnection,
|
||||
device_ids: list[str],
|
||||
before: datetime,
|
||||
per_device_limit: int,
|
||||
) -> list[dict]:
|
||||
"""Return the latest raw values before a replay starts for detector warmup."""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"SELECT d.device_id,h.monitored_value FROM unnest(%s::text[]) d(device_id) "
|
||||
"CROSS JOIN LATERAL (SELECT monitored_value,time FROM scada.measurements "
|
||||
"WHERE device_id=d.device_id AND time<%s ORDER BY time DESC LIMIT %s) h "
|
||||
"ORDER BY d.device_id,h.time",
|
||||
(device_ids, before, per_device_limit),
|
||||
)
|
||||
return [dict(row) for row in await cur.fetchall()]
|
||||
|
||||
@staticmethod
|
||||
async def get_raw_page(
|
||||
conn: AsyncConnection, device_ids: list[str], start: datetime,
|
||||
end: datetime, through: datetime, limit: int,
|
||||
after: tuple[datetime, str] | None = None, *, frame: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Read raw only, with a stable composite key and a bounded frame/page."""
|
||||
predicate = "device_id=ANY(%s) AND time>=%s AND time<%s AND time<=%s"
|
||||
params: tuple = (device_ids, start, end, through)
|
||||
if after is not None:
|
||||
predicate += " AND (time,device_id)>(%s,%s)"
|
||||
params += after
|
||||
if frame:
|
||||
statement = (
|
||||
"WITH next_time AS (SELECT min(time) AS time FROM scada.measurements WHERE "
|
||||
+ predicate + ") SELECT m.time,m.device_id,m.monitored_value,"
|
||||
"md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements m WHERE m.device_id=ANY(%s) "
|
||||
"AND m.time=(SELECT time FROM next_time) ORDER BY m.time,m.device_id LIMIT %s"
|
||||
)
|
||||
params += (device_ids, limit)
|
||||
else:
|
||||
statement = (
|
||||
"SELECT time,device_id,monitored_value,"
|
||||
"md5(COALESCE(monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements WHERE " + predicate
|
||||
+ " ORDER BY time,device_id LIMIT %s"
|
||||
)
|
||||
params += (limit,)
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(statement, params)
|
||||
return [dict(row) for row in await cur.fetchall()]
|
||||
|
||||
@staticmethod
|
||||
async def apply_replay_cleaning(conn: AsyncConnection, rows: list[dict]) -> None:
|
||||
"""Guard every raw version; update corrected values without inserting raw."""
|
||||
if not rows:
|
||||
return
|
||||
async with conn.transaction():
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"SELECT m.time,m.device_id,md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
|
||||
"FROM scada.measurements m JOIN unnest(%s::timestamptz[],%s::text[]) "
|
||||
"b(time,device_id) USING(time,device_id) FOR UPDATE OF m",
|
||||
([r["time"] for r in rows], [r["device_id"] for r in rows]),
|
||||
)
|
||||
existing = {(r["time"].isoformat(), r["device_id"]): r["source_hash"]
|
||||
for r in await cur.fetchall()}
|
||||
if len(existing) != len(rows) or any(
|
||||
existing.get((datetime.fromisoformat(r["time"]).isoformat(), r["device_id"]))
|
||||
!= r["source_hash"] for r in rows
|
||||
):
|
||||
raise ValueError("SOURCE_CHANGED: SCADA 原始值在处理期间发生变化")
|
||||
await cur.execute(
|
||||
"UPDATE scada.measurements m SET cleaned_value=b.value "
|
||||
"FROM unnest(%s::timestamptz[],%s::text[],%s::double precision[]) "
|
||||
"b(time,device_id,value) WHERE m.time=b.time AND m.device_id=b.device_id "
|
||||
"AND m.cleaned_value IS DISTINCT FROM b.value",
|
||||
([r["time"] for r in rows], [r["device_id"] for r in rows],
|
||||
[r["cleaned_value"] for r in rows]),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
|
||||
if not data:
|
||||
|
||||
@@ -98,3 +98,19 @@ async def bounded_calculation(function, *args, **kwargs):
|
||||
finally:
|
||||
_slots.release()
|
||||
return await asyncio.to_thread(run)
|
||||
|
||||
|
||||
async def calculate_replay_burst_location(
|
||||
parameters: dict[str, Any], *, network: str, username: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Run location with Server-built frozen replay series and no live SCADA query."""
|
||||
from app.services.burst_location import run_burst_location_by_network
|
||||
|
||||
return await bounded_calculation(
|
||||
run_burst_location_by_network,
|
||||
network=network,
|
||||
username=username,
|
||||
scheme_name=None,
|
||||
final_candidates_csv_path=None,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ async def verify_task(scope: QueryScope, binding: BoundContext) -> None:
|
||||
task = await repo.get_task(str(binding.task_id))
|
||||
if not task:
|
||||
raise HTTPException(404, "Task is not in this project")
|
||||
binding.replay_context = task.get("source", {}).get("replay_context")
|
||||
if binding.purpose == "task_collaboration":
|
||||
return
|
||||
if OPERATIONAL_TASK_EXECUTE not in permissions:
|
||||
|
||||
@@ -291,5 +291,6 @@ class BoundContext(BaseModel):
|
||||
task_id: UUID | None = None
|
||||
run_id: UUID | None = None
|
||||
execution_request_id: UUID | None = None
|
||||
replay_context: dict | None = None
|
||||
expires_at: datetime
|
||||
revoked_at: datetime | None = None
|
||||
|
||||
@@ -66,8 +66,23 @@ def page_result(rows, args):
|
||||
async def authorize_query(
|
||||
scope: QueryScope, binding: BoundContext, tool: QueryTool, args: BaseModel
|
||||
) -> None:
|
||||
replay_context = getattr(binding, "replay_context", None)
|
||||
if tool.name not in binding.tool_cap:
|
||||
raise HTTPException(403, "Query is outside the captured tool catalog")
|
||||
if replay_context:
|
||||
from app.services.scada_replay_queries import check_replay_window
|
||||
allowed = {"get_task_context", "get_task_evidence", "list_scada_devices", "get_scada_device",
|
||||
"query_scada_readings", "get_network_schema", "run_burst_location",
|
||||
"get_calculation_job", "get_calculation_result", "list_calculation_jobs"}
|
||||
if tool.name not in allowed:
|
||||
raise HTTPException(403, "回放调查当前仅开放冻结 SCADA 证据与模型目录查询;请使用 query_scada_readings")
|
||||
if tool.name == "query_scada_readings":
|
||||
check_replay_window(replay_context, args.start, args.end)
|
||||
if tool.name == "run_burst_location":
|
||||
if args.data_source != "monitoring" or args.scada_normal_start is None or args.scada_normal_end is None:
|
||||
raise HTTPException(422, "回放定位只支持显式异常窗和正常对照窗的 monitoring 输入")
|
||||
check_replay_window(replay_context, args.scada_burst_start, args.scada_burst_end)
|
||||
check_replay_window(replay_context, args.scada_normal_start, args.scada_normal_end)
|
||||
permissions = permissions_for_context(scope.project)
|
||||
if tool.mutating:
|
||||
if binding.purpose != "ordinary_chat" or binding.task_id is not None or scope.project.project_role != "member":
|
||||
@@ -149,7 +164,37 @@ async def execute_query(
|
||||
gaps = []
|
||||
source_ref = "mcp:" + str(uuid4())
|
||||
evidence_id = None
|
||||
if tool.computation:
|
||||
replay_context = getattr(binding, "replay_context", None)
|
||||
if name == "query_scada_readings" and replay_context:
|
||||
from app.services.scada_replay_queries import replay_snapshot_rows
|
||||
rows = await replay_snapshot_rows(conn, scope.project.project_id, replay_context,
|
||||
args.device_ids, args.start, args.end, args.limit + 1, args.offset)
|
||||
data, page, truncated = page_result(rows, args)
|
||||
origin = "synthetic"
|
||||
gaps.append("历史数据回放证据,仅覆盖该轮调查已提交的批次,不表示现场实时观测。")
|
||||
elif name == "run_burst_location" and replay_context:
|
||||
from app.mcp.analysis_service import calculate_replay_burst_location
|
||||
from app.services.scada_replay_queries import replay_burst_location_inputs
|
||||
|
||||
parameters = await replay_burst_location_inputs(
|
||||
conn, scope.project.project_id, replay_context, args,
|
||||
)
|
||||
await conn.commit()
|
||||
async with scope.simulation_routing():
|
||||
result = await calculate_replay_burst_location(
|
||||
parameters,
|
||||
network=scope.project.project_code,
|
||||
username=scope.user.username,
|
||||
)
|
||||
data = {
|
||||
"parameters": args.model_dump(mode="json", exclude={"context_handle"}),
|
||||
"result": result,
|
||||
}
|
||||
origin = "synthetic"
|
||||
gaps = [
|
||||
"定位只使用任务知识截止前已固化的回放清洗输出;候选排序依赖漏量假设、模型和测点覆盖,仍需现场核实。"
|
||||
]
|
||||
elif tool.computation:
|
||||
from app.mcp.analysis_service import TIMESCALE_CALCULATIONS, calculate
|
||||
|
||||
# Context/receipt reads may have started an implicit transaction. Release
|
||||
|
||||
@@ -9,6 +9,7 @@ import yaml
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[2] / "resources" / "work-order-skills"
|
||||
WORKFLOW_SKILLS = {
|
||||
"scada_anomaly": "scada-anomaly",
|
||||
"burst_report": "burst-report",
|
||||
"water_quality_complaint": "water-quality-complaint",
|
||||
"general": "appointment-follow-up",
|
||||
|
||||
@@ -161,6 +161,14 @@ class OperationalTaskService:
|
||||
)
|
||||
|
||||
async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail:
|
||||
if command.source.replay_context:
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
replay_context = command.source.replay_context
|
||||
replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id)
|
||||
require(replay_doc.get("delivered_until") is not None
|
||||
and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"])
|
||||
and replay_context.batch_sequence <= replay_doc["batch_sequence"],
|
||||
"REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422)
|
||||
body = _json(command)
|
||||
request_id = str(command.request_id)
|
||||
scope = "task:create"
|
||||
@@ -219,6 +227,27 @@ class OperationalTaskService:
|
||||
await self._task_event(task_document, now)
|
||||
return await self.detail(task_id)
|
||||
|
||||
async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None:
|
||||
"""A recovery is new evidence, coalesced through the existing run lifecycle."""
|
||||
scope = f"task:{task_id}:replay-evidence"
|
||||
async with self.repo.transaction():
|
||||
if await self._request_replay(scope, request_id, replay_context):
|
||||
return
|
||||
await self.repo.lock(f"task:{task_id}")
|
||||
task = await self._task(task_id)
|
||||
prior = task["source"].get("replay_context")
|
||||
require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID",
|
||||
"Replay evidence belongs to another investigation", status=409)
|
||||
require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]),
|
||||
"REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409)
|
||||
task["source"]["replay_context"] = replay_context
|
||||
task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1
|
||||
now = self._now()
|
||||
await self._requeue_task(task, now)
|
||||
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
||||
await self._task_event(task, now)
|
||||
await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)})
|
||||
|
||||
async def list(
|
||||
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
|
||||
peripheral_work_order_id: UUID | None = None,
|
||||
@@ -336,6 +365,9 @@ class OperationalTaskService:
|
||||
selected_report=await self.report(task_id, report_revision) if report_revision is not None else None,
|
||||
)
|
||||
|
||||
async def replay_context(self, task_id: UUID) -> dict | None:
|
||||
return (await self._task(task_id)).get("source", {}).get("replay_context")
|
||||
|
||||
async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool:
|
||||
task = await self._task(task_id)
|
||||
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
|
||||
@@ -1037,6 +1069,8 @@ class OperationalTaskService:
|
||||
await self.repo.lock(f"run:{command.run_id}")
|
||||
task = await self._task(task_id)
|
||||
self._require_actionable_task(task)
|
||||
require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED",
|
||||
"SCADA 回放调查仅生成分析与报告,不执行真实工单或设备动作", status=409)
|
||||
run = await self._active_run(task)
|
||||
require(run is not None and str(run["run_id"]) == str(command.run_id)
|
||||
and run["generation"] == task["execution_generation"],
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Read-only chronological SCADA paging shared by REST and replay processing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime
|
||||
|
||||
from app.domain.schemas.scada import ScadaRawPage, ScadaRawReading
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
|
||||
|
||||
def validate_window(start: datetime, end: datetime) -> None:
|
||||
if start.tzinfo is None or end.tzinfo is None or start >= end:
|
||||
raise ValueError("需要带时区、结束晚于开始的 SCADA 时间区间")
|
||||
|
||||
|
||||
def raw_reading(row: dict) -> ScadaRawReading:
|
||||
value = row["monitored_value"]
|
||||
invalid = value is not None and not math.isfinite(float(value))
|
||||
return ScadaRawReading(
|
||||
time=row["time"], device_id=row["device_id"],
|
||||
monitored_value=None if invalid else value,
|
||||
quality="invalid" if invalid else "missing" if value is None else "unvalidated",
|
||||
)
|
||||
|
||||
|
||||
async def read_raw_page(
|
||||
conn, *, project_id: str, device_ids: list[str], start: datetime,
|
||||
end: datetime, limit: int = 500, cursor: str | None = None,
|
||||
through: datetime | None = None,
|
||||
) -> ScadaRawPage:
|
||||
validate_window(start, end)
|
||||
ids = sorted(set(device_ids))
|
||||
if not ids or len(ids) > 500 or not 1 <= limit <= 500:
|
||||
raise ValueError("请选择 1 至 500 个设备,每页最多 500 条")
|
||||
cap = min(through or end, end)
|
||||
if cap.tzinfo is None or cap < start:
|
||||
raise ValueError("模拟时间不在可读取区间内")
|
||||
signature = hashlib.sha256(json.dumps(
|
||||
[project_id, ids, start.isoformat(), end.isoformat()], separators=(",", ":")
|
||||
).encode()).hexdigest()
|
||||
after = None
|
||||
if cursor:
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
frozen = datetime.fromisoformat(data["through"])
|
||||
last = datetime.fromisoformat(data["time"])
|
||||
if (data["query"] != signature or frozen.tzinfo is None or last.tzinfo is None
|
||||
or frozen > cap or not start <= last <= frozen or data["device"] not in ids):
|
||||
raise ValueError()
|
||||
cap, after = frozen, (last, data["device"])
|
||||
except (ValueError, KeyError, TypeError, UnicodeError) as exc:
|
||||
raise ValueError("SCADA 分页游标无效或查询范围已改变") from exc
|
||||
rows = await ScadaRepository.get_raw_page(conn, ids, start, end, cap, limit + 1, after)
|
||||
selected = rows[:limit]
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = selected[-1]
|
||||
next_cursor = base64.urlsafe_b64encode(json.dumps({
|
||||
"query": signature, "through": cap.isoformat(),
|
||||
"time": last["time"].isoformat(), "device": last["device_id"],
|
||||
}, separators=(",", ":")).encode()).decode()
|
||||
return ScadaRawPage(items=[raw_reading(row) for row in selected],
|
||||
next_cursor=next_cursor, scanned_through=cap)
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from fastapi import HTTPException
|
||||
from app.auth.permissions import permissions_for_context
|
||||
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.content_digest import digest
|
||||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||||
from app.services.scada_replay_monitoring import evaluate_monitoring_period
|
||||
|
||||
|
||||
def parse_time(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def snapshot(document: dict) -> ScadaReplaySnapshot:
|
||||
return ScadaReplaySnapshot(
|
||||
**{key: document[key] for key in (
|
||||
"replay_id", "project_id", "revision", "status", "start_time", "end_time",
|
||||
"virtual_now", "created_at", "updated_at",
|
||||
)},
|
||||
delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"),
|
||||
processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0),
|
||||
devices=document["devices"], rules=document["config"]["rules"],
|
||||
latest=list(document.get("latest", {}).values()),
|
||||
active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")],
|
||||
error=document.get("error"),
|
||||
monitoring_interval_seconds=document.get("monitoring_interval_seconds"),
|
||||
step_pending=document.get("step_pending", False),
|
||||
monitoring_count=document.get("monitoring_count", 0),
|
||||
last_monitoring=document.get("last_monitoring"),
|
||||
)
|
||||
|
||||
|
||||
class ScadaReplayService:
|
||||
def __init__(self, repository, context, timescale=None, *, clock=None):
|
||||
self.repo, self.context, self.timescale = repository, context, timescale
|
||||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||||
|
||||
async def ensure_storage(self):
|
||||
if not await self.repo.available():
|
||||
raise HTTPException(503, "SCADA 回放存储未配置,需要应用 031_scada_replays.sql")
|
||||
|
||||
async def create(
|
||||
self,
|
||||
command: ScadaReplayCreate,
|
||||
*,
|
||||
monitoring_interval_seconds: int | None,
|
||||
monitoring_config: ScadaMonitoringSettings | None = None,
|
||||
):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.control():
|
||||
previous = await self.repo.by_request(command.request_id)
|
||||
config = command.model_dump(mode="json")
|
||||
if previous:
|
||||
if previous["config"] != config:
|
||||
raise HTTPException(409, "创建请求号已用于不同回放配置")
|
||||
return snapshot(previous)
|
||||
if monitoring_interval_seconds is None:
|
||||
raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds,再创建演示")
|
||||
replay_id = str(uuid4())
|
||||
if await self.repo.active_other(replay_id):
|
||||
raise HTTPException(409, "当前项目已有回放,请先停止原回放再新建")
|
||||
metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)}
|
||||
if not set(command.device_ids) <= metadata.keys():
|
||||
raise HTTPException(422, "设备不属于当前项目")
|
||||
for rule in command.rules:
|
||||
if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids):
|
||||
raise HTTPException(422, "规则单位与设备单位不一致,请分组配置")
|
||||
if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip():
|
||||
raise HTTPException(422, "水质规则必须明确指标名称,不能仅凭单位解释读数")
|
||||
if monitoring_config:
|
||||
role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids)
|
||||
if not role_ids <= metadata.keys():
|
||||
raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备")
|
||||
if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids):
|
||||
raise HTTPException(422, "压力监测角色必须引用 pressure 设备")
|
||||
if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"}
|
||||
for device in monitoring_config.outlet_flow_device_ids):
|
||||
raise HTTPException(422, "出厂边界流量角色必须引用流量设备")
|
||||
now = self.clock().isoformat()
|
||||
document = {
|
||||
"replay_id": replay_id, "project_id": str(self.context.project_id),
|
||||
"created_by": str(self.context.user_id), "config": config,
|
||||
"revision": 1, "status": "ready", "start_time": config["start_time"],
|
||||
"end_time": config["end_time"], "virtual_now": config["start_time"],
|
||||
"created_at": now, "updated_at": now,
|
||||
"monitoring_interval_seconds": monitoring_interval_seconds,
|
||||
"monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None,
|
||||
"devices": [metadata[device] for device in command.device_ids],
|
||||
"batch_sequence": 0, "processed_count": 0, "anomaly_count": 0,
|
||||
"monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {},
|
||||
}
|
||||
await self.repo.insert(document)
|
||||
return snapshot(document)
|
||||
|
||||
async def command(self, replay_id, command: ScadaReplayCommand):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.control():
|
||||
document = await self.repo.get(replay_id)
|
||||
key, body = str(command.request_id), command.model_dump(mode="json")
|
||||
receipt = document.setdefault("receipts", {}).get(key)
|
||||
if receipt:
|
||||
if receipt != body:
|
||||
raise HTTPException(409, "命令请求号已用于不同内容")
|
||||
return snapshot(document)
|
||||
if command.expected_revision != document["revision"]:
|
||||
raise HTTPException(409, "回放控制状态已更新,请刷新后重试")
|
||||
action, status = command.action, document["status"]
|
||||
if document.get("step_pending") and action == "step":
|
||||
raise HTTPException(409, "当前时间步正在处理,请完成后继续")
|
||||
if action in {"step", "retry"} and await self.repo.active_other(replay_id):
|
||||
raise HTTPException(409, "当前项目已有另一条活动回放")
|
||||
if action == "step" and status in {"ready", "paused"}:
|
||||
interval = document.get("monitoring_interval_seconds")
|
||||
if interval is None:
|
||||
raise HTTPException(409, "旧回放未固定监测周期,请停止后新建演示")
|
||||
start = parse_time(document["start_time"])
|
||||
elapsed = (parse_time(document["virtual_now"]) - start).total_seconds()
|
||||
target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval)
|
||||
document["step_start"] = document["virtual_now"]
|
||||
document["step_target"] = min(parse_time(document["end_time"]), target).isoformat()
|
||||
document["status"], document["step_pending"] = "paused", True
|
||||
elif action == "stop" and status not in {"completed", "stopped"}:
|
||||
document["status"], document["step_pending"] = "stopped", False
|
||||
document.pop("step_target", None)
|
||||
document.pop("step_start", None)
|
||||
elif action == "retry" and status == "failed":
|
||||
if "SOURCE_CHANGED" in (document.get("error") or ""):
|
||||
raise HTTPException(409, "原始数据发生变化,需要停止并新建回放")
|
||||
document["status"], document["error"] = "paused", None
|
||||
document["step_pending"] = bool(document.get("step_target"))
|
||||
else:
|
||||
raise HTTPException(409, "当前回放状态不支持此操作")
|
||||
document["revision"] += 1
|
||||
document["receipts"][key] = body
|
||||
if len(document["receipts"]) > 256:
|
||||
del document["receipts"][next(iter(document["receipts"]))]
|
||||
await self.repo.save(document)
|
||||
return snapshot(document)
|
||||
|
||||
async def process_pending_step(self, replay_id):
|
||||
await self.ensure_storage()
|
||||
async with self.repo.processing() as owned:
|
||||
if not owned:
|
||||
raise HTTPException(409, "当前时间步正在由另一请求处理")
|
||||
while True:
|
||||
document = await self.repo.get(replay_id)
|
||||
if not document.get("step_pending"):
|
||||
return snapshot(document)
|
||||
try:
|
||||
await self._advance(document)
|
||||
except Exception as exc:
|
||||
# Failure never discards the prepared batch; retry resumes it.
|
||||
document = await self.repo.get(replay_id)
|
||||
document["status"], document["step_pending"] = "failed", False
|
||||
document["error"] = (
|
||||
"SOURCE_CHANGED: 原始观测已改变,需要新建回放"
|
||||
if "SOURCE_CHANGED" in str(exc)
|
||||
else "SCADA 处理失败,已保留批次与进度;请检查服务日志后重试"
|
||||
)
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
raise
|
||||
|
||||
async def _advance(self, document):
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
replay_id = document["replay_id"]
|
||||
if not document.get("warmed"):
|
||||
import math
|
||||
window = max(rule.window_samples for rule in config.rules)
|
||||
async with self.timescale.transaction():
|
||||
warmup = await ScadaRepository.get_warmup_values(
|
||||
self.timescale, config.device_ids, config.start_time, window,
|
||||
)
|
||||
for row in warmup:
|
||||
value = row["monitored_value"]
|
||||
bounds = config.quality_ranges.get(row["device_id"])
|
||||
if value is not None and math.isfinite(value) and not (bounds and (
|
||||
bounds.lower is not None and value < bounds.lower
|
||||
or bounds.upper is not None and value > bounds.upper
|
||||
)):
|
||||
document["histories"].setdefault(row["device_id"], []).append(value)
|
||||
document["warmed"] = True
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
batch = await self.repo.pending_batch(replay_id)
|
||||
if batch is None:
|
||||
step = document.get("step_pending", False)
|
||||
virtual = parse_time(document["virtual_now"])
|
||||
through = parse_time(document["step_target"]) if step else virtual
|
||||
after = ((parse_time(document["cursor"][0]), document["cursor"][1])
|
||||
if document.get("cursor") else None)
|
||||
async with self.timescale.transaction():
|
||||
rows = await ScadaRepository.get_raw_page(
|
||||
self.timescale, config.device_ids, config.start_time, config.end_time,
|
||||
through, 501, after, frame=True,
|
||||
)
|
||||
if len(rows) > 500:
|
||||
raise ValueError("SCADA frame exceeds selected device bound")
|
||||
at = rows[0]["time"] if rows else through
|
||||
if step:
|
||||
virtual = through
|
||||
document["virtual_now"] = virtual.isoformat()
|
||||
if not rows and document.get("processed_until") == at.isoformat() and not step:
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save(document)
|
||||
return
|
||||
sequence = document["batch_sequence"] + 1
|
||||
refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||||
"source_hash": row["source_hash"]} for row in rows]
|
||||
batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(),
|
||||
"refs": refs, "input_digest": digest(refs), "events": []}
|
||||
async with self.repo.connection.transaction():
|
||||
document["batch_sequence"] = sequence
|
||||
document["delivered_until"] = at.isoformat()
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
await self.repo.save(document)
|
||||
else:
|
||||
rows = None
|
||||
if batch["stage"] == "selected":
|
||||
if rows is None:
|
||||
# Only crash recovery reloads the selected raw frame.
|
||||
at = parse_time(batch["at"])
|
||||
async with self.timescale.transaction():
|
||||
rows = await ScadaRepository.get_raw_page(
|
||||
self.timescale, config.device_ids, at, config.end_time, at, 501,
|
||||
) if batch["refs"] else []
|
||||
actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"],
|
||||
"source_hash": row["source_hash"]} for row in rows]
|
||||
if digest(actual) != batch["input_digest"]:
|
||||
raise ValueError("SOURCE_CHANGED: selected raw frame differs")
|
||||
batch["outputs"] = prepare_cleaning(rows, config)
|
||||
batch["stage"] = "clean_prepared"
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
if batch["stage"] == "clean_prepared":
|
||||
await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"])
|
||||
batch["stage"] = "cleaned"
|
||||
async with self.repo.connection.transaction():
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
if batch["stage"] == "cleaned":
|
||||
result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"]))
|
||||
result["processed_until"] = batch["at"]
|
||||
target = result.get("step_target")
|
||||
was_step = bool(target)
|
||||
result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target))
|
||||
monitoring = None
|
||||
if was_step and not result["step_pending"]:
|
||||
period_start = parse_time(result.get("step_start", result["start_time"]))
|
||||
period_end = parse_time(target)
|
||||
role = result.get("monitoring_config") or {}
|
||||
role_ids = list(dict.fromkeys(
|
||||
role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", [])
|
||||
))
|
||||
previous = await self.repo.monitoring_outputs(
|
||||
replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)),
|
||||
period_end, batch["sequence"],
|
||||
) if role_ids else []
|
||||
from app.mcp.analysis_service import bounded_calculation
|
||||
monitoring = await bounded_calculation(
|
||||
evaluate_monitoring_period,
|
||||
result,
|
||||
previous + batch["outputs"],
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
batch_sequence=batch["sequence"],
|
||||
)
|
||||
result["last_monitoring"] = monitoring
|
||||
result["monitoring_count"] = result.get("monitoring_count", 0) + 1
|
||||
if not result["step_pending"]:
|
||||
result.pop("step_target", None)
|
||||
result.pop("step_start", None)
|
||||
if batch["refs"]:
|
||||
last = batch["refs"][-1]
|
||||
result["cursor"] = [last["time"], last["device_id"]]
|
||||
if parse_time(batch["at"]) >= config.end_time:
|
||||
result["status"] = "completed"
|
||||
batch["stage"], batch["events"] = "detected", changes
|
||||
if monitoring is not None:
|
||||
batch["monitoring"] = monitoring
|
||||
async with self.repo.connection.transaction():
|
||||
await self._create_investigations(result, changes, batch, create_tasks=not was_step)
|
||||
if monitoring is not None:
|
||||
await self._create_monitoring_task(result, monitoring, batch)
|
||||
batch["monitoring"] = monitoring
|
||||
result["last_monitoring"] = monitoring
|
||||
await self.repo.save_batch(replay_id, batch)
|
||||
await self.repo.save(result)
|
||||
|
||||
async def _create_investigations(self, document, changes, batch, *, create_tasks=True):
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
from app.domain.schemas.task_alert import TaskAlertEvent
|
||||
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
||||
from app.services.task_alerts import TaskAlertService
|
||||
from uuid import UUID
|
||||
producer = "scada-replay:" + document["replay_id"]
|
||||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, internal_producer_id=producer)
|
||||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context))
|
||||
for event in changes:
|
||||
if event["change"] == "raised":
|
||||
event["batch_sequence"] = batch["sequence"]
|
||||
document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"]
|
||||
await alerts.push(TaskAlertEvent(
|
||||
event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]),
|
||||
alert_id=event["event_id"], producer_id=producer, event_type=event["change"],
|
||||
occurred_at=event["occurred_at"], condition_id=None,
|
||||
source={"kind": "scada_replay", "replay_id": document["replay_id"],
|
||||
"rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]},
|
||||
title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning",
|
||||
))
|
||||
if event["change"] != "raised":
|
||||
if event.get("task_id"):
|
||||
await tasks.advance_replay_investigation(UUID(event["task_id"]), {
|
||||
"replay_id": document["replay_id"], "knowledge_cutoff": batch["at"],
|
||||
"batch_sequence": batch["sequence"],
|
||||
}, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery"))
|
||||
continue
|
||||
if event.get("category") == "data_quality":
|
||||
continue
|
||||
if not create_tasks:
|
||||
continue
|
||||
external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}"
|
||||
command = OperationalTaskCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation",
|
||||
workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}",
|
||||
summary=event["detail"], source=TaskSource(
|
||||
kind="external_system", external_id=external, external_revision=1,
|
||||
event_kind="scada_anomaly",
|
||||
replay_context={"replay_id": document["replay_id"],
|
||||
"knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]},
|
||||
),
|
||||
trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号,原因尚待调查"),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
event["task_id"] = str(task.task_id)
|
||||
document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"]
|
||||
|
||||
async def _create_monitoring_task(self, document, monitoring, batch):
|
||||
if monitoring["status"] != "suspected_burst":
|
||||
return
|
||||
from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger
|
||||
from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
|
||||
from app.services.operational_tasks import OperationalTaskService
|
||||
tasks = OperationalTaskService(
|
||||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context,
|
||||
permissions_for_context(self.context),
|
||||
)
|
||||
external = (
|
||||
f"scada_replay:{document['replay_id']}:monitor:"
|
||||
f"{monitoring['period_end']}:{monitoring['rule_version']}"
|
||||
)
|
||||
pressure = monitoring["pressure"]
|
||||
flow = monitoring["outlet_flow"]
|
||||
detector = monitoring["burst_detection"]
|
||||
summary = (
|
||||
f"步长监测疑似爆管:爆管侦测得分 {detector['target_score']:.4f};"
|
||||
f"压力下降测点 {len(pressure['hit_device_ids'])} 个;"
|
||||
f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。"
|
||||
)
|
||||
command = OperationalTaskCreate(
|
||||
request_id=uuid5(NAMESPACE_URL, external),
|
||||
task_type="event_investigation",
|
||||
workflow="burst_report",
|
||||
title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}",
|
||||
summary=summary,
|
||||
source=TaskSource(
|
||||
kind="external_system",
|
||||
external_id=external,
|
||||
external_revision=1,
|
||||
event_kind="scada_burst_suspected",
|
||||
replay_context={
|
||||
"replay_id": document["replay_id"],
|
||||
"knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||||
"batch_sequence": batch["sequence"],
|
||||
},
|
||||
),
|
||||
trigger=TaskTrigger(
|
||||
kind="event",
|
||||
occurred_at=monitoring["period_end"],
|
||||
detail="主动步长监测的多项指标相互支持,进入爆管调查;尚未现场确诊",
|
||||
),
|
||||
)
|
||||
task = await tasks.create(command)
|
||||
monitoring["task_id"] = str(task.task_id)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Deterministic monitoring summaries built only from frozen replay outputs."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from statistics import median
|
||||
from typing import Any
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.burst_detection import (
|
||||
TARGET_MU,
|
||||
TARGET_N_ESTIMATORS,
|
||||
TARGET_RANDOM_STATE,
|
||||
TARGET_SCORE_THRESHOLD,
|
||||
run_burst_detection,
|
||||
)
|
||||
|
||||
|
||||
def _time(value: str | datetime) -> datetime:
|
||||
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def _valid_rows(rows: list[dict], device_id: str) -> list[dict]:
|
||||
return sorted(
|
||||
(
|
||||
row for row in rows
|
||||
if row.get("device_id") == device_id
|
||||
and row.get("quality") in {"valid", "corrected"}
|
||||
and row.get("cleaned_value") is not None
|
||||
),
|
||||
key=lambda row: _time(row["time"]),
|
||||
)
|
||||
|
||||
|
||||
def _relative_indicator(
|
||||
rows: list[dict],
|
||||
config: ScadaReplayCreate,
|
||||
device_ids: list[str],
|
||||
period_start: datetime,
|
||||
period_end: datetime,
|
||||
*,
|
||||
direction: str,
|
||||
) -> dict[str, Any]:
|
||||
details: list[dict[str, Any]] = []
|
||||
gaps: list[dict[str, str]] = []
|
||||
for device_id in device_ids:
|
||||
rule = next((
|
||||
item for item in config.rules
|
||||
if item.kind == "relative_change" and device_id in item.device_ids
|
||||
and item.threshold is not None
|
||||
), None)
|
||||
if rule is None:
|
||||
gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"})
|
||||
continue
|
||||
valid = _valid_rows(rows, device_id)
|
||||
baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:]
|
||||
current = [row for row in valid if period_start <= _time(row["time"]) <= period_end]
|
||||
if len(baseline) < rule.minimum_samples:
|
||||
gaps.append({"device_id": device_id, "reason": "baseline_not_ready"})
|
||||
continue
|
||||
if not current:
|
||||
gaps.append({"device_id": device_id, "reason": "period_reading_missing"})
|
||||
continue
|
||||
reference = median(float(row["cleaned_value"]) for row in baseline)
|
||||
if reference == 0:
|
||||
gaps.append({"device_id": device_id, "reason": "zero_baseline"})
|
||||
continue
|
||||
latest = current[-1]
|
||||
value = float(latest["cleaned_value"])
|
||||
change = (value - reference) / abs(reference) * 100
|
||||
hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold
|
||||
details.append({
|
||||
"device_id": device_id,
|
||||
"observed_at": _time(latest["time"]).isoformat(),
|
||||
"baseline": reference,
|
||||
"value": value,
|
||||
"change_percent": change,
|
||||
"threshold_percent": rule.threshold,
|
||||
"hit": hit,
|
||||
})
|
||||
return {
|
||||
"status": "ready" if not gaps else "insufficient_data",
|
||||
"hit": any(item["hit"] for item in details),
|
||||
"hit_device_ids": [item["device_id"] for item in details if item["hit"]],
|
||||
"devices": details,
|
||||
"gaps": gaps,
|
||||
}
|
||||
|
||||
|
||||
def _burst_detection(
|
||||
rows: list[dict],
|
||||
pressure_device_ids: list[str],
|
||||
interval_seconds: int,
|
||||
period_end: datetime,
|
||||
) -> dict[str, Any]:
|
||||
if len(pressure_device_ids) < 5:
|
||||
return {"status": "sensor_coverage_insufficient", "hit": False,
|
||||
"required_sensors": 5, "available_sensors": len(pressure_device_ids)}
|
||||
if interval_seconds % 60 or 86400 % interval_seconds:
|
||||
return {"status": "unsupported_sampling_interval", "hit": False,
|
||||
"interval_seconds": interval_seconds}
|
||||
points_per_day = 86400 // interval_seconds
|
||||
count = 15 * points_per_day
|
||||
series_by_device: dict[str, dict[datetime, float]] = {}
|
||||
common_times: set[datetime] | None = None
|
||||
for device_id in pressure_device_ids:
|
||||
values = {_time(row["time"]): float(row["cleaned_value"])
|
||||
for row in _valid_rows(rows, device_id) if _time(row["time"]) <= period_end}
|
||||
series_by_device[device_id] = values
|
||||
common_times = set(values) if common_times is None else common_times & set(values)
|
||||
if not common_times:
|
||||
return {"status": "baseline_not_ready", "hit": False, "required_samples": count}
|
||||
target = max(common_times)
|
||||
expected = [target - timedelta(seconds=interval_seconds * offset)
|
||||
for offset in range(count - 1, -1, -1)]
|
||||
incomplete = [device_id for device_id, values in series_by_device.items()
|
||||
if any(at not in values for at in expected)]
|
||||
if incomplete:
|
||||
return {"status": "baseline_not_ready", "hit": False, "required_samples": count,
|
||||
"incomplete_device_ids": incomplete}
|
||||
frame = pd.DataFrame({device_id: [series_by_device[device_id][at] for at in expected]
|
||||
for device_id in pressure_device_ids}, index=expected)
|
||||
try:
|
||||
result = run_burst_detection(
|
||||
network="scada-replay",
|
||||
username="scada-replay",
|
||||
observed_pressure_data=frame,
|
||||
points_per_day=points_per_day,
|
||||
mu=TARGET_MU,
|
||||
iforest_params={
|
||||
"n_estimators": TARGET_N_ESTIMATORS,
|
||||
"random_state": TARGET_RANDOM_STATE,
|
||||
"contamination": "auto",
|
||||
},
|
||||
sensor_nodes=pressure_device_ids,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return {"status": "analysis_failed", "hit": False, "reason": str(exc)[:300]}
|
||||
latest = result["rows"][-1]
|
||||
score = float(latest["Score"])
|
||||
return {
|
||||
"status": "ready",
|
||||
"hit": score <= TARGET_SCORE_THRESHOLD,
|
||||
"target_time": target.isoformat(),
|
||||
"target_score": score,
|
||||
"score_threshold": TARGET_SCORE_THRESHOLD,
|
||||
"sensor_count": len(pressure_device_ids),
|
||||
"reference_day_count": 14,
|
||||
"latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_monitoring_period(
|
||||
document: dict,
|
||||
rows: list[dict],
|
||||
*,
|
||||
period_start: datetime,
|
||||
period_end: datetime,
|
||||
batch_sequence: int,
|
||||
) -> dict[str, Any]:
|
||||
config = ScadaReplayCreate.model_validate(document["config"])
|
||||
monitoring = document.get("monitoring_config")
|
||||
rule_version = (monitoring or {}).get("rule_version", "unconfigured")
|
||||
monitoring_id = str(uuid5(
|
||||
NAMESPACE_URL,
|
||||
f"{document['replay_id']}:{period_end.isoformat()}:{rule_version}",
|
||||
))
|
||||
base = {
|
||||
"monitoring_id": monitoring_id,
|
||||
"period_start": period_start.isoformat(),
|
||||
"period_end": period_end.isoformat(),
|
||||
"knowledge_cutoff": period_end.isoformat(),
|
||||
"batch_sequence": batch_sequence,
|
||||
"rule_version": rule_version,
|
||||
"task_id": None,
|
||||
}
|
||||
if not monitoring:
|
||||
return {**base, "status": "insufficient_data", "detail": "项目未配置 SCADA 监测设备角色",
|
||||
"pressure": {"status": "unconfigured", "hit": False},
|
||||
"outlet_flow": {"status": "unconfigured", "hit": False},
|
||||
"burst_detection": {"status": "not_run", "hit": False}}
|
||||
pressure_ids = monitoring.get("pressure_device_ids", [])
|
||||
flow_ids = monitoring.get("outlet_flow_device_ids", [])
|
||||
selected = set(config.device_ids)
|
||||
role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected)
|
||||
if not pressure_ids or not flow_ids or role_gaps:
|
||||
return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备",
|
||||
"role_gaps": role_gaps,
|
||||
"pressure": {"status": "unconfigured", "hit": False},
|
||||
"outlet_flow": {"status": "unconfigured", "hit": False},
|
||||
"burst_detection": {"status": "not_run", "hit": False}}
|
||||
pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease")
|
||||
outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase")
|
||||
detector = _burst_detection(
|
||||
rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end,
|
||||
)
|
||||
complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready"
|
||||
hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"]))
|
||||
if not complete:
|
||||
status, detail = "insufficient_data", "监测数据、设备角色或爆管侦测基线不足"
|
||||
elif all(hits):
|
||||
status, detail = "suspected_burst", "爆管侦测、压力下降与出厂边界流量增加相互支持"
|
||||
elif any(hits):
|
||||
status, detail = "attention", "仅部分爆管相关指标命中,需要继续观察或人工核查"
|
||||
else:
|
||||
status, detail = "normal", "本周期爆管相关指标均未命中"
|
||||
return {**base, "status": status, "detail": detail, "pressure": pressure,
|
||||
"outlet_flow": outlet_flow, "burst_detection": detector}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Causal batch quality checks and grouped anomaly episodes, no database IO."""
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from math import isfinite
|
||||
from statistics import median
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
import re
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
|
||||
|
||||
def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]:
|
||||
result = []
|
||||
for row in rows:
|
||||
value = row["monitored_value"]
|
||||
quality, reason = "valid", "原始值通过数值与已配置量程校验"
|
||||
if value is None:
|
||||
quality, reason = "missing", "原始读数缺失"
|
||||
elif not isfinite(float(value)):
|
||||
quality, reason = "invalid", "原始读数不是有限数值"
|
||||
else:
|
||||
bounds = config.quality_ranges.get(row["device_id"])
|
||||
if bounds and ((bounds.lower is not None and value < bounds.lower)
|
||||
or (bounds.upper is not None and value > bounds.upper)):
|
||||
quality, reason = "invalid", "超出本次明确配置的仪表量程"
|
||||
result.append({
|
||||
"time": row["time"].isoformat() if isinstance(row["time"], datetime) else row["time"],
|
||||
"device_id": row["device_id"], "source_hash": row["source_hash"],
|
||||
"monitored_value": value if value is not None and isfinite(float(value)) else None,
|
||||
"cleaned_value": value if quality == "valid" else None,
|
||||
"quality": quality, "reason": reason,
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at: datetime) -> tuple[dict, list[dict]]:
|
||||
"""Consume a complete timestamp; never treat missing data as recovery."""
|
||||
result = deepcopy(document)
|
||||
histories = result.setdefault("histories", {})
|
||||
states = result.setdefault("rule_states", {})
|
||||
latest = result.setdefault("latest", {})
|
||||
for row in rows:
|
||||
latest[row["device_id"]] = row
|
||||
by_id = {row["device_id"]: row for row in rows}
|
||||
changes = []
|
||||
devices = {device["device_id"]: device for device in result.get("devices", [])}
|
||||
for device_id in config.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
invalid = row is not None and row["quality"] != "valid"
|
||||
previous = latest.get(device_id)
|
||||
frequency = str(devices.get(device_id, {}).get("transmission_frequency", ""))
|
||||
match = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency)
|
||||
if match:
|
||||
seconds = int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3])
|
||||
else:
|
||||
minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency)
|
||||
seconds = int(minutes[1]) * 60 if minutes else 0
|
||||
prior_time = datetime.fromisoformat(previous["time"]) if previous else config.start_time
|
||||
missing = not row and seconds > 0 and (at - prior_time).total_seconds() >= seconds * 2
|
||||
key = "quality-" + device_id
|
||||
state = states.setdefault(key, {"event": None, "recoveries": 0})
|
||||
if invalid or missing:
|
||||
state["recoveries"] = 0
|
||||
if state["event"] is None:
|
||||
event = {"event_id": str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{key}:{at.isoformat()}")),
|
||||
"rule_id": key, "device_ids": [device_id], "occurred_at": at.isoformat(),
|
||||
"last_observed_at": at.isoformat(), "status": "active", "category": "data_quality",
|
||||
"detail": row["reason"] if invalid else "连续两个设备周期未读取到观测",
|
||||
"task_id": None}
|
||||
state["event"] = event
|
||||
changes.append({**event, "change": "raised"})
|
||||
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
|
||||
elif row and row["quality"] == "valid" and state["event"]:
|
||||
state["recoveries"] += 1
|
||||
if state["recoveries"] >= 2:
|
||||
changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"})
|
||||
state["event"] = None
|
||||
for rule in config.rules:
|
||||
state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None})
|
||||
hits, usable, reasons = [], [], []
|
||||
for device_id in rule.device_ids:
|
||||
row = by_id.get(device_id)
|
||||
if not row or row["quality"] != "valid":
|
||||
continue
|
||||
value = row["cleaned_value"]
|
||||
if rule.kind == "bounds":
|
||||
triggered = ((rule.lower is not None and value < rule.lower)
|
||||
or (rule.upper is not None and value > rule.upper))
|
||||
reason = f"{device_id}={value} {rule.unit},超出配置范围"
|
||||
else:
|
||||
previous = histories.get(device_id, [])[-rule.window_samples:]
|
||||
if len(previous) < rule.minimum_samples:
|
||||
continue
|
||||
baseline = median(previous)
|
||||
if baseline == 0:
|
||||
continue
|
||||
change = (value - baseline) / abs(baseline) * 100
|
||||
triggered = (abs(change) >= rule.threshold if rule.direction == "both"
|
||||
else change >= rule.threshold if rule.direction == "increase"
|
||||
else change <= -rule.threshold)
|
||||
reason = f"{device_id} 较历史中位值变化 {change:.1f}%,阈值 {rule.threshold}%"
|
||||
usable.append(device_id)
|
||||
if triggered:
|
||||
hits.append(device_id)
|
||||
reasons.append(reason)
|
||||
if len(hits) >= rule.minimum_devices:
|
||||
state["hits"] += 1
|
||||
state["recoveries"] = 0
|
||||
if state["event"] is None and state["hits"] >= rule.consecutive_samples:
|
||||
event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}"))
|
||||
event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits,
|
||||
"occurred_at": at.isoformat(), "last_observed_at": at.isoformat(),
|
||||
"status": "active", "detail": ";".join(reasons)[:2000], "task_id": None}
|
||||
state["event"] = event
|
||||
changes.append({**event, "change": "raised"})
|
||||
result["anomaly_count"] = result.get("anomaly_count", 0) + 1
|
||||
elif state["event"]:
|
||||
state["event"]["last_observed_at"] = at.isoformat()
|
||||
elif len(usable) == len(rule.device_ids):
|
||||
state["hits"] = 0
|
||||
if state["event"]:
|
||||
state["recoveries"] += 1
|
||||
if state["recoveries"] >= rule.recovery_samples:
|
||||
event = {**state["event"], "status": "resolved", "resolved_at": at.isoformat(),
|
||||
"change": "resolved"}
|
||||
changes.append(event)
|
||||
state["event"] = None
|
||||
state["recoveries"] = 0
|
||||
# Incomplete frames cannot count as a consecutive hit/recovery.
|
||||
elif not usable and any(device in by_id for device in rule.device_ids):
|
||||
state["hits"] = state["recoveries"] = 0
|
||||
frozen = {device for rule in config.rules if states[rule.rule_id]["event"]
|
||||
for device in rule.device_ids}
|
||||
for row in rows:
|
||||
if row["quality"] == "valid" and row["device_id"] not in frozen:
|
||||
history = histories.setdefault(row["device_id"], [])
|
||||
history.append(row["cleaned_value"])
|
||||
histories[row["device_id"]] = history[-96:]
|
||||
result["processed_count"] = result.get("processed_count", 0) + len(rows)
|
||||
return result, changes
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Server-enforced replay knowledge bounds and immutable corrected snapshots."""
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
|
||||
|
||||
def check_replay_window(context, start, end):
|
||||
cap = datetime.fromisoformat(context["knowledge_cutoff"])
|
||||
if start is None or end is None or start.tzinfo is None or end.tzinfo is None or start > end or end > cap:
|
||||
raise HTTPException(422, f"回放查询需要明确时间范围,结束不得晚于 {cap.isoformat()}")
|
||||
|
||||
|
||||
async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0):
|
||||
check_replay_window(context, start, end)
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if not set(device_ids) <= set(run["config"]["device_ids"]):
|
||||
raise HTTPException(403, "设备或回放不属于当前调查")
|
||||
if start < datetime.fromisoformat(run["start_time"]):
|
||||
raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测,回放前基线已用于后台初始化")
|
||||
rows = await repository.detected_outputs(
|
||||
context["replay_id"], device_ids, start, end, context["batch_sequence"],
|
||||
limit=limit, offset=offset,
|
||||
)
|
||||
return [{key: row[key] for key in (
|
||||
"time", "device_id", "monitored_value", "cleaned_value", "quality", "reason"
|
||||
)} for row in rows]
|
||||
|
||||
|
||||
async def replay_burst_location_inputs(connection, project_id, context, args):
|
||||
"""Build location series from immutable replay outputs, never live TSDB rows."""
|
||||
if args.data_source != "monitoring":
|
||||
raise HTTPException(422, "回放定位只支持 monitoring 数据源")
|
||||
if args.scada_normal_start is None or args.scada_normal_end is None:
|
||||
raise HTTPException(422, "回放定位必须显式提供已处理范围内的正常对照窗口")
|
||||
check_replay_window(context, args.scada_burst_start, args.scada_burst_end)
|
||||
check_replay_window(context, args.scada_normal_start, args.scada_normal_end)
|
||||
repository = ScadaReplayRepository(connection, project_id)
|
||||
run = await repository.get(context["replay_id"])
|
||||
if run:
|
||||
replay_start = datetime.fromisoformat(run["start_time"])
|
||||
if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start:
|
||||
raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内")
|
||||
devices = {item["device_id"]: item for item in run.get("devices", [])}
|
||||
monitoring = run.get("monitoring_config") or {}
|
||||
pressure_devices = monitoring.get("pressure_device_ids", [])
|
||||
pressure_map = {str(devices[item]["node_id"]): item for item in pressure_devices
|
||||
if item in devices and devices[item].get("node_id")}
|
||||
requested_pressure = args.pressure_scada_ids or list(pressure_map)
|
||||
if not requested_pressure or not set(requested_pressure) <= set(pressure_map):
|
||||
raise HTTPException(422, "压力节点必须来自本轮已配置的回放监测角色")
|
||||
flow_devices = monitoring.get("outlet_flow_device_ids", [])
|
||||
flow_map = {str(devices[item]["link_id"]): item for item in flow_devices
|
||||
if item in devices and devices[item].get("link_id")}
|
||||
requested_flow = args.flow_scada_ids or []
|
||||
if not set(requested_flow) <= set(flow_map):
|
||||
raise HTTPException(422, "流量要素必须来自本轮已配置的出厂边界流量角色")
|
||||
selected = [pressure_map[item] for item in requested_pressure] + [flow_map[item] for item in requested_flow]
|
||||
rows = await repository.detected_outputs(
|
||||
context["replay_id"], selected,
|
||||
min(args.scada_burst_start, args.scada_normal_start),
|
||||
max(args.scada_burst_end, args.scada_normal_end),
|
||||
context["batch_sequence"],
|
||||
)
|
||||
|
||||
def averages(mapping, start, end):
|
||||
result = {}
|
||||
for element_id, device_id in mapping.items():
|
||||
values = [float(row["cleaned_value"]) for row in rows
|
||||
if row.get("device_id") == device_id
|
||||
and row.get("quality") in {"valid", "corrected"}
|
||||
and row.get("cleaned_value") is not None
|
||||
and start <= datetime.fromisoformat(row["time"]) <= end]
|
||||
if not values:
|
||||
raise HTTPException(422, f"回放窗口缺少 {device_id} 的有效清洗观测")
|
||||
result[element_id] = sum(values) / len(values)
|
||||
return result
|
||||
|
||||
pressure_mapping = {item: pressure_map[item] for item in requested_pressure}
|
||||
flow_mapping = {item: flow_map[item] for item in requested_flow}
|
||||
return {
|
||||
"data_source": "monitoring",
|
||||
"burst_leakage": args.burst_leakage,
|
||||
"pressure_scada_ids": requested_pressure,
|
||||
"burst_pressure": averages(pressure_mapping, args.scada_burst_start, args.scada_burst_end),
|
||||
"normal_pressure": averages(pressure_mapping, args.scada_normal_start, args.scada_normal_end),
|
||||
**({
|
||||
"flow_scada_ids": requested_flow,
|
||||
"burst_flow": averages(flow_mapping, args.scada_burst_start, args.scada_burst_end),
|
||||
"normal_flow": averages(flow_mapping, args.scada_normal_start, args.scada_normal_end),
|
||||
"use_scada_flow": True,
|
||||
} if requested_flow else {}),
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
"contracts": {
|
||||
"server": {
|
||||
"file": "server-v1.openapi.json",
|
||||
"sha256": "2c1d757ccf06ae72b10c592e1e9b3d32b4f72cda8489e224cf2c5f2690fc6972"
|
||||
"sha256": "b039042be13243a88691309cb74eee105a3b71765c342edb9222374e41c0458e"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="720" height="362" viewBox="0 0 720 362" role="img" aria-labelledby="title description">
|
||||
<title id="title">SCADA 演示时间轴 FM 调频盘样式</title>
|
||||
<desc id="description">静态设计示意。选定区间为十月八日八点至十四点,当前时间十点十五分。半透明玻璃刻度窗内细密时间刻度横向排列,中央红色指示线固定,控制栏包含暂停、下一批和六十倍速。</desc>
|
||||
<defs>
|
||||
<linearGradient id="glass" x1="0" y1="0" x2="0" y2="1">
|
||||
<stop offset="0" stop-color="#fff" stop-opacity=".72"/>
|
||||
<stop offset=".5" stop-color="#f7faff" stop-opacity=".4"/>
|
||||
<stop offset="1" stop-color="#fff" stop-opacity=".64"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="edge" x1="0" y1="0" x2="1" y2="0">
|
||||
<stop offset="0" stop-color="#f7f9fc"/>
|
||||
<stop offset=".12" stop-color="#f7f9fc" stop-opacity="0"/>
|
||||
<stop offset=".88" stop-color="#f7f9fc" stop-opacity="0"/>
|
||||
<stop offset="1" stop-color="#f7f9fc"/>
|
||||
</linearGradient>
|
||||
<filter id="blur"><feGaussianBlur stdDeviation="16"/></filter>
|
||||
<filter id="shadow" x="-.1" y="-.3" width="1.2" height="1.6">
|
||||
<feDropShadow dx="0" dy="2" stdDeviation="4" flood-color="#243852" flood-opacity=".1"/>
|
||||
</filter>
|
||||
<clipPath id="window"><rect x="48" y="130" width="624" height="110" rx="12"/></clipPath>
|
||||
</defs>
|
||||
<rect width="720" height="362" rx="18" fill="#e9edf2"/>
|
||||
<rect x="24" y="22" width="672" height="318" rx="16" fill="#fafbfd" stroke="#d7dee7"/>
|
||||
<g font-family="'Noto Sans SC Variable', -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', 'Noto Sans CJK SC', sans-serif" fill="#263347">
|
||||
<text x="48" y="54" font-size="13" font-weight="600">SCADA 时段演示</text>
|
||||
<text x="672" y="54" text-anchor="end" font-size="11" fill="#778398">静态样式示意</text>
|
||||
<text x="48" y="79" font-size="12" fill="#68758a">演示区间</text>
|
||||
<text x="114" y="79" font-size="12">10 月 8 日 08:00 — 14:00</text>
|
||||
<text x="360" y="118" text-anchor="middle" font-size="28" font-weight="500" font-variant-numeric="tabular-nums" letter-spacing="1">10:15:00</text>
|
||||
<rect x="48" y="130" width="624" height="110" rx="12" fill="#edf2f7" filter="url(#shadow)"/>
|
||||
<g clip-path="url(#window)">
|
||||
<g filter="url(#blur)" opacity=".6">
|
||||
<path d="M12 234 L280 132 L423 247 L749 142" fill="none" stroke="#c7d8e7" stroke-width="35"/>
|
||||
<path d="M63 134 L221 219 L438 128 L647 212" fill="none" stroke="#fff" stroke-width="38"/>
|
||||
</g>
|
||||
<rect x="48" y="130" width="624" height="110" fill="url(#glass)"/>
|
||||
<path d="M65 131 H655" stroke="#fff" stroke-width="2" opacity=".9"/>
|
||||
<path d="M49 138 V231 M671 138 V231" stroke="#fff" opacity=".6"/>
|
||||
<path d="M60 237 H660" stroke="#a8b6c8" opacity=".25"/>
|
||||
<g fill="none" stroke="#67778b" stroke-width="1">
|
||||
<path d="M90 166 V197 M180 166 V197 M270 166 V197 M360 166 V197 M450 166 V197 M540 166 V197 M630 166 V197"/>
|
||||
<path d="M60 179 V196 M120 179 V196 M150 179 V196 M210 179 V196 M240 179 V196 M300 179 V196 M330 179 V196 M390 179 V196 M420 179 V196 M480 179 V196 M510 179 V196 M570 179 V196 M600 179 V196 M660 179 V196" opacity=".65"/>
|
||||
</g>
|
||||
<g font-size="11" text-anchor="middle" font-variant-numeric="tabular-nums">
|
||||
<text x="90" y="216">09:30</text><text x="180" y="216">09:45</text>
|
||||
<text x="270" y="216">10:00</text><text x="360" y="216" font-weight="600">10:15</text>
|
||||
<g fill="#7b889a"><text x="450" y="216">10:30</text><text x="540" y="216">10:45</text><text x="630" y="216">11:00</text></g>
|
||||
</g>
|
||||
<rect x="48" y="131" width="624" height="108" fill="url(#edge)"/>
|
||||
</g>
|
||||
<rect x="48.5" y="130.5" width="623" height="109" rx="11.5" fill="none" stroke="#fff" stroke-opacity=".8"/>
|
||||
<path d="M356 141 H364 L360 147 Z" fill="#cf453f"/>
|
||||
<path d="M360 147 V201" stroke="#cf453f" stroke-width="2"/>
|
||||
<rect x="48" y="256" width="88" height="38" rx="8" fill="#4662ba"/>
|
||||
<path d="M65 269 V281 M70 269 V281" stroke="#fff" stroke-width="2"/>
|
||||
<text x="98" y="280" text-anchor="middle" fill="#fff" font-size="12">暂停</text>
|
||||
<rect x="148" y="256" width="88" height="38" rx="8" fill="#fff" stroke="#d7dee7"/>
|
||||
<text x="192" y="280" text-anchor="middle" font-size="12" fill="#7b8799">下一批</text>
|
||||
<rect x="578" y="256" width="94" height="38" rx="8" fill="#fff" stroke="#d7dee7"/>
|
||||
<text x="601" y="280" font-size="12">60×</text>
|
||||
<path d="M651 273 L655 277 L659 273" fill="none" stroke="#64738a" stroke-width="1.5"/>
|
||||
<text x="48" y="319" font-size="11" fill="#68758a">已读取 10:15 · 已检测 10:10</text>
|
||||
<text x="672" y="319" text-anchor="end" font-size="11" fill="#68758a">红线对应当前时间 · 横向拨动可回看</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.7 KiB |
@@ -39,6 +39,8 @@ Forward Hostname 后追加 `/api/sample/v1`。本机开发和预览模式也将
|
||||
- `/api/sample/v1/monitoring/points`、`observations`、`alerts`
|
||||
- `/api/sample/v1/demo-data`
|
||||
|
||||
SCADA 按月数据集、5 分钟时间定位、主动周期监测、质量校验、爆管调查与定位边界见 [SCADA 外围系统时段回放、主动监测与爆管调查设计](scada-peripheral-replay-design.md)。部署回放存储需在项目 BizDB 执行 `031_scada_replays.sql`;项目通用配置需在 MetaDB 执行 `033_metadata_project_settings.sql`。监测频率保存于 `projects.settings.scada_monitoring_interval_seconds`,压力与出厂边界流量角色保存在类型化 `projects.settings.scada_monitoring` 中;本阶段不注册定时任务,也不部署 SCADA 回放 worker。代码和结构迁移完成不等于月度数据或 Agent 联调已经验收。
|
||||
|
||||
演示数据通过 `scripts/seed_peripheral_sample.py` 写入 `tjwater_v2` 的 `peripheral_sample` schema。当前固定窗口为 2026-09-17,所有记录标记为 `synthetic`,并使用 `DEMO-` 前缀。
|
||||
|
||||
## 工单事件调查
|
||||
|
||||
@@ -0,0 +1,476 @@
|
||||
# SCADA 外围系统时段回放、主动监测与爆管调查设计
|
||||
|
||||
状态:时段回放、主动周期监测、条件式 `burst_report` 调查及回放安全的爆管定位适配已实现。单步处理由用户操作触发并在当前 API 请求内完成,不部署 SCADA 回放 worker。`tjwater_v2` 与 `tjwater2_v2` 尚未确认出厂边界流量角色;仍待完整月度数据和真实 Agent 联调。更新日期:2026-11-03。下面明确区分已实现范围、后续目标与验收要求。
|
||||
|
||||
本次将外围 SCADA 定义为一个模拟外部数据源:本地 TSDB 已保存原始观测,通过只读 API 按选定时间区间和后台时钟分批读取 `scada.measurements.monitored_value`,将这一批直接交给清洗流水线,保存 `cleaned_value` 后执行异常检测及后续 Agent 调查。原始值、修正值、真实工况异常和处理结论分别保留。浏览器负责控制和展示,关闭页面不停止后台回放。
|
||||
|
||||
演示主路径为“TSDB 已存 raw → 用户点击下一步并监测 → 顺序读取一个步长内的观测 → 清洗并更新修正值 → 在步长边界执行多指标监测 → 满足条件时进入爆管调查与定位”。真实外部系统新增观测时,仍先通过入库接口保存 raw,再进入相同的清洗检测流程。演示读取已有数据不再次调用原始值插入接口,也不把同一批数据从一个内部服务读出后再写回原表。
|
||||
|
||||
演示操作以用户选定的时间区间为中心:一个月是可选择的数据范围,单次演示可以只覆盖其中几小时或几天。时间轴采用 FM 调频盘样式,玻璃刻度窗内有固定红色指示线,时间刻度带随拨动横向移动。时间轴使用 5 分钟时间戳定位;项目监测频率只决定“下一步并监测”覆盖的业务周期,两者互不替代。
|
||||
|
||||
## 首版实现范围与限制
|
||||
|
||||
已实现只读 raw 分页、持久回放控制、按项目监测频率推进、逐时刻清洗检测、质量告警与运行异常事件、周期监测回执、组合判定及调查任务。主动单步只有组合结果为 `suspected_burst` 时才创建 `burst_report`。前端改为后端状态投影,主界面只保留选定区间、5 分钟 FM 时间定位、主动周期控制和本周期监测结论,不重复展示 SCADA 设备目录与逐测点读数表。主要代码为 [回放服务](../app/services/scada_replay.py)、[组合监测](../app/services/scada_replay_monitoring.py)、[质量与检测](../app/services/scada_replay_processing.py)、[持久状态](../app/infra/db/postgresql/scada_replays.py) 和 [前端组件](../../next-tjwater-frontend/src/features/workbench/peripheral-sample/scada-replay-section.tsx)。
|
||||
|
||||
本版采用每项目最多 500 个设备、一个完整时间戳一批、至多一个未完成批次的串行处理,不实现 20 批并发队列或大于 500 个设备的分片接收。处理变慢时逐批追赶且不跳过中间观测;后台停机期间最多补算两秒现实时间,不一次性追赶全部停机时长。数据质量以有限数值和显式仪表量程为依据,不自动插值或平滑;缺测判断使用设备档案的采样周期。分组规则的恢复需要该组设备在同一帧都有可信观测,不同采样周期或采样相位应拆成规则。
|
||||
|
||||
本版冻结设备、区间、规则及批次输出,按批次原始值摘要校验源变更。完整月度数据清单、全月源版本摘要、独立事件真值和检测效果评估仍是后续数据准备工作;能力接口仅判断回放存储是否存在与当前操作权限,不声明月度数据已经就绪。现有 14 天数据不能当作一个月验收。
|
||||
|
||||
Agent 查询本轮已检测的固定批次快照,受任务的 replay_id、knowledge_cutoff、batch_sequence 约束。开始前历史只供后台基线初始化,本版调查查询不返回该部分;爆管定位已支持受回放截止约束的冻结输入适配,其他未适配计算工具继续拒绝。新 [SCADA 调查 Skill](../resources/work-order-skills/scada-anomaly/SKILL.md) 要求据实说明这些证据限制。运行异常自动建立调查,数据质量异常仅建立告警,不触发模型;恢复时推进调查代次。真实模型推理与报告效果尚未联调。
|
||||
|
||||
控制进度和事件状态使用有界轮询,沿用告警流发布业务告警;本版未新增 scada.replay.changed SSE 主题。演示面板只显示最近一个周期的组合结论,逐测点输出和完整事件历史由分页 API 保留给诊断与调查入口,不在主面板重复铺开。
|
||||
|
||||
## 已确定的设计边界
|
||||
|
||||
- 数据集覆盖一个月,包含多次独立异常及各次事件发生、持续、恢复的过程。
|
||||
- 项目配置一个 SCADA 监测频率,只用于一次主动监测所覆盖的周期;时间轴独立使用 5 分钟时间戳,它不关联设备或设备类型,本阶段不据此创建定时任务。
|
||||
- 设备采样周期仍属于设备档案。回放实际传入时间以原始观测时间戳为准,不使用项目监测任务间隔生成、重采样或判断设备缺测。
|
||||
- 复用 `scada.measurements.monitored_value` 与 `cleaned_value`;不另建一套正式 SCADA 原始值和修正值表。
|
||||
- 回放读取 API 只返回原始观测,修正值由本地清洗流水线计算和保存;GET 不更新游标、不清洗、不创建检测任务。
|
||||
- 真实压力、流量、水质异常保留在历史序列中。事件恢复或调查结束不删除、不抹平历史异常。
|
||||
- 数据处理顺序为原始观测持久化、数据质量校验、必要的修正、业务异常检测、事件与调查。
|
||||
- 步长内的数据批次按原始时间驱动清洗和基础规则检测;到达步长边界后,用户触发的本周期监测汇总多项指标。正常周期不启动 Agent。
|
||||
- 时间范围、设备、数据集版本和检测规则在一次回放创建后固定;改变这些条件需要新建回放。
|
||||
- 页面首先选择演示起止时间,再建立回放。月度数据集、设备配置和规则作为来源与设置,不占据主播放区。
|
||||
- 本次演示对已存 raw 只读,清洗阶段仅更新同一条观测的 cleaned_value;不导出整月副本、不重复插入原始值。每次回放的质量结论、检测结果和调查证据按回放身份隔离。
|
||||
|
||||
### 项目通用配置与监测频率
|
||||
|
||||
MetaDB 的 `projects.settings` 为项目级通用 JSONB 对象,默认 `{}`,用于保存可扩展的业务参数,不存放密钥。新增参数可直接增加 JSON 键,无需为每项配置新增数据库列;后端为已知参数提供类型校验,并保留其他 JSON 键。
|
||||
|
||||
其中 `settings.scada_monitoring_interval_seconds` 表示项目的 SCADA 监测频率,单位为秒。本阶段只用于主动处理边界和监测周期标识,不控制时间轴刻度,不注册 cron、后台 schedule 或周期性 Agent 任务。它与设备目录、设备类型、设备采样周期和页面刷新周期均无关联。省略或设为 `null` 表示未配置;有效范围为 60 秒至 365 天。
|
||||
|
||||
管理员可通过 `PATCH /api/v1/admin/projects/{project_id}` 配置:
|
||||
|
||||
```json
|
||||
{
|
||||
"settings": {
|
||||
"scada_monitoring_interval_seconds": 900
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
更新时按 `settings` 顶层键合并:未传入的键保留,传入的嵌套对象整体替换,`settings: {}` 不清空已有配置,`settings: null` 会被拒绝。读取项目详情和管理员项目列表均返回 `settings`。
|
||||
|
||||
组合监测仍复用该 JSON 对象,不新增固定数据库列。后端类型化嵌套对象显式声明项目实际具备的监测角色:
|
||||
|
||||
```json
|
||||
{
|
||||
"settings": {
|
||||
"scada_monitoring_interval_seconds": 900,
|
||||
"scada_monitoring": {
|
||||
"rule_version": "burst-monitor-v1",
|
||||
"pressure_device_ids": ["明确的压力设备 ID"],
|
||||
"outlet_flow_device_ids": ["明确的出厂边界流量设备 ID"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
当前代码已类型校验并在创建回放时冻结 `scada_monitoring`。项目没有出厂流量或不足 5 个完整压力点时保持缺项,能力接口说明原因,监测回执返回 `insufficient_data`;不能为了让演示通过而把其他流量点冒充出厂流量或虚构设备。`tjwater_v2` 现有目录只有通用 `pipe_flow` 类型及模型管段映射,没有足以证明“出厂边界”的角色字段,因此本轮未写入该嵌套配置。
|
||||
|
||||
部署前需在 MetaDB 执行 `resources/sql/033_metadata_project_settings.sql`。迁移会将实验版本可能存在的独立监测间隔列搬入 `settings`,核对冲突后移除旧列;重复执行不丢失已有配置。当前 `tjwater_v2` 与 `tjwater2_v2` 均配置 900 秒,即 15 分钟的主动监测周期。该参数不改变 5 分钟时间轴定位,本阶段明确不实现通用定时任务调度。
|
||||
|
||||
## 实施前基础与原设计差距
|
||||
|
||||
下表记录实施前基础,用于说明改动缘由;本次实际实现以首版范围及后文接口表为准。设备目录采样周期为本轮讨论中 2026-11-02 的只读核查快照,与项目监测任务间隔是两个概念。
|
||||
|
||||
| 位置 | 当前能力 | 本次需要补齐 |
|
||||
| --- | --- | --- |
|
||||
| [外围 API](../samples/peripheral/api.py)、[存储路由](../samples/peripheral/routing.py) | 挂载在 `/peripheral`,使用 Bearer 与 `X-Project-Id`,按已授权项目解析 BizDB 和 TimescaleDB | 回放控制、源批次、回放状态与能力声明 |
|
||||
| [历史观测接口](../samples/peripheral/store.py) | 单点、最多 7 天、最多返回 1000 条并声明截断;当前使用 `api_query_id` 查历史 | 回放读取使用规范 `device_id`,按时间和设备联合游标分页;不能直接拼接现有单点接口来装载一个月 |
|
||||
| [SCADA API](../app/api/v1/endpoints/timeseries/scada.py)、[仓储](../app/infra/db/timescaledb/repositories/scada.py) | 范围读取、最新值读取、批量插入;当前范围读取返回 raw 和 corrected,缺少回放游标及完整流水线 | 增加只读 raw 分页契约;复用存储查询,补齐处理检查点、清洗与检测调度 |
|
||||
| [清洗服务](../app/services/timeseries_analysis.py) | 从原始值计算并更新 `cleaned_value`;目前支持压力和流量 | 在线质量标记、因果修正及选择值依据;水质不能直接套用现有清洗 |
|
||||
| [压力清洗](../app/algorithms/scada_cleaning/pressure_series.py)、[流量清洗](../app/algorithms/scada_cleaning/flow_series.py) | 包含居中窗口、后向补值、Kalman 平滑;流量路径还会补为 1 分钟序列 | 在线路径只使用已到达的数据,保留设备频率,不整月预清洗后再模拟检测 |
|
||||
| [外围工单事件同步](peripheral-work-order-events.md) | 持久事件、通知唤醒、后台同步和定期对账 | 复用可靠处理方式,SCADA 的批次有自己的持久状态 |
|
||||
| [调查 worker](../scripts/run_operational_worker.py) | 可靠投递 `event_investigation`,恢复执行租约,管理 Agent 执行代次 | 接收由 SCADA 异常事件建立的调查;不承担每条观测的检测 |
|
||||
| [任务告警](../app/domain/schemas/task_alert.py) | 支持 raised、acknowledged、resolved;当前来源必须携带任务和执行标识 | 增加真实的回放检测来源,不能为调查前告警伪造 `task_id`、`run_id` |
|
||||
| [MCP 上下文](../app/mcp/contexts.py)、[SCADA 查询](../app/mcp/query_service.py) | 校验任务、项目、执行代次和会话,保存查询证据 | 在所有回放调查读取入口执行统一的时间与数据版本约束 |
|
||||
| [业务 Skill 映射](../app/mcp/work_order_skills.py) | 目前映射爆管上报、水质投诉、预约跟进 | 增加 SCADA 回放调查流程,不将检测信号直接等同于已确认爆管或污染 |
|
||||
| [前端 SCADA 区域](../../next-tjwater-frontend/src/features/workbench/peripheral-sample/scada-system-section.tsx) | 旧版同时展示设备表、逐点读数、重复概览和手工合成读数 | 改为服务端状态投影;主界面只承担时间定位、主动单步和周期结论,详细数据留在诊断与调查接口 |
|
||||
|
||||
实施前只读核查中,`tjwater_v2` 有压力、流量和水质监测类型,其他项目的类型集合与历史周期并不相同。项目通用配置保存一个监测任务周期,不与设备类型绑定,也不把 5/15 分钟写成枚举;设备原有采样信息仍用于缺测判断。
|
||||
|
||||
现有 [演示记录](../../present.md) 描述的是 14 天合成观测,不能作为一个月数据已经准备好的证明。一个月数据准备与本设计实现分别验收。
|
||||
|
||||
## 运行链路与职责
|
||||
|
||||
```text
|
||||
本地 TSDB 已存月度原始观测;项目 BizDB 保存设备目录
|
||||
|
|
||||
v
|
||||
只读 raw API / 同一读取服务 <--- 持久回放状态 <--- 页面“下一步并监测”
|
||||
|
|
||||
| 返回一批原始观测,不再写入 raw
|
||||
v
|
||||
后台固定本批记录引用、摘要和处理身份
|
||||
|
|
||||
v
|
||||
后台质量校验与清洗
|
||||
|
|
||||
v
|
||||
本地 TSDB:仅更新 cleaned_value
|
||||
|
|
||||
v
|
||||
用同批清洗结果执行基础规则检测
|
||||
|
|
||||
v
|
||||
到达步长边界,汇总爆管侦测、压力下降、出厂流量变化
|
||||
|
|
||||
v
|
||||
疑似爆管且定位输入具备 ---> burst_report 任务与 MCP Skill
|
||||
|
|
||||
v
|
||||
工作台告警、调查进度与结果
|
||||
```
|
||||
|
||||
SCADA 单步命令在当前 API 请求内按原始时间顺序完成 raw 读取、清洗、基础规则检测和周期组合监测。每个阶段仍写入持久批次,网络中断后的同一幂等请求可以继续未完成步骤,但页面关闭后不会自动推进时间,也不需要单独的 SCADA 回放进程。
|
||||
|
||||
现有调查 worker 只处理已经创建的调查执行,计算 worker 继续处理长计算。当前阶段不增加消息中间件,不扩展通用 schedule API,也不注册 SCADA 定时任务。页面的明确单步命令写入持久 `step_target`,API 只完成这次已授权的周期处理;不能用“每隔几分钟创建一条 Agent 任务”替代。
|
||||
|
||||
最小可交付版本必须包含主动周期处理、批次可靠保存、至少一种明确配置的检测规则和结果展示;只移动页面时间轴不能代表数据已经处理。
|
||||
|
||||
## 一个自然月的数据集
|
||||
|
||||
数据集用带时区的半开区间定义。例如北京时间 `[2026-10-01 00:00, 2026-11-01 00:00)` 是 31 天,不能固定按 30 天计算。内部统一保存带时区时间,界面按 Asia/Shanghai 展示。
|
||||
|
||||
准备数据时冻结清单:`dataset_id`、`dataset_revision`、源项目、观测区间、可用历史基线区间、设备规范 ID、类型、单位、设备采样周期、采样相位、指标含义、raw 分片摘要和规则配置版本。由服务端读取项目路由,页面不传物理库名、SQL 或连接信息。摘要只覆盖原始观测与必要元数据,不覆盖会被本轮清洗更新的 cleaned_value。月度摘要由数据集准备流程预先生成,创建回放的 HTTP 请求只校验清单,不同步扫描和加载整个月。
|
||||
|
||||
首版演示直接读取已准备在本地 TSDB 的 raw,不另导出一套完整原始数据源。月份数据如果尚未齐备,由单独授权的数据准备过程补齐并生成版本清单;创建或播放回放本身不生成、删除、搬移或重复插入 SCADA 原始观测。演示批次是对已有观测的读取与处理身份,不能在统计中称为本轮新插入条数。
|
||||
|
||||
设备档案采样周期与实际观测间隔不一致时,记录配置差异和缺测情况,仍按实际时间戳读取。不会将 15 分钟数据插值成 5 分钟观测,也不会把没有记录的时刻当作正常值。项目监测任务间隔不参与这项判断。
|
||||
|
||||
多次异常应同时覆盖:真实压力异常、真实流量变化、真实水质变化、仪表或通信故障、正常调度引起的变化,以及恢复后的再次异常。数据集真值清单独立保存,用于事后计算命中、误报、漏报、重复事件数和发现延迟。真值不得出现在检测输入、Agent 上下文、预告型时间轴标记或普通业务报告中。
|
||||
|
||||
水质必须声明具体指标。当前演示中的化学示踪物浓度不能直接解释为余氯、浊度,也不能只凭 `mg/L` 选择检测阈值。流量保留方向,压力确认 m、kPa、MPa 等单位后再应用规则。
|
||||
|
||||
数据集准备负责人需要在运行前提交版本化设备清单、规则参数和独立真值。这是新数据集的输入要求,不表示本次已选定新的 5 分钟设备分组或生成了观测。
|
||||
|
||||
## 时间轴和读取语义
|
||||
|
||||
### 时间与进度
|
||||
|
||||
| 名称 | 含义 |
|
||||
| --- | --- |
|
||||
| `observed_at` | 原始观测时间,不随回放改写 |
|
||||
| `virtual_now` | 已完成主动监测的周期边界时间 |
|
||||
| `delivered_until` | 本轮源区间已读完,且所有分片的记录引用已登记为待处理的时间上限;表示模拟传入进度,不是原始值新插入时间 |
|
||||
| `processed_until` | 质量校验和检测已完整提交的时间上限 |
|
||||
| `read_at`、`cleaned_at`、`processed_at` | 实际批次读取、清洗提交和检测完成时间,用于延迟、故障与审计 |
|
||||
|
||||
始终满足 `processed_until <= delivered_until <= virtual_now <= end_at`。还未处理任何批次时,读取和处理进度为空,不能伪装成起点已经处理。GET 返回数据不自动推进任何进度;后台确认本批处理身份后才更新读取进度。表格可显示本轮已读取但仍在清洗的 raw 并标明状态,检测只能消费已完成清洗阶段的批次。主界面的次级进度使用“已读取”和“已检测”。
|
||||
|
||||
后台仅在播放状态累计模拟时间。暂停期间不推进;恢复时重新锚定现实时间。进程重启从持久进度重新锚定,不把停机时长一次性折算成需要跳过的数据。
|
||||
|
||||
### 顺序读取
|
||||
|
||||
1. 整次回放选取 `[start_at, end_at)`;首批包含 `start_at` 的观测。
|
||||
2. 时间推进到 T 时,候选记录必须不晚于 T、早于 `end_at`,且在已成功接收的联合游标 `(observed_at, device_id)` 之后。
|
||||
3. 所有选中设备合并后按 `observed_at, device_id` 排序;优先按同一观测时刻成批,不按设备分别跑独立时钟。
|
||||
4. 每个传输片段最多 500 条、序列化正文最多 2 MiB;超限继续分片。记录片段序号与该时刻是否完整,不在半个时刻接收成功时提前推进时间上限;同一时刻的设备分组检测等待该时刻所有分片收齐。
|
||||
5. 后台将本批记录引用和原始摘要登记成功后,才推进读取游标。重试同一批使用相同编号和摘要;清洗或检测失败从该阶段恢复,不重新选择一批、不重新插入 raw。正常路径复用本次读取的 raw 对象计算清洗结果,清洗提交后直接将同批结果用于检测,不再重新查询该批 raw 或 corrected。
|
||||
6. 没有观测的时间段按设备档案采样周期与采样相位识别应到时刻。后台提交记录数为零的区间检查批次,推进覆盖进度并触发缺测判断;它不生成伪造观测。不能仅依赖“下一条数据到来”唤醒,否则连续断流永远不会被发现,终点也无法排空完成。
|
||||
|
||||
例如设备档案中的压力采样周期为 5 分钟、流量采样周期为 15 分钟。时间轴到 09:10 时,压力设备可传入 09:00、09:05、09:10,流量设备只有 09:00;到 09:15 时,再传入两类设备对应的 09:15 记录。
|
||||
|
||||
首版至多保留一个未完成批次,清洗检测提交后才选择下一批。单次请求不跳过周期内的中间观测、质量检查、异常开始和恢复,也不提供播放倍率。
|
||||
|
||||
### 页面操作
|
||||
|
||||
| 操作 | 约定 |
|
||||
| --- | --- |
|
||||
| 创建 | 固定数据集、范围、设备和规则,进入 ready |
|
||||
| 播放 | ready 或 paused 进入 running,由后台推进 |
|
||||
| 暂停 | 停止释放新批次;已持久接收的批次继续处理。返回实际生效进度,在途接收与暂停命令通过同一回放版本串行裁决 |
|
||||
| 下一步并监测 | 在 ready 或 paused 状态推进到下一个项目监测周期边界;周期内的所有观测按原始时间顺序处理,到边界后提交一次本周期监测回执并保持暂停,尾步可短于完整周期 |
|
||||
| 调速 | 原子更新速度和时间锚点,不改变观测顺序 |
|
||||
| 拖动时间轴 | 只查看已释放的数据与已产生的事件;未来区间不预览读数,不触发事件 |
|
||||
| 从指定时刻重播 | 新建 `replay_id`,独立检测与调查;新起点之前仅作为有声明的历史基线,跳过段不声称已经检测 |
|
||||
| 到达终点 | 停止新批次,进入 draining;待已接收批次完成后 completed。Agent 调查是否完成另行展示 |
|
||||
| 停止 | 终止继续传入,保留已接收批次、事件和证据;不自动撤销已经建立的调查 |
|
||||
| 重试故障 | 修复依赖后校验固定配置,failed 进入 paused,重试原未完成批次;源版本或批次内容冲突需要新建回放,不能靠重试忽略 |
|
||||
|
||||
生命周期为 ready、running、paused、draining、completed、stopped、failed。队列等待、基线不足和依赖暂不可用是处理状态或原因字段,不再增加一套并行生命周期。
|
||||
|
||||
首次运行遇到可重试投递错误,按现实时间退避重试同一批次;权限错误、数据集版本变化、同批编号内容冲突和结构缺失进入 failed,明确原因。用户恢复前必须解决原因;原批次事实继续保留。
|
||||
|
||||
### 选定演示区间
|
||||
|
||||
用户先在月度数据可用范围内选择起止时间。创建回放后,将该区间固定为当前播放范围;进度比例与剩余时长都相对该区间计算。例如选择 10 月 8 日 08:00 至 14:00,时间轴只覆盖这 6 小时,不能将月内 8 日的位置作为本轮进度。
|
||||
|
||||
主要界面常驻显示选定区间和当前监测时间。月份、数据版本、设备范围及规则收进来源与设置区域;需要校验的配置差异仍在开始前明确显示。不存在上一次选择时不自动处理整个月,等待用户确定区间。
|
||||
|
||||
演示建立后改变时间区间需要停止并创建新回放;保留旧回放的事件和证据,不把旧回放时间游标直接重设。已完成区间需要再次演示时,新建独立回放;不自动循环处理和重复触发调查。
|
||||
|
||||
选定区间之前的可用历史仅用于已声明的基线初始化,不占用主时间轴、不作为新到达批次、不预触发此前事件。基线不足时在当前区间显示具体可用性,不能悄悄扩大演示范围或读取区间之后的数据。
|
||||
|
||||
处理到选定终点后停在终点,显示数据处理是否完成及剩余调查数量;不会自动继续处理该月剩余数据。空区间明确显示没有观测,不能通过控件动画假装已经收到 SCADA 数据。
|
||||
|
||||
### 5 分钟时间定位与项目监测周期
|
||||
|
||||
时间轴以 5 分钟时间戳组织刻度、标签和选择值,只负责定位和回看,不读取 `projects.settings.scada_monitoring_interval_seconds`。方向键按 5 分钟移动;鼠标和触摸的拖动轨迹保持连续调频手感,选定值仍按最接近的 5 分钟时间戳归档。标签密度可以随面板宽度合并,不会改变拖动手感或时间选择精度。
|
||||
|
||||
能力接口另行返回当前项目监测周期;创建回放时由服务端固定到回放文档及快照的 `monitoring_interval_seconds`,页面不能提交不同周期。项目配置后续调整只影响新回放,当前演示继续使用固定值;未配置时提示设置。
|
||||
|
||||
| 交互 | 行为 |
|
||||
| --- | --- |
|
||||
| 方向键 | 前后移动一个 5 分钟时间戳 |
|
||||
| PageUp / PageDown | 移动一个当前可见的主刻度,主刻度始终为 5 分钟的整数倍 |
|
||||
| 拖动回看 | 拖动轨迹连续无缝,选择值按 5 分钟量化;只读定位不推进后台 |
|
||||
| 主刻度 | 宽屏可每 5 分钟标注,窄屏按 10、15 分钟等密度显示,细刻线保留 5 分钟定位 |
|
||||
| 下一步并监测 | 不读取时间轴位置;如后台已监测至 08:07,15 分钟周期处理到 08:22 |
|
||||
|
||||
“下一步并监测”通过持久化 `step_target` 与 `step_pending` 管理。单步开始后,API 按原观测时刻读取、清洗和检测到目标边界,完成前禁用重复单步。若周期内有 5 分钟观测,15 分钟监测周期仍处理其中每个时刻;没有观测则提交空区间检查,不补造数据。尾步到达 end_time 时,仍遵守原始区间 `[start_time, end_time)`。
|
||||
|
||||
时间轴以 5 分钟时间戳定位,不能控制或触发周期处理;“下一步并监测”按项目监测频率推进后台周期,不将原始观测重采样。旧回放没有固定周期时,页面要求停止后重新创建,不能把后来修改的项目周期追补为历史事实。
|
||||
|
||||
### 步长边界主动监测与爆管流程
|
||||
|
||||
一次主动监测的身份为 `(replay_id, period_end, monitor_rule_version)`。同一步骤重试复用原回执,不能重复创建告警、调查或计算。监测回执保存在该步最后一个回放批次的 `monitoring` 中,并将最近一次结果投影到回放快照;内容包含周期起止、知识截止、输入批次上限、数据完整性、各指标状态、组合结论和关联任务。它不是 `scheduled_flow`,也不需要通用定时器。
|
||||
|
||||
步长边界按以下顺序执行:
|
||||
|
||||
1. 确认本周期内所有已到达观测均已完成质量校验和清洗;缺测、无效值和单位冲突进入数据完整性结果,不能按正常值处理。
|
||||
2. 汇总压力下降幅度。只使用映射明确、单位一致、质量合格的压力测点,并同时保留异常测点数、覆盖测点数、基线窗口和最大或中位下降幅度。
|
||||
3. 汇总出厂流量变化。设备必须在项目配置或设备档案中明确标注为“出厂边界流量”,不能把任意 `pipe_flow` 测点当作出厂流量;保留方向、单位、基线和增量。
|
||||
4. 条件满足时运行爆管侦测。现有侦测要求至少 5 个完整压力点和目标时刻之前 15 天同周期数据;不满足时返回 `baseline_not_ready` 或 `sensor_coverage_insufficient`,不能将“未运行”解释为“无异常”。
|
||||
5. 组合上述证据形成监测结论;只有疑似爆管才创建 `event_investigation`,工作流使用现有 `burst_report`,并绑定本轮 `replay_context`。正常周期只保存回执,不启动 Agent;数据不足周期保存缺口并在界面展示。
|
||||
6. `burst_report` Skill 核对证据后决定是否调用爆管定位。定位必须有 m³/s 的有依据漏量估计或明确情景假设,以及模型节点映射、异常窗和正常对照窗;输入不具备时保持“疑似爆管、定位待补充”,不能填默认漏量。
|
||||
|
||||
组合判定采用可版本化策略,首版演示建议如下:
|
||||
|
||||
| 监测结果 | 判定条件 | 后续动作 |
|
||||
| --- | --- | --- |
|
||||
| `insufficient_data` | 必需点位缺测、质量无效、单位或角色未配置,或爆管侦测基线不满足 | 保存缺口,不声明正常,不启动定位 |
|
||||
| `normal` | 数据完整,爆管侦测未命中,压力与出厂流量规则均未命中 | 保存周期回执,不创建 Agent 任务 |
|
||||
| `attention` | 只有单项指标命中,或指标间方向不一致 | 保存观察结果;下一步继续监测,必要时由人发起调查 |
|
||||
| `suspected_burst` | 爆管侦测命中,且压力下降与出厂流量增加在同一业务时间窗内相互支持 | 创建一次 `burst_report` 调查;定位输入具备后进入计算 |
|
||||
|
||||
爆管侦测和压力下降存在相关性,因此回执不能把两者描述成完全独立的两票。出厂流量也会受泵站调度、储水设施和正常用水变化影响。组合结论只是调查触发条件,不是现场确诊。
|
||||
|
||||
回放中的 MCP 定位已走 Server 持有的安全适配层:按 `replay_id`、`knowledge_cutoff` 和 `batch_sequence` 从已固化清洗输出组装压力与可选流量序列,再调用现有计算队列。回放只允许 monitoring 模式,并要求显式异常窗、正常对照窗和有依据的 `burst_leakage`;不能直接查询 TSDB 中尚未回放的未来记录。步长爆管侦测同样只读取最近 15 天已固化输出。计算结果保存输入、算法与证据引用,任务恢复或重试仍使用相同知识上限。
|
||||
|
||||
### FM 调频盘样式
|
||||
|
||||
用户明确选择电台调频盘的刻度、红色指示线和玻璃质感。这里的 FM 表示调谐外观,不增加电台、音频、频段切换或拟物旋钮功能。
|
||||
|
||||
视觉方向:工作台内一块冷白半透明的玻璃刻度窗,红色细线作为唯一醒目的时间定位标记;文字、控件与下方数据沿用现有工作台主题。内容顺序为选定区间、当前时间、刻度窗、控制栏、传入与检测进度。交互特征为固定指示线与横向移动的刻度带,手动拨动直接跟手。
|
||||
|
||||

|
||||
|
||||
图中时间和状态仅为静态样式示意,不是运行结果。玻璃下的底色用于说明透光与边缘高光,运行时仍需验证实际工作台背景下的对比度。
|
||||
|
||||
| 元素 | 设计约定 |
|
||||
| --- | --- |
|
||||
| 演示区间 | 常驻显示完整起止日期和时间;更换区间是次要入口,月度来源收在设置中 |
|
||||
| 当前时间 | 刻度窗上方突出显示红线对应的时间,数字等宽;跨日时同时显示日期 |
|
||||
| 时间刻度 | 主刻度有时间标签,次刻度细密且高低有别;实际密度按区间与容器宽度计算,窄屏优先减少标签而非缩小文字 |
|
||||
| 红色指示线 | 固定在刻度窗水平中央,1 至 2px 宽,带小型顶端指示;这是时间定位色,不代表告警严重度 |
|
||||
| 玻璃刻度窗 | 局部半透明、背景柔化、细边缘高光和轻微内阴影,玻璃层不影响数字、刻度和红线的清晰度 |
|
||||
| 控制栏 | 播放或暂停是唯一主操作,下一批与倍速为辅助;回看时才出现回到当前 |
|
||||
| 次级进度 | 单行显示已传入与已检测时间,积压时显示等待处理;不增加三块同等醒目的仪表 |
|
||||
|
||||
玻璃效果是本次按用户要求对 SCADA 调频刻度窗的局部材质例外,外围面板外壳、表格、表单、菜单和 Agent 面板仍沿用现有 shadcn/ui 实体设计。新增时间指示颜色使用独立语义变量,不能让正常播放状态套用危险告警颜色。关闭透明效果或不支持背景模糊时,回退到清晰的实体刻度窗。
|
||||
|
||||
固定红线表示当前定位时间,刻度带按 5 分钟时间戳移动;只在选定区间内标注时间,两端之外留白,不带出整个月的刻度。该窗是选定区间的只读定位视窗,完整起止范围始终由上方文字明确表达;拨动不改变传入或检测进度。
|
||||
|
||||
手动拖动和触屏横拨改变查看位置,不立即改写后台时间。拨动到已释放时间时显示历史快照并标明回看;拨动到尚未释放时间时只显示待传入时间,不展示未来读数,若需从该位置开始则明确新建回放。回到当前只恢复跟随,不补发旧数据、告警或调查。
|
||||
|
||||
时间拨动按 slider 语义提供中文名称、时间范围、当前时间文本和键盘操作:方向键按 5 分钟刻度调整,PageUp/PageDown 调整一个主刻度,Home/End 到选定范围两端。键盘操作即时生效;鼠标和触摸拖动轨迹连续无缝、不带惯性,选定时间按 5 分钟量化。减少动态效果时关闭装饰性过渡,保留真实时间定位。状态不用颜色单独表达。
|
||||
|
||||
## 数据质量与业务异常
|
||||
|
||||
真实外部系统新增观测时,先通过现有入库边界将 raw 保存到 TSDB。本次演示使用已经入库的原始值,读取 API 校验身份、项目、设备与时间范围后返回 raw,直接进入质量校验和清洗;不为了模拟传入而重复执行原始值 INSERT。清洗只写回同一条观测的 cleaned_value,monitored_value 始终保留。
|
||||
|
||||
清洗完成后才启动业务异常检测。valid 可将 cleaned_value 设为原始值,corrected 保存有依据的修正值,无法可靠修正的 invalid 或 missing 保持空值并保存质量结论。suspect 保留 raw 与不确定性,不能自动替换为正常值。是否已经清洗以阶段回执和质量结论为准,不能仅用 cleaned_value 是否为空判断。
|
||||
|
||||
| 质量结论 | 选择值与后续处理 |
|
||||
| --- | --- |
|
||||
| valid | 使用 raw,包括可靠采集的突降、突升、低压和浓度变化 |
|
||||
| corrected | 使用有依据的修正值,同时保留 raw、修正值、规则版本、原因和可用时间 |
|
||||
| suspect | 保留 raw 并声明不确定性,结合其他测点和设备状态复核;不自动改成正常值 |
|
||||
| invalid 或 missing | 保留原始记录或缺测事实,退出要求可信观测的规则输入,形成数据质量事件 |
|
||||
|
||||
质量校验依据结构、有限数值、单位、设备身份、观测顺序、重复记录、量程与设备状态;仅偏离日常水平不能直接判 invalid。负流量、零浓度、低压均需按设备和业务含义解释。缺少仪表状态时明确证据不足,不能凭一次尖峰确认仪表损坏。
|
||||
|
||||
当前正式表中的 `cleaned_value` 可以保留,但不等于本轮已经完成清洗。读取 raw 时不清空已有 corrected,也不将旧 corrected 当作清洗输入。清洗阶段使用本批 raw 和本轮已知历史重新判断,提交本轮清洗版本并保存处理结果;检测只能使用该版本已提交的结果,不能直接取一个来源和时间不明的旧 corrected 当作已校验输入。
|
||||
|
||||
在线修正只使用当前已到达数据。当前居中窗口、后向补值、整段 Kalman 平滑和整段统计阈值保留为离线分析能力;不能一次性清洗整个月后将结果送入时间轴。在线清洗更新目标 TSDB 的修正值投影,原始值不变;原修正版本、当前修正版本和依据保留在处理证据中,旧回放仍读取自己的固定版本。
|
||||
|
||||
每条参与判断的数据记录其 `value_source`、质量结论、选用值、原始记录引用、规则版本和计算时间。回放过程不为未知错误强制生成一个修正数值。
|
||||
|
||||
这一区分对应 [EPA 在线水质监测指南第 6.1、6.2 节](https://nepis.epa.gov/Exe/ZyPURL.cgi?Dockey=P100UJYF.txt) 的“先校验、再异常检测,并保留原始值”原则;历史修订留痕可参考 [OPC UA Historical Access 第 4.4 节](https://reference.opcfoundation.org/specs/OPC-10000-11/4)。本项目的具体质量判定仍由指标和设备证据决定。
|
||||
|
||||
## 检测规则和多次事件
|
||||
|
||||
首版实时检测使用有版本的类型化规则:压力变化与上下限、流量变化与方向、水质具体指标的上下限或变化,以及数据质量规则。规则输入包含质量结论,输出只表示异常信号及依据,不确认爆管位置或污染原因。
|
||||
|
||||
每项启用的规则必须声明:`rule_id`、指标、方向、单位、阈值、使用的历史窗口、最少有效样本、连续命中点数、恢复阈值、连续恢复点数,以及事件组和最少同时命中的设备数。参数在创建回放前校验并冻结,至少需要一种已配置规则才可创建检测回放。缺失参数时该规则为 not_configured,不能借用统一 30% 或把零到非零统一解释为 100% 污染风险。
|
||||
|
||||
同一个事件组由明确的设备集合组成,不使用预设异常真值自动分组。同组、同规则、同一持续异常归并为一个事件;不同事件组可以同时产生不同事件。恢复需要可信的新观测满足恢复条件,缺测不构成恢复。恢复后的再次命中建立新事件。
|
||||
|
||||
事件保存首次异常、检测触发、最后异常、恢复时刻、证据引用和调查结论。传感器恢复正常只表示观测恢复,不代表现场维修、污染处置或业务调查已经完成。
|
||||
|
||||
事件身份由 `replay_id + rule_id + event_group_id + first_trigger_at` 确定;同一事件更新不创建多条调查。批次重试不重复创建事件,新一次回放的事件与旧回放独立。
|
||||
|
||||
现有 [爆管侦测服务](../app/services/burst_detection.py) 可作为满足条件后的进一步分析工具,不能直接等同于逐批在线检测器。当前 target 模式需要 15 天数据及至少 5 个完整压力测点。回放前没有足够历史时,展示 baseline_not_ready;支持短历史的规则仍可运行,不能用未来几天补足参考窗口。
|
||||
|
||||
正常基线只使用当时已知、质量合格且适用于该规则的历史。已确认异常可以从正常基线中排除,原始记录仍保留。事后确认或重新标注产生新版本,不改写此前检测“当时知道了什么”。
|
||||
|
||||
## 持久化和事务
|
||||
|
||||
观测数据以项目目标 TSDB 的 `scada.measurements` 为权威存储,沿用 time、device_id、monitored_value 和 cleaned_value。业务库不替代 TSDB 保存正式观测。
|
||||
|
||||
建议在 BizDB 增加以下处理对象:
|
||||
|
||||
| 对象 | 权威信息 |
|
||||
| --- | --- |
|
||||
| `peripheral_sample.scada_replays` | 回放身份、项目、创建者、冻结配置、生命周期、版本、模拟时间锚点、读取与处理进度、工作租约和恢复检查点 |
|
||||
| `peripheral_sample.scada_replay_batches` | 本批 TSDB 记录键、raw 摘要、固定规则版本、清洗输出、质量结论、检测结果、处理阶段和重试信息 |
|
||||
|
||||
两个对象分别具有控制生命周期与批次处理生命周期。现有 test_data 是手工单条演练 CRUD,缺少顺序、批次和消费语义;任务表表示业务调查,也不能存正常观测批次。这里保存记录引用和处理结果,不复制一整套 raw;需要固化异常证据或修正依据时,只保存必要的原始片段。由于本次不重新摄入 raw,不新增 TSDB 原始值收件表。表和列同时写清数据库注释。
|
||||
|
||||
事件及当前告警复用 `workbench.alert_events`、`workbench.alerts`,调查及其证据复用现有 operational 表。告警来源扩展为判别联合:保留现有任务来源,新增 `kind=scada_replay`、`replay_id`、`dataset_revision`、规则组与触发批次,避免伪造调查身份。质量故障也用同一事件存储,按类别区分。
|
||||
|
||||
### 批次处理与提交
|
||||
|
||||
不使用一个长期跨库事务。批次按以下阶段单调推进,重试保持原批次身份:
|
||||
|
||||
1. `selected`:后台读取 raw 后,BizDB 固定记录键、输入摘要、规则与知识截止时间,并在同一事务登记处理身份和读取检查点。GET 本身不做此写入。
|
||||
2. `clean_prepared`:直接对已读入的这一批 raw 和已加载的历史状态计算质量与修正结果,先将确定的输出及输入摘要保存在 BizDB。正常路径不调用会重新查询同批 raw 的整段清洗入口。
|
||||
3. `cleaned`:TSDB 用一个有界事务按记录键更新 cleaned_value,并校验 raw 仍与本批输入一致;不更新 monitored_value。只有 TSDB 提交成功且 BizDB 记录已提交阶段后,才允许检测。单行结果相同时不必重写字段。
|
||||
4. 若 TSDB 已提交但 BizDB 阶段尚未更新,恢复时核验预存的输入与输出;已经匹配则补记 cleaned,未写入则幂等应用预存结果,raw 已变则停止该批次。只在这种恢复或校验冲突路径增加读取,不为每批正常处理增加一次重复查询。
|
||||
5. `detected`:直接使用本批已提交的清洗结果做检测,BizDB 在同一事务保存检测结果、事件状态、调查意图、检查点和 processed_until,然后发布业务事件。检测失败只重试本阶段,不重新清洗或重写 raw。
|
||||
|
||||
以上阶段保存在批次中。页面继续突出模拟时间、已读取进度和已检测进度,清洗中或失败作为当前批次状态展示。正常路径对新批次读取一次 raw、必要时批量更新一次修正值,再复用这批处理结果完成检测;界面后续查看与故障恢复属于独立的必要读取。
|
||||
|
||||
BizDB 批次状态是持久待处理队列,通知只负责唤醒。后台启动、通知中断与定期对账均扫描未结束批次,按已提交阶段恢复;同库状态与事件在同一事务保存,不能假定两个数据库天然原子。[PostgreSQL NOTIFY](https://www.postgresql.org/docs/15/sql-notify.html) 和 [SELECT 锁定语义](https://www.postgresql.org/docs/15/sql-select.html) 提供通知及领取基础。
|
||||
|
||||
调用慢算法和外部服务前结束数据库事务,不持有事务锁等待 HTTP。清洗结果未提交,不执行异常检测;中断后从已有检查点恢复,不丢弃批次身份重新处理一遍。
|
||||
|
||||
### 重复读取与重复回放
|
||||
|
||||
读取 API 的同一次游标请求可以安全重试,但相同数据不能被当作新的处理批次。后台以项目、回放、批次序号和分片序号登记幂等身份,记录级使用规范设备 ID 与原始时间;同键同摘要复用现有处理状态,不同摘要返回冲突。
|
||||
|
||||
新一轮演示按新的 replay_id 建立独立处理状态,可以重新读取和分析选定区间,但仍不插入 raw。界面统计已读取条数和已处理条数,不能称为新写入 SCADA 条数。
|
||||
|
||||
首版同一项目只允许一条活动的读取清洗流水线,避免不同回放同时改写同一 cleaned_value。历史结果仍可同时查看。修正字段表示最近一次已发布清洗结果,旧版结果和依据保留在批次输出及必要的原始证据中;旧 Agent 执行和历史回看不直接读取被后续处理替换的修正投影。
|
||||
|
||||
工作台事件流只发布回放控制状态变化、故障与业务告警;不为每个测点值发布 SSE,也不把全部原始数据放入事件日志。未来若新增 `scada.replay.changed` 主题,只用于控制状态,并需同步主题模型、约束和消费契约;当前控制进度由轮询读取。
|
||||
|
||||
未读取数据所在 raw 分片摘要变化时停止该回放,要求新数据集版本;本轮 cleaned_value 更新不计作源版本漂移。空检查批次也需要保存处理阶段,记录数为零不代表可以绕过缺测判断。
|
||||
|
||||
每个回放同一时刻只有一个有效处理租约,采用数据库会话锁阻止两个请求重复推进同一周期。每个批次提交后再读取下一原始时间戳,不持久保存进程计时器值。
|
||||
|
||||
源数据和相关回放证据在本次演示中不自动过期;一个月是数据覆盖范围,不是保存期限。事件结束、暂停或重播不清理已有证据。生产保留策略单独制定。
|
||||
|
||||
## Agent 调查与时间隔离
|
||||
|
||||
主动单步在边界形成组合监测回执:一般关注和数据不足只保存回执,只有 `suspected_burst` 进入 `burst_report`。SCADA 演示请求只负责建立持久调查任务,不要求先生成一条合成 test_data,也不在请求内执行 Agent。
|
||||
|
||||
告警事实与“待建立调查”意图在事件事务中保存。创建调查失败时后台对账重试;组合监测调查的来源稳定标识使用 `scada_replay:<replay_id>:monitor:<period_end>:<rule_version>`,一般事件继续使用 `scada_replay:<replay_id>:event:<event_id>`,均沿用现有外部来源幂等约束。事件不因 Agent 不可用而丢失。
|
||||
|
||||
已新增 `scada_anomaly` 工作流与对应 Skill,要求先核验数据质量、点位和单位,再读取事件窗口及正常参考。疑似爆管复用现有 `burst_report` Skill;它会区分侦测、定位候选和现场确诊,并要求漏量依据。`run_burst_location` 在回放任务中通过冻结输出适配执行,其他未适配计算继续拒绝;不能仅修改 Skill 文本或 MCP allowlist 绕过知识截止。重计算沿用 [后台计算队列](calculation-queue.md),保留外层 HTTP 限制。
|
||||
|
||||
新增 Server 持有的回放查询上下文:`replay_id`、数据集版本、知识截止时刻 `knowledge_cutoff`、已接收批次上限、处理版本和来源事件。它与任务及执行代次绑定,不能由模型或客户端自行扩大。
|
||||
|
||||
每代调查开始时冻结其可用批次与时间上限;同一执行中后续回放推进不会偷偷改变证据。确有新的重要证据时,按现有 [调查轮次规则](operational-tasks.md) 创建或合并后续调查轮次,而不是每批次追加一次模型执行。
|
||||
|
||||
时间隔离必须覆盖 MCP 的 SCADA 查询、要素历史、任务数据快照、外围观测,以及检测算法内部读取。raw 源读取 API 只允许读取不晚于模拟时钟的观测;检测只使用已完成本轮清洗的批次结果,原始表格可以显示本轮已读取且尚在清洗的 raw。历史回看和旧代调查读取对应固定版本或原始记录引用,异常证据使用已经固化的必要片段。即使 TSDB 存有整个月记录,也必须按本轮读取检查点、设备集合和知识截止时间过滤,不能只在提示词中要求遵守。
|
||||
|
||||
所有回放调查证据保留真实来源:合成数据仍为 synthetic,历史回放有明确标识,不能因为读取的是正式表就改称现场实时观测。普通历史数据管理仍可按原权限查看数据集,但不能作为回放会话绕过截止时间的途径。
|
||||
|
||||
回放调查只生成演示分析、计划和报告;真实外围工单和现场控制不属于自动回放动作。Server 动作执行边界必须识别回放上下文,不能仅靠 Agent 提示阻止将演示事件发成真实业务工单。
|
||||
|
||||
## 接口与界面
|
||||
|
||||
以下为已经加入 Server OpenAPI 的路径,统一使用 Bearer 与 X-Project-Id。保留原有 `/peripheral/api/sample/v1` 工单与手工演练接口,回放不增加一层外围代理。
|
||||
|
||||
| 接口 | 语义 |
|
||||
| --- | --- |
|
||||
| `GET /api/v1/scada-replays/capabilities` | 回放表可用性、写权限、监测频率、爆管组合监测角色是否就绪及最近一轮状态;不代表完整月度数据就绪 |
|
||||
| `POST /api/v1/scada-replays` | 固定区间、设备、规则和质量量程,返回 ready;携带稳定 request_id |
|
||||
| `GET /api/v1/scada-replays/{replay_id}` | 权威控制、时间、计数、设备与固定规则、当前输出及故障 |
|
||||
| `POST /api/v1/scada-replays/{replay_id}/commands` | play、pause、step、set_speed、stop、retry;request_id + expected_revision |
|
||||
| `GET /api/v1/scada-replays/{replay_id}/observations?through=...` | 仅本轮已读取时间范围内的历史快照,每设备最近一条、最多 500 个 |
|
||||
| `GET /api/v1/timeseries/scada-readings/raw` | 当前项目 TSDB 原始值联合游标分页;不插入、不隐式清洗 |
|
||||
| `GET /api/v1/scada-replays/{replay_id}/events?offset=0&limit=100` | 检测变化与调查关联,返回 items、next_offset |
|
||||
|
||||
raw API 允许参数为 start_time、end_time、device_ids、cursor、limit;回放请求携带经服务端验证的 replay_id,任务请求由已有绑定解析回放身份。有效范围为所选区间与模拟时间上限的交集,不能因省略 replay_id 绕过已绑定的任务限制。返回 items(time、device_id、monitored_value、quality)、next_cursor 和 scanned_through;本版不返回尚未准备的 dataset_revision,不返回 corrected 作为源输入。
|
||||
|
||||
next_cursor 绑定项目、设备、查询区间并固定本轮分页时间上限;本版不为整个查询提供 MVCC 源版本快照,按 time、device_id 排序,每页最多 500 条。读取窗口未结束但当前页为空时,scanned_through 表示已经确认没有观测的区间上限,供后台登记空检查批次。分页、刷新和网络重试均是无副作用读取。
|
||||
|
||||
查询和检查点更新使用类型化服务方法及已验证项目上下文。清洗函数接受已经读入的批次,不再调用当前会重新查询 raw 的整段清洗入口;检测函数接受这批已提交清洗结果。页面的单步 POST 命令同步完成一个监测周期,GET 预览和回看不启动任务。
|
||||
|
||||
回放读取要求 scada.view,创建及控制要求 scada.clean;普通 raw API 沿用 timeseries SCADA 路由的读权限。API 不自动创建表,不回退默认项目,也不隐式枚举全部活跃项目。
|
||||
|
||||
前端 SCADA 页签按以下顺序组织:
|
||||
|
||||
1. 选定演示区间:开始时间、结束时间和项目监测步长;设备范围、规则及基线收纳在折叠的检测参数区域。
|
||||
2. FM 调频式时间轴:局部玻璃刻度窗、固定红色指示线、两侧渐暗及中央放大;按 5 分钟时间戳定位,不读取逐测点列表,也不触发监测。
|
||||
3. 主动监测操作:以“开始并监测”或“下一步并监测”为唯一主操作,按项目频率同步处理一个周期。
|
||||
4. 本周期结论:显示正常、关注、数据不足或疑似爆管,以及是否已经进入爆管调查与定位流程。
|
||||
|
||||
设备目录、raw 与修正值明细、完整事件列表和手工单点异常录入不在演示主面板展示。底层数据与 API 继续保留,供诊断、追溯和调查任务按权限读取。
|
||||
|
||||
沿用现有 workbench-shadcn-surface、日期控件和实体表面,调频刻度窗采用上文明确的局部玻璃例外;时间轴支持键盘操作和中文可访问名称。拖动回看不触发后台写操作。
|
||||
|
||||
页面可见时每 5 秒读取一次权威控制快照;单步请求执行期间由操作本身返回最终快照。页面隐藏时停止界面采样。演示面板不轮询完整事件列表,调查入口需要时再按分页 API 查询。
|
||||
|
||||
## 实施范围与验证
|
||||
|
||||
这是跨 Server、供水前端和 Agent 契约的变更,不新增后台进程。数据、存储、身份及网络复用现有部署。
|
||||
|
||||
| 仓库 | 主要落点 |
|
||||
| --- | --- |
|
||||
| Server | `app/api/v1/endpoints/scada_replay.py` 的回放控制与能力;`app/services/` 和 `app/infra/db/` 的批次、质量和事件;现有 task_alert、operational_task、MCP 上下文、查询和 Skill |
|
||||
| Server 契约与存储 | `resources/sql/` 的 BizDB 两张回放表迁移;`contracts/` 的 raw 读取、Server OpenAPI 和清单;外围协议不变;TSDB 沿用已有观测表 |
|
||||
| 前端 | `src/features/workbench/peripheral-sample/` 的回放控制与状态 hook;SCADA 客户端与生成契约;告警来源和事件解析 |
|
||||
| Agent | `src/product/` 的生成契约及任务上下文适配,保留已有原生 MCP 与执行权限边界 |
|
||||
|
||||
验收用例必须覆盖:
|
||||
|
||||
- 5/15 分钟设备混合、乱序源记录、同一时间戳跨页、空片段、结束边界和多次异常。
|
||||
- 重复批次、相同编号不同内容、响应丢失、请求中断和恢复重试;不漏读、不重复事件。
|
||||
- GET 重试、分页重复请求、清洗输出预存后中断、TSDB 更新提交而 BizDB 阶段未更新、检测失败;逐阶段恢复,不提前检测。
|
||||
- 正常处理每批只读一次 raw,清洗后检测复用结果;回放不执行 raw INSERT,不在读取时清空旧 corrected,空数据不伪造观测。
|
||||
- 周期内多批次单步、重复单步拒绝、并发请求互斥和尾步完成状态。
|
||||
- 5 分钟时间定位与 15 分钟监测周期相互独立、非整点起点、周期配置变更不影响当前回放、尾步不足完整周期和未配置时禁止开始。
|
||||
- 每个步长边界只产生一份幂等监测回执;正常、数据不足、单指标关注和疑似爆管分别处理,不为正常周期创建 Agent。
|
||||
- 爆管侦测基线不足、压力下降、出厂流量变化、设备角色错误、指标方向冲突和多指标同时命中;只有疑似爆管建立 `burst_report`。
|
||||
- 回放 MCP 侦测与定位只能读取固定批次和知识截止前数据;定位缺少漏量依据时停止在待补充,不调用算法。
|
||||
- 选定小时区间、跨日区间和再次演示;刻度与进度只对应选定区间,基线初始化不释放旧事件,到终点不自动播放整月。
|
||||
- FM 红线固定、刻度带移动、拖动回看不重复投递、未来位置不泄露读数;桌面与 375px、320px 窄屏下的长日期、键盘、减少动态效果及玻璃回退。
|
||||
- 真实突降不被当作采集误差修掉;负流量、零水质值、空值、错误单位和缺测恢复分别处理。
|
||||
- 首个异常、持续异常、可信恢复、再次异常及两个事件组并发;不会逐点创建调查。
|
||||
- 缺少基线、缺少水质指标配置、历史 corrected 无可用时间,以及 source_revision 变化。
|
||||
- MCP、任务数据接口及算法内部的未来查询均被拒绝;下一代调查才能使用新增证据。
|
||||
- 合成来源、历史回放来源、事件真值隔离;回放不发布真实现场动作。
|
||||
- 合成测试链路、真实项目 API、常驻后台、真实 Agent 执行和浏览器分别记录验证结果。
|
||||
|
||||
后续实现的验证命令从各仓库执行:Server 运行新增回放、批次、质量、事件与查询边界的聚焦 pytest,并保留现有 SCADA/告警/调查回归;前端运行 `pnpm lint`、`pnpm typecheck`、`pnpm test` 和聚焦 `pnpm test:browser`;Agent 修改后运行 `bun run check` 及对应聚焦测试。具体已执行结果见下方,不把待部署链路计为通过。
|
||||
|
||||
数据库应用遵守 [工作台数据库变更门禁](workbench-database-rollout.md):先核对目标项目,备份和回滚演练,再按明确范围应用及验收;不得默认向全部项目写入。本地 MetaDB 已对明确的演示项目应用并复验 `033_metadata_project_settings.sql`,其他项目保持原值;`tjwater2_v2` BizDB 已应用 `031_scada_replays.sql`,不据此推断其他项目已迁移。本次没有生成一个月观测或启动常驻进程。
|
||||
|
||||
回退应用前暂停新批次读取,核对 clean_prepared、cleaned 与 detected 阶段,保留原始观测、已提交修正值、回放表、告警与调查证据;升级恢复后从原阶段继续。BizDB 流程记录和 TSDB 修正值不在同一事务,回滚应用不能假定另一库结果也被撤回。不能通过删除事件、重置时间游标或清空 SCADA 原始表来伪装成从未处理。若需要撤回数据库变更,另按备份与新增业务写入情况制定恢复步骤,不在仍有写入时覆盖数据库。
|
||||
|
||||
本方案依赖的关键前提是源历史能提供稳定设备身份、可信时间戳和固定数据版本。缺少完整月度观测时,能力接口只能声明实际可用范围;缺少修正来源时采用 raw;缺少检测参数时该类检测不启用。任何这些缺口都不能由时间轴、插值或预设事件答案掩盖。
|
||||
|
||||
## 部署准备与验证记录
|
||||
|
||||
迁移脚本为 [031_scada_replays.sql](../resources/sql/031_scada_replays.sql),显式限制物理数据库 `tjwater_v2` 与 `tjwater2_v2`,依赖既有 peripheral_sample schema、告警及调查结构。仅新增两张 BizDB 表,不新增 TSDB 观测表。`tjwater2_v2` 已应用该迁移并配置 `settings.scada_monitoring_interval_seconds=900`。
|
||||
|
||||
真实数据库验证使用 `SCADA_REPLAY_STORAGE_TEST=1 conda run -n server python -m pytest tests/integration/test_scada_replay_storage.py -q`。该测试在项目路由选出的 BizDB 与 TSDB 双回滚事务中临时建表、写测试观测,覆盖真实 SQL 的 cleaned 更新、事件持久化和主动周期推进,退出全部回滚;它不证明身份认证 HTTP 和真实 Agent 推理已经验收。浏览器测试以 mock Server 验证交互与窄屏布局,不能作为现场检测效果证明。
|
||||
|
||||
本轮验证结果(2026-11-03,当前工作区):
|
||||
|
||||
| 层级 | 结果与边界 |
|
||||
| --- | --- |
|
||||
| Server | 主动监测、条件式爆管调查、冻结定位适配、MCP、项目配置、raw、处理及 OpenAPI 聚焦回归共 120 项通过 |
|
||||
| 数据库与契约 | 本地 MetaDB 的 033 迁移已单项目应用并重复执行验证;OpenAPI 279 条路径与清单摘要一致;本轮未运行 031 的真实双库集成测试 |
|
||||
| 前端 | lint、typecheck、全量 148 文件共 884 项 Vitest 通过;设备列表、手工单点演练及对应触发适配已移除 |
|
||||
| 浏览器 | 1 项 mock Server 场景通过,包含 5 分钟刻度、选择值与键盘步进、连续拖动轨迹、15 分钟主动监测、中央放大、两侧渐暗、固定红针及 1280/375/320px 布局;未作为真实检测效果证明 |
|
||||
| Agent | 同步 Server OpenAPI 并生成 server-api.d.ts,bun run check 通过;无真实模型推理 |
|
||||
| 仓库 | Server、供水前端、Agent 的 git diff --check 通过;未提交、推送或部署 |
|
||||
|
||||
生产联调仍需核验真实 HTTP 接口、调查执行链路和 Agent 报告,并单独准备完整月度数据及事件真值。不能将以上局部测试解释为这些环节已经完成。
|
||||
@@ -0,0 +1,67 @@
|
||||
# tjwater2_v2 爆管模拟校准设计
|
||||
|
||||
状态:参数与只读校验已准备,尚未运行水力模拟。目标项目为 `tjwater2_v2`(演示项目2(新版)),不是 `tjwater_v2`。
|
||||
|
||||
## 目标
|
||||
|
||||
先用现有管网、阀门和监测点选择可观测的事故管段,再在隔离评测数据库中校准破口面积。首版使用管道截面积的 1/32、1/16、1/8;1/8 是上限。若 1/8 仍不能形成明确降压信号,该管段退出 SCADA 爆管用例,不继续扩大破口。
|
||||
|
||||
模拟调用链为:
|
||||
|
||||
```text
|
||||
只读源数据快照 -> 隔离 BizDB/模型模板 -> 正常对照模拟
|
||||
-> 爆管三档校准 -> 选择最小通过面积
|
||||
-> 保存正式分析结果 -> 映射 SCADA 数据
|
||||
```
|
||||
|
||||
当前 `temporary_project_database` 会从项目模型模板创建临时数据库,同时短暂禁止模板连接并终止模板会话。因此即使 `run_burst_simulation` 不保存结果,也不能直接针对原项目执行。正式运行必须把只读逻辑快照恢复到独立评测数据库及其模板,Server、worker 和 Agent 全部使用评测路由。
|
||||
|
||||
## 方法参数
|
||||
|
||||
`run_burst_simulation` 接收带时区的 `start_time`、60 至 86400 秒的 `duration`、1 至 20 个输出节点、至多 20 条输出管段、1 至 20 条爆管管段,以及与管段一一对应的正数孔口面积 `orifice_areas_cm2`。本次每个调用只模拟一条爆管管段,等待预算使用 240 秒;排队成功只表示已接受,仍需查询计算任务和最终结果。
|
||||
|
||||
`tjwater2_v2` 使用 LPS 流量单位、米制压力、0.5 emitter 指数,水力和报告步长均为 15 分钟。压力设备值使用 m,流量设备值使用 m³/h。3600 秒模拟必须产生 0、15、30、45、60 分钟五个结果时刻;缺少任一时刻即失败。爆管服务把 emitter 放在管段末端 junction,切换临时模型为 PDA,并设置 `REQUIRED PRESSURE=10 m`。
|
||||
|
||||
孔口面积是输入,不是漏水流量。服务的 emitter 系数为:
|
||||
|
||||
```text
|
||||
coefficient[L/s/sqrt(m)] = 0.65 * area[cm²] / 10000 * sqrt(19.6) * 1000
|
||||
```
|
||||
|
||||
正式运行后,定位使用的泄漏量按事故节点相对正常对照的 demand 增量计算,并从 L/s 除以 1000 转成 m³/s。流量监测设备单位为 m³/h,模型流量结果为 L/s,映射 SCADA 时乘以 3.6。
|
||||
|
||||
## 管段与场景
|
||||
|
||||
主场景管段 `541720` 为 DN800、长 483.038 m,破口节点 `502046`。附近两个压力点沿管网约 1.2 km,最近流量点约 1.2 km;两座边界阀可隔离 11 个节点。其破口面积为 157.079633、314.159265、628.318531 cm²。
|
||||
|
||||
清单同时保存 SCADA 设备 ID 与其实际水力模型对象 ID,不能假定两者相同。例如压力设备 `447382`、`444985` 分别绑定节点 `7543252226207002012`、`14141146287589625673`,流量设备 `542121` 绑定管段 `pipe-eaa5c512bd8f4649b37bdbe3d`。模拟输出使用模型对象 ID,得分与数据映射使用设备 ID。
|
||||
|
||||
启动校验还要求所选压力设备到绑定节点、流量设备到绑定管段的本地坐标距离均不超过 1 m。全量复核中待人工确认的 `542017`、`547063` 和 `DEMO-WQ-551571` 不进入本套爆管 benchmark;不得为通过校验而自动吸附设备坐标或猜测改绑对象。
|
||||
|
||||
次级场景管段 `520323` 为 DN300、长 33.5951 m,破口节点 `91055`。最近压力点沿管网约 262 m,五座边界阀可隔离 19 个节点。其破口面积为 22.089323、44.178647、88.357293 cm²。
|
||||
|
||||
两条管段分别在北京时间 2026-10-08 02:00 和 12:00 模拟,形成 B01 至 B04 四个物理场景。每个场景都必须有同时间、同持续时长、相同模型状态的无爆管正常对照。严重档先验证是否可观测;通过后依次测试中度和轻度,最终选择满足要求的最小面积。最坏情况下执行 12 次爆管校准模拟,正常对照按项目、时间和模型版本复用。
|
||||
|
||||
## 通过条件
|
||||
|
||||
一个面积只有同时满足以下条件才可冻结:
|
||||
|
||||
1. 至少一个指定压力点连续两个 15 分钟时刻较正常对照下降 30%。
|
||||
2. 第二压力点下降至少 10%,或指定流量点相对正常对照变化至少 20%。
|
||||
3. 正常对照不触发相同规则。
|
||||
4. 指定监测节点不出现负压,模拟正常收敛,所有请求输出完整。
|
||||
5. 同一管段的夜间和白天场景使用同一面积;两者都通过时才能进入正式 benchmark。
|
||||
|
||||
压力与流量变化都按相同时间戳的正常对照计算:`abs(事故值 - 正常值) / max(abs(正常值), 1e-6)`。压力规则只取下降方向,流量规则取绝对变化。不得用其他时段均值代替同时间戳对照。
|
||||
|
||||
压力下降、流量变化、实际泄漏量、计算状态和缺失输出都保存到校准报告。失败的档位保留,不重试到通过。若严重档失败,该管段标记为不适用;若较小档位失败而较大档位通过,选择最小通过档位。
|
||||
|
||||
## 校验与后续执行
|
||||
|
||||
清单保存在 `resources/benchmarks/tjwater2-v2-burst-simulation.json`。以下命令使用本地占位 context handle,把全部 12 组计划参数送入当前 `BurstSimulationInput` 做契约校验,再只读检查 Metadata 项目身份、当前 BizDB、模型模板、管段属性、模型单位、步长、监测点、设备到绑定对象的距离、边界阀和既有 emitter;它不创建运行上下文,也不调用模拟工具:
|
||||
|
||||
```bash
|
||||
conda run -n server python scripts/check_burst_simulation_design.py
|
||||
```
|
||||
|
||||
隔离评测数据库准备完成后,runner 先按清单调用非持久 `run_burst_simulation` 校准。面积冻结后,通过 `save_simulation_scenario` 在评测数据库保存全量节点和管段结果,生成固定的运行 ID 和数据摘要。模型分析结果与模拟 SCADA 保留明确来源,不标记为实测数据。
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"version": "burst-simulation-design@1",
|
||||
"suite_id": "tjwater2-v2-burst-calibration",
|
||||
"project_code": "tjwater2_v2",
|
||||
"project_id": "871d7538-589c-47de-a6d5-3185ccb9f0e1",
|
||||
"model_expectations": {
|
||||
"node_count": 92392,
|
||||
"pipe_count": 91049,
|
||||
"valve_count": 4489,
|
||||
"flow_unit": "LPS",
|
||||
"pressure_unit": "METERS",
|
||||
"pressure_measurement_unit": "m",
|
||||
"flow_measurement_unit": "m3/h",
|
||||
"emitter_exponent": 0.5,
|
||||
"hydraulic_timestep_seconds": 900,
|
||||
"report_timestep_seconds": 900
|
||||
},
|
||||
"duration_seconds": 3600,
|
||||
"event_times": [
|
||||
"2026-10-08T02:00:00+08:00",
|
||||
"2026-10-08T12:00:00+08:00"
|
||||
],
|
||||
"severity_levels": [
|
||||
{"level": "light", "cross_section_fraction": 0.03125},
|
||||
{"level": "medium", "cross_section_fraction": 0.0625},
|
||||
{"level": "severe", "cross_section_fraction": 0.125}
|
||||
],
|
||||
"calibration_order": ["severe", "medium", "light"],
|
||||
"selection_rule": {
|
||||
"primary_pressure_drop_percent": 30,
|
||||
"corroborating_pressure_drop_percent": 10,
|
||||
"flow_change_percent": 20,
|
||||
"consecutive_samples": 2,
|
||||
"expected_sample_count": 5,
|
||||
"relative_change_epsilon": 0.000001,
|
||||
"normal_control_must_not_trigger": true,
|
||||
"negative_monitor_pressure_allowed": false,
|
||||
"choose_smallest_passing_area": true
|
||||
},
|
||||
"binding_validation": {
|
||||
"max_pressure_distance_m": 1.0,
|
||||
"max_flow_distance_m": 1.0
|
||||
},
|
||||
"cases": [
|
||||
{"case_id": "B01", "pipe_id": "541720", "occurred_at": "2026-10-08T02:00:00+08:00"},
|
||||
{"case_id": "B02", "pipe_id": "541720", "occurred_at": "2026-10-08T12:00:00+08:00"},
|
||||
{"case_id": "B03", "pipe_id": "520323", "occurred_at": "2026-10-08T02:00:00+08:00"},
|
||||
{"case_id": "B04", "pipe_id": "520323", "occurred_at": "2026-10-08T12:00:00+08:00"}
|
||||
],
|
||||
"pipes": [
|
||||
{
|
||||
"role": "primary",
|
||||
"pipe_id": "541720",
|
||||
"diameter_mm": 800,
|
||||
"length_m": 483.038,
|
||||
"start_node_id": "532586",
|
||||
"emitter_node_id": "502046",
|
||||
"longitude": 121.483586,
|
||||
"latitude": 30.996219,
|
||||
"candidate_orifice_areas_cm2": {
|
||||
"light": 157.079633,
|
||||
"medium": 314.159265,
|
||||
"severe": 628.318531
|
||||
},
|
||||
"pressure_bindings": [
|
||||
{"device_id": "447382", "node_id": "7543252226207002012"},
|
||||
{"device_id": "444985", "node_id": "14141146287589625673"},
|
||||
{"device_id": "100026", "node_id": "100026"},
|
||||
{"device_id": "115039", "node_id": "115039"},
|
||||
{"device_id": "107672", "node_id": "107672"}
|
||||
],
|
||||
"flow_bindings": [
|
||||
{"device_id": "542121", "link_id": "pipe-eaa5c512bd8f4649b37bdbe3d"},
|
||||
{"device_id": "449417", "link_id": "449417"},
|
||||
{"device_id": "516234", "link_id": "516234"}
|
||||
],
|
||||
"boundary_valve_ids": [
|
||||
"valve-6fd58ab5b5714fbab738a544",
|
||||
"valve-8aea6a1876d8475b9f35a845"
|
||||
],
|
||||
"output_node_ids": [
|
||||
"12290448554375617026", "1371402016460607520", "400946", "413270",
|
||||
"414851", "465564", "466633", "482531", "493166", "502046",
|
||||
"532586", "7543252226207002012", "14141146287589625673",
|
||||
"100026", "115039", "107672"
|
||||
],
|
||||
"output_link_ids": [
|
||||
"541720", "pipe-eaa5c512bd8f4649b37bdbe3d", "449417", "516234",
|
||||
"valve-6fd58ab5b5714fbab738a544",
|
||||
"valve-8aea6a1876d8475b9f35a845"
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "secondary",
|
||||
"pipe_id": "520323",
|
||||
"diameter_mm": 300,
|
||||
"length_m": 33.5951,
|
||||
"start_node_id": "91054",
|
||||
"emitter_node_id": "91055",
|
||||
"longitude": 121.468305,
|
||||
"latitude": 30.932363,
|
||||
"candidate_orifice_areas_cm2": {
|
||||
"light": 22.089323,
|
||||
"medium": 44.178647,
|
||||
"severe": 88.357293
|
||||
},
|
||||
"pressure_bindings": [
|
||||
{"device_id": "91100", "node_id": "91100"},
|
||||
{"device_id": "98367", "node_id": "98367"},
|
||||
{"device_id": "69693", "node_id": "69693"},
|
||||
{"device_id": "78085", "node_id": "78085"},
|
||||
{"device_id": "105833", "node_id": "105833"}
|
||||
],
|
||||
"flow_bindings": [
|
||||
{"device_id": "527865", "link_id": "527865"},
|
||||
{"device_id": "489082", "link_id": "489082"},
|
||||
{"device_id": "531564", "link_id": "531564"}
|
||||
],
|
||||
"boundary_valve_ids": [
|
||||
"valve-18b95b02a7564e7dac737102",
|
||||
"valve-3a8fd38228d74583a08b00d9",
|
||||
"valve-3b453e4e901049d1a1346689",
|
||||
"valve-ad39de2f15f24ade9d18c836",
|
||||
"valve-fc1164434a954f418a6f11d6"
|
||||
],
|
||||
"output_node_ids": ["91054", "91055", "91100", "98367", "69693", "78085", "105833"],
|
||||
"output_link_ids": [
|
||||
"520323", "527865", "489082", "531564",
|
||||
"valve-18b95b02a7564e7dac737102",
|
||||
"valve-3a8fd38228d74583a08b00d9",
|
||||
"valve-3b453e4e901049d1a1346689",
|
||||
"valve-ad39de2f15f24ade9d18c836",
|
||||
"valve-fc1164434a954f418a6f11d6"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -19,13 +19,18 @@ CREATE TABLE IF NOT EXISTS projects (
|
||||
gs_workspace VARCHAR(100) UNIQUE NOT NULL,
|
||||
map_extent JSONB,
|
||||
map_config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
settings JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status VARCHAR(20) DEFAULT 'active' NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
CONSTRAINT projects_status_check CHECK (status IN ('active', 'inactive', 'archived'))
|
||||
CONSTRAINT projects_status_check CHECK (status IN ('active', 'inactive', 'archived')),
|
||||
CONSTRAINT projects_settings_object_check CHECK (jsonb_typeof(settings) = 'object')
|
||||
);
|
||||
|
||||
ALTER TABLE projects ADD COLUMN IF NOT EXISTS map_config JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
ALTER TABLE projects ADD COLUMN IF NOT EXISTS settings JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
COMMENT ON COLUMN projects.settings IS
|
||||
'项目级通用配置(JSON 对象,不存放密钥);scada_monitoring_interval_seconds 为监测任务间隔秒数。';
|
||||
COMMENT ON COLUMN projects.map_config IS
|
||||
'项目级地图配置(JSON);place_search 保存地名搜索启用状态、服务商、行政区编码及区域名称;保留既有图例与坐标参考配置。';
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
BEGIN;
|
||||
DO $$ BEGIN
|
||||
IF current_database() NOT IN ('tjwater_v2', 'tjwater2_v2') THEN
|
||||
RAISE EXCEPTION '031_scada_replays requires an explicitly verified SCADA demo project';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS peripheral_sample.scada_replays (
|
||||
project_id uuid NOT NULL,
|
||||
replay_id uuid NOT NULL,
|
||||
request_id uuid NOT NULL,
|
||||
status text NOT NULL CHECK (status IN ('ready','paused','completed','stopped','failed')),
|
||||
document jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (project_id,replay_id),
|
||||
UNIQUE (project_id,request_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS scada_replays_status_idx
|
||||
ON peripheral_sample.scada_replays(project_id,status,created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS peripheral_sample.scada_replay_batches (
|
||||
project_id uuid NOT NULL,
|
||||
replay_id uuid NOT NULL,
|
||||
sequence bigint NOT NULL,
|
||||
stage text NOT NULL CHECK(stage IN ('selected','clean_prepared','cleaned','detected')),
|
||||
document jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY(project_id,replay_id,sequence),
|
||||
FOREIGN KEY(project_id,replay_id) REFERENCES peripheral_sample.scada_replays(project_id,replay_id)
|
||||
);
|
||||
COMMENT ON TABLE peripheral_sample.scada_replays IS 'SCADA 演示回放的权威控制状态与处理检查点,不保存正式原始观测';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.project_id IS '经认证与路由校验的所属项目';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.replay_id IS '独立回放身份,不等同 Agent 执行编号';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.request_id IS '创建请求的稳定幂等编号';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.status IS '供领取和约束使用的 document.status 查询投影';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.document IS '冻结配置、设备快照、时钟、检查点、规则状态和命令回执';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.created_at IS '实际创建时间';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replays.updated_at IS '实际控制或进度更新时间';
|
||||
COMMENT ON TABLE peripheral_sample.scada_replay_batches IS 'SCADA raw 引用、清洗输出及检测阶段的恢复凭证,禁止重写 raw';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.project_id IS '所属项目';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.replay_id IS '所属回放';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.sequence IS '回放内单调批次序号';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.stage IS '已提交处理阶段';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.document IS '观测键、raw 摘要、确定的清洗结果与检测事件';
|
||||
COMMENT ON COLUMN peripheral_sample.scada_replay_batches.created_at IS '实际登记批次时间';
|
||||
COMMIT;
|
||||
@@ -0,0 +1,54 @@
|
||||
BEGIN;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('public.project_databases') IS NULL
|
||||
OR to_regclass('public.user_project_membership') IS NULL THEN
|
||||
RAISE EXCEPTION '033_metadata_project_settings must run in the project metadata database';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
ALTER TABLE public.projects
|
||||
ADD COLUMN IF NOT EXISTS settings jsonb NOT NULL DEFAULT '{}'::jsonb;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid = 'public.projects'::regclass
|
||||
AND conname = 'projects_settings_object_check'
|
||||
) THEN
|
||||
ALTER TABLE public.projects
|
||||
ADD CONSTRAINT projects_settings_object_check
|
||||
CHECK (jsonb_typeof(settings) = 'object');
|
||||
END IF;
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'projects'
|
||||
AND column_name = 'scada_monitoring_interval_seconds'
|
||||
) THEN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM public.projects
|
||||
WHERE scada_monitoring_interval_seconds IS NOT NULL
|
||||
AND settings ? 'scada_monitoring_interval_seconds'
|
||||
AND settings->'scada_monitoring_interval_seconds'
|
||||
IS DISTINCT FROM to_jsonb(scada_monitoring_interval_seconds)
|
||||
) THEN
|
||||
RAISE EXCEPTION 'Conflicting SCADA monitoring intervals; resolve before migration';
|
||||
END IF;
|
||||
|
||||
UPDATE public.projects
|
||||
SET settings = settings || jsonb_build_object(
|
||||
'scada_monitoring_interval_seconds', scada_monitoring_interval_seconds
|
||||
)
|
||||
WHERE scada_monitoring_interval_seconds IS NOT NULL;
|
||||
|
||||
ALTER TABLE public.projects DROP COLUMN scada_monitoring_interval_seconds;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
COMMENT ON COLUMN public.projects.settings IS
|
||||
'项目级通用配置(JSON 对象,不存放密钥);scada_monitoring_interval_seconds 为监测任务间隔秒数。';
|
||||
|
||||
COMMIT;
|
||||
@@ -2,7 +2,7 @@
|
||||
name: burst-report
|
||||
description: 调查供水爆管上报,形成有依据的候选位置、隔离阀门、影响范围和后续处置建议。
|
||||
metadata:
|
||||
version: "2.0.6"
|
||||
version: "2.0.8"
|
||||
---
|
||||
|
||||
# 爆管上报处置
|
||||
@@ -13,6 +13,8 @@ metadata:
|
||||
|
||||
读取当前任务、事件时刻、地点、来源、已有报告、证据、动作和反馈。新一轮优先检查新增事实和上轮未决项,按时间、来源、对象及输入比较新旧结果,不按 ID 推断先后,也不因原工单未更新就断言没有新证据。保留本流程名称和版本。
|
||||
|
||||
若任务来源带有 `replay_context`,这是 SCADA 历史回放中的疑似爆管演示。先读取任务摘要中的步长监测结论,再用 `query_scada_readings` 核对冻结批次;不得查询知识截止之后的数据。回放中的 `run_burst_location` 只接受 monitoring 模式,并要求显式提供本轮已处理范围内的异常窗和正常对照窗;Server 会从冻结清洗输出构造计算输入。缺少有依据的 `burst_leakage` 时保留“定位待补充”,不得用出厂流量增量、压力下降或默认值替代。回放结果保持 synthetic,不发布真实工单或设备动作。
|
||||
|
||||
## 定位与影响分析
|
||||
|
||||
根据现有线索自主选择必要步骤:
|
||||
@@ -26,9 +28,19 @@ metadata:
|
||||
|
||||
查询错误、权限拒绝或接口故障不是缺数证据。必要分析无法执行时,说明失败步骤和已经完成的部分,不借有限摘要宣布调查完成。
|
||||
|
||||
## 事件前后影响说明
|
||||
|
||||
每轮按实际证据说明以下阶段的影响,给出结果或具体限制;尚未发生的处置列为预测和待验证项:
|
||||
|
||||
1. **事件前与爆管后、隔离前**:核对正常基线和异常窗口,说明压力、流量变化及已覆盖的异常点位、资产和区域。候选管段及破口面积有依据时,可用 `run_burst_simulation` 补充情景水力影响,标明假设。漏量估计须说明边界流量、正常用水及储量变化等依据;有可靠漏量序列和时间覆盖时可累计漏水量,保留单位、方法和不确定性。定位工具的 `burst_leakage` 是输入,不能当成反演所得漏量。
|
||||
2. **关阀隔离方案**:说明止漏目标、直接隔离节点和关阀后新增失去水源连接的节点,分别保留 `affected_nodes` 与 `supply_affected_nodes` 及其计数。结合关阀水力结果说明预计停水、低压或供水不足的范围和覆盖限制;有用户关联资料时说明受影响用户、重要设施及保障需求。拓扑失源、模型预测和现场实际停水分别标注,不把节点数当户数。
|
||||
3. **实际隔离与恢复供水后**:后续轮次按实际关阀、抢修和恢复时间读取前后监测与现场反馈,对照预测核查止漏效果、实际影响范围和恢复情况,说明持续低压、异常流量、再次异常及尚待验证的影响。核对正常基线、观察窗口和判定依据,工单状态或单个正常读数不足以证明影响消除。
|
||||
|
||||
比较须核对对象、单位、时间对齐、模型版本、需水与水源边界、泵阀状态及计算设置;区分实测变化与模型方案差异。关阀工具不会自动继承爆管情景,条件不同的独立计算不能直接作为同一事故前后效果。可比时给出变化量和方案差异,缺少可比基准时说明当前结果及缺口,不补造改善比例。必要的紧急核验、通知和供水保障不等待可选的完整模拟。
|
||||
|
||||
## 处置与报告
|
||||
|
||||
报告分别给出位置候选及依据、隔离阀门及可用性、替代方案、拓扑与水力影响、用户影响边界,以及处置和闭环缺口。只有初步结果时明确范围,不宣称已经定位、确诊或闭环。位置、候选、监测、影响和阀门方案按实际调查步骤同步展示,遵循后台展示规则,不用报告定位链接代替过程展示。覆盖面保留分组、缺失几何和示意边界,不能当作实际停水、水力影响或用户服务边界。无数据或展示失败时说明具体限制;展示不替代证据,也不要求等待浏览器。
|
||||
报告分别给出位置候选及依据、隔离阀门及可用性、替代方案、事件前后分阶段的拓扑与水力影响、用户影响边界,以及处置和闭环缺口。影响说明保留时间窗、对象、证据、实测或预测属性及限制。只有初步结果时明确范围,不宣称已经定位、确诊或闭环。位置、候选、监测、影响和阀门方案按实际调查步骤同步展示,遵循后台展示规则,不用报告定位链接代替过程展示。覆盖面保留分组、缺失几何和示意边界,不能当作实际停水、水力影响或用户服务边界。无数据或展示失败时说明具体限制;展示不替代证据,也不要求等待浏览器。
|
||||
|
||||
形成具体建议后,先保存阶段结论,再提议必要动作。人工核验、通知、定位隔离、抢修、适用的清洗复测和恢复回访使用关联的 `work_order_type=general` 草稿;原爆管事件保持独立。草稿写明位置、任务、依赖、执行前核验和回执要求。没有通知适配器时明确由人工实施。设备命令仅用于真实可控目标,审批和执行由 Server 控制。
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
name: scada-anomaly
|
||||
description: 调查 SCADA 回放中的异常信号,区分读数质量与运行异常,形成可追溯的演示报告。
|
||||
metadata:
|
||||
version: "1.0.1"
|
||||
---
|
||||
|
||||
# SCADA 回放异常调查
|
||||
|
||||
先读取任务上下文和来源中的 replay_context,核对规则、设备、单位、异常时刻及知识截止时间。数据来自已存 SCADA 的演示回放,不代表新采集的现场故障。
|
||||
|
||||
通过 query_scada_readings 读取事件前后本轮已处理的观测;查询结束时间不得晚于 knowledge_cutoff。结合 monitored_value 与 cleaned_value 判断数值变化,不将被可靠采集的低压或水质变化当作数据错误。缺测、无效值和未完成清洗应明确记录。
|
||||
|
||||
报告说明事件前后影响:在已开放且已处理的观测范围内,核对异常前参考、异常期间及已有恢复观测,列出压力、流量或水质变化、持续时间、涉及测点和已核实的资产关联。只有对象、单位、时间与运行条件可比时才给出变化量;说明参考窗口、证据和覆盖限制。尚无恢复观测时列为待验证,读数恢复不能证明现场修复或用户影响消除。停水范围、关阀影响及泄漏量需要相应拓扑、水力、流量平衡或现场依据;当前证据无法支持时明确待分析,不由单点低压推定停水区、受影响户数或漏量。
|
||||
|
||||
当前回放调查仅开放冻结 SCADA 证据、设备与模型目录;不支持原生计算或绕过回放上限的其他数据入口。回放前历史用于后台基线初始化,当前证据接口不返回该部分。需要更多历史或进一步计算时,报告具体缺口,不能读取未来或虚构分析结果。
|
||||
|
||||
报告列出观测事实、规则信号、质量问题、尚未确认的原因和建议核查。压力异常不等于已确认爆管,浓度变化不等于已确认污染。回放只生成演示分析与报告,不创建真实现场工单、不下发设备命令。
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Strict contracts for a burst-simulation calibration design."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
Severity = Literal["light", "medium", "severe"]
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
|
||||
|
||||
|
||||
class ModelExpectations(StrictModel):
|
||||
node_count: int = Field(gt=0)
|
||||
pipe_count: int = Field(gt=0)
|
||||
valve_count: int = Field(gt=0)
|
||||
flow_unit: Literal["LPS"]
|
||||
pressure_unit: Literal["METERS"]
|
||||
pressure_measurement_unit: Literal["m"]
|
||||
flow_measurement_unit: Literal["m3/h"]
|
||||
emitter_exponent: float = Field(gt=0)
|
||||
hydraulic_timestep_seconds: int = Field(gt=0)
|
||||
report_timestep_seconds: int = Field(gt=0)
|
||||
|
||||
|
||||
class SeverityLevel(StrictModel):
|
||||
level: Severity
|
||||
cross_section_fraction: float = Field(gt=0, le=0.125)
|
||||
|
||||
|
||||
class SelectionRule(StrictModel):
|
||||
primary_pressure_drop_percent: float = Field(gt=0, le=100)
|
||||
corroborating_pressure_drop_percent: float = Field(gt=0, le=100)
|
||||
flow_change_percent: float = Field(gt=0, le=1000)
|
||||
consecutive_samples: int = Field(ge=1, le=20)
|
||||
expected_sample_count: int = Field(ge=2, le=100)
|
||||
relative_change_epsilon: float = Field(gt=0, le=0.001)
|
||||
normal_control_must_not_trigger: Literal[True]
|
||||
negative_monitor_pressure_allowed: Literal[False]
|
||||
choose_smallest_passing_area: Literal[True]
|
||||
|
||||
|
||||
class BindingValidation(StrictModel):
|
||||
max_pressure_distance_m: float = Field(gt=0, le=10)
|
||||
max_flow_distance_m: float = Field(gt=0, le=10)
|
||||
|
||||
|
||||
class SimulationCase(StrictModel):
|
||||
case_id: str = Field(pattern=r"^B[0-9]{2}$")
|
||||
pipe_id: str = Field(min_length=1)
|
||||
occurred_at: datetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def aware_time(self):
|
||||
if self.occurred_at.tzinfo is None:
|
||||
raise ValueError("case time must include a timezone")
|
||||
return self
|
||||
|
||||
|
||||
class PressureBinding(StrictModel):
|
||||
device_id: str = Field(min_length=1)
|
||||
node_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class FlowBinding(StrictModel):
|
||||
device_id: str = Field(min_length=1)
|
||||
link_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class PipeCalibration(StrictModel):
|
||||
role: Literal["primary", "secondary"]
|
||||
pipe_id: str = Field(min_length=1)
|
||||
diameter_mm: float = Field(gt=0)
|
||||
length_m: float = Field(gt=0)
|
||||
start_node_id: str = Field(min_length=1)
|
||||
emitter_node_id: str = Field(min_length=1)
|
||||
longitude: float = Field(ge=-180, le=180)
|
||||
latitude: float = Field(gt=-90, lt=90)
|
||||
candidate_orifice_areas_cm2: dict[Severity, float]
|
||||
pressure_bindings: list[PressureBinding] = Field(min_length=1, max_length=20)
|
||||
flow_bindings: list[FlowBinding] = Field(min_length=1, max_length=20)
|
||||
boundary_valve_ids: list[str] = Field(min_length=1, max_length=20)
|
||||
output_node_ids: list[str] = Field(min_length=1, max_length=20)
|
||||
output_link_ids: list[str] = Field(default_factory=list, max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def complete_outputs(self):
|
||||
if set(self.candidate_orifice_areas_cm2) != {"light", "medium", "severe"}:
|
||||
raise ValueError("all three severity areas are required")
|
||||
pressure_device_ids = [value.device_id for value in self.pressure_bindings]
|
||||
pressure_node_ids = [value.node_id for value in self.pressure_bindings]
|
||||
flow_device_ids = [value.device_id for value in self.flow_bindings]
|
||||
flow_link_ids = [value.link_id for value in self.flow_bindings]
|
||||
for values, label in (
|
||||
(pressure_device_ids, "pressure devices"),
|
||||
(pressure_node_ids, "pressure nodes"),
|
||||
(flow_device_ids, "flow devices"),
|
||||
(flow_link_ids, "flow links"),
|
||||
(self.boundary_valve_ids, "boundary valves"),
|
||||
(self.output_node_ids, "output nodes"),
|
||||
(self.output_link_ids, "output links"),
|
||||
):
|
||||
if len(values) != len(set(values)):
|
||||
raise ValueError(f"{label} must be distinct")
|
||||
if not {self.start_node_id, self.emitter_node_id, *pressure_node_ids}.issubset(
|
||||
self.output_node_ids
|
||||
):
|
||||
raise ValueError("pipe endpoints and pressure nodes must be returned")
|
||||
if not {self.pipe_id, *flow_link_ids, *self.boundary_valve_ids}.issubset(
|
||||
self.output_link_ids
|
||||
):
|
||||
raise ValueError("target, flow and boundary links must be returned")
|
||||
return self
|
||||
|
||||
|
||||
class BurstSimulationDesign(StrictModel):
|
||||
version: Literal["burst-simulation-design@1"]
|
||||
suite_id: Literal["tjwater2-v2-burst-calibration"]
|
||||
project_code: Literal["tjwater2_v2"]
|
||||
project_id: UUID
|
||||
model_expectations: ModelExpectations
|
||||
duration_seconds: int = Field(ge=60, le=86400)
|
||||
event_times: list[datetime] = Field(min_length=1, max_length=8)
|
||||
severity_levels: list[SeverityLevel] = Field(min_length=3, max_length=3)
|
||||
calibration_order: list[Severity] = Field(min_length=3, max_length=3)
|
||||
selection_rule: SelectionRule
|
||||
binding_validation: BindingValidation
|
||||
cases: list[SimulationCase] = Field(min_length=1, max_length=20)
|
||||
pipes: list[PipeCalibration] = Field(min_length=1, max_length=4)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def coherent_design(self):
|
||||
if any(value.tzinfo is None for value in self.event_times):
|
||||
raise ValueError("event times must include a timezone")
|
||||
if len(set(self.event_times)) != len(self.event_times):
|
||||
raise ValueError("event times must be distinct")
|
||||
levels = {entry.level: entry.cross_section_fraction for entry in self.severity_levels}
|
||||
if set(levels) != {"light", "medium", "severe"}:
|
||||
raise ValueError("light, medium and severe levels are required")
|
||||
if not levels["light"] < levels["medium"] < levels["severe"]:
|
||||
raise ValueError("severity fractions must increase")
|
||||
expected_levels = {"light": 1 / 32, "medium": 1 / 16, "severe": 1 / 8}
|
||||
if any(
|
||||
not math.isclose(levels[level], fraction, rel_tol=0, abs_tol=1e-12)
|
||||
for level, fraction in expected_levels.items()
|
||||
):
|
||||
raise ValueError("severity fractions must remain 1/32, 1/16 and 1/8")
|
||||
if self.calibration_order != ["severe", "medium", "light"]:
|
||||
raise ValueError("calibration order must be severe, medium, light")
|
||||
report_step = self.model_expectations.report_timestep_seconds
|
||||
if self.duration_seconds % report_step:
|
||||
raise ValueError("duration must contain whole report timesteps")
|
||||
expected_samples = self.duration_seconds // report_step + 1
|
||||
if self.selection_rule.expected_sample_count != expected_samples:
|
||||
raise ValueError("expected sample count does not match duration and report step")
|
||||
pipes = {pipe.pipe_id: pipe for pipe in self.pipes}
|
||||
if len(pipes) != len(self.pipes):
|
||||
raise ValueError("pipe IDs must be distinct")
|
||||
if len({case.case_id for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("case IDs must be distinct")
|
||||
expected_cases = {(pipe_id, at) for pipe_id in pipes for at in self.event_times}
|
||||
actual_cases = {(case.pipe_id, case.occurred_at) for case in self.cases}
|
||||
if actual_cases != expected_cases:
|
||||
raise ValueError("cases must cover every pipe and event time exactly once")
|
||||
for pipe in self.pipes:
|
||||
cross_section_cm2 = math.pi * (pipe.diameter_mm / 10) ** 2 / 4
|
||||
for level, fraction in levels.items():
|
||||
expected = cross_section_cm2 * fraction
|
||||
actual = pipe.candidate_orifice_areas_cm2[level]
|
||||
if not math.isclose(actual, expected, rel_tol=0, abs_tol=0.001):
|
||||
raise ValueError(
|
||||
f"{pipe.pipe_id} {level} area does not match its diameter fraction"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
def load_simulation_design(path: Path) -> BurstSimulationDesign:
|
||||
return BurstSimulationDesign.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only validation of the tjwater2_v2 burst-simulation design."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg import connect
|
||||
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
|
||||
def readonly(dsn: str):
|
||||
return connect(
|
||||
make_conninfo(
|
||||
dsn,
|
||||
options="-c default_transaction_read_only=on -c statement_timeout=20000",
|
||||
application_name="tjwater-burst-design-readonly",
|
||||
connect_timeout="5",
|
||||
),
|
||||
row_factory=dict_row,
|
||||
)
|
||||
|
||||
|
||||
async def resolve_project(design):
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.infra.db.metadb.database import SessionLocal, engine
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
)
|
||||
repo = MetadataRepository(session)
|
||||
project = await repo.get_project_by_code(design.project_code)
|
||||
if (
|
||||
project is None
|
||||
or project.id != design.project_id
|
||||
or project.status != "active"
|
||||
):
|
||||
raise ValueError("project identity or active status does not match")
|
||||
biz = await repo.get_project_db_routing(project.id, "biz_data")
|
||||
iot = await repo.get_project_db_routing(project.id, "iot_data")
|
||||
if biz is None or iot is None:
|
||||
raise ValueError("project database routing is incomplete")
|
||||
await session.rollback()
|
||||
return biz.dsn, iot.dsn
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def validate_database(connection, design, *, template: bool) -> None:
|
||||
connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
if connection.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] != "on":
|
||||
raise ValueError("database transaction is not read-only")
|
||||
expected = design.model_expectations
|
||||
counts = connection.execute(
|
||||
"SELECT (SELECT count(*) FROM network.nodes) AS nodes, "
|
||||
"(SELECT count(*) FROM network.pipes) AS pipes, "
|
||||
"(SELECT count(*) FROM network.links WHERE link_type='valve') AS valves"
|
||||
).fetchone()
|
||||
if (counts["nodes"], counts["pipes"], counts["valves"]) != (
|
||||
expected.node_count,
|
||||
expected.pipe_count,
|
||||
expected.valve_count,
|
||||
):
|
||||
raise ValueError("model counts do not match the frozen design")
|
||||
|
||||
settings = {
|
||||
row["key"]: row["value"]
|
||||
for row in connection.execute(
|
||||
"SELECT key,value FROM network.simulation_settings "
|
||||
"WHERE engine_version='legacy' AND key=ANY(%s)",
|
||||
(["UNITS", "PRESSURE", "EMITTER EXPONENT"],),
|
||||
)
|
||||
}
|
||||
times = {
|
||||
row["key"]: row["value"]
|
||||
for row in connection.execute(
|
||||
"SELECT key,value FROM network.time_settings "
|
||||
"WHERE key=ANY(%s)",
|
||||
(["HYDRAULIC TIMESTEP", "REPORT TIMESTEP"],),
|
||||
)
|
||||
}
|
||||
if settings != {
|
||||
"UNITS": expected.flow_unit,
|
||||
"PRESSURE": expected.pressure_unit,
|
||||
"EMITTER EXPONENT": f"{expected.emitter_exponent:.4f}",
|
||||
}:
|
||||
raise ValueError("simulation settings do not match the frozen design")
|
||||
if times != {"HYDRAULIC TIMESTEP": "0:15:00", "REPORT TIMESTEP": "0:15:00"}:
|
||||
raise ValueError("simulation timesteps do not match the frozen design")
|
||||
|
||||
for pipe in design.pipes:
|
||||
row = connection.execute(
|
||||
"SELECT l.start_node_id,l.end_node_id,p.length,p.diameter,p.status "
|
||||
"FROM network.links l JOIN network.pipes p ON p.link_id=l.id WHERE l.id=%s",
|
||||
(pipe.pipe_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ValueError(f"pipe {pipe.pipe_id} is missing")
|
||||
if (
|
||||
str(row["start_node_id"]) != pipe.start_node_id
|
||||
or str(row["end_node_id"]) != pipe.emitter_node_id
|
||||
or abs(float(row["length"]) - pipe.length_m) > 0.001
|
||||
or abs(float(row["diameter"]) - pipe.diameter_mm) > 0.001
|
||||
or str(row["status"]).upper() != "OPEN"
|
||||
):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} properties changed")
|
||||
node_rows = connection.execute(
|
||||
"SELECT id,node_type FROM network.nodes WHERE id=ANY(%s)",
|
||||
(pipe.output_node_ids,),
|
||||
).fetchall()
|
||||
if {str(row["id"]) for row in node_rows} != set(pipe.output_node_ids):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} output nodes are incomplete")
|
||||
node_types = {str(row["id"]): str(row["node_type"]) for row in node_rows}
|
||||
if node_types.get(pipe.emitter_node_id) != "junction":
|
||||
raise ValueError(f"pipe {pipe.pipe_id} emitter endpoint is not a junction")
|
||||
link_rows = connection.execute(
|
||||
"SELECT id,link_type FROM network.links WHERE id=ANY(%s)",
|
||||
(pipe.output_link_ids,),
|
||||
).fetchall()
|
||||
if {str(row["id"]) for row in link_rows} != set(pipe.output_link_ids):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} output links are incomplete")
|
||||
link_types = {str(row["id"]): str(row["link_type"]) for row in link_rows}
|
||||
if any(link_types.get(value) != "valve" for value in pipe.boundary_valve_ids):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} boundary valve changed")
|
||||
emitter = connection.execute(
|
||||
"SELECT coefficient FROM network.emitters WHERE junction_id=%s",
|
||||
(pipe.emitter_node_id,),
|
||||
).fetchone()
|
||||
if emitter is not None:
|
||||
raise ValueError(f"pipe {pipe.pipe_id} endpoint already has an emitter")
|
||||
|
||||
if not template:
|
||||
pressure_bindings = {
|
||||
(
|
||||
str(row["device_id"]),
|
||||
str(row["node_id"]),
|
||||
str(row["measurement_unit"]),
|
||||
str(row["transmission_frequency"]),
|
||||
)
|
||||
for row in connection.execute(
|
||||
"SELECT device_id,node_id,measurement_unit,transmission_frequency "
|
||||
"FROM asset.scada_devices "
|
||||
"WHERE device_type='pressure' AND device_id=ANY(%s)",
|
||||
([value.device_id for value in pipe.pressure_bindings],),
|
||||
)
|
||||
}
|
||||
flow_bindings = {
|
||||
(
|
||||
str(row["device_id"]),
|
||||
str(row["link_id"]),
|
||||
str(row["measurement_unit"]),
|
||||
str(row["transmission_frequency"]),
|
||||
)
|
||||
for row in connection.execute(
|
||||
"SELECT device_id,link_id,measurement_unit,transmission_frequency "
|
||||
"FROM asset.scada_devices "
|
||||
"WHERE device_type IN ('pipe_flow','flow') AND device_id=ANY(%s)",
|
||||
([value.device_id for value in pipe.flow_bindings],),
|
||||
)
|
||||
}
|
||||
expected_pressure_bindings = {
|
||||
(
|
||||
value.device_id,
|
||||
value.node_id,
|
||||
expected.pressure_measurement_unit,
|
||||
str(timedelta(seconds=expected.report_timestep_seconds)),
|
||||
)
|
||||
for value in pipe.pressure_bindings
|
||||
}
|
||||
expected_flow_bindings = {
|
||||
(
|
||||
value.device_id,
|
||||
value.link_id,
|
||||
expected.flow_measurement_unit,
|
||||
str(timedelta(seconds=expected.report_timestep_seconds)),
|
||||
)
|
||||
for value in pipe.flow_bindings
|
||||
}
|
||||
if pressure_bindings != expected_pressure_bindings:
|
||||
raise ValueError(f"pipe {pipe.pipe_id} pressure bindings changed")
|
||||
if flow_bindings != expected_flow_bindings:
|
||||
raise ValueError(f"pipe {pipe.pipe_id} flow bindings changed")
|
||||
pressure_distances = {
|
||||
str(row["device_id"]): row["distance_m"]
|
||||
for row in connection.execute(
|
||||
"SELECT d.device_id,ST_Distance(d.geom,n.geom) AS distance_m "
|
||||
"FROM asset.scada_devices d "
|
||||
"JOIN gis.node_geometries n ON n.node_id=d.node_id "
|
||||
"WHERE d.device_id=ANY(%s)",
|
||||
([value.device_id for value in pipe.pressure_bindings],),
|
||||
)
|
||||
}
|
||||
flow_distances = {
|
||||
str(row["device_id"]): row["distance_m"]
|
||||
for row in connection.execute(
|
||||
"SELECT d.device_id,"
|
||||
"ST_Distance(d.geom,ST_Transform(p.geom,900914)) AS distance_m "
|
||||
"FROM asset.scada_devices d "
|
||||
"JOIN gis.pipes p ON p.id=d.link_id "
|
||||
"WHERE d.device_id=ANY(%s)",
|
||||
([value.device_id for value in pipe.flow_bindings],),
|
||||
)
|
||||
}
|
||||
if len(pressure_distances) != len(pipe.pressure_bindings) or any(
|
||||
distance is None
|
||||
or float(distance) > design.binding_validation.max_pressure_distance_m
|
||||
for distance in pressure_distances.values()
|
||||
):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} pressure binding geometry changed")
|
||||
if len(flow_distances) != len(pipe.flow_bindings) or any(
|
||||
distance is None
|
||||
or float(distance) > design.binding_validation.max_flow_distance_m
|
||||
for distance in flow_distances.values()
|
||||
):
|
||||
raise ValueError(f"pipe {pipe.pipe_id} flow binding geometry changed")
|
||||
connection.rollback()
|
||||
|
||||
|
||||
def validate_tool_inputs(design) -> None:
|
||||
from app.mcp.extended_models import BurstSimulationInput
|
||||
|
||||
pipes = {pipe.pipe_id: pipe for pipe in design.pipes}
|
||||
for case in design.cases:
|
||||
pipe = pipes[case.pipe_id]
|
||||
for severity in design.calibration_order:
|
||||
BurstSimulationInput(
|
||||
context_handle="benchmark-design-context-handle-0001",
|
||||
timeout_seconds=240,
|
||||
start_time=case.occurred_at,
|
||||
duration=design.duration_seconds,
|
||||
output_node_ids=pipe.output_node_ids,
|
||||
output_link_ids=pipe.output_link_ids,
|
||||
burst_pipe_ids=[pipe.pipe_id],
|
||||
orifice_areas_cm2=[pipe.candidate_orifice_areas_cm2[severity]],
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--design",
|
||||
type=Path,
|
||||
default=Path("resources/benchmarks/tjwater2-v2-burst-simulation.json"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
try:
|
||||
from scripts.benchmark.simulation_design import load_simulation_design
|
||||
|
||||
design = load_simulation_design(args.design)
|
||||
validate_tool_inputs(design)
|
||||
biz_dsn, _ = asyncio.run(resolve_project(design))
|
||||
physical_name = conninfo_to_dict(biz_dsn).get("dbname")
|
||||
if not physical_name:
|
||||
raise ValueError("business route has no database name")
|
||||
template_dsn = make_conninfo(biz_dsn, dbname=physical_name + "_template")
|
||||
with readonly(biz_dsn) as connection:
|
||||
validate_database(connection, design, template=False)
|
||||
with readonly(template_dsn) as connection:
|
||||
validate_database(connection, design, template=True)
|
||||
except Exception as exc:
|
||||
frames = traceback.extract_tb(exc.__traceback__)
|
||||
location = frames[-1] if frames else None
|
||||
suffix = f" at {location.filename}:{location.lineno}" if location else ""
|
||||
print(f"burst simulation design check failed ({type(exc).__name__}){suffix}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
maximum_runs = len(design.cases) * len(design.severity_levels)
|
||||
print(
|
||||
f"burst simulation design ready: {len(design.pipes)} pipes, "
|
||||
f"{len(design.cases)} cases, at most {maximum_runs} calibration runs; "
|
||||
"tool inputs, SCADA bindings, BizDB and model template were read-only verified"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -616,3 +616,52 @@ def test_project_map_config_accepts_search_and_preserves_extra_options():
|
||||
AdminProjectUpdateRequest(map_config=None)
|
||||
with pytest.raises(ValidationError):
|
||||
AdminProjectUpdateRequest(map_config={'place_search': {'enabled': True, 'admin_code': 'wrong', 'region': '凤阳县'}})
|
||||
|
||||
|
||||
def test_project_scada_monitoring_interval_is_a_strict_schedule_parameter():
|
||||
from pydantic import ValidationError
|
||||
from app.domain.schemas.admin_metadata import AdminProjectUpdateRequest
|
||||
|
||||
config = {
|
||||
"scada_monitoring_interval_seconds": 900,
|
||||
"future_option": {"enabled": True, "values": [1, "demo", None]},
|
||||
}
|
||||
request = AdminProjectUpdateRequest(settings=config)
|
||||
assert request.model_dump(exclude_unset=True) == {"settings": config}
|
||||
for value in (0, 59, True, 900.0, "900", 31_536_001):
|
||||
with pytest.raises(ValidationError):
|
||||
AdminProjectUpdateRequest(settings={"scada_monitoring_interval_seconds": value})
|
||||
for value in (None, [], "settings", {"future_option": object()}):
|
||||
with pytest.raises(ValidationError):
|
||||
AdminProjectUpdateRequest(settings=value)
|
||||
assert AdminProjectUpdateRequest().model_dump(exclude_unset=True) == {}
|
||||
assert AdminProjectUpdateRequest(settings={}).model_dump(exclude_unset=True) == {"settings": {}}
|
||||
assert AdminProjectUpdateRequest(
|
||||
settings={"scada_monitoring_interval_seconds": None}
|
||||
).model_dump(exclude_unset=True) == {"settings": {"scada_monitoring_interval_seconds": None}}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_project_passes_only_requested_settings_and_returns_extra_keys(monkeypatch):
|
||||
from app.domain.schemas.admin_metadata import AdminProjectUpdateRequest
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
project = _project(
|
||||
code="demo", description=None, gs_workspace="demo", map_extent=None,
|
||||
settings={"scada_monitoring_interval_seconds": 900, "future_option": {"enabled": True}},
|
||||
status="active", created_at=now, updated_at=now,
|
||||
)
|
||||
repo = SimpleNamespace(session=object(), update_project=AsyncMock(return_value=project))
|
||||
monkeypatch.setattr(admin_metadata, "log_audit_event", AsyncMock())
|
||||
|
||||
response = await admin_metadata.update_admin_project(
|
||||
AdminProjectUpdateRequest(settings={"future_option": {"enabled": True}}),
|
||||
project_id=project.id,
|
||||
current_user=_user(role="admin", is_superuser=True),
|
||||
metadata_repo=repo,
|
||||
)
|
||||
|
||||
repo.update_project.assert_awaited_once_with(
|
||||
project.id, updates={"settings": {"future_option": {"enabled": True}}}
|
||||
)
|
||||
assert response.settings.model_dump() == {**project.settings, "scada_monitoring": None}
|
||||
|
||||
@@ -86,6 +86,7 @@ async def test_preconfigured_service_access_avoids_server_delegation(monkeypatch
|
||||
source_ref=snapshot.source_ref, observed_at=now, recorded_at=now,
|
||||
recorded_by="service", has_snapshot=True)
|
||||
service = SimpleNamespace(can_read_source=AsyncMock(return_value=True),
|
||||
replay_context=AsyncMock(return_value=None),
|
||||
record_data_snapshot=AsyncMock(return_value=(evidence, snapshot)))
|
||||
source = AsyncMock(return_value=snapshot)
|
||||
monkeypatch.setattr(endpoints, "_service", lambda _ctx, _conn: service)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""HTTP boundary checks without project database access."""
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1.endpoints.timeseries import scada
|
||||
from app.domain.schemas.scada import ScadaRawPage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def raw_api(monkeypatch):
|
||||
app = FastAPI()
|
||||
app.include_router(scada.router)
|
||||
project = uuid4()
|
||||
app.dependency_overrides[scada.get_timescale_connection] = lambda: object()
|
||||
app.dependency_overrides[scada.get_postgres_connection] = lambda: object()
|
||||
app.dependency_overrides[scada.get_project_context] = lambda: SimpleNamespace(project_id=project)
|
||||
devices = AsyncMock(return_value={"P1"})
|
||||
monkeypatch.setattr(scada.ScadaInfoRepository, "get_existing_device_ids", devices)
|
||||
read = AsyncMock(return_value=ScadaRawPage(items=[], next_cursor=None, scanned_through=datetime(2026, 10, 2, tzinfo=timezone.utc)))
|
||||
monkeypatch.setattr(scada, "read_raw_page", read)
|
||||
with TestClient(app) as client:
|
||||
yield client, devices, read, project
|
||||
|
||||
|
||||
def test_raw_http_returns_only_read_projection_and_binds_project(raw_api):
|
||||
client, devices, read, project = raw_api
|
||||
response = client.get("/timeseries/scada-readings/raw", params={
|
||||
"start_time": "2026-10-01T00:00:00Z", "end_time": "2026-10-02T00:00:00Z", "device_ids": "P1,P1",
|
||||
})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["items"] == []
|
||||
assert read.await_args.kwargs["project_id"] == str(project)
|
||||
assert read.await_args.kwargs["device_ids"] == ["P1"]
|
||||
devices.assert_awaited_once()
|
||||
|
||||
|
||||
def test_raw_http_rejects_naive_time_before_replay_lookup(raw_api):
|
||||
client, devices, read, _ = raw_api
|
||||
response = client.get("/timeseries/scada-readings/raw", params={
|
||||
"start_time": "2026-10-01T00:00:00", "end_time": "2026-10-02T00:00:00Z",
|
||||
"device_ids": "P1", "replay_id": str(uuid4()),
|
||||
})
|
||||
assert response.status_code == 422
|
||||
devices.assert_not_awaited()
|
||||
read.assert_not_awaited()
|
||||
|
||||
|
||||
def test_raw_http_rejects_cross_project_device_before_tsdb(raw_api):
|
||||
client, _, read, _ = raw_api
|
||||
response = client.get("/timeseries/scada-readings/raw", params={
|
||||
"start_time": "2026-10-01T00:00:00Z", "end_time": "2026-10-02T00:00:00Z", "device_ids": "other",
|
||||
})
|
||||
assert response.status_code == 422
|
||||
read.assert_not_awaited()
|
||||
@@ -0,0 +1,113 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1.endpoints import scada_replay
|
||||
from app.auth.project_dependencies import ProjectContext, get_project_context
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def replay_api(monkeypatch):
|
||||
context = ProjectContext(uuid4(), "demo", uuid4(), "member")
|
||||
project = SimpleNamespace(settings={"scada_monitoring_interval_seconds": 900, "other": True})
|
||||
metadata = SimpleNamespace(get_project_by_id=AsyncMock(return_value=project))
|
||||
repo = SimpleNamespace(available=AsyncMock(return_value=True), current=AsyncMock(return_value=None))
|
||||
create = AsyncMock(side_effect=RuntimeError("creation mock must be configured"))
|
||||
monkeypatch.setattr(scada_replay.ScadaReplayService, "create", create)
|
||||
app = FastAPI()
|
||||
app.include_router(scada_replay.router)
|
||||
app.dependency_overrides[get_project_context] = lambda: context
|
||||
app.dependency_overrides[scada_replay.repository] = lambda: repo
|
||||
app.dependency_overrides[scada_replay.get_metadata_repository] = lambda: metadata
|
||||
with TestClient(app) as client:
|
||||
yield client, project, metadata, context, create
|
||||
|
||||
|
||||
def test_capabilities_reads_frequency_from_current_project_settings(replay_api):
|
||||
client, project, metadata, context, _ = replay_api
|
||||
response = client.get("/scada-replays/capabilities")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["monitoring_interval_seconds"] == 900
|
||||
assert response.json()["burst_monitoring_ready"] is False
|
||||
assert "尚未配置" in response.json()["burst_monitoring_reason"]
|
||||
metadata.get_project_by_id.assert_awaited_once_with(context.project_id)
|
||||
project.settings = {}
|
||||
assert client.get("/scada-replays/capabilities").json()["monitoring_interval_seconds"] is None
|
||||
|
||||
|
||||
def test_capabilities_reports_ready_only_with_five_pressures_and_outlet_flow(replay_api):
|
||||
client, project, *_ = replay_api
|
||||
project.settings = {
|
||||
"scada_monitoring_interval_seconds": 900,
|
||||
"scada_monitoring": {
|
||||
"pressure_device_ids": ["P1", "P2", "P3", "P4", "P5"],
|
||||
"outlet_flow_device_ids": ["F1"],
|
||||
},
|
||||
}
|
||||
|
||||
response = client.get("/scada-replays/capabilities")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["burst_monitoring_ready"] is True
|
||||
assert response.json()["burst_monitoring_reason"] is None
|
||||
|
||||
|
||||
def test_capabilities_reports_only_missing_storage_not_worker():
|
||||
repo = SimpleNamespace(available=AsyncMock(return_value=False))
|
||||
context = ProjectContext(uuid4(), "demo", uuid4(), "member")
|
||||
|
||||
result = asyncio.run(scada_replay.replay_capabilities(
|
||||
repo=repo,
|
||||
ctx=context,
|
||||
settings=scada_replay.ProjectSettings(scada_monitoring_interval_seconds=900),
|
||||
))
|
||||
|
||||
assert result.available is False
|
||||
assert result.reason == "需先部署 SCADA 回放存储结构"
|
||||
|
||||
|
||||
def test_step_command_processes_pending_period_in_same_request(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Service:
|
||||
def __init__(self, repo, context, timescale):
|
||||
calls.append((repo, context, timescale))
|
||||
|
||||
async def command(self, replay_id, command):
|
||||
calls.append((replay_id, command.action))
|
||||
return SimpleNamespace(step_pending=True)
|
||||
|
||||
async def process_pending_step(self, replay_id):
|
||||
calls.append((replay_id, "processed"))
|
||||
return "completed-period"
|
||||
|
||||
monkeypatch.setattr(scada_replay, "ScadaReplayService", Service)
|
||||
replay_id = uuid4()
|
||||
context = ProjectContext(uuid4(), "demo", uuid4(), "member")
|
||||
command = scada_replay.ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"
|
||||
)
|
||||
|
||||
result = asyncio.run(scada_replay.replay_command(
|
||||
replay_id, command, repo="repo", ctx=context, timescale="timescale"
|
||||
))
|
||||
|
||||
assert result == "completed-period"
|
||||
assert calls[-2:] == [(replay_id, "step"), (replay_id, "processed")]
|
||||
|
||||
|
||||
def test_client_cannot_override_project_frequency_on_creation(replay_api):
|
||||
client, _, _, _, create = replay_api
|
||||
response = client.post("/scada-replays", json={
|
||||
"request_id": str(uuid4()), "start_time": "2026-10-08T00:00:00Z",
|
||||
"end_time": "2026-10-08T01:00:00Z", "device_ids": ["P1"],
|
||||
"rules": [{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}],
|
||||
"monitoring_interval_seconds": 300,
|
||||
})
|
||||
assert response.status_code == 422
|
||||
create.assert_not_awaited()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Real project SQL, with all schema, observations, events and tasks rolled back."""
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
import pytest
|
||||
|
||||
from app.auth.project_dependencies import ProjectContext
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
|
||||
from app.infra.db.metadb.database import SessionLocal, engine
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.services.scada_replay import ScadaReplayService
|
||||
|
||||
pytestmark = pytest.mark.skipif(os.getenv("SCADA_REPLAY_STORAGE_TEST") != "1", reason="explicit rollback-only SQL test required")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend(): return "asyncio"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_real_manual_steps_clean_raw_and_persist_events_roll_back(monkeypatch):
|
||||
actual_project = UUID("871d7538-589c-47de-a6d5-3185ccb9f0e1")
|
||||
async with SessionLocal() as session:
|
||||
metadata = MetadataRepository(session)
|
||||
business = await metadata.get_project_db_routing(actual_project, "biz_data")
|
||||
timeseries = await metadata.get_project_db_routing(actual_project, "iot_data")
|
||||
await engine.dispose()
|
||||
pg = await psycopg.AsyncConnection.connect(business.dsn, row_factory=dict_row, connect_timeout=5)
|
||||
ts = await psycopg.AsyncConnection.connect(timeseries.dsn, row_factory=dict_row, connect_timeout=5)
|
||||
try:
|
||||
async with pg.transaction(force_rollback=True), ts.transaction(force_rollback=True):
|
||||
actual = await (await pg.execute("SELECT current_database() AS db")).fetchone()
|
||||
assert actual["db"] == "tjwater2_v2"
|
||||
migration = (Path(__file__).parents[2] / "resources/sql/031_scada_replays.sql").read_text()
|
||||
migration = migration.replace("BEGIN;\n", "", 1).removesuffix("COMMIT;\n")
|
||||
await pg.execute(migration)
|
||||
device = "SCADA_TEST_" + uuid4().hex[:12]
|
||||
start = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
for minute, value in ((0, 10), (15, 20), (30, 20), (45, 10), (60, 10)):
|
||||
await ts.execute("INSERT INTO scada.measurements(time,device_id,monitored_value) VALUES(%s,%s,%s)",
|
||||
(start + timedelta(minutes=minute), device, value))
|
||||
monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
|
||||
"device_id": device, "device_type": "pressure", "measurement_unit": "m",
|
||||
"transmission_frequency": "0:15:00", "transmission_mode": "non_realtime"
|
||||
}]))
|
||||
context = ProjectContext(
|
||||
project_id=uuid4(),
|
||||
project_code="tjwater2_v2",
|
||||
user_id=uuid4(),
|
||||
project_role="member",
|
||||
)
|
||||
repo = ScadaReplayRepository(pg, context.project_id)
|
||||
wall = [start]
|
||||
service = ScadaReplayService(repo, context, ts, clock=lambda: wall[0])
|
||||
config = ScadaReplayCreate(request_id=uuid4(), start_time=start, end_time=start + timedelta(minutes=75),
|
||||
device_ids=[device],
|
||||
rules=[{"rule_id": "pressure", "device_ids": [device], "kind": "bounds", "upper": 15, "unit": "m"}])
|
||||
result = await service.create(config, monitoring_interval_seconds=900)
|
||||
while result.status != "completed":
|
||||
await service.command(result.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=result.revision, action="step"
|
||||
))
|
||||
result = await service.process_pending_step(result.replay_id)
|
||||
assert result.status == "completed"
|
||||
assert result.processed_count == 5
|
||||
assert result.anomaly_count == 1
|
||||
events = await repo.events(result.replay_id)
|
||||
assert {event["change"] for event in events} == {"raised", "resolved"}
|
||||
assert all(event["task_id"] is None for event in events)
|
||||
raw = await (await ts.execute("SELECT monitored_value,cleaned_value FROM scada.measurements WHERE device_id=%s ORDER BY time", (device,))).fetchall()
|
||||
assert [row["monitored_value"] for row in raw] == [10, 20, 20, 10, 10]
|
||||
assert [row["cleaned_value"] for row in raw] == [10, 20, 20, 10, 10]
|
||||
finally:
|
||||
await pg.close(); await ts.close()
|
||||
@@ -300,6 +300,27 @@ def binding(task=False):
|
||||
return value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_name", ["run_burst_detection", "query_element_history"])
|
||||
def test_replay_binding_denies_unscoped_computations_and_history(tool_name):
|
||||
bound = binding(task=True)
|
||||
bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00+00:00", "batch_sequence": 1}
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(query_service.authorize_query(SimpleNamespace(), bound, TOOLS[tool_name], SimpleNamespace()))
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
def test_replay_binding_denies_future_and_allows_device_metadata(monkeypatch):
|
||||
bound = binding(task=True)
|
||||
bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00+00:00", "batch_sequence": 1}
|
||||
args = SimpleNamespace(start=datetime(2026, 10, 8, tzinfo=timezone.utc),
|
||||
end=datetime(2026, 10, 8, 2, tzinfo=timezone.utc))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
asyncio.run(query_service.authorize_query(SimpleNamespace(), bound, TOOLS["query_scada_readings"], args))
|
||||
assert error.value.status_code == 422
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: PERMISSIONS)
|
||||
asyncio.run(query_service.authorize_query(SimpleNamespace(project=object()), bound, TOOLS["list_scada_devices"], SimpleNamespace()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seconds", [30, 120, 240])
|
||||
def test_all_tools_advertise_the_effective_host_request_budget(runtime, seconds):
|
||||
app, client, _, _ = runtime
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlalchemy.dialects import postgresql
|
||||
|
||||
from app.infra.db.metadb.repositories import metadata_repository
|
||||
from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository
|
||||
from app.domain.schemas.project_settings import ProjectSettings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -99,3 +100,60 @@ async def test_update_map_config_preserves_legacy_settings_and_other_projects():
|
||||
repository.get_project_by_id.assert_awaited_once_with(project_id)
|
||||
assert project.map_config == {'legend': 'original', 'coordinate_reference': {'source_srid': 990001}, 'place_search': search}
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_project_settings_preserves_unmentioned_parameters():
|
||||
original = {"scada_monitoring_interval_seconds": 300, "future_option": {"enabled": True}}
|
||||
project = SimpleNamespace(settings=original)
|
||||
session = SimpleNamespace(commit=AsyncMock(), refresh=AsyncMock())
|
||||
repository = MetadataRepository(session)
|
||||
repository.get_project_by_id = AsyncMock(return_value=project)
|
||||
project_id = uuid4()
|
||||
|
||||
await repository.update_project(
|
||||
project_id,
|
||||
updates={"settings": {"scada_monitoring_interval_seconds": 900}},
|
||||
)
|
||||
|
||||
assert project.settings == {"scada_monitoring_interval_seconds": 900, "future_option": {"enabled": True}}
|
||||
assert original["scada_monitoring_interval_seconds"] == 300
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_empty_settings_update_preserves_values_and_explicit_null_clears_interval():
|
||||
project = SimpleNamespace(settings={"scada_monitoring_interval_seconds": 900, "other": True})
|
||||
session = SimpleNamespace(commit=AsyncMock(), refresh=AsyncMock())
|
||||
repository = MetadataRepository(session)
|
||||
repository.get_project_by_id = AsyncMock(return_value=project)
|
||||
project_id = uuid4()
|
||||
|
||||
await repository.update_project(project_id, updates={"settings": {}})
|
||||
assert project.settings == {"scada_monitoring_interval_seconds": 900, "other": True}
|
||||
await repository.update_project(
|
||||
project_id, updates={"settings": {"scada_monitoring_interval_seconds": None}}
|
||||
)
|
||||
assert project.settings == {"scada_monitoring_interval_seconds": None, "other": True}
|
||||
|
||||
|
||||
def test_scada_monitoring_roles_are_typed_inside_extensible_project_settings():
|
||||
settings = ProjectSettings.model_validate({
|
||||
"scada_monitoring_interval_seconds": 900,
|
||||
"scada_monitoring": {
|
||||
"rule_version": "burst-monitor-v1",
|
||||
"pressure_device_ids": ["P1", "P2"],
|
||||
"outlet_flow_device_ids": ["F1"],
|
||||
},
|
||||
"future_option": True,
|
||||
})
|
||||
|
||||
assert settings.scada_monitoring.pressure_device_ids == ["P1", "P2"]
|
||||
assert settings.model_dump()["future_option"] is True
|
||||
with pytest.raises(ValueError):
|
||||
ProjectSettings.model_validate({
|
||||
"scada_monitoring": {
|
||||
"pressure_device_ids": ["D1"],
|
||||
"outlet_flow_device_ids": ["D1"],
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.scada_raw import read_raw_page
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def reading(minutes, device="P1", value=1):
|
||||
return {"time": START + timedelta(minutes=minutes), "device_id": device,
|
||||
"monitored_value": value, "source_hash": "hash"}
|
||||
|
||||
|
||||
def test_pagination_keeps_same_timestamp_devices_and_freezes_clock(monkeypatch):
|
||||
fetch = AsyncMock(side_effect=[[reading(0), reading(0, "P2")], [reading(0, "P2")]])
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch)
|
||||
async def run():
|
||||
args = dict(project_id="project", device_ids=["P1", "P2"], start=START,
|
||||
end=START + timedelta(hours=1), limit=1)
|
||||
first = await read_raw_page(None, **args, through=START + timedelta(minutes=15))
|
||||
second = await read_raw_page(None, **args, cursor=first.next_cursor, through=START + timedelta(minutes=30))
|
||||
assert first.items[0].device_id == "P1"
|
||||
assert second.items[0].device_id == "P2"
|
||||
assert second.scanned_through == START + timedelta(minutes=15)
|
||||
assert fetch.await_args.args[-1] == (START, "P1")
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_cursor_cannot_expand_query_or_move_into_future(monkeypatch):
|
||||
fetch = AsyncMock(return_value=[reading(0), reading(5)])
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch)
|
||||
async def run():
|
||||
args = dict(project_id="project", device_ids=["P1"], start=START,
|
||||
end=START + timedelta(hours=1), limit=1)
|
||||
first = await read_raw_page(None, **args, through=START + timedelta(minutes=15))
|
||||
for change in ({"project_id": "other"}, {"device_ids": ["P2"]}, {"through": START}):
|
||||
with pytest.raises(ValueError, match="游标"):
|
||||
await read_raw_page(None, **(args | change), cursor=first.next_cursor)
|
||||
asyncio.run(run())
|
||||
assert fetch.await_count == 1
|
||||
|
||||
|
||||
def test_raw_response_does_not_expose_corrected_values_and_marks_bad_numbers(monkeypatch):
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", AsyncMock(return_value=[
|
||||
reading(0, value=float("nan")), reading(5, value=None), reading(10, value=0),
|
||||
]))
|
||||
page = asyncio.run(read_raw_page(None, project_id="project", device_ids=["P1"], start=START,
|
||||
end=START + timedelta(hours=1)))
|
||||
assert [item.quality for item in page.items] == ["invalid", "missing", "unvalidated"]
|
||||
assert page.items[2].monitored_value == 0
|
||||
assert "cleaned_value" not in page.model_dump_json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cursor", ["not-base64", "e30=", "W10="])
|
||||
def test_malformed_cursor_is_rejected_without_query(monkeypatch, cursor):
|
||||
fetch = AsyncMock()
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch)
|
||||
with pytest.raises(ValueError):
|
||||
asyncio.run(read_raw_page(None, project_id="project", device_ids=["P1"], start=START,
|
||||
end=START + timedelta(hours=1), cursor=cursor))
|
||||
fetch.assert_not_awaited()
|
||||
@@ -0,0 +1,108 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services import scada_replay_monitoring
|
||||
|
||||
|
||||
TARGET = datetime(2026, 10, 16, tzinfo=timezone.utc)
|
||||
PRESSURE_IDS = [f"P{index}" for index in range(1, 6)]
|
||||
FLOW_IDS = ["F1"]
|
||||
|
||||
|
||||
def _document():
|
||||
config = ScadaReplayCreate.model_validate({
|
||||
"request_id": str(uuid4()),
|
||||
"start_time": TARGET - timedelta(days=15),
|
||||
"end_time": TARGET + timedelta(hours=1),
|
||||
"device_ids": PRESSURE_IDS + FLOW_IDS,
|
||||
"rules": [
|
||||
{"rule_id": "pressure_drop", "device_ids": PRESSURE_IDS,
|
||||
"kind": "relative_change", "direction": "decrease", "threshold": 20, "unit": "m"},
|
||||
{"rule_id": "outlet_flow", "device_ids": FLOW_IDS,
|
||||
"kind": "relative_change", "direction": "increase", "threshold": 20, "unit": "m3/h"},
|
||||
],
|
||||
})
|
||||
return {
|
||||
"replay_id": str(uuid4()),
|
||||
"config": config.model_dump(mode="json"),
|
||||
"monitoring_interval_seconds": 900,
|
||||
"monitoring_config": {
|
||||
"rule_version": "burst-monitor-v1",
|
||||
"pressure_device_ids": PRESSURE_IDS,
|
||||
"outlet_flow_device_ids": FLOW_IDS,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _rows():
|
||||
rows = []
|
||||
count = 15 * 96
|
||||
start = TARGET - timedelta(minutes=15 * (count - 1))
|
||||
for index in range(count):
|
||||
at = start + timedelta(minutes=15 * index)
|
||||
for device_id in PRESSURE_IDS:
|
||||
rows.append({"time": at.isoformat(), "device_id": device_id,
|
||||
"cleaned_value": 70 if at == TARGET else 100,
|
||||
"quality": "valid"})
|
||||
rows.append({"time": at.isoformat(), "device_id": "F1",
|
||||
"cleaned_value": 150 if at == TARGET else 100,
|
||||
"quality": "valid"})
|
||||
return rows
|
||||
|
||||
|
||||
def test_multiple_indicators_create_suspected_burst_result(monkeypatch):
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
|
||||
"rows": [{"Score": -0.05}],
|
||||
"summary": {"latest_sensor_rankings": []},
|
||||
})
|
||||
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
_document(), _rows(),
|
||||
period_start=TARGET - timedelta(minutes=15),
|
||||
period_end=TARGET,
|
||||
batch_sequence=1440,
|
||||
)
|
||||
|
||||
assert result["status"] == "suspected_burst"
|
||||
assert result["pressure"]["hit_device_ids"] == PRESSURE_IDS
|
||||
assert result["outlet_flow"]["hit_device_ids"] == FLOW_IDS
|
||||
assert result["burst_detection"]["hit"] is True
|
||||
|
||||
|
||||
def test_single_indicator_stays_attention(monkeypatch):
|
||||
monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: {
|
||||
"rows": [{"Score": 0.1}],
|
||||
"summary": {"latest_sensor_rankings": []},
|
||||
})
|
||||
rows = _rows()
|
||||
for row in rows:
|
||||
if row["device_id"] == "F1" and row["time"] == TARGET.isoformat():
|
||||
row["cleaned_value"] = 100
|
||||
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
_document(), rows,
|
||||
period_start=TARGET - timedelta(minutes=15),
|
||||
period_end=TARGET,
|
||||
batch_sequence=1440,
|
||||
)
|
||||
|
||||
assert result["status"] == "attention"
|
||||
assert result["pressure"]["hit"] is True
|
||||
assert result["outlet_flow"]["hit"] is False
|
||||
assert result["burst_detection"]["hit"] is False
|
||||
|
||||
|
||||
def test_missing_project_roles_is_data_gap_not_normal():
|
||||
document = _document()
|
||||
document["monitoring_config"] = None
|
||||
|
||||
result = scada_replay_monitoring.evaluate_monitoring_period(
|
||||
document, [],
|
||||
period_start=TARGET - timedelta(minutes=15),
|
||||
period_end=TARGET,
|
||||
batch_sequence=1,
|
||||
)
|
||||
|
||||
assert result["status"] == "insufficient_data"
|
||||
assert result["burst_detection"]["status"] == "not_run"
|
||||
@@ -0,0 +1,66 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate
|
||||
from app.services.scada_replay_processing import prepare_cleaning, detect_frame
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def config(**changes):
|
||||
return ScadaReplayCreate.model_validate({
|
||||
"request_id": str(uuid4()), "start_time": START, "end_time": START + timedelta(hours=2),
|
||||
"device_ids": ["P1"], "rules": [{"rule_id": "pressure", "device_ids": ["P1"],
|
||||
"kind": "relative_change", "direction": "decrease", "threshold": 30, "unit": "m"}],
|
||||
**changes,
|
||||
})
|
||||
|
||||
|
||||
def rows(value, minute=0):
|
||||
return [{"time": START + timedelta(minutes=minute), "device_id": "P1", "monitored_value": value, "source_hash": "h"}]
|
||||
|
||||
|
||||
def test_quality_does_not_repair_a_real_pressure_drop_or_reverse_flow():
|
||||
for value in (0, -5, 1):
|
||||
result = prepare_cleaning(rows(value), config())
|
||||
assert result[0]["quality"] == "valid"
|
||||
assert result[0]["cleaned_value"] == value
|
||||
invalid = prepare_cleaning(rows(-5), config(quality_ranges={"P1": {"lower": 0}}))
|
||||
assert invalid[0]["quality"] == "invalid"
|
||||
assert invalid[0]["monitored_value"] == -5
|
||||
assert invalid[0]["cleaned_value"] is None
|
||||
|
||||
|
||||
def test_persistent_drop_is_one_episode_and_recovery_then_drop_is_new_episode():
|
||||
cfg = config()
|
||||
doc = {"replay_id": "run", "histories": {"P1": [10, 10, 10, 10]}}
|
||||
all_events = []
|
||||
for minute, value in enumerate((1, 1, None, 10, 10, 1)):
|
||||
doc, events = detect_frame(doc, prepare_cleaning(rows(value, minute), cfg), cfg, START + timedelta(minutes=minute))
|
||||
all_events.extend(events)
|
||||
business = [event for event in all_events if event.get("category") != "data_quality"]
|
||||
quality = [event for event in all_events if event.get("category") == "data_quality"]
|
||||
assert [event["change"] for event in business] == ["raised", "resolved", "raised"]
|
||||
assert [event["change"] for event in quality] == ["raised", "resolved"]
|
||||
assert business[0]["event_id"] != business[2]["event_id"]
|
||||
assert doc["anomaly_count"] == 3
|
||||
assert doc["processed_count"] == 6
|
||||
|
||||
|
||||
def test_water_quality_zero_baseline_uses_explicit_bounds():
|
||||
cfg = config(rules=[{"rule_id": "tracer", "device_ids": ["P1"], "kind": "bounds", "upper": 0.1, "unit": "m"}])
|
||||
doc = {"replay_id": "run"}
|
||||
doc, events = detect_frame(doc, prepare_cleaning(rows(0), cfg), cfg, START)
|
||||
assert events == []
|
||||
_, events = detect_frame(doc, prepare_cleaning(rows(0.4, 15), cfg), cfg, START + timedelta(minutes=15))
|
||||
assert events[0]["change"] == "raised"
|
||||
|
||||
|
||||
def test_empty_clock_ticks_do_not_break_consecutive_observations():
|
||||
cfg = config(rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10,
|
||||
"unit": "m", "consecutive_samples": 2}])
|
||||
doc, events = detect_frame({"replay_id": "run"}, prepare_cleaning(rows(20), cfg), cfg, START)
|
||||
assert events == []
|
||||
doc, _ = detect_frame(doc, [], cfg, START + timedelta(minutes=1))
|
||||
_, events = detect_frame(doc, prepare_cleaning(rows(20, 15), cfg), cfg, START + timedelta(minutes=15))
|
||||
assert events[0]["change"] == "raised"
|
||||
@@ -0,0 +1,145 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.mcp.analysis_models import BurstLocationInput
|
||||
from app.mcp.catalog import TOOLS
|
||||
from app.mcp.models import BoundContext
|
||||
from app.mcp import query_service
|
||||
from app.services.scada_replay_queries import replay_burst_location_inputs
|
||||
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class Cursor:
|
||||
def __init__(self, rows): self.rows = rows
|
||||
async def fetchone(self): return self.rows[0] if self.rows else None
|
||||
async def fetchall(self): return self.rows
|
||||
|
||||
|
||||
class Connection:
|
||||
def __init__(self, results):
|
||||
self.results = list(results)
|
||||
self.statements = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self): yield
|
||||
|
||||
async def execute(self, statement, params):
|
||||
self.statements.append((statement, params))
|
||||
return Cursor(self.results.pop(0))
|
||||
|
||||
async def commit(self): pass
|
||||
|
||||
|
||||
def _args(**changes):
|
||||
return BurstLocationInput.model_validate({
|
||||
"context_handle": "a" * 43,
|
||||
"burst_leakage": 0.02,
|
||||
"pressure_scada_ids": ["J1"],
|
||||
"flow_scada_ids": ["L1"],
|
||||
"scada_burst_start": START + timedelta(minutes=15),
|
||||
"scada_burst_end": START + timedelta(minutes=30),
|
||||
"scada_normal_start": START,
|
||||
"scada_normal_end": START + timedelta(minutes=14),
|
||||
**changes,
|
||||
})
|
||||
|
||||
|
||||
def test_replay_location_builds_series_only_from_frozen_outputs():
|
||||
replay_id = str(uuid4())
|
||||
run = {
|
||||
"start_time": START.isoformat(),
|
||||
"devices": [
|
||||
{"device_id": "P1", "node_id": "J1", "link_id": None},
|
||||
{"device_id": "F1", "node_id": None, "link_id": "L1"},
|
||||
],
|
||||
"monitoring_config": {
|
||||
"pressure_device_ids": ["P1"],
|
||||
"outlet_flow_device_ids": ["F1"],
|
||||
},
|
||||
}
|
||||
readings = [
|
||||
{"reading": {"device_id": "P1", "time": START.isoformat(), "cleaned_value": 100, "quality": "valid"}},
|
||||
{"reading": {"device_id": "F1", "time": START.isoformat(), "cleaned_value": 50, "quality": "valid"}},
|
||||
{"reading": {"device_id": "P1", "time": (START + timedelta(minutes=15)).isoformat(), "cleaned_value": 70, "quality": "valid"}},
|
||||
{"reading": {"device_id": "F1", "time": (START + timedelta(minutes=15)).isoformat(), "cleaned_value": 80, "quality": "valid"}},
|
||||
]
|
||||
connection = Connection([[{"document": run}], readings])
|
||||
context = {"replay_id": replay_id, "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(),
|
||||
"batch_sequence": 9}
|
||||
|
||||
result = asyncio.run(replay_burst_location_inputs(connection, uuid4(), context, _args()))
|
||||
|
||||
assert result["burst_pressure"] == {"J1": 70}
|
||||
assert result["normal_pressure"] == {"J1": 100}
|
||||
assert result["burst_flow"] == {"L1": 80}
|
||||
assert result["normal_flow"] == {"L1": 50}
|
||||
assert connection.statements[1][1][2] == 9
|
||||
|
||||
|
||||
def test_replay_location_authorization_requires_explicit_bounded_normal_window(monkeypatch):
|
||||
binding = BoundContext(
|
||||
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
|
||||
tool_cap=["run_burst_location"], permission_cap=["burst.run", "scada.view"],
|
||||
native_session_id="s", purpose="task_execution", task_id=uuid4(),
|
||||
expires_at=START + timedelta(days=1),
|
||||
replay_context={"replay_id": str(uuid4()),
|
||||
"knowledge_cutoff": (START + timedelta(hours=1)).isoformat(),
|
||||
"batch_sequence": 9},
|
||||
)
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"burst.run", "scada.view"})
|
||||
scope = SimpleNamespace(project=SimpleNamespace(project_role="member"))
|
||||
|
||||
asyncio.run(query_service.authorize_query(scope, binding, TOOLS["run_burst_location"], _args()))
|
||||
|
||||
missing_normal = _args().model_copy(update={"scada_normal_start": None, "scada_normal_end": None})
|
||||
with pytest.raises(HTTPException) as caught:
|
||||
asyncio.run(query_service.authorize_query(scope, binding, TOOLS["run_burst_location"], missing_normal))
|
||||
assert caught.value.status_code == 422
|
||||
|
||||
|
||||
def test_replay_location_execution_uses_frozen_adapter_instead_of_live_scada(monkeypatch):
|
||||
from app.mcp import analysis_service
|
||||
from app.services import scada_replay_queries
|
||||
|
||||
binding = BoundContext(
|
||||
context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host",
|
||||
tool_cap=["run_burst_location"], permission_cap=["burst.run", "scada.view"],
|
||||
native_session_id="s", purpose="task_execution", task_id=uuid4(),
|
||||
expires_at=START + timedelta(days=1),
|
||||
replay_context={"replay_id": str(uuid4()),
|
||||
"knowledge_cutoff": (START + timedelta(hours=1)).isoformat(),
|
||||
"batch_sequence": 9},
|
||||
)
|
||||
monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"burst.run", "scada.view"})
|
||||
frozen = {"burst_leakage": 0.02, "burst_pressure": {"J1": 70}, "normal_pressure": {"J1": 100}}
|
||||
build = AsyncMock(return_value=frozen)
|
||||
calculate = AsyncMock(return_value={"candidates": [{"id": "PIPE-1"}]})
|
||||
monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", build)
|
||||
monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", calculate)
|
||||
|
||||
@asynccontextmanager
|
||||
async def routing(): yield
|
||||
|
||||
scope = SimpleNamespace(
|
||||
connection=Connection([]),
|
||||
project=SimpleNamespace(project_id=binding.project_id, project_code="demo", project_role="member"),
|
||||
user=SimpleNamespace(username="tester"),
|
||||
simulation_routing=routing,
|
||||
)
|
||||
result = asyncio.run(query_service.execute_query(
|
||||
scope, binding, TOOLS["run_burst_location"], _args(),
|
||||
))
|
||||
|
||||
build.assert_awaited_once()
|
||||
calculate.assert_awaited_once_with(frozen, network="demo", username="tester")
|
||||
assert result.origin == "synthetic"
|
||||
assert result.data["result"]["candidates"][0]["id"] == "PIPE-1"
|
||||
@@ -0,0 +1,242 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand
|
||||
from app.infra.db.postgresql.scada import ScadaInfoRepository
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
from app.services.scada_replay import ScadaReplayService
|
||||
|
||||
START = datetime(2026, 10, 8, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class Connection:
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
yield
|
||||
|
||||
|
||||
class Repository:
|
||||
def __init__(self):
|
||||
self.project_id = str(uuid4())
|
||||
self.connection = Connection()
|
||||
self.document = None
|
||||
self.batches = {}
|
||||
|
||||
async def available(self): return True
|
||||
@asynccontextmanager
|
||||
async def control(self): yield
|
||||
@asynccontextmanager
|
||||
async def processing(self): yield True
|
||||
async def by_request(self, key):
|
||||
return deepcopy(self.document) if self.document and self.document["config"]["request_id"] == str(key) else None
|
||||
async def active_other(self, key): return False
|
||||
async def insert(self, document): self.document = deepcopy(document)
|
||||
async def save(self, document): self.document = deepcopy(document)
|
||||
async def get(self, _): return deepcopy(self.document)
|
||||
async def current(self): return deepcopy(self.document)
|
||||
async def pending_batch(self, _):
|
||||
return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None)
|
||||
async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch)
|
||||
async def monitoring_outputs(self, *_args): return []
|
||||
|
||||
|
||||
def setup(monkeypatch):
|
||||
repo = Repository()
|
||||
context = SimpleNamespace(
|
||||
project_id=repo.project_id,
|
||||
user_id=uuid4(),
|
||||
)
|
||||
service = ScadaReplayService(repo, context, Connection(), clock=lambda: START + timedelta(seconds=2))
|
||||
monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{
|
||||
"device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00"
|
||||
}]))
|
||||
async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_):
|
||||
return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}]
|
||||
if after is None else [])
|
||||
query = AsyncMock(side_effect=raw_page)
|
||||
cleaning = AsyncMock()
|
||||
monkeypatch.setattr(ScadaRepository, "get_raw_page", query)
|
||||
monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning)
|
||||
monkeypatch.setattr(service, "_create_investigations", AsyncMock())
|
||||
create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1),
|
||||
device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}])
|
||||
return repo, service, query, cleaning, create
|
||||
|
||||
|
||||
def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
restored = await service.create(create, monitoring_interval_seconds=300)
|
||||
assert restored.replay_id == first.replay_id
|
||||
assert restored.monitoring_interval_seconds == 900
|
||||
command = ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")
|
||||
played = await service.command(first.replay_id, command)
|
||||
assert played.status == "paused" and played.step_pending and played.revision == 2
|
||||
assert (await service.command(first.replay_id, command)).revision == 2
|
||||
with pytest.raises(HTTPException) as conflict:
|
||||
await service.command(first.replay_id, command.model_copy(update={"request_id": uuid4()}))
|
||||
assert conflict.value.status_code == 409
|
||||
asyncio.run(run())
|
||||
query.assert_not_awaited(); cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.processed_count == 1
|
||||
assert result.anomaly_count == 1
|
||||
assert repo.batches[1]["stage"] == "detected"
|
||||
asyncio.run(run())
|
||||
assert query.await_count == 2
|
||||
assert cleaning.await_count == 2
|
||||
assert cleaning.await_args_list[0].args[1][0]["cleaned_value"] == 20
|
||||
|
||||
|
||||
def test_detection_failure_resumes_cleaned_batch_without_duplicate_reads_or_writes(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
publish = AsyncMock(side_effect=[RuntimeError("Agent storage unavailable"), None, None])
|
||||
monkeypatch.setattr(service, "_create_investigations", publish)
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
|
||||
assert repo.document["status"] == "failed"
|
||||
assert repo.batches[1]["stage"] == "cleaned"
|
||||
assert repo.document["processed_count"] == 0
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry"))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.processed_count == 1 and result.anomaly_count == 1
|
||||
asyncio.run(run())
|
||||
assert query.await_count == 2
|
||||
assert cleaning.await_count == 2
|
||||
|
||||
|
||||
def test_cleaning_failure_never_runs_detection_and_preserves_prepared_result(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
cleaning.side_effect = RuntimeError("Timescale unavailable")
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step"))
|
||||
with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id)
|
||||
assert repo.batches[1]["stage"] == "clean_prepared"
|
||||
assert repo.document["processed_count"] == 0
|
||||
asyncio.run(run())
|
||||
service._create_investigations.assert_not_awaited()
|
||||
|
||||
|
||||
def test_creation_requires_explicit_project_monitoring_interval(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
|
||||
async def run():
|
||||
with pytest.raises(HTTPException) as missing:
|
||||
await service.create(create, monitoring_interval_seconds=None)
|
||||
assert missing.value.status_code == 422
|
||||
assert repo.document is None
|
||||
|
||||
asyncio.run(run())
|
||||
query.assert_not_awaited()
|
||||
cleaning.assert_not_awaited()
|
||||
|
||||
|
||||
def test_step_processes_every_observation_within_one_monitoring_interval(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
rows = [{"time": START + timedelta(minutes=minute), "device_id": "P1",
|
||||
"monitored_value": 20., "source_hash": f"h{minute}"} for minute in (0, 5, 10, 15, 20)]
|
||||
|
||||
async def raw_page(_connection, _devices, _start, _end, through, _limit, after=None, **_):
|
||||
return next(([row] for row in rows if row["time"] <= through
|
||||
and (after is None or (row["time"], row["device_id"]) > after)), [])
|
||||
|
||||
query.side_effect = raw_page
|
||||
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document["warmed"] = True
|
||||
stepping = await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"
|
||||
))
|
||||
assert stepping.status == "paused" and stepping.step_pending
|
||||
with pytest.raises(HTTPException):
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=2, action="step"
|
||||
))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.processed_count == 4
|
||||
assert result.virtual_now == START + timedelta(minutes=15)
|
||||
assert result.status == "paused"
|
||||
assert not result.step_pending
|
||||
assert result.processed_until == START + timedelta(minutes=15)
|
||||
assert result.monitoring_count == 1
|
||||
assert result.last_monitoring.status == "insufficient_data"
|
||||
assert "step_target" not in repo.document
|
||||
assert all(call.args[4] == START + timedelta(minutes=15) for call in query.await_args_list)
|
||||
|
||||
asyncio.run(run())
|
||||
assert cleaning.await_count == 4
|
||||
|
||||
|
||||
def test_partial_final_step_checks_empty_interval_and_completes_without_fabricated_rows(monkeypatch):
|
||||
repo, service, query, cleaning, create = setup(monkeypatch)
|
||||
create = create.model_copy(update={"end_time": START + timedelta(minutes=22)})
|
||||
query.side_effect = None
|
||||
query.return_value = []
|
||||
|
||||
async def run():
|
||||
first = await service.create(create, monitoring_interval_seconds=900)
|
||||
repo.document.update(warmed=True, virtual_now=(START + timedelta(minutes=15)).isoformat())
|
||||
await service.command(first.replay_id, ScadaReplayCommand(
|
||||
request_id=uuid4(), expected_revision=1, action="step"
|
||||
))
|
||||
result = await service.process_pending_step(first.replay_id)
|
||||
assert result.virtual_now == create.end_time
|
||||
assert result.processed_until == create.end_time
|
||||
assert result.status == "completed" and not result.step_pending
|
||||
assert result.processed_count == 0
|
||||
|
||||
asyncio.run(run())
|
||||
assert query.await_args.args[4] == create.end_time
|
||||
|
||||
|
||||
def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch):
|
||||
repo, service, *_ = setup(monkeypatch)
|
||||
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
|
||||
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
|
||||
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
|
||||
monkeypatch.setattr(
|
||||
"app.services.operational_tasks.OperationalTaskService",
|
||||
lambda *_: SimpleNamespace(create=create_task),
|
||||
)
|
||||
monitoring = {
|
||||
"status": "suspected_burst",
|
||||
"period_end": (START + timedelta(minutes=15)).isoformat(),
|
||||
"knowledge_cutoff": (START + timedelta(minutes=15)).isoformat(),
|
||||
"rule_version": "burst-monitor-v1",
|
||||
"pressure": {"hit_device_ids": ["P1", "P2"]},
|
||||
"outlet_flow": {"hit_device_ids": ["F1"]},
|
||||
"burst_detection": {"target_score": -0.05},
|
||||
"task_id": None,
|
||||
}
|
||||
document = {"replay_id": str(uuid4())}
|
||||
|
||||
asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8}))
|
||||
|
||||
command = create_task.await_args.args[0]
|
||||
assert command.workflow == "burst_report"
|
||||
assert command.source.replay_context.batch_sequence == 8
|
||||
assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15)
|
||||
assert monitoring["task_id"] == str(create_task.return_value.task_id)
|
||||
Reference in New Issue
Block a user