diff --git a/app/api/v1/endpoints/admin_metadata.py b/app/api/v1/endpoints/admin_metadata.py index 872499c..3fc19b6 100644 --- a/app/api/v1/endpoints/admin_metadata.py +++ b/app/api/v1/endpoints/admin_metadata.py @@ -49,6 +49,7 @@ def _project_response(project: models.Project) -> AdminProjectResponse: gs_workspace=project.gs_workspace, map_extent=project.map_extent, map_config=getattr(project, "map_config", {}) or {}, + settings=getattr(project, "settings", {}) or {}, status=project.status, created_at=project.created_at, updated_at=project.updated_at, @@ -276,6 +277,7 @@ async def create_admin_project( gs_workspace=payload.gs_workspace, map_extent=payload.map_extent, map_config=payload.map_config.model_dump(exclude_unset=True), + settings=payload.settings.model_dump(exclude_unset=True), status=payload.status, creator_user_id=current_user.id, ) diff --git a/app/api/v1/endpoints/meta.py b/app/api/v1/endpoints/meta.py index 7b8be7d..96fcba6 100644 --- a/app/api/v1/endpoints/meta.py +++ b/app/api/v1/endpoints/meta.py @@ -45,6 +45,7 @@ async def get_project_metadata( gs_workspace=project.gs_workspace, map_extent=project.map_extent, map_config=getattr(project, "map_config", {}) or {}, + settings=getattr(project, "settings", {}) or {}, status=project.status, project_role=ctx.project_role, ) diff --git a/app/api/v1/endpoints/model_import.py b/app/api/v1/endpoints/model_import.py index 329ba4a..f1cd2f9 100644 --- a/app/api/v1/endpoints/model_import.py +++ b/app/api/v1/endpoints/model_import.py @@ -320,6 +320,7 @@ async def provision_project( gs_workspace=project.gs_workspace, map_extent=project.map_extent, map_config=getattr(project, "map_config", {}) or {}, + settings=getattr(project, "settings", {}) or {}, status=project.status, created_at=project.created_at, updated_at=project.updated_at, diff --git a/app/api/v1/endpoints/operational_tasks.py b/app/api/v1/endpoints/operational_tasks.py index 5c31c8b..68a2540 100644 --- a/app/api/v1/endpoints/operational_tasks.py +++ b/app/api/v1/endpoints/operational_tasks.py @@ -567,10 +567,24 @@ async def query_operational_task_data( if not await service.can_read_source(task_id, query.source): raise DomainError("PERMISSION_DENIED", "Task has no approved access to this data source", status=403) source_permission = SCADA_VIEW if query.source.startswith("scada_") else SIMULATION_VIEW - snapshot = await read_task_data( - context=ctx, query=query, credential=authorization, - delegated=source_permission not in _task_permissions(ctx), - ) + from app.services.scada_replay_queries import replay_snapshot_rows + replay_context = await service.replay_context(task_id) + if replay_context: + if query.source != "scada_readings": + raise HTTPException(403, "回放调查数据接口仅开放本轮 SCADA 快照，请通过 MCP 查询设备目录") + rows = await replay_snapshot_rows(conn, ctx.project_id, replay_context, query.device_ids, + query.start, query.end, 1001) + from app.services.content_digest import digest + from datetime import datetime, timezone + snapshot = TaskDataSnapshot( + source="scada_readings", source_ref=f"scada_replay:{replay_context['replay_id']}:{digest(rows)}", + queried_at=datetime.now(timezone.utc), origin="synthetic", data=rows[:1000], truncated=len(rows) > 1000, + ) + else: + snapshot = await read_task_data( + context=ctx, query=query, credential=authorization, + delegated=source_permission not in _task_permissions(ctx), + ) evidence, stored_snapshot = await service.record_data_snapshot(task_id, snapshot) return TaskDataQueryResult(evidence=evidence, snapshot=stored_snapshot) except TaskDataError as exc: diff --git a/app/api/v1/endpoints/project.py b/app/api/v1/endpoints/project.py index de2d599..3f2f4a1 100644 --- a/app/api/v1/endpoints/project.py +++ b/app/api/v1/endpoints/project.py @@ -35,6 +35,7 @@ async def get_project_info_endpoint( gs_workspace=project_detail.gs_workspace, map_extent=project_detail.map_extent, map_config=getattr(project_detail, "map_config", {}) or {}, + settings=getattr(project_detail, "settings", {}) or {}, status=project_detail.status, project_role="viewer", # Default role for public access ) diff --git a/app/api/v1/endpoints/scada_replay.py b/app/api/v1/endpoints/scada_replay.py new file mode 100644 index 0000000..4c39215 --- /dev/null +++ b/app/api/v1/endpoints/scada_replay.py @@ -0,0 +1,114 @@ +"""SCADA replay controls. Manual steps are processed by the command request.""" +from uuid import UUID +from datetime import datetime + +from fastapi import APIRouter, Depends, Query, HTTPException + +from app.auth.metadata_dependencies import get_metadata_repository +from app.auth.permissions import SCADA_VIEW, SCADA_CLEAN, require_permission, permissions_for_context +from app.auth.project_dependencies import ProjectContext, get_project_pg_connection, get_project_timescale_connection +from app.domain.schemas.scada_replay import ( + ScadaReplayCapabilities, ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot, + ScadaReplayEventsPage, ScadaReplayReadings, +) +from app.infra.db.postgresql.scada_replays import ScadaReplayRepository +from app.domain.schemas.project_settings import ProjectSettings +from app.services.scada_replay import ScadaReplayService, snapshot + +router = APIRouter(prefix="/scada-replays", tags=["SCADA Replay"]) + + +async def repository( + ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)), + connection=Depends(get_project_pg_connection), +): + return ScadaReplayRepository(connection, ctx.project_id) + + +async def project_settings( + ctx: ProjectContext = Depends(require_permission(SCADA_VIEW)), + metadata_repo=Depends(get_metadata_repository), +) -> ProjectSettings: + project = await metadata_repo.get_project_by_id(ctx.project_id) + if project is None: + raise HTTPException(404, "当前项目不存在") + return ProjectSettings.model_validate(project.settings or {}) + + +@router.get("/capabilities", response_model=ScadaReplayCapabilities) +async def replay_capabilities(repo=Depends(repository), ctx=Depends(require_permission(SCADA_VIEW)), + settings=Depends(project_settings)): + available = await repo.available() + current = await repo.current() if available else None + monitoring = settings.scada_monitoring + monitoring_ready = bool( + monitoring + and len(monitoring.pressure_device_ids) >= 5 + and monitoring.outlet_flow_device_ids + ) + if monitoring_ready: + monitoring_reason = None + elif monitoring is None: + monitoring_reason = "项目尚未配置压力与出厂边界流量监测角色" + elif len(monitoring.pressure_device_ids) < 5: + monitoring_reason = "爆管侦测至少需要 5 个压力监测设备" + else: + monitoring_reason = "项目尚未配置出厂边界流量设备" + return ScadaReplayCapabilities( + available=available, can_write=SCADA_CLEAN in permissions_for_context(ctx), + reason=None if available else "需先部署 SCADA 回放存储结构", + monitoring_interval_seconds=settings.scada_monitoring_interval_seconds, + burst_monitoring_ready=monitoring_ready, + burst_monitoring_reason=monitoring_reason, + current=snapshot(current) if current else None, + ) + + +@router.post("", response_model=ScadaReplaySnapshot, status_code=201) +async def create_replay(command: ScadaReplayCreate, repo=Depends(repository), + ctx=Depends(require_permission(SCADA_CLEAN)), settings=Depends(project_settings)): + return await ScadaReplayService(repo, ctx).create( + command, + monitoring_interval_seconds=settings.scada_monitoring_interval_seconds, + monitoring_config=settings.scada_monitoring, + ) + + +@router.get("/{replay_id}", response_model=ScadaReplaySnapshot) +async def get_replay(replay_id: UUID, repo=Depends(repository)): + await ScadaReplayService(repo, None).ensure_storage() + return snapshot(await repo.get(replay_id)) + + +@router.post("/{replay_id}/commands", response_model=ScadaReplaySnapshot) +async def replay_command(replay_id: UUID, command: ScadaReplayCommand, repo=Depends(repository), + ctx=Depends(require_permission(SCADA_CLEAN)), + timescale=Depends(get_project_timescale_connection)): + service = ScadaReplayService(repo, ctx, timescale) + result = await service.command(replay_id, command) + if command.action in {"step", "retry"} and result.step_pending: + return await service.process_pending_step(replay_id) + return result + + +@router.get("/{replay_id}/events", response_model=ScadaReplayEventsPage) +async def replay_events(replay_id: UUID, offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=200), + repo=Depends(repository)): + await ScadaReplayService(repo, None).ensure_storage() + await repo.get(replay_id) + items = await repo.events(replay_id, offset, limit + 1) + return ScadaReplayEventsPage(items=items[:limit], next_offset=offset + limit if len(items) > limit else None) + + +@router.get("/{replay_id}/observations", response_model=ScadaReplayReadings) +async def replay_observations(replay_id: UUID, through: datetime, repo=Depends(repository)): + await ScadaReplayService(repo, None).ensure_storage() + document = await repo.get(replay_id) + if (through.tzinfo is None or not document.get("delivered_until") + or through > datetime.fromisoformat(document["delivered_until"]) + or through < datetime.fromisoformat(document["start_time"])): + raise HTTPException(422, "回看仅允许本轮已读取的时间区间") + return ScadaReplayReadings( + items=await repo.latest_outputs(replay_id, through), + through=through, + ) diff --git a/app/api/v1/endpoints/timeseries/scada.py b/app/api/v1/endpoints/timeseries/scada.py index f02976a..e4951f3 100644 --- a/app/api/v1/endpoints/timeseries/scada.py +++ b/app/api/v1/endpoints/timeseries/scada.py @@ -1,10 +1,13 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body from typing import List from datetime import datetime +from uuid import UUID from psycopg import AsyncConnection from pydantic import BaseModel, Field, field_validator -from app.domain.schemas.scada import ScadaLatestReadingsResponse +from app.domain.schemas.scada import ScadaLatestReadingsResponse, ScadaRawPage +from app.auth.project_dependencies import get_project_context, ProjectContext +from app.services.scada_raw import read_raw_page, validate_window from app.infra.db.postgresql.scada import ScadaInfoRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository from .dependencies import get_postgres_connection, get_timescale_connection @@ -13,6 +16,46 @@ router = APIRouter() SCADA_BATCH_MAX_ITEMS = 10_000 +@router.get("/timeseries/scada-readings/raw", response_model=ScadaRawPage, + summary="按时间顺序分页读取 SCADA 原始值") +async def get_scada_raw( + start_time: datetime, end_time: datetime, device_ids: str, + limit: int = Query(500, ge=1, le=500), + cursor: str | None = Query(None, max_length=2048), + replay_id: UUID | None = None, + conn: AsyncConnection = Depends(get_timescale_connection), + postgres_conn: AsyncConnection = Depends(get_postgres_connection), + ctx: ProjectContext = Depends(get_project_context), +) -> ScadaRawPage: + try: + validate_window(start_time, end_time) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + ids = list(dict.fromkeys(item.strip() for item in device_ids.split(",") if item.strip())) + if not ids or len(ids) > 500: + raise HTTPException(422, "请选择 1 至 500 个 SCADA 设备") + existing = await ScadaInfoRepository.get_existing_device_ids(postgres_conn, ids) + if set(ids) != existing: + raise HTTPException(422, "查询包含当前项目未配置的 SCADA 设备") + try: + through = None + if replay_id is not None: + from app.infra.db.postgresql.scada_replays import ScadaReplayRepository + from app.services.scada_replay import ScadaReplayService + repo = ScadaReplayRepository(postgres_conn, ctx.project_id) + await ScadaReplayService(repo, ctx).ensure_storage() + replay = await repo.get(replay_id) + if not set(ids) <= set(replay["config"]["device_ids"]): + raise HTTPException(403, "设备超出回放范围") + if start_time < datetime.fromisoformat(replay["start_time"]) or end_time > datetime.fromisoformat(replay["end_time"]): + raise HTTPException(422, "查询超出选定回放区间") + through = datetime.fromisoformat(replay["virtual_now"]) + return await read_raw_page(conn, project_id=str(ctx.project_id), device_ids=ids, + start=start_time, end=end_time, limit=limit, cursor=cursor, through=through) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + + class ScadaReadingBatchItem(BaseModel): time: datetime device_id: str = Field(min_length=1) diff --git a/app/api/v1/router.py b/app/api/v1/router.py index 7108288..3c5311f 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -61,8 +61,10 @@ from app.auth.permissions import ( require_method_permission, require_permission, ) +from app.api.v1.endpoints import scada_replay api_router = APIRouter() +api_router.include_router(scada_replay.router) webgis_access = Depends( require_method_permission( diff --git a/app/domain/schemas/admin_metadata.py b/app/domain/schemas/admin_metadata.py index eef0e5b..8d87326 100644 --- a/app/domain/schemas/admin_metadata.py +++ b/app/domain/schemas/admin_metadata.py @@ -2,9 +2,10 @@ from datetime import datetime from typing import Literal from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from app.domain.schemas.project_map_config import ProjectMapConfig, ProjectMapConfigUpdate +from app.domain.schemas.project_settings import ProjectSettings BusinessRole = Literal["admin", "service", "user"] @@ -72,6 +73,7 @@ class ProjectMemberResponse(BaseModel): class AdminProjectCreateRequest(BaseModel): + settings: ProjectSettings = Field(default_factory=ProjectSettings) name: str = Field(..., min_length=1, max_length=100) code: str = Field(..., min_length=1, max_length=50) description: str | None = None @@ -82,6 +84,10 @@ class AdminProjectCreateRequest(BaseModel): class AdminProjectUpdateRequest(ProjectMapConfigUpdate): + settings: ProjectSettings | None = Field( + default=None, + description="按顶层键合并项目配置；省略的键保持原值，嵌套对象整体替换", + ) name: str | None = Field(default=None, min_length=1, max_length=100) code: str | None = Field(default=None, min_length=1, max_length=50) description: str | None = None @@ -89,8 +95,16 @@ class AdminProjectUpdateRequest(ProjectMapConfigUpdate): map_extent: dict | None = None status: ProjectStatus | None = None + @field_validator("settings") + @classmethod + def reject_null_settings(cls, value: ProjectSettings | None) -> ProjectSettings: + if value is None: + raise ValueError("settings must be an object") + return value + class AdminProjectResponse(BaseModel): + settings: ProjectSettings = Field(default_factory=ProjectSettings) project_id: UUID name: str code: str diff --git a/app/domain/schemas/metadata.py b/app/domain/schemas/metadata.py index b358edd..7c72d3b 100644 --- a/app/domain/schemas/metadata.py +++ b/app/domain/schemas/metadata.py @@ -4,9 +4,11 @@ from uuid import UUID from pydantic import BaseModel, Field from app.domain.schemas.project_map_config import ProjectMapConfig +from app.domain.schemas.project_settings import ProjectSettings class ProjectMetaResponse(BaseModel): + settings: ProjectSettings = Field(default_factory=ProjectSettings) project_id: UUID name: str code: str diff --git a/app/domain/schemas/operational_task.py b/app/domain/schemas/operational_task.py index 9bdeaeb..c64231a 100644 --- a/app/domain/schemas/operational_task.py +++ b/app/domain/schemas/operational_task.py @@ -41,12 +41,19 @@ ExecutionStatus = Literal[ ] +class ScadaReplayTaskContext(StrictModel): + replay_id: UUID + knowledge_cutoff: WorkbenchDateTime + batch_sequence: PositiveInt + + class TaskSource(StrictModel): kind: TaskSourceKind external_id: NonEmptyStr | None = None external_revision: PositiveInt | None = None event_kind: NonEmptyStr | None = None deleted_at: WorkbenchDateTime | None = None + replay_context: ScadaReplayTaskContext | None = None class TaskTrigger(StrictModel): diff --git a/app/domain/schemas/project_settings.py b/app/domain/schemas/project_settings.py new file mode 100644 index 0000000..a5d84e5 --- /dev/null +++ b/app/domain/schemas/project_settings.py @@ -0,0 +1,44 @@ +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + + +class ScadaMonitoringSettings(BaseModel): + """Project-owned device roles used by the manual SCADA monitoring demo.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + rule_version: str = Field( + default="burst-monitor-v1", + min_length=1, + max_length=80, + pattern=r"^[a-zA-Z0-9_.-]+$", + ) + pressure_device_ids: list[str] = Field(default_factory=list, max_length=20) + outlet_flow_device_ids: list[str] = Field(default_factory=list, max_length=20) + + @model_validator(mode="after") + def distinct_roles(self): + for values in (self.pressure_device_ids, self.outlet_flow_device_ids): + if len(set(values)) != len(values) or any(not value.strip() for value in values): + raise ValueError("SCADA 监测角色中的设备标识不能为空或重复") + if set(self.pressure_device_ids) & set(self.outlet_flow_device_ids): + raise ValueError("同一设备不能同时作为压力与出厂边界流量") + return self + + +class ProjectSettings(BaseModel): + """Extensible, non-secret project parameters stored in MetaDB as JSON.""" + + model_config = ConfigDict(extra="allow", allow_inf_nan=False) + + __pydantic_extra__: dict[str, JsonValue] = Field(init=False) + scada_monitoring_interval_seconds: int | None = Field( + default=None, + strict=True, + ge=60, + le=31_536_000, + description="SCADA 主动监测演示步长，单位秒；null 表示未配置", + ) + scada_monitoring: ScadaMonitoringSettings | None = Field( + default=None, + description="主动监测使用的项目设备角色；不创建定时任务", + ) diff --git a/app/domain/schemas/scada.py b/app/domain/schemas/scada.py index 1b0b379..4873146 100644 --- a/app/domain/schemas/scada.py +++ b/app/domain/schemas/scada.py @@ -1,8 +1,8 @@ from datetime import datetime +from typing import Literal from pydantic import BaseModel, ConfigDict, Field - class ScadaDeviceResponse(BaseModel): """Project SCADA metadata keyed by the canonical device identifier.""" @@ -47,3 +47,20 @@ class ScadaLatestReadingsResponse(BaseModel): offset: int = Field(ge=0) observed_count: int = Field(ge=0) latest_observed_at: datetime | None = None + + +class ScadaRawReading(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + time: datetime + device_id: str + monitored_value: float | None + quality: Literal["unvalidated", "missing", "invalid"] = "unvalidated" + + +class ScadaRawPage(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + items: list[ScadaRawReading] + next_cursor: str | None + scanned_through: datetime diff --git a/app/domain/schemas/scada_replay.py b/app/domain/schemas/scada_replay.py new file mode 100644 index 0000000..9fe2962 --- /dev/null +++ b/app/domain/schemas/scada_replay.py @@ -0,0 +1,142 @@ +"""Persisted replay controls and explicit, unit-aware demonstration rules.""" +from datetime import datetime, timedelta +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ReplayModel(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + +class ScadaReplayRule(ReplayModel): + rule_id: str = Field(min_length=1, max_length=80, pattern=r"^[a-zA-Z0-9_-]+$") + device_ids: list[str] = Field(min_length=1, max_length=500) + kind: Literal["relative_change", "bounds"] + direction: Literal["both", "increase", "decrease"] = "both" + threshold: float | None = Field(None, gt=0) + lower: float | None = None + upper: float | None = None + unit: str = Field(min_length=1, max_length=40) + metric_name: str = Field(default="", max_length=80) + minimum_samples: int = Field(3, ge=1, le=96) + window_samples: int = Field(4, ge=1, le=96) + consecutive_samples: int = Field(1, ge=1, le=20) + recovery_samples: int = Field(2, ge=1, le=20) + minimum_devices: int = Field(1, ge=1, le=500) + + @model_validator(mode="after") + def parameters(self): + if len(set(self.device_ids)) != len(self.device_ids): + raise ValueError("规则内设备不能重复") + if self.minimum_devices > len(self.device_ids): + raise ValueError("同时异常设备数超过规则设备数") + if self.kind == "relative_change" and (self.threshold is None or self.minimum_samples > self.window_samples): + raise ValueError("变化率规则需要阈值和足够的历史窗口") + if self.kind == "bounds" and self.lower is None and self.upper is None: + raise ValueError("上下限规则至少配置一个阈值") + if self.lower is not None and self.upper is not None and self.lower >= self.upper: + raise ValueError("规则下限必须小于上限") + return self + + +class ScadaQualityRange(ReplayModel): + lower: float | None = None + upper: float | None = None + + @model_validator(mode="after") + def bounds(self): + if self.lower is not None and self.upper is not None and self.lower >= self.upper: + raise ValueError("仪表量程下限必须小于上限") + return self + + +class ScadaReplayCreate(ReplayModel): + request_id: UUID + start_time: datetime + end_time: datetime + device_ids: list[str] = Field(min_length=1, max_length=500) + rules: list[ScadaReplayRule] = Field(min_length=1, max_length=30) + quality_ranges: dict[str, ScadaQualityRange] = Field(default_factory=dict) + + @model_validator(mode="after") + def configuration(self): + if (self.start_time.tzinfo is None or self.end_time.tzinfo is None + or not timedelta(0) < self.end_time - self.start_time <= timedelta(days=31)): + raise ValueError("请选择带时区、最长 31 天的有效时间区间") + ids = set(self.device_ids) + if len(ids) != len(self.device_ids) or any(not value.strip() for value in ids): + raise ValueError("设备标识不能为空或重复") + if len({rule.rule_id for rule in self.rules}) != len(self.rules): + raise ValueError("规则标识不能重复") + if any(not set(rule.device_ids) <= ids for rule in self.rules) or not set(self.quality_ranges) <= ids: + raise ValueError("规则及量程设备必须属于本次回放") + return self + + +class ScadaReplayCommand(ReplayModel): + request_id: UUID + expected_revision: int = Field(ge=1) + action: Literal["step", "stop", "retry"] + + +class ScadaMonitoringResult(ReplayModel): + monitoring_id: UUID + period_start: datetime + period_end: datetime + knowledge_cutoff: datetime + batch_sequence: int = Field(ge=1) + rule_version: str = Field(min_length=1, max_length=80) + status: Literal["normal", "attention", "insufficient_data", "suspected_burst"] + detail: str = Field(min_length=1, max_length=1000) + task_id: UUID | None = None + pressure: dict + outlet_flow: dict + burst_detection: dict + role_gaps: list[str] = Field(default_factory=list) + + +class ScadaReplaySnapshot(ReplayModel): + replay_id: UUID + project_id: UUID + revision: int + status: Literal["ready", "paused", "completed", "stopped", "failed"] + start_time: datetime + end_time: datetime + monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000) + step_pending: bool = False + monitoring_count: int = 0 + last_monitoring: ScadaMonitoringResult | None = None + virtual_now: datetime + delivered_until: datetime | None = None + processed_until: datetime | None = None + processed_count: int = 0 + anomaly_count: int = 0 + devices: list[dict] + rules: list[ScadaReplayRule] + latest: list[dict] + active_events: list[dict] + error: str | None = None + created_at: datetime + updated_at: datetime + + +class ScadaReplayCapabilities(ReplayModel): + available: bool + can_write: bool + reason: str | None = None + monitoring_interval_seconds: int | None = Field(None, ge=60, le=31_536_000) + burst_monitoring_ready: bool = False + burst_monitoring_reason: str | None = None + current: ScadaReplaySnapshot | None = None + + +class ScadaReplayEventsPage(ReplayModel): + items: list[dict] + next_offset: int | None + + +class ScadaReplayReadings(ReplayModel): + items: list[dict] + through: datetime diff --git a/app/domain/schemas/task_alert.py b/app/domain/schemas/task_alert.py index 3b8a5b4..b5ab710 100644 --- a/app/domain/schemas/task_alert.py +++ b/app/domain/schemas/task_alert.py @@ -5,7 +5,7 @@ from typing import Literal from uuid import UUID from app.domain.schemas.shared import NonEmptyStr, PositiveInt, StrictModel -from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchDateTime +from app.domain.schemas.workbench_runtime import WorkbenchAlertSource, WorkbenchScadaReplaySource, WorkbenchDateTime class TaskAlertCommand(StrictModel): @@ -14,7 +14,7 @@ class TaskAlertCommand(StrictModel): event_type: Literal["raised", "acknowledged", "resolved"] occurred_at: WorkbenchDateTime condition_id: NonEmptyStr | None - source: WorkbenchAlertSource + source: WorkbenchAlertSource | WorkbenchScadaReplaySource title: NonEmptyStr description: NonEmptyStr severity: Literal["warning", "critical"] @@ -29,7 +29,7 @@ class TaskAlertState(StrictModel): revision: PositiveInt producer_id: NonEmptyStr condition_id: NonEmptyStr | None - source: WorkbenchAlertSource + source: WorkbenchAlertSource | WorkbenchScadaReplaySource title: NonEmptyStr description: NonEmptyStr occurred_at: WorkbenchDateTime diff --git a/app/domain/schemas/workbench_runtime.py b/app/domain/schemas/workbench_runtime.py index a25080b..edc07b3 100644 --- a/app/domain/schemas/workbench_runtime.py +++ b/app/domain/schemas/workbench_runtime.py @@ -49,10 +49,17 @@ class WorkbenchAlertSource(StrictModel): run_id: NonEmptyStr +class WorkbenchScadaReplaySource(StrictModel): + kind: Literal["scada_replay"] + replay_id: NonEmptyStr + rule_id: NonEmptyStr + batch_sequence: int = Field(ge=1) + + class WorkbenchRuntimeAlert(StrictModel): id: NonEmptyStr condition_id: NonEmptyStr | None - source: WorkbenchAlertSource + source: WorkbenchAlertSource | WorkbenchScadaReplaySource title: NonEmptyStr description: NonEmptyStr occurred_at: WorkbenchDateTime @@ -62,7 +69,7 @@ class WorkbenchRuntimeAlert(StrictModel): class WorkbenchPersistedAlert(StrictModel): alert_id: NonEmptyStr condition_id: NonEmptyStr | None - source: WorkbenchAlertSource + source: WorkbenchAlertSource | WorkbenchScadaReplaySource title: NonEmptyStr description: NonEmptyStr occurred_at: WorkbenchDateTime diff --git a/app/infra/db/metadb/models.py b/app/infra/db/metadb/models.py index c510e6d..72df5b4 100644 --- a/app/infra/db/metadb/models.py +++ b/app/infra/db/metadb/models.py @@ -44,6 +44,7 @@ class Project(Base): gs_workspace: Mapped[str] = mapped_column(String(100), unique=True) map_extent: Mapped[dict | None] = mapped_column(JSONB, nullable=True) map_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}") + settings: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}") status: Mapped[str] = mapped_column(String(20), default="active") created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=datetime.utcnow diff --git a/app/infra/db/metadb/repositories/metadata_repository.py b/app/infra/db/metadb/repositories/metadata_repository.py index 41b1f07..945a6a7 100644 --- a/app/infra/db/metadb/repositories/metadata_repository.py +++ b/app/infra/db/metadb/repositories/metadata_repository.py @@ -67,6 +67,7 @@ class ProjectDetail: map_extent: Optional[dict] status: str map_config: dict = field(default_factory=dict) + settings: dict = field(default_factory=dict) @dataclass(frozen=True) @@ -223,6 +224,7 @@ class MetadataRepository: status: str, creator_user_id: UUID | None = None, map_config: dict | None = None, + settings: dict | None = None, ) -> models.Project: project = models.Project( id=uuid4(), @@ -233,6 +235,7 @@ class MetadataRepository: map_extent=map_extent, status=status, map_config=map_config or {}, + settings=settings or {}, created_at=_utcnow(), updated_at=_utcnow(), ) @@ -263,6 +266,7 @@ class MetadataRepository: timescale_dsn: str, pool_min_size: int = 1, pool_max_size: int = 4, + settings: dict | None = None, ) -> models.Project: """Atomically expose a fully provisioned project and both DB routes.""" business_secret = _encrypt_database_secret(business_dsn) @@ -276,6 +280,7 @@ class MetadataRepository: map_extent=map_extent, status="active", map_config={}, + settings=settings or {}, created_at=_utcnow(), updated_at=_utcnow(), ) @@ -333,6 +338,8 @@ class MetadataRepository: setattr(project, field, updates[field]) if "map_config" in updates: project.map_config = {**(project.map_config or {}), **updates["map_config"]} + if "settings" in updates: + project.settings = {**(project.settings or {}), **updates["settings"]} project.updated_at = _utcnow() await self.session.commit() await self.session.refresh(project) @@ -350,6 +357,7 @@ class MetadataRepository: gs_workspace=project.gs_workspace, map_extent=project.map_extent, map_config=project.map_config or {}, + settings=project.settings or {}, status=project.status, ) diff --git a/app/infra/db/postgresql/scada_replays.py b/app/infra/db/postgresql/scada_replays.py new file mode 100644 index 0000000..21724c3 --- /dev/null +++ b/app/infra/db/postgresql/scada_replays.py @@ -0,0 +1,188 @@ +"""Short-transaction storage for replay control and resumable batches.""" +from contextlib import asynccontextmanager +from datetime import datetime, timezone + +from fastapi import HTTPException +from psycopg.types.json import Jsonb + + +class ScadaReplayRepository: + def __init__(self, connection, project_id): + self.connection, self.project_id = connection, str(project_id) + self.lock_key = f"scada-replay:{self.project_id}" + + async def available(self) -> bool: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT to_regclass('peripheral_sample.scada_replays') IS NOT NULL " + "AND to_regclass('peripheral_sample.scada_replay_batches') IS NOT NULL AS ready" + ) + return (await cur.fetchone())["ready"] + + @asynccontextmanager + async def control(self): + async with self.connection.transaction(): + await self.connection.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (self.lock_key,)) + yield + + @asynccontextmanager + async def processing(self): + # Session lock outlives short transactions, never an open SQL transaction + # across Timescale IO. A dead connection automatically releases ownership. + async with self.connection.transaction(): + cur = await self.connection.execute("SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS owned", (self.lock_key,)) + owned = (await cur.fetchone())["owned"] + try: + yield owned + finally: + if owned: + async with self.connection.transaction(): + await self.connection.execute("SELECT pg_advisory_unlock(hashtextextended(%s,0))", (self.lock_key,)) + + async def get(self, replay_id) -> dict: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id=%s", + (self.project_id, str(replay_id)), + ) + row = await cur.fetchone() + if not row: + raise HTTPException(404, "当前项目不存在此 SCADA 回放") + return row["document"] + + async def current(self) -> dict | None: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s" + " ORDER BY created_at DESC LIMIT 1", (self.project_id,), + ) + row = await cur.fetchone() + return row["document"] if row else None + + async def by_request(self, request_id) -> dict | None: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND request_id=%s", + (self.project_id, str(request_id)), + ) + row = await cur.fetchone() + return row["document"] if row else None + + async def active_other(self, replay_id) -> bool: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT 1 FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id<>%s " + "AND status IN ('ready','paused') LIMIT 1", + (self.project_id, str(replay_id)), + ) + return await cur.fetchone() is not None + + async def insert(self, document: dict): + await self.connection.execute( + "INSERT INTO peripheral_sample.scada_replays(project_id,replay_id,request_id,status,document) " + "VALUES(%s,%s,%s,%s,%s)", + (self.project_id, document["replay_id"], document["config"]["request_id"], document["status"], Jsonb(document)), + ) + + async def save(self, document: dict): + document["updated_at"] = datetime.now(timezone.utc).isoformat() + await self.connection.execute( + "UPDATE peripheral_sample.scada_replays SET status=%s,document=%s,updated_at=now() " + "WHERE project_id=%s AND replay_id=%s", + (document["status"], Jsonb(document), self.project_id, document["replay_id"]), + ) + + async def pending_batch(self, replay_id) -> dict | None: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT document FROM peripheral_sample.scada_replay_batches " + "WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1", + (self.project_id, str(replay_id)), + ) + row = await cur.fetchone() + return row["document"] if row else None + + async def save_batch(self, replay_id, batch: dict): + await self.connection.execute( + "INSERT INTO peripheral_sample.scada_replay_batches(project_id,replay_id,sequence,stage,document) " + "VALUES(%s,%s,%s,%s,%s) ON CONFLICT(project_id,replay_id,sequence) " + "DO UPDATE SET stage=excluded.stage,document=excluded.document", + (self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)), + ) + + async def events(self, replay_id, offset=0, limit=100) -> list[dict]: + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e " + "WHERE b.project_id=%s AND b.replay_id=%s ORDER BY b.sequence DESC LIMIT %s OFFSET %s", + (self.project_id, str(replay_id), limit, offset), + ) + return [r["event"] for r in await cur.fetchall()] + + async def monitoring_outputs( + self, + replay_id, + device_ids: list[str], + start: datetime, + end: datetime, + before_sequence: int, + ) -> list[dict]: + """Read only committed replay outputs available before the current batch.""" + if not device_ids: + return [] + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' AND b.sequence<%s " + "AND r.value->>'device_id'=ANY(%s) " + "AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s " + "ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'", + (self.project_id, str(replay_id), before_sequence, device_ids, start, end), + ) + return [row["reading"] for row in await cur.fetchall()] + + async def detected_outputs( + self, + replay_id, + device_ids: list[str], + start: datetime, + end: datetime, + before_sequence: int, + *, + limit: int | None = None, + offset: int = 0, + ) -> list[dict]: + """Read immutable detector outputs available to a bounded replay investigation.""" + statement = ( + "SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence<=%s AND b.stage='detected' " + "AND r.value->>'device_id'=ANY(%s) " + "AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s " + "ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'" + ) + params: tuple = ( + self.project_id, str(replay_id), before_sequence, device_ids, start, end, + ) + if limit is not None: + statement += " LIMIT %s OFFSET %s" + params += (limit, offset) + async with self.connection.transaction(): + cur = await self.connection.execute(statement, params) + return [row["reading"] for row in await cur.fetchall()] + + async def latest_outputs(self, replay_id, through: datetime, limit: int = 500) -> list[dict]: + """Return the latest committed cleaned output for each replay device.""" + async with self.connection.transaction(): + cur = await self.connection.execute( + "SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading " + "FROM peripheral_sample.scada_replay_batches b " + "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r " + "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') " + "AND (r.value->>'time')::timestamptz<=%s " + "ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s", + (self.project_id, str(replay_id), through, limit), + ) + return [row["reading"] for row in await cur.fetchall()] diff --git a/app/infra/db/timescaledb/repositories/scada.py b/app/infra/db/timescaledb/repositories/scada.py index 89c047d..d5f0ae5 100644 --- a/app/infra/db/timescaledb/repositories/scada.py +++ b/app/infra/db/timescaledb/repositories/scada.py @@ -9,6 +9,86 @@ class ScadaRepository: VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"}) RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value" + @staticmethod + async def get_warmup_values( + conn: AsyncConnection, + device_ids: list[str], + before: datetime, + per_device_limit: int, + ) -> list[dict]: + """Return the latest raw values before a replay starts for detector warmup.""" + async with conn.cursor() as cur: + await cur.execute( + "SELECT d.device_id,h.monitored_value FROM unnest(%s::text[]) d(device_id) " + "CROSS JOIN LATERAL (SELECT monitored_value,time FROM scada.measurements " + "WHERE device_id=d.device_id AND time<%s ORDER BY time DESC LIMIT %s) h " + "ORDER BY d.device_id,h.time", + (device_ids, before, per_device_limit), + ) + return [dict(row) for row in await cur.fetchall()] + + @staticmethod + async def get_raw_page( + conn: AsyncConnection, device_ids: list[str], start: datetime, + end: datetime, through: datetime, limit: int, + after: tuple[datetime, str] | None = None, *, frame: bool = False, + ) -> list[dict]: + """Read raw only, with a stable composite key and a bounded frame/page.""" + predicate = "device_id=ANY(%s) AND time>=%s AND time<%s AND time<=%s" + params: tuple = (device_ids, start, end, through) + if after is not None: + predicate += " AND (time,device_id)>(%s,%s)" + params += after + if frame: + statement = ( + "WITH next_time AS (SELECT min(time) AS time FROM scada.measurements WHERE " + + predicate + ") SELECT m.time,m.device_id,m.monitored_value," + "md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash " + "FROM scada.measurements m WHERE m.device_id=ANY(%s) " + "AND m.time=(SELECT time FROM next_time) ORDER BY m.time,m.device_id LIMIT %s" + ) + params += (device_ids, limit) + else: + statement = ( + "SELECT time,device_id,monitored_value," + "md5(COALESCE(monitored_value::text,'NULL')) AS source_hash " + "FROM scada.measurements WHERE " + predicate + + " ORDER BY time,device_id LIMIT %s" + ) + params += (limit,) + async with conn.cursor() as cur: + await cur.execute(statement, params) + return [dict(row) for row in await cur.fetchall()] + + @staticmethod + async def apply_replay_cleaning(conn: AsyncConnection, rows: list[dict]) -> None: + """Guard every raw version; update corrected values without inserting raw.""" + if not rows: + return + async with conn.transaction(): + async with conn.cursor() as cur: + await cur.execute( + "SELECT m.time,m.device_id,md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash " + "FROM scada.measurements m JOIN unnest(%s::timestamptz[],%s::text[]) " + "b(time,device_id) USING(time,device_id) FOR UPDATE OF m", + ([r["time"] for r in rows], [r["device_id"] for r in rows]), + ) + existing = {(r["time"].isoformat(), r["device_id"]): r["source_hash"] + for r in await cur.fetchall()} + if len(existing) != len(rows) or any( + existing.get((datetime.fromisoformat(r["time"]).isoformat(), r["device_id"])) + != r["source_hash"] for r in rows + ): + raise ValueError("SOURCE_CHANGED: SCADA 原始值在处理期间发生变化") + await cur.execute( + "UPDATE scada.measurements m SET cleaned_value=b.value " + "FROM unnest(%s::timestamptz[],%s::text[],%s::double precision[]) " + "b(time,device_id,value) WHERE m.time=b.time AND m.device_id=b.device_id " + "AND m.cleaned_value IS DISTINCT FROM b.value", + ([r["time"] for r in rows], [r["device_id"] for r in rows], + [r["cleaned_value"] for r in rows]), + ) + @staticmethod async def insert_scada_batch(conn: AsyncConnection, data: List[dict]): if not data: diff --git a/app/mcp/analysis_service.py b/app/mcp/analysis_service.py index b346b14..4fed2d5 100644 --- a/app/mcp/analysis_service.py +++ b/app/mcp/analysis_service.py @@ -98,3 +98,19 @@ async def bounded_calculation(function, *args, **kwargs): finally: _slots.release() return await asyncio.to_thread(run) + + +async def calculate_replay_burst_location( + parameters: dict[str, Any], *, network: str, username: str, +) -> dict[str, Any]: + """Run location with Server-built frozen replay series and no live SCADA query.""" + from app.services.burst_location import run_burst_location_by_network + + return await bounded_calculation( + run_burst_location_by_network, + network=network, + username=username, + scheme_name=None, + final_candidates_csv_path=None, + **parameters, + ) diff --git a/app/mcp/contexts.py b/app/mcp/contexts.py index 8f95a41..0ab3141 100644 --- a/app/mcp/contexts.py +++ b/app/mcp/contexts.py @@ -30,6 +30,7 @@ async def verify_task(scope: QueryScope, binding: BoundContext) -> None: task = await repo.get_task(str(binding.task_id)) if not task: raise HTTPException(404, "Task is not in this project") + binding.replay_context = task.get("source", {}).get("replay_context") if binding.purpose == "task_collaboration": return if OPERATIONAL_TASK_EXECUTE not in permissions: diff --git a/app/mcp/models.py b/app/mcp/models.py index 109d73e..2b6ea23 100644 --- a/app/mcp/models.py +++ b/app/mcp/models.py @@ -291,5 +291,6 @@ class BoundContext(BaseModel): task_id: UUID | None = None run_id: UUID | None = None execution_request_id: UUID | None = None + replay_context: dict | None = None expires_at: datetime revoked_at: datetime | None = None diff --git a/app/mcp/query_service.py b/app/mcp/query_service.py index cc5676a..23dcde7 100644 --- a/app/mcp/query_service.py +++ b/app/mcp/query_service.py @@ -66,8 +66,23 @@ def page_result(rows, args): async def authorize_query( scope: QueryScope, binding: BoundContext, tool: QueryTool, args: BaseModel ) -> None: + replay_context = getattr(binding, "replay_context", None) if tool.name not in binding.tool_cap: raise HTTPException(403, "Query is outside the captured tool catalog") + if replay_context: + from app.services.scada_replay_queries import check_replay_window + allowed = {"get_task_context", "get_task_evidence", "list_scada_devices", "get_scada_device", + "query_scada_readings", "get_network_schema", "run_burst_location", + "get_calculation_job", "get_calculation_result", "list_calculation_jobs"} + if tool.name not in allowed: + raise HTTPException(403, "回放调查当前仅开放冻结 SCADA 证据与模型目录查询；请使用 query_scada_readings") + if tool.name == "query_scada_readings": + check_replay_window(replay_context, args.start, args.end) + if tool.name == "run_burst_location": + if args.data_source != "monitoring" or args.scada_normal_start is None or args.scada_normal_end is None: + raise HTTPException(422, "回放定位只支持显式异常窗和正常对照窗的 monitoring 输入") + check_replay_window(replay_context, args.scada_burst_start, args.scada_burst_end) + check_replay_window(replay_context, args.scada_normal_start, args.scada_normal_end) permissions = permissions_for_context(scope.project) if tool.mutating: if binding.purpose != "ordinary_chat" or binding.task_id is not None or scope.project.project_role != "member": @@ -149,7 +164,37 @@ async def execute_query( gaps = [] source_ref = "mcp:" + str(uuid4()) evidence_id = None - if tool.computation: + replay_context = getattr(binding, "replay_context", None) + if name == "query_scada_readings" and replay_context: + from app.services.scada_replay_queries import replay_snapshot_rows + rows = await replay_snapshot_rows(conn, scope.project.project_id, replay_context, + args.device_ids, args.start, args.end, args.limit + 1, args.offset) + data, page, truncated = page_result(rows, args) + origin = "synthetic" + gaps.append("历史数据回放证据，仅覆盖该轮调查已提交的批次，不表示现场实时观测。") + elif name == "run_burst_location" and replay_context: + from app.mcp.analysis_service import calculate_replay_burst_location + from app.services.scada_replay_queries import replay_burst_location_inputs + + parameters = await replay_burst_location_inputs( + conn, scope.project.project_id, replay_context, args, + ) + await conn.commit() + async with scope.simulation_routing(): + result = await calculate_replay_burst_location( + parameters, + network=scope.project.project_code, + username=scope.user.username, + ) + data = { + "parameters": args.model_dump(mode="json", exclude={"context_handle"}), + "result": result, + } + origin = "synthetic" + gaps = [ + "定位只使用任务知识截止前已固化的回放清洗输出；候选排序依赖漏量假设、模型和测点覆盖，仍需现场核实。" + ] + elif tool.computation: from app.mcp.analysis_service import TIMESCALE_CALCULATIONS, calculate # Context/receipt reads may have started an implicit transaction. Release diff --git a/app/mcp/work_order_skills.py b/app/mcp/work_order_skills.py index 12ad30b..237c1ad 100644 --- a/app/mcp/work_order_skills.py +++ b/app/mcp/work_order_skills.py @@ -9,6 +9,7 @@ import yaml SKILL_ROOT = Path(__file__).resolve().parents[2] / "resources" / "work-order-skills" WORKFLOW_SKILLS = { + "scada_anomaly": "scada-anomaly", "burst_report": "burst-report", "water_quality_complaint": "water-quality-complaint", "general": "appointment-follow-up", diff --git a/app/services/operational_tasks.py b/app/services/operational_tasks.py index ec80da7..48faf8c 100644 --- a/app/services/operational_tasks.py +++ b/app/services/operational_tasks.py @@ -161,6 +161,14 @@ class OperationalTaskService: ) async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail: + if command.source.replay_context: + from app.infra.db.postgresql.scada_replays import ScadaReplayRepository + replay_context = command.source.replay_context + replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id) + require(replay_doc.get("delivered_until") is not None + and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"]) + and replay_context.batch_sequence <= replay_doc["batch_sequence"], + "REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422) body = _json(command) request_id = str(command.request_id) scope = "task:create" @@ -219,6 +227,27 @@ class OperationalTaskService: await self._task_event(task_document, now) return await self.detail(task_id) + async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None: + """A recovery is new evidence, coalesced through the existing run lifecycle.""" + scope = f"task:{task_id}:replay-evidence" + async with self.repo.transaction(): + if await self._request_replay(scope, request_id, replay_context): + return + await self.repo.lock(f"task:{task_id}") + task = await self._task(task_id) + prior = task["source"].get("replay_context") + require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID", + "Replay evidence belongs to another investigation", status=409) + require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]), + "REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409) + task["source"]["replay_context"] = replay_context + task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1 + now = self._now() + await self._requeue_task(task, now) + await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task))) + await self._task_event(task, now) + await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)}) + async def list( self, *, task_type: str | None, status: str | None, limit: int, offset: int, peripheral_work_order_id: UUID | None = None, @@ -336,6 +365,9 @@ class OperationalTaskService: selected_report=await self.report(task_id, report_revision) if report_revision is not None else None, ) + async def replay_context(self, task_id: UUID) -> dict | None: + return (await self._task(task_id)).get("source", {}).get("replay_context") + async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool: task = await self._task(task_id) permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW @@ -1037,6 +1069,8 @@ class OperationalTaskService: await self.repo.lock(f"run:{command.run_id}") task = await self._task(task_id) self._require_actionable_task(task) + require(not task.get("source", {}).get("replay_context"), "REPLAY_ACTION_DISABLED", + "SCADA 回放调查仅生成分析与报告，不执行真实工单或设备动作", status=409) run = await self._active_run(task) require(run is not None and str(run["run_id"]) == str(command.run_id) and run["generation"] == task["execution_generation"], diff --git a/app/services/scada_raw.py b/app/services/scada_raw.py new file mode 100644 index 0000000..01f3ff8 --- /dev/null +++ b/app/services/scada_raw.py @@ -0,0 +1,66 @@ +"""Read-only chronological SCADA paging shared by REST and replay processing.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import math +from datetime import datetime + +from app.domain.schemas.scada import ScadaRawPage, ScadaRawReading +from app.infra.db.timescaledb.repositories.scada import ScadaRepository + + +def validate_window(start: datetime, end: datetime) -> None: + if start.tzinfo is None or end.tzinfo is None or start >= end: + raise ValueError("需要带时区、结束晚于开始的 SCADA 时间区间") + + +def raw_reading(row: dict) -> ScadaRawReading: + value = row["monitored_value"] + invalid = value is not None and not math.isfinite(float(value)) + return ScadaRawReading( + time=row["time"], device_id=row["device_id"], + monitored_value=None if invalid else value, + quality="invalid" if invalid else "missing" if value is None else "unvalidated", + ) + + +async def read_raw_page( + conn, *, project_id: str, device_ids: list[str], start: datetime, + end: datetime, limit: int = 500, cursor: str | None = None, + through: datetime | None = None, +) -> ScadaRawPage: + validate_window(start, end) + ids = sorted(set(device_ids)) + if not ids or len(ids) > 500 or not 1 <= limit <= 500: + raise ValueError("请选择 1 至 500 个设备，每页最多 500 条") + cap = min(through or end, end) + if cap.tzinfo is None or cap < start: + raise ValueError("模拟时间不在可读取区间内") + signature = hashlib.sha256(json.dumps( + [project_id, ids, start.isoformat(), end.isoformat()], separators=(",", ":") + ).encode()).hexdigest() + after = None + if cursor: + try: + data = json.loads(base64.urlsafe_b64decode(cursor.encode())) + frozen = datetime.fromisoformat(data["through"]) + last = datetime.fromisoformat(data["time"]) + if (data["query"] != signature or frozen.tzinfo is None or last.tzinfo is None + or frozen > cap or not start <= last <= frozen or data["device"] not in ids): + raise ValueError() + cap, after = frozen, (last, data["device"]) + except (ValueError, KeyError, TypeError, UnicodeError) as exc: + raise ValueError("SCADA 分页游标无效或查询范围已改变") from exc + rows = await ScadaRepository.get_raw_page(conn, ids, start, end, cap, limit + 1, after) + selected = rows[:limit] + next_cursor = None + if len(rows) > limit: + last = selected[-1] + next_cursor = base64.urlsafe_b64encode(json.dumps({ + "query": signature, "through": cap.isoformat(), + "time": last["time"].isoformat(), "device": last["device_id"], + }, separators=(",", ":")).encode()).decode() + return ScadaRawPage(items=[raw_reading(row) for row in selected], + next_cursor=next_cursor, scanned_through=cap) diff --git a/app/services/scada_replay.py b/app/services/scada_replay.py new file mode 100644 index 0000000..13c8c23 --- /dev/null +++ b/app/services/scada_replay.py @@ -0,0 +1,397 @@ +"""Durable, project-scoped SCADA playback; raw is never inserted or rewritten.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from fastapi import HTTPException +from app.auth.permissions import permissions_for_context + +from app.domain.schemas.project_settings import ScadaMonitoringSettings +from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand, ScadaReplaySnapshot +from app.infra.db.postgresql.scada import ScadaInfoRepository +from app.infra.db.timescaledb.repositories.scada import ScadaRepository +from app.services.content_digest import digest +from app.services.scada_replay_processing import prepare_cleaning, detect_frame +from app.services.scada_replay_monitoring import evaluate_monitoring_period + + +def parse_time(value: str) -> datetime: + return datetime.fromisoformat(value) + + +def snapshot(document: dict) -> ScadaReplaySnapshot: + return ScadaReplaySnapshot( + **{key: document[key] for key in ( + "replay_id", "project_id", "revision", "status", "start_time", "end_time", + "virtual_now", "created_at", "updated_at", + )}, + delivered_until=document.get("delivered_until"), processed_until=document.get("processed_until"), + processed_count=document.get("processed_count", 0), anomaly_count=document.get("anomaly_count", 0), + devices=document["devices"], rules=document["config"]["rules"], + latest=list(document.get("latest", {}).values()), + active_events=[state["event"] for state in document.get("rule_states", {}).values() if state.get("event")], + error=document.get("error"), + monitoring_interval_seconds=document.get("monitoring_interval_seconds"), + step_pending=document.get("step_pending", False), + monitoring_count=document.get("monitoring_count", 0), + last_monitoring=document.get("last_monitoring"), + ) + + +class ScadaReplayService: + def __init__(self, repository, context, timescale=None, *, clock=None): + self.repo, self.context, self.timescale = repository, context, timescale + self.clock = clock or (lambda: datetime.now(timezone.utc)) + + async def ensure_storage(self): + if not await self.repo.available(): + raise HTTPException(503, "SCADA 回放存储未配置，需要应用 031_scada_replays.sql") + + async def create( + self, + command: ScadaReplayCreate, + *, + monitoring_interval_seconds: int | None, + monitoring_config: ScadaMonitoringSettings | None = None, + ): + await self.ensure_storage() + async with self.repo.control(): + previous = await self.repo.by_request(command.request_id) + config = command.model_dump(mode="json") + if previous: + if previous["config"] != config: + raise HTTPException(409, "创建请求号已用于不同回放配置") + return snapshot(previous) + if monitoring_interval_seconds is None: + raise HTTPException(422, "请先配置项目 settings.scada_monitoring_interval_seconds，再创建演示") + replay_id = str(uuid4()) + if await self.repo.active_other(replay_id): + raise HTTPException(409, "当前项目已有回放，请先停止原回放再新建") + metadata = {row["device_id"]: row for row in await ScadaInfoRepository.get_scadas(self.repo.connection)} + if not set(command.device_ids) <= metadata.keys(): + raise HTTPException(422, "设备不属于当前项目") + for rule in command.rules: + if any(metadata[device]["measurement_unit"] != rule.unit for device in rule.device_ids): + raise HTTPException(422, "规则单位与设备单位不一致，请分组配置") + if any(metadata[device]["device_type"] == "quality" for device in rule.device_ids) and not rule.metric_name.strip(): + raise HTTPException(422, "水质规则必须明确指标名称，不能仅凭单位解释读数") + if monitoring_config: + role_ids = set(monitoring_config.pressure_device_ids) | set(monitoring_config.outlet_flow_device_ids) + if not role_ids <= metadata.keys(): + raise HTTPException(422, "项目 SCADA 监测角色包含不存在的设备") + if any(metadata[device]["device_type"] != "pressure" for device in monitoring_config.pressure_device_ids): + raise HTTPException(422, "压力监测角色必须引用 pressure 设备") + if any(metadata[device]["device_type"] not in {"pipe_flow", "flow"} + for device in monitoring_config.outlet_flow_device_ids): + raise HTTPException(422, "出厂边界流量角色必须引用流量设备") + now = self.clock().isoformat() + document = { + "replay_id": replay_id, "project_id": str(self.context.project_id), + "created_by": str(self.context.user_id), "config": config, + "revision": 1, "status": "ready", "start_time": config["start_time"], + "end_time": config["end_time"], "virtual_now": config["start_time"], + "created_at": now, "updated_at": now, + "monitoring_interval_seconds": monitoring_interval_seconds, + "monitoring_config": monitoring_config.model_dump(mode="json") if monitoring_config else None, + "devices": [metadata[device] for device in command.device_ids], + "batch_sequence": 0, "processed_count": 0, "anomaly_count": 0, + "monitoring_count": 0, "latest": {}, "histories": {}, "rule_states": {}, "receipts": {}, + } + await self.repo.insert(document) + return snapshot(document) + + async def command(self, replay_id, command: ScadaReplayCommand): + await self.ensure_storage() + async with self.repo.control(): + document = await self.repo.get(replay_id) + key, body = str(command.request_id), command.model_dump(mode="json") + receipt = document.setdefault("receipts", {}).get(key) + if receipt: + if receipt != body: + raise HTTPException(409, "命令请求号已用于不同内容") + return snapshot(document) + if command.expected_revision != document["revision"]: + raise HTTPException(409, "回放控制状态已更新，请刷新后重试") + action, status = command.action, document["status"] + if document.get("step_pending") and action == "step": + raise HTTPException(409, "当前时间步正在处理，请完成后继续") + if action in {"step", "retry"} and await self.repo.active_other(replay_id): + raise HTTPException(409, "当前项目已有另一条活动回放") + if action == "step" and status in {"ready", "paused"}: + interval = document.get("monitoring_interval_seconds") + if interval is None: + raise HTTPException(409, "旧回放未固定监测周期，请停止后新建演示") + start = parse_time(document["start_time"]) + elapsed = (parse_time(document["virtual_now"]) - start).total_seconds() + target = start + timedelta(seconds=(int(elapsed // interval) + 1) * interval) + document["step_start"] = document["virtual_now"] + document["step_target"] = min(parse_time(document["end_time"]), target).isoformat() + document["status"], document["step_pending"] = "paused", True + elif action == "stop" and status not in {"completed", "stopped"}: + document["status"], document["step_pending"] = "stopped", False + document.pop("step_target", None) + document.pop("step_start", None) + elif action == "retry" and status == "failed": + if "SOURCE_CHANGED" in (document.get("error") or ""): + raise HTTPException(409, "原始数据发生变化，需要停止并新建回放") + document["status"], document["error"] = "paused", None + document["step_pending"] = bool(document.get("step_target")) + else: + raise HTTPException(409, "当前回放状态不支持此操作") + document["revision"] += 1 + document["receipts"][key] = body + if len(document["receipts"]) > 256: + del document["receipts"][next(iter(document["receipts"]))] + await self.repo.save(document) + return snapshot(document) + + async def process_pending_step(self, replay_id): + await self.ensure_storage() + async with self.repo.processing() as owned: + if not owned: + raise HTTPException(409, "当前时间步正在由另一请求处理") + while True: + document = await self.repo.get(replay_id) + if not document.get("step_pending"): + return snapshot(document) + try: + await self._advance(document) + except Exception as exc: + # Failure never discards the prepared batch; retry resumes it. + document = await self.repo.get(replay_id) + document["status"], document["step_pending"] = "failed", False + document["error"] = ( + "SOURCE_CHANGED: 原始观测已改变，需要新建回放" + if "SOURCE_CHANGED" in str(exc) + else "SCADA 处理失败，已保留批次与进度；请检查服务日志后重试" + ) + async with self.repo.connection.transaction(): + await self.repo.save(document) + raise + + async def _advance(self, document): + config = ScadaReplayCreate.model_validate(document["config"]) + replay_id = document["replay_id"] + if not document.get("warmed"): + import math + window = max(rule.window_samples for rule in config.rules) + async with self.timescale.transaction(): + warmup = await ScadaRepository.get_warmup_values( + self.timescale, config.device_ids, config.start_time, window, + ) + for row in warmup: + value = row["monitored_value"] + bounds = config.quality_ranges.get(row["device_id"]) + if value is not None and math.isfinite(value) and not (bounds and ( + bounds.lower is not None and value < bounds.lower + or bounds.upper is not None and value > bounds.upper + )): + document["histories"].setdefault(row["device_id"], []).append(value) + document["warmed"] = True + async with self.repo.connection.transaction(): + await self.repo.save(document) + batch = await self.repo.pending_batch(replay_id) + if batch is None: + step = document.get("step_pending", False) + virtual = parse_time(document["virtual_now"]) + through = parse_time(document["step_target"]) if step else virtual + after = ((parse_time(document["cursor"][0]), document["cursor"][1]) + if document.get("cursor") else None) + async with self.timescale.transaction(): + rows = await ScadaRepository.get_raw_page( + self.timescale, config.device_ids, config.start_time, config.end_time, + through, 501, after, frame=True, + ) + if len(rows) > 500: + raise ValueError("SCADA frame exceeds selected device bound") + at = rows[0]["time"] if rows else through + if step: + virtual = through + document["virtual_now"] = virtual.isoformat() + if not rows and document.get("processed_until") == at.isoformat() and not step: + async with self.repo.connection.transaction(): + await self.repo.save(document) + return + sequence = document["batch_sequence"] + 1 + refs = [{"time": row["time"].isoformat(), "device_id": row["device_id"], + "source_hash": row["source_hash"]} for row in rows] + batch = {"sequence": sequence, "stage": "selected", "at": at.isoformat(), + "refs": refs, "input_digest": digest(refs), "events": []} + async with self.repo.connection.transaction(): + document["batch_sequence"] = sequence + document["delivered_until"] = at.isoformat() + await self.repo.save_batch(replay_id, batch) + await self.repo.save(document) + else: + rows = None + if batch["stage"] == "selected": + if rows is None: + # Only crash recovery reloads the selected raw frame. + at = parse_time(batch["at"]) + async with self.timescale.transaction(): + rows = await ScadaRepository.get_raw_page( + self.timescale, config.device_ids, at, config.end_time, at, 501, + ) if batch["refs"] else [] + actual = [{"time": row["time"].isoformat(), "device_id": row["device_id"], + "source_hash": row["source_hash"]} for row in rows] + if digest(actual) != batch["input_digest"]: + raise ValueError("SOURCE_CHANGED: selected raw frame differs") + batch["outputs"] = prepare_cleaning(rows, config) + batch["stage"] = "clean_prepared" + async with self.repo.connection.transaction(): + await self.repo.save_batch(replay_id, batch) + if batch["stage"] == "clean_prepared": + await ScadaRepository.apply_replay_cleaning(self.timescale, batch["outputs"]) + batch["stage"] = "cleaned" + async with self.repo.connection.transaction(): + await self.repo.save_batch(replay_id, batch) + if batch["stage"] == "cleaned": + result, changes = detect_frame(document, batch["outputs"], config, parse_time(batch["at"])) + result["processed_until"] = batch["at"] + target = result.get("step_target") + was_step = bool(target) + result["step_pending"] = bool(target and parse_time(batch["at"]) < parse_time(target)) + monitoring = None + if was_step and not result["step_pending"]: + period_start = parse_time(result.get("step_start", result["start_time"])) + period_end = parse_time(target) + role = result.get("monitoring_config") or {} + role_ids = list(dict.fromkeys( + role.get("pressure_device_ids", []) + role.get("outlet_flow_device_ids", []) + )) + previous = await self.repo.monitoring_outputs( + replay_id, role_ids, max(config.start_time, period_end - timedelta(days=15)), + period_end, batch["sequence"], + ) if role_ids else [] + from app.mcp.analysis_service import bounded_calculation + monitoring = await bounded_calculation( + evaluate_monitoring_period, + result, + previous + batch["outputs"], + period_start=period_start, + period_end=period_end, + batch_sequence=batch["sequence"], + ) + result["last_monitoring"] = monitoring + result["monitoring_count"] = result.get("monitoring_count", 0) + 1 + if not result["step_pending"]: + result.pop("step_target", None) + result.pop("step_start", None) + if batch["refs"]: + last = batch["refs"][-1] + result["cursor"] = [last["time"], last["device_id"]] + if parse_time(batch["at"]) >= config.end_time: + result["status"] = "completed" + batch["stage"], batch["events"] = "detected", changes + if monitoring is not None: + batch["monitoring"] = monitoring + async with self.repo.connection.transaction(): + await self._create_investigations(result, changes, batch, create_tasks=not was_step) + if monitoring is not None: + await self._create_monitoring_task(result, monitoring, batch) + batch["monitoring"] = monitoring + result["last_monitoring"] = monitoring + await self.repo.save_batch(replay_id, batch) + await self.repo.save(result) + + async def _create_investigations(self, document, changes, batch, *, create_tasks=True): + from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger + from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository + from app.services.operational_tasks import OperationalTaskService + from app.domain.schemas.task_alert import TaskAlertEvent + from app.infra.db.postgresql.task_alerts import TaskAlertRepository + from app.services.task_alerts import TaskAlertService + from uuid import UUID + producer = "scada-replay:" + document["replay_id"] + alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id), + self.context, internal_producer_id=producer) + tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id), + self.context, permissions_for_context(self.context)) + for event in changes: + if event["change"] == "raised": + event["batch_sequence"] = batch["sequence"] + document["rule_states"][event["rule_id"]]["event"]["batch_sequence"] = batch["sequence"] + await alerts.push(TaskAlertEvent( + event_id=uuid5(NAMESPACE_URL, event["event_id"] + ":" + event["change"]), + alert_id=event["event_id"], producer_id=producer, event_type=event["change"], + occurred_at=event["occurred_at"], condition_id=None, + source={"kind": "scada_replay", "replay_id": document["replay_id"], + "rule_id": event["rule_id"], "batch_sequence": event["batch_sequence"]}, + title=f"SCADA 回放异常 · {event['rule_id']}", description=event["detail"], severity="warning", + )) + if event["change"] != "raised": + if event.get("task_id"): + await tasks.advance_replay_investigation(UUID(event["task_id"]), { + "replay_id": document["replay_id"], "knowledge_cutoff": batch["at"], + "batch_sequence": batch["sequence"], + }, uuid5(NAMESPACE_URL, event["event_id"] + ":recovery")) + continue + if event.get("category") == "data_quality": + continue + if not create_tasks: + continue + external = f"scada_replay:{document['replay_id']}:event:{event['event_id']}" + command = OperationalTaskCreate( + request_id=uuid5(NAMESPACE_URL, external), task_type="event_investigation", + workflow="scada_anomaly", title=f"SCADA 回放调查 · {event['rule_id']}", + summary=event["detail"], source=TaskSource( + kind="external_system", external_id=external, external_revision=1, + event_kind="scada_anomaly", + replay_context={"replay_id": document["replay_id"], + "knowledge_cutoff": batch["at"], "batch_sequence": batch["sequence"]}, + ), + trigger=TaskTrigger(kind="event", occurred_at=event["occurred_at"], detail="已存 SCADA 历史的回放检测信号，原因尚待调查"), + ) + task = await tasks.create(command) + event["task_id"] = str(task.task_id) + document["rule_states"][event["rule_id"]]["event"]["task_id"] = event["task_id"] + + async def _create_monitoring_task(self, document, monitoring, batch): + if monitoring["status"] != "suspected_burst": + return + from app.domain.schemas.operational_task import OperationalTaskCreate, TaskSource, TaskTrigger + from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository + from app.services.operational_tasks import OperationalTaskService + tasks = OperationalTaskService( + OperationalTaskRepository(self.repo.connection, self.context.project_id), + self.context, + permissions_for_context(self.context), + ) + external = ( + f"scada_replay:{document['replay_id']}:monitor:" + f"{monitoring['period_end']}:{monitoring['rule_version']}" + ) + pressure = monitoring["pressure"] + flow = monitoring["outlet_flow"] + detector = monitoring["burst_detection"] + summary = ( + f"步长监测疑似爆管：爆管侦测得分 {detector['target_score']:.4f}；" + f"压力下降测点 {len(pressure['hit_device_ids'])} 个；" + f"出厂边界流量增加测点 {len(flow['hit_device_ids'])} 个。" + ) + command = OperationalTaskCreate( + request_id=uuid5(NAMESPACE_URL, external), + task_type="event_investigation", + workflow="burst_report", + title=f"SCADA 回放疑似爆管 · {monitoring['period_end']}", + summary=summary, + source=TaskSource( + kind="external_system", + external_id=external, + external_revision=1, + event_kind="scada_burst_suspected", + replay_context={ + "replay_id": document["replay_id"], + "knowledge_cutoff": monitoring["knowledge_cutoff"], + "batch_sequence": batch["sequence"], + }, + ), + trigger=TaskTrigger( + kind="event", + occurred_at=monitoring["period_end"], + detail="主动步长监测的多项指标相互支持，进入爆管调查；尚未现场确诊", + ), + ) + task = await tasks.create(command) + monitoring["task_id"] = str(task.task_id) diff --git a/app/services/scada_replay_monitoring.py b/app/services/scada_replay_monitoring.py new file mode 100644 index 0000000..eebbd5a --- /dev/null +++ b/app/services/scada_replay_monitoring.py @@ -0,0 +1,210 @@ +"""Deterministic monitoring summaries built only from frozen replay outputs.""" +from __future__ import annotations + +from datetime import datetime, timedelta +from statistics import median +from typing import Any +from uuid import NAMESPACE_URL, uuid5 + +import pandas as pd + +from app.domain.schemas.scada_replay import ScadaReplayCreate +from app.services.burst_detection import ( + TARGET_MU, + TARGET_N_ESTIMATORS, + TARGET_RANDOM_STATE, + TARGET_SCORE_THRESHOLD, + run_burst_detection, +) + + +def _time(value: str | datetime) -> datetime: + return value if isinstance(value, datetime) else datetime.fromisoformat(value) + + +def _valid_rows(rows: list[dict], device_id: str) -> list[dict]: + return sorted( + ( + row for row in rows + if row.get("device_id") == device_id + and row.get("quality") in {"valid", "corrected"} + and row.get("cleaned_value") is not None + ), + key=lambda row: _time(row["time"]), + ) + + +def _relative_indicator( + rows: list[dict], + config: ScadaReplayCreate, + device_ids: list[str], + period_start: datetime, + period_end: datetime, + *, + direction: str, +) -> dict[str, Any]: + details: list[dict[str, Any]] = [] + gaps: list[dict[str, str]] = [] + for device_id in device_ids: + rule = next(( + item for item in config.rules + if item.kind == "relative_change" and device_id in item.device_ids + and item.threshold is not None + ), None) + if rule is None: + gaps.append({"device_id": device_id, "reason": "relative_change_rule_missing"}) + continue + valid = _valid_rows(rows, device_id) + baseline = [row for row in valid if _time(row["time"]) < period_start][-rule.window_samples:] + current = [row for row in valid if period_start <= _time(row["time"]) <= period_end] + if len(baseline) < rule.minimum_samples: + gaps.append({"device_id": device_id, "reason": "baseline_not_ready"}) + continue + if not current: + gaps.append({"device_id": device_id, "reason": "period_reading_missing"}) + continue + reference = median(float(row["cleaned_value"]) for row in baseline) + if reference == 0: + gaps.append({"device_id": device_id, "reason": "zero_baseline"}) + continue + latest = current[-1] + value = float(latest["cleaned_value"]) + change = (value - reference) / abs(reference) * 100 + hit = change <= -rule.threshold if direction == "decrease" else change >= rule.threshold + details.append({ + "device_id": device_id, + "observed_at": _time(latest["time"]).isoformat(), + "baseline": reference, + "value": value, + "change_percent": change, + "threshold_percent": rule.threshold, + "hit": hit, + }) + return { + "status": "ready" if not gaps else "insufficient_data", + "hit": any(item["hit"] for item in details), + "hit_device_ids": [item["device_id"] for item in details if item["hit"]], + "devices": details, + "gaps": gaps, + } + + +def _burst_detection( + rows: list[dict], + pressure_device_ids: list[str], + interval_seconds: int, + period_end: datetime, +) -> dict[str, Any]: + if len(pressure_device_ids) < 5: + return {"status": "sensor_coverage_insufficient", "hit": False, + "required_sensors": 5, "available_sensors": len(pressure_device_ids)} + if interval_seconds % 60 or 86400 % interval_seconds: + return {"status": "unsupported_sampling_interval", "hit": False, + "interval_seconds": interval_seconds} + points_per_day = 86400 // interval_seconds + count = 15 * points_per_day + series_by_device: dict[str, dict[datetime, float]] = {} + common_times: set[datetime] | None = None + for device_id in pressure_device_ids: + values = {_time(row["time"]): float(row["cleaned_value"]) + for row in _valid_rows(rows, device_id) if _time(row["time"]) <= period_end} + series_by_device[device_id] = values + common_times = set(values) if common_times is None else common_times & set(values) + if not common_times: + return {"status": "baseline_not_ready", "hit": False, "required_samples": count} + target = max(common_times) + expected = [target - timedelta(seconds=interval_seconds * offset) + for offset in range(count - 1, -1, -1)] + incomplete = [device_id for device_id, values in series_by_device.items() + if any(at not in values for at in expected)] + if incomplete: + return {"status": "baseline_not_ready", "hit": False, "required_samples": count, + "incomplete_device_ids": incomplete} + frame = pd.DataFrame({device_id: [series_by_device[device_id][at] for at in expected] + for device_id in pressure_device_ids}, index=expected) + try: + result = run_burst_detection( + network="scada-replay", + username="scada-replay", + observed_pressure_data=frame, + points_per_day=points_per_day, + mu=TARGET_MU, + iforest_params={ + "n_estimators": TARGET_N_ESTIMATORS, + "random_state": TARGET_RANDOM_STATE, + "contamination": "auto", + }, + sensor_nodes=pressure_device_ids, + ) + except (TypeError, ValueError) as exc: + return {"status": "analysis_failed", "hit": False, "reason": str(exc)[:300]} + latest = result["rows"][-1] + score = float(latest["Score"]) + return { + "status": "ready", + "hit": score <= TARGET_SCORE_THRESHOLD, + "target_time": target.isoformat(), + "target_score": score, + "score_threshold": TARGET_SCORE_THRESHOLD, + "sensor_count": len(pressure_device_ids), + "reference_day_count": 14, + "latest_sensor_rankings": result["summary"].get("latest_sensor_rankings", []), + } + + +def evaluate_monitoring_period( + document: dict, + rows: list[dict], + *, + period_start: datetime, + period_end: datetime, + batch_sequence: int, +) -> dict[str, Any]: + config = ScadaReplayCreate.model_validate(document["config"]) + monitoring = document.get("monitoring_config") + rule_version = (monitoring or {}).get("rule_version", "unconfigured") + monitoring_id = str(uuid5( + NAMESPACE_URL, + f"{document['replay_id']}:{period_end.isoformat()}:{rule_version}", + )) + base = { + "monitoring_id": monitoring_id, + "period_start": period_start.isoformat(), + "period_end": period_end.isoformat(), + "knowledge_cutoff": period_end.isoformat(), + "batch_sequence": batch_sequence, + "rule_version": rule_version, + "task_id": None, + } + if not monitoring: + return {**base, "status": "insufficient_data", "detail": "项目未配置 SCADA 监测设备角色", + "pressure": {"status": "unconfigured", "hit": False}, + "outlet_flow": {"status": "unconfigured", "hit": False}, + "burst_detection": {"status": "not_run", "hit": False}} + pressure_ids = monitoring.get("pressure_device_ids", []) + flow_ids = monitoring.get("outlet_flow_device_ids", []) + selected = set(config.device_ids) + role_gaps = sorted((set(pressure_ids) | set(flow_ids)) - selected) + if not pressure_ids or not flow_ids or role_gaps: + return {**base, "status": "insufficient_data", "detail": "监测角色为空或不属于本轮回放设备", + "role_gaps": role_gaps, + "pressure": {"status": "unconfigured", "hit": False}, + "outlet_flow": {"status": "unconfigured", "hit": False}, + "burst_detection": {"status": "not_run", "hit": False}} + pressure = _relative_indicator(rows, config, pressure_ids, period_start, period_end, direction="decrease") + outlet_flow = _relative_indicator(rows, config, flow_ids, period_start, period_end, direction="increase") + detector = _burst_detection( + rows, pressure_ids, int(document["monitoring_interval_seconds"]), period_end, + ) + complete = pressure["status"] == outlet_flow["status"] == detector["status"] == "ready" + hits = (bool(detector["hit"]), bool(pressure["hit"]), bool(outlet_flow["hit"])) + if not complete: + status, detail = "insufficient_data", "监测数据、设备角色或爆管侦测基线不足" + elif all(hits): + status, detail = "suspected_burst", "爆管侦测、压力下降与出厂边界流量增加相互支持" + elif any(hits): + status, detail = "attention", "仅部分爆管相关指标命中，需要继续观察或人工核查" + else: + status, detail = "normal", "本周期爆管相关指标均未命中" + return {**base, "status": status, "detail": detail, "pressure": pressure, + "outlet_flow": outlet_flow, "burst_detection": detector} diff --git a/app/services/scada_replay_processing.py b/app/services/scada_replay_processing.py new file mode 100644 index 0000000..6d9fe99 --- /dev/null +++ b/app/services/scada_replay_processing.py @@ -0,0 +1,139 @@ +"""Causal batch quality checks and grouped anomaly episodes, no database IO.""" +from copy import deepcopy +from datetime import datetime +from math import isfinite +from statistics import median +from uuid import NAMESPACE_URL, uuid5 +import re +from app.domain.schemas.scada_replay import ScadaReplayCreate + + +def prepare_cleaning(rows: list[dict], config: ScadaReplayCreate) -> list[dict]: + result = [] + for row in rows: + value = row["monitored_value"] + quality, reason = "valid", "原始值通过数值与已配置量程校验" + if value is None: + quality, reason = "missing", "原始读数缺失" + elif not isfinite(float(value)): + quality, reason = "invalid", "原始读数不是有限数值" + else: + bounds = config.quality_ranges.get(row["device_id"]) + if bounds and ((bounds.lower is not None and value < bounds.lower) + or (bounds.upper is not None and value > bounds.upper)): + quality, reason = "invalid", "超出本次明确配置的仪表量程" + result.append({ + "time": row["time"].isoformat() if isinstance(row["time"], datetime) else row["time"], + "device_id": row["device_id"], "source_hash": row["source_hash"], + "monitored_value": value if value is not None and isfinite(float(value)) else None, + "cleaned_value": value if quality == "valid" else None, + "quality": quality, "reason": reason, + }) + return result + + +def detect_frame(document: dict, rows: list[dict], config: ScadaReplayCreate, at: datetime) -> tuple[dict, list[dict]]: + """Consume a complete timestamp; never treat missing data as recovery.""" + result = deepcopy(document) + histories = result.setdefault("histories", {}) + states = result.setdefault("rule_states", {}) + latest = result.setdefault("latest", {}) + for row in rows: + latest[row["device_id"]] = row + by_id = {row["device_id"]: row for row in rows} + changes = [] + devices = {device["device_id"]: device for device in result.get("devices", [])} + for device_id in config.device_ids: + row = by_id.get(device_id) + invalid = row is not None and row["quality"] != "valid" + previous = latest.get(device_id) + frequency = str(devices.get(device_id, {}).get("transmission_frequency", "")) + match = re.fullmatch(r"(\d+):(\d+):(\d+)", frequency) + if match: + seconds = int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3]) + else: + minutes = re.fullmatch(r"(\d+)\s*(?:min|分钟)", frequency) + seconds = int(minutes[1]) * 60 if minutes else 0 + prior_time = datetime.fromisoformat(previous["time"]) if previous else config.start_time + missing = not row and seconds > 0 and (at - prior_time).total_seconds() >= seconds * 2 + key = "quality-" + device_id + state = states.setdefault(key, {"event": None, "recoveries": 0}) + if invalid or missing: + state["recoveries"] = 0 + if state["event"] is None: + event = {"event_id": str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{key}:{at.isoformat()}")), + "rule_id": key, "device_ids": [device_id], "occurred_at": at.isoformat(), + "last_observed_at": at.isoformat(), "status": "active", "category": "data_quality", + "detail": row["reason"] if invalid else "连续两个设备周期未读取到观测", + "task_id": None} + state["event"] = event + changes.append({**event, "change": "raised"}) + result["anomaly_count"] = result.get("anomaly_count", 0) + 1 + elif row and row["quality"] == "valid" and state["event"]: + state["recoveries"] += 1 + if state["recoveries"] >= 2: + changes.append({**state["event"], "status": "resolved", "resolved_at": at.isoformat(), "change": "resolved"}) + state["event"] = None + for rule in config.rules: + state = states.setdefault(rule.rule_id, {"hits": 0, "recoveries": 0, "event": None}) + hits, usable, reasons = [], [], [] + for device_id in rule.device_ids: + row = by_id.get(device_id) + if not row or row["quality"] != "valid": + continue + value = row["cleaned_value"] + if rule.kind == "bounds": + triggered = ((rule.lower is not None and value < rule.lower) + or (rule.upper is not None and value > rule.upper)) + reason = f"{device_id}={value} {rule.unit}，超出配置范围" + else: + previous = histories.get(device_id, [])[-rule.window_samples:] + if len(previous) < rule.minimum_samples: + continue + baseline = median(previous) + if baseline == 0: + continue + change = (value - baseline) / abs(baseline) * 100 + triggered = (abs(change) >= rule.threshold if rule.direction == "both" + else change >= rule.threshold if rule.direction == "increase" + else change <= -rule.threshold) + reason = f"{device_id} 较历史中位值变化 {change:.1f}%，阈值 {rule.threshold}%" + usable.append(device_id) + if triggered: + hits.append(device_id) + reasons.append(reason) + if len(hits) >= rule.minimum_devices: + state["hits"] += 1 + state["recoveries"] = 0 + if state["event"] is None and state["hits"] >= rule.consecutive_samples: + event_id = str(uuid5(NAMESPACE_URL, f"{result['replay_id']}:{rule.rule_id}:{at.isoformat()}")) + event = {"event_id": event_id, "rule_id": rule.rule_id, "device_ids": hits, + "occurred_at": at.isoformat(), "last_observed_at": at.isoformat(), + "status": "active", "detail": "；".join(reasons)[:2000], "task_id": None} + state["event"] = event + changes.append({**event, "change": "raised"}) + result["anomaly_count"] = result.get("anomaly_count", 0) + 1 + elif state["event"]: + state["event"]["last_observed_at"] = at.isoformat() + elif len(usable) == len(rule.device_ids): + state["hits"] = 0 + if state["event"]: + state["recoveries"] += 1 + if state["recoveries"] >= rule.recovery_samples: + event = {**state["event"], "status": "resolved", "resolved_at": at.isoformat(), + "change": "resolved"} + changes.append(event) + state["event"] = None + state["recoveries"] = 0 + # Incomplete frames cannot count as a consecutive hit/recovery. + elif not usable and any(device in by_id for device in rule.device_ids): + state["hits"] = state["recoveries"] = 0 + frozen = {device for rule in config.rules if states[rule.rule_id]["event"] + for device in rule.device_ids} + for row in rows: + if row["quality"] == "valid" and row["device_id"] not in frozen: + history = histories.setdefault(row["device_id"], []) + history.append(row["cleaned_value"]) + histories[row["device_id"]] = history[-96:] + result["processed_count"] = result.get("processed_count", 0) + len(rows) + return result, changes diff --git a/app/services/scada_replay_queries.py b/app/services/scada_replay_queries.py new file mode 100644 index 0000000..d088527 --- /dev/null +++ b/app/services/scada_replay_queries.py @@ -0,0 +1,95 @@ +"""Server-enforced replay knowledge bounds and immutable corrected snapshots.""" +from datetime import datetime + +from fastapi import HTTPException + +from app.infra.db.postgresql.scada_replays import ScadaReplayRepository + + +def check_replay_window(context, start, end): + cap = datetime.fromisoformat(context["knowledge_cutoff"]) + if start is None or end is None or start.tzinfo is None or end.tzinfo is None or start > end or end > cap: + raise HTTPException(422, f"回放查询需要明确时间范围，结束不得晚于 {cap.isoformat()}") + + +async def replay_snapshot_rows(connection, project_id, context, device_ids, start, end, limit, offset=0): + check_replay_window(context, start, end) + repository = ScadaReplayRepository(connection, project_id) + run = await repository.get(context["replay_id"]) + if not set(device_ids) <= set(run["config"]["device_ids"]): + raise HTTPException(403, "设备或回放不属于当前调查") + if start < datetime.fromisoformat(run["start_time"]): + raise HTTPException(422, "当前调查证据接口仅返回本轮已处理观测，回放前基线已用于后台初始化") + rows = await repository.detected_outputs( + context["replay_id"], device_ids, start, end, context["batch_sequence"], + limit=limit, offset=offset, + ) + return [{key: row[key] for key in ( + "time", "device_id", "monitored_value", "cleaned_value", "quality", "reason" + )} for row in rows] + + +async def replay_burst_location_inputs(connection, project_id, context, args): + """Build location series from immutable replay outputs, never live TSDB rows.""" + if args.data_source != "monitoring": + raise HTTPException(422, "回放定位只支持 monitoring 数据源") + if args.scada_normal_start is None or args.scada_normal_end is None: + raise HTTPException(422, "回放定位必须显式提供已处理范围内的正常对照窗口") + check_replay_window(context, args.scada_burst_start, args.scada_burst_end) + check_replay_window(context, args.scada_normal_start, args.scada_normal_end) + repository = ScadaReplayRepository(connection, project_id) + run = await repository.get(context["replay_id"]) + if run: + replay_start = datetime.fromisoformat(run["start_time"]) + if args.scada_burst_start < replay_start or args.scada_normal_start < replay_start: + raise HTTPException(422, "定位窗口必须位于本轮已处理回放范围内") + devices = {item["device_id"]: item for item in run.get("devices", [])} + monitoring = run.get("monitoring_config") or {} + pressure_devices = monitoring.get("pressure_device_ids", []) + pressure_map = {str(devices[item]["node_id"]): item for item in pressure_devices + if item in devices and devices[item].get("node_id")} + requested_pressure = args.pressure_scada_ids or list(pressure_map) + if not requested_pressure or not set(requested_pressure) <= set(pressure_map): + raise HTTPException(422, "压力节点必须来自本轮已配置的回放监测角色") + flow_devices = monitoring.get("outlet_flow_device_ids", []) + flow_map = {str(devices[item]["link_id"]): item for item in flow_devices + if item in devices and devices[item].get("link_id")} + requested_flow = args.flow_scada_ids or [] + if not set(requested_flow) <= set(flow_map): + raise HTTPException(422, "流量要素必须来自本轮已配置的出厂边界流量角色") + selected = [pressure_map[item] for item in requested_pressure] + [flow_map[item] for item in requested_flow] + rows = await repository.detected_outputs( + context["replay_id"], selected, + min(args.scada_burst_start, args.scada_normal_start), + max(args.scada_burst_end, args.scada_normal_end), + context["batch_sequence"], + ) + + def averages(mapping, start, end): + result = {} + for element_id, device_id in mapping.items(): + values = [float(row["cleaned_value"]) for row in rows + if row.get("device_id") == device_id + and row.get("quality") in {"valid", "corrected"} + and row.get("cleaned_value") is not None + and start <= datetime.fromisoformat(row["time"]) <= end] + if not values: + raise HTTPException(422, f"回放窗口缺少 {device_id} 的有效清洗观测") + result[element_id] = sum(values) / len(values) + return result + + pressure_mapping = {item: pressure_map[item] for item in requested_pressure} + flow_mapping = {item: flow_map[item] for item in requested_flow} + return { + "data_source": "monitoring", + "burst_leakage": args.burst_leakage, + "pressure_scada_ids": requested_pressure, + "burst_pressure": averages(pressure_mapping, args.scada_burst_start, args.scada_burst_end), + "normal_pressure": averages(pressure_mapping, args.scada_normal_start, args.scada_normal_end), + **({ + "flow_scada_ids": requested_flow, + "burst_flow": averages(flow_mapping, args.scada_burst_start, args.scada_burst_end), + "normal_flow": averages(flow_mapping, args.scada_normal_start, args.scada_normal_end), + "use_scada_flow": True, + } if requested_flow else {}), + } diff --git a/contracts/manifest.json b/contracts/manifest.json index feaef88..00c349d 100644 --- a/contracts/manifest.json +++ b/contracts/manifest.json @@ -3,7 +3,7 @@ "contracts": { "server": { "file": "server-v1.openapi.json", - "sha256": "2c1d757ccf06ae72b10c592e1e9b3d32b4f72cda8489e224cf2c5f2690fc6972" + "sha256": "b039042be13243a88691309cb74eee105a3b71765c342edb9222374e41c0458e" } } } diff --git a/contracts/server-v1.openapi.json b/contracts/server-v1.openapi.json index 0c157ae..dc5c813 100644 --- a/contracts/server-v1.openapi.json +++ b/contracts/server-v1.openapi.json @@ -304,6 +304,9 @@ "title": "Name", "type": "string" }, + "settings": { + "$ref": "#/components/schemas/ProjectSettings" + }, "status": { "default": "active", "enum": [ @@ -373,6 +376,9 @@ "title": "Project Id", "type": "string" }, + "settings": { + "$ref": "#/components/schemas/ProjectSettings" + }, "status": { "title": "Status", "type": "string" @@ -469,6 +475,17 @@ ], "title": "Name" }, + "settings": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProjectSettings" + }, + { + "type": "null" + } + ], + "description": "按顶层键合并项目配置；省略的键保持原值，嵌套对象整体替换" + }, "status": { "anyOf": [ { @@ -6670,6 +6687,9 @@ "title": "Project Role", "type": "string" }, + "settings": { + "$ref": "#/components/schemas/ProjectSettings" + }, "status": { "title": "Status", "type": "string" @@ -6726,6 +6746,41 @@ "title": "ProjectProvisionResponse", "type": "object" }, + "ProjectSettings": { + "additionalProperties": { + "$ref": "#/components/schemas/JsonValue" + }, + "description": "Extensible, non-secret project parameters stored in MetaDB as JSON.", + "properties": { + "scada_monitoring": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScadaMonitoringSettings" + }, + { + "type": "null" + } + ], + "description": "主动监测使用的项目设备角色；不创建定时任务" + }, + "scada_monitoring_interval_seconds": { + "anyOf": [ + { + "maximum": 31536000.0, + "minimum": 60.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "SCADA 主动监测演示步长，单位秒；null 表示未配置", + "title": "Scada Monitoring Interval Seconds" + } + }, + "title": "ProjectSettings", + "type": "object" + }, "ProjectSummaryResponse": { "properties": { "code": { @@ -7327,6 +7382,245 @@ "title": "ScadaLatestReadingsResponse", "type": "object" }, + "ScadaMonitoringResult": { + "additionalProperties": false, + "properties": { + "batch_sequence": { + "minimum": 1.0, + "title": "Batch Sequence", + "type": "integer" + }, + "burst_detection": { + "additionalProperties": true, + "title": "Burst Detection", + "type": "object" + }, + "detail": { + "maxLength": 1000, + "minLength": 1, + "title": "Detail", + "type": "string" + }, + "knowledge_cutoff": { + "format": "date-time", + "title": "Knowledge Cutoff", + "type": "string" + }, + "monitoring_id": { + "format": "uuid", + "title": "Monitoring Id", + "type": "string" + }, + "outlet_flow": { + "additionalProperties": true, + "title": "Outlet Flow", + "type": "object" + }, + "period_end": { + "format": "date-time", + "title": "Period End", + "type": "string" + }, + "period_start": { + "format": "date-time", + "title": "Period Start", + "type": "string" + }, + "pressure": { + "additionalProperties": true, + "title": "Pressure", + "type": "object" + }, + "role_gaps": { + "items": { + "type": "string" + }, + "title": "Role Gaps", + "type": "array" + }, + "rule_version": { + "maxLength": 80, + "minLength": 1, + "title": "Rule Version", + "type": "string" + }, + "status": { + "enum": [ + "normal", + "attention", + "insufficient_data", + "suspected_burst" + ], + "title": "Status", + "type": "string" + }, + "task_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Task Id" + } + }, + "required": [ + "monitoring_id", + "period_start", + "period_end", + "knowledge_cutoff", + "batch_sequence", + "rule_version", + "status", + "detail", + "pressure", + "outlet_flow", + "burst_detection" + ], + "title": "ScadaMonitoringResult", + "type": "object" + }, + "ScadaMonitoringSettings": { + "additionalProperties": false, + "description": "Project-owned device roles used by the manual SCADA monitoring demo.", + "properties": { + "outlet_flow_device_ids": { + "items": { + "type": "string" + }, + "maxItems": 20, + "title": "Outlet Flow Device Ids", + "type": "array" + }, + "pressure_device_ids": { + "items": { + "type": "string" + }, + "maxItems": 20, + "title": "Pressure Device Ids", + "type": "array" + }, + "rule_version": { + "default": "burst-monitor-v1", + "maxLength": 80, + "minLength": 1, + "pattern": "^[a-zA-Z0-9_.-]+$", + "title": "Rule Version", + "type": "string" + } + }, + "title": "ScadaMonitoringSettings", + "type": "object" + }, + "ScadaQualityRange": { + "additionalProperties": false, + "properties": { + "lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Lower" + }, + "upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Upper" + } + }, + "title": "ScadaQualityRange", + "type": "object" + }, + "ScadaRawPage": { + "additionalProperties": false, + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/ScadaRawReading" + }, + "title": "Items", + "type": "array" + }, + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Cursor" + }, + "scanned_through": { + "format": "date-time", + "title": "Scanned Through", + "type": "string" + } + }, + "required": [ + "items", + "next_cursor", + "scanned_through" + ], + "title": "ScadaRawPage", + "type": "object" + }, + "ScadaRawReading": { + "additionalProperties": false, + "properties": { + "device_id": { + "title": "Device Id", + "type": "string" + }, + "monitored_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Monitored Value" + }, + "quality": { + "default": "unvalidated", + "enum": [ + "unvalidated", + "missing", + "invalid" + ], + "title": "Quality", + "type": "string" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + } + }, + "required": [ + "time", + "device_id", + "monitored_value" + ], + "title": "ScadaRawReading", + "type": "object" + }, "ScadaReadingBatchItem": { "properties": { "cleaned_value": { @@ -7369,6 +7663,550 @@ "title": "ScadaReadingBatchItem", "type": "object" }, + "ScadaReplayCapabilities": { + "additionalProperties": false, + "properties": { + "available": { + "title": "Available", + "type": "boolean" + }, + "burst_monitoring_ready": { + "default": false, + "title": "Burst Monitoring Ready", + "type": "boolean" + }, + "burst_monitoring_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Burst Monitoring Reason" + }, + "can_write": { + "title": "Can Write", + "type": "boolean" + }, + "current": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScadaReplaySnapshot" + }, + { + "type": "null" + } + ] + }, + "monitoring_interval_seconds": { + "anyOf": [ + { + "maximum": 31536000.0, + "minimum": 60.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Monitoring Interval Seconds" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + } + }, + "required": [ + "available", + "can_write" + ], + "title": "ScadaReplayCapabilities", + "type": "object" + }, + "ScadaReplayCommand": { + "additionalProperties": false, + "properties": { + "action": { + "enum": [ + "step", + "stop", + "retry" + ], + "title": "Action", + "type": "string" + }, + "expected_revision": { + "minimum": 1.0, + "title": "Expected Revision", + "type": "integer" + }, + "request_id": { + "format": "uuid", + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "expected_revision", + "action" + ], + "title": "ScadaReplayCommand", + "type": "object" + }, + "ScadaReplayCreate": { + "additionalProperties": false, + "properties": { + "device_ids": { + "items": { + "type": "string" + }, + "maxItems": 500, + "minItems": 1, + "title": "Device Ids", + "type": "array" + }, + "end_time": { + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "quality_ranges": { + "additionalProperties": { + "$ref": "#/components/schemas/ScadaQualityRange" + }, + "title": "Quality Ranges", + "type": "object" + }, + "request_id": { + "format": "uuid", + "title": "Request Id", + "type": "string" + }, + "rules": { + "items": { + "$ref": "#/components/schemas/ScadaReplayRule" + }, + "maxItems": 30, + "minItems": 1, + "title": "Rules", + "type": "array" + }, + "start_time": { + "format": "date-time", + "title": "Start Time", + "type": "string" + } + }, + "required": [ + "request_id", + "start_time", + "end_time", + "device_ids", + "rules" + ], + "title": "ScadaReplayCreate", + "type": "object" + }, + "ScadaReplayEventsPage": { + "additionalProperties": false, + "properties": { + "items": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Items", + "type": "array" + }, + "next_offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Next Offset" + } + }, + "required": [ + "items", + "next_offset" + ], + "title": "ScadaReplayEventsPage", + "type": "object" + }, + "ScadaReplayReadings": { + "additionalProperties": false, + "properties": { + "items": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Items", + "type": "array" + }, + "through": { + "format": "date-time", + "title": "Through", + "type": "string" + } + }, + "required": [ + "items", + "through" + ], + "title": "ScadaReplayReadings", + "type": "object" + }, + "ScadaReplayRule": { + "additionalProperties": false, + "properties": { + "consecutive_samples": { + "default": 1, + "maximum": 20.0, + "minimum": 1.0, + "title": "Consecutive Samples", + "type": "integer" + }, + "device_ids": { + "items": { + "type": "string" + }, + "maxItems": 500, + "minItems": 1, + "title": "Device Ids", + "type": "array" + }, + "direction": { + "default": "both", + "enum": [ + "both", + "increase", + "decrease" + ], + "title": "Direction", + "type": "string" + }, + "kind": { + "enum": [ + "relative_change", + "bounds" + ], + "title": "Kind", + "type": "string" + }, + "lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Lower" + }, + "metric_name": { + "default": "", + "maxLength": 80, + "title": "Metric Name", + "type": "string" + }, + "minimum_devices": { + "default": 1, + "maximum": 500.0, + "minimum": 1.0, + "title": "Minimum Devices", + "type": "integer" + }, + "minimum_samples": { + "default": 3, + "maximum": 96.0, + "minimum": 1.0, + "title": "Minimum Samples", + "type": "integer" + }, + "recovery_samples": { + "default": 2, + "maximum": 20.0, + "minimum": 1.0, + "title": "Recovery Samples", + "type": "integer" + }, + "rule_id": { + "maxLength": 80, + "minLength": 1, + "pattern": "^[a-zA-Z0-9_-]+$", + "title": "Rule Id", + "type": "string" + }, + "threshold": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Threshold" + }, + "unit": { + "maxLength": 40, + "minLength": 1, + "title": "Unit", + "type": "string" + }, + "upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Upper" + }, + "window_samples": { + "default": 4, + "maximum": 96.0, + "minimum": 1.0, + "title": "Window Samples", + "type": "integer" + } + }, + "required": [ + "rule_id", + "device_ids", + "kind", + "unit" + ], + "title": "ScadaReplayRule", + "type": "object" + }, + "ScadaReplaySnapshot": { + "additionalProperties": false, + "properties": { + "active_events": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Active Events", + "type": "array" + }, + "anomaly_count": { + "default": 0, + "title": "Anomaly Count", + "type": "integer" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "delivered_until": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delivered Until" + }, + "devices": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Devices", + "type": "array" + }, + "end_time": { + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "last_monitoring": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScadaMonitoringResult" + }, + { + "type": "null" + } + ] + }, + "latest": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Latest", + "type": "array" + }, + "monitoring_count": { + "default": 0, + "title": "Monitoring Count", + "type": "integer" + }, + "monitoring_interval_seconds": { + "anyOf": [ + { + "maximum": 31536000.0, + "minimum": 60.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Monitoring Interval Seconds" + }, + "processed_count": { + "default": 0, + "title": "Processed Count", + "type": "integer" + }, + "processed_until": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Processed Until" + }, + "project_id": { + "format": "uuid", + "title": "Project Id", + "type": "string" + }, + "replay_id": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + }, + "revision": { + "title": "Revision", + "type": "integer" + }, + "rules": { + "items": { + "$ref": "#/components/schemas/ScadaReplayRule" + }, + "title": "Rules", + "type": "array" + }, + "start_time": { + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "status": { + "enum": [ + "ready", + "paused", + "completed", + "stopped", + "failed" + ], + "title": "Status", + "type": "string" + }, + "step_pending": { + "default": false, + "title": "Step Pending", + "type": "boolean" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "virtual_now": { + "format": "date-time", + "title": "Virtual Now", + "type": "string" + } + }, + "required": [ + "replay_id", + "project_id", + "revision", + "status", + "start_time", + "end_time", + "virtual_now", + "devices", + "rules", + "latest", + "active_events", + "created_at", + "updated_at" + ], + "title": "ScadaReplaySnapshot", + "type": "object" + }, + "ScadaReplayTaskContext": { + "additionalProperties": false, + "properties": { + "batch_sequence": { + "minimum": 1.0, + "title": "Batch Sequence", + "type": "integer" + }, + "knowledge_cutoff": { + "format": "date-time", + "title": "Knowledge Cutoff", + "type": "string" + }, + "replay_id": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + "required": [ + "replay_id", + "knowledge_cutoff", + "batch_sequence" + ], + "title": "ScadaReplayTaskContext", + "type": "object" + }, "SensorPlacementExportRequest": { "properties": { "adjustment_status": { @@ -7960,7 +8798,15 @@ "type": "string" }, "source": { - "$ref": "#/components/schemas/WorkbenchAlertSource" + "anyOf": [ + { + "$ref": "#/components/schemas/WorkbenchAlertSource" + }, + { + "$ref": "#/components/schemas/WorkbenchScadaReplaySource" + } + ], + "title": "Source" }, "title": { "minLength": 1, @@ -8031,7 +8877,15 @@ "type": "string" }, "source": { - "$ref": "#/components/schemas/WorkbenchAlertSource" + "anyOf": [ + { + "$ref": "#/components/schemas/WorkbenchAlertSource" + }, + { + "$ref": "#/components/schemas/WorkbenchScadaReplaySource" + } + ], + "title": "Source" }, "status": { "enum": [ @@ -9373,6 +10227,16 @@ ], "title": "Kind", "type": "string" + }, + "replay_context": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScadaReplayTaskContext" + }, + { + "type": "null" + } + ] } }, "required": [ @@ -9777,7 +10641,15 @@ "type": "string" }, "source": { - "$ref": "#/components/schemas/WorkbenchAlertSource" + "anyOf": [ + { + "$ref": "#/components/schemas/WorkbenchAlertSource" + }, + { + "$ref": "#/components/schemas/WorkbenchScadaReplaySource" + } + ], + "title": "Source" }, "title": { "minLength": 1, @@ -10385,6 +11257,39 @@ "title": "WorkbenchRuntimeWorkOrder", "type": "object" }, + "WorkbenchScadaReplaySource": { + "additionalProperties": false, + "properties": { + "batch_sequence": { + "minimum": 1.0, + "title": "Batch Sequence", + "type": "integer" + }, + "kind": { + "const": "scada_replay", + "title": "Kind", + "type": "string" + }, + "replay_id": { + "minLength": 1, + "title": "Replay Id", + "type": "string" + }, + "rule_id": { + "minLength": 1, + "title": "Rule Id", + "type": "string" + } + }, + "required": [ + "kind", + "replay_id", + "rule_id", + "batch_sequence" + ], + "title": "WorkbenchScadaReplaySource", + "type": "object" + }, "WorkbenchSyncSnapshot": { "additionalProperties": false, "properties": { @@ -41258,6 +42163,681 @@ ] } }, + "/api/v1/scada-replays": { + "post": { + "operationId": "post_scada_replays", + "parameters": [ + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplayCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplaySnapshot" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Create Replay", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/capabilities": { + "get": { + "operationId": "get_scada_replays_capabilities", + "parameters": [ + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplayCapabilities" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Replay Capabilities", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/{replay_id}": { + "get": { + "operationId": "get_scada_replays_replay_id", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplaySnapshot" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Get Replay", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/{replay_id}/commands": { + "post": { + "operationId": "post_scada_replays_replay_id_commands", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplayCommand" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplaySnapshot" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Replay Command", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/{replay_id}/events": { + "get": { + "operationId": "get_scada_replays_replay_id_events", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "default": 0, + "minimum": 0, + "title": "Offset", + "type": "integer" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 100, + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplayEventsPage" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Replay Events", + "tags": [ + "SCADA Replay" + ] + } + }, + "/api/v1/scada-replays/{replay_id}/observations": { + "get": { + "operationId": "get_scada_replays_replay_id_observations", + "parameters": [ + { + "in": "path", + "name": "replay_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Replay Id", + "type": "string" + } + }, + { + "in": "query", + "name": "through", + "required": true, + "schema": { + "format": "date-time", + "title": "Through", + "type": "string" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaReplayReadings" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "Replay Observations", + "tags": [ + "SCADA Replay" + ] + } + }, "/api/v1/sensor-placement-candidates/{node_id}": { "get": { "operationId": "get_sensor_placement_candidates_node_id", @@ -49361,6 +50941,178 @@ ] } }, + "/api/v1/timeseries/scada-readings/raw": { + "get": { + "operationId": "get_timeseries_scada_readings_raw", + "parameters": [ + { + "in": "query", + "name": "start_time", + "required": true, + "schema": { + "format": "date-time", + "title": "Start Time", + "type": "string" + } + }, + { + "in": "query", + "name": "end_time", + "required": true, + "schema": { + "format": "date-time", + "title": "End Time", + "type": "string" + } + }, + { + "in": "query", + "name": "device_ids", + "required": true, + "schema": { + "title": "Device Ids", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 500, + "maximum": 500, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "query", + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cursor" + } + }, + { + "in": "query", + "name": "replay_id", + "required": false, + "schema": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Replay Id" + } + }, + { + "in": "header", + "name": "X-Project-Id", + "required": true, + "schema": { + "title": "X-Project-Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScadaRawPage" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Authentication required" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Insufficient permission" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource not found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Resource conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Validation error" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + }, + "description": "Dependency unavailable" + } + }, + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], + "summary": "按时间顺序分页读取 SCADA 原始值", + "tags": [ + "TimescaleDB - SCADA" + ] + } + }, "/api/v1/timeseries/scada-readings/{device_id}/field": { "patch": { "description": "更新指定设备的字段值\n\n更新SCADA设备在特定时间的某个字段监测数据。\n\nArgs:\n device_id: 设备ID\n time: 数据时间戳\n field: 字段名称\n value: 字段新值\n\nReturns:\n 更新结果信息\n\nRaises:\n HTTPException: 当字段不存在或更新失败时返回400错误", diff --git a/docs/assets/scada-fm-timeline.svg b/docs/assets/scada-fm-timeline.svg new file mode 100644 index 0000000..dd3840b --- /dev/null +++ b/docs/assets/scada-fm-timeline.svg @@ -0,0 +1,65 @@ + + SCADA 演示时间轴 FM 调频盘样式 + 静态设计示意。选定区间为十月八日八点至十四点，当前时间十点十五分。半透明玻璃刻度窗内细密时间刻度横向排列，中央红色指示线固定，控制栏包含暂停、下一批和六十倍速。 + + + + + + + + + + + + + + + + + + + + + + SCADA 时段演示 + 静态样式示意 + 演示区间 + 10 月 8 日 08:00 — 14:00 + 10:15:00 + + + + + + + + + + + + + + + + 09:3009:45 + 10:0010:15 + 10:3010:4511:00 + + + + + + + + + 暂停 + + 下一批 + + 60× + + 已读取 10:15 · 已检测 10:10 + 红线对应当前时间 · 横向拨动可回看 + + diff --git a/docs/peripheral-sample.md b/docs/peripheral-sample.md index c918603..b832e73 100644 --- a/docs/peripheral-sample.md +++ b/docs/peripheral-sample.md @@ -39,6 +39,8 @@ Forward Hostname 后追加 `/api/sample/v1`。本机开发和预览模式也将 - `/api/sample/v1/monitoring/points`、`observations`、`alerts` - `/api/sample/v1/demo-data` +SCADA 按月数据集、5 分钟时间定位、主动周期监测、质量校验、爆管调查与定位边界见 [SCADA 外围系统时段回放、主动监测与爆管调查设计](scada-peripheral-replay-design.md)。部署回放存储需在项目 BizDB 执行 `031_scada_replays.sql`；项目通用配置需在 MetaDB 执行 `033_metadata_project_settings.sql`。监测频率保存于 `projects.settings.scada_monitoring_interval_seconds`，压力与出厂边界流量角色保存在类型化 `projects.settings.scada_monitoring` 中；本阶段不注册定时任务，也不部署 SCADA 回放 worker。代码和结构迁移完成不等于月度数据或 Agent 联调已经验收。 + 演示数据通过 `scripts/seed_peripheral_sample.py` 写入 `tjwater_v2` 的 `peripheral_sample` schema。当前固定窗口为 2026-09-17，所有记录标记为 `synthetic`，并使用 `DEMO-` 前缀。 ## 工单事件调查 diff --git a/docs/scada-peripheral-replay-design.md b/docs/scada-peripheral-replay-design.md new file mode 100644 index 0000000..ecb4ad9 --- /dev/null +++ b/docs/scada-peripheral-replay-design.md @@ -0,0 +1,476 @@ +# SCADA 外围系统时段回放、主动监测与爆管调查设计 + +状态：时段回放、主动周期监测、条件式 `burst_report` 调查及回放安全的爆管定位适配已实现。单步处理由用户操作触发并在当前 API 请求内完成，不部署 SCADA 回放 worker。`tjwater_v2` 与 `tjwater2_v2` 尚未确认出厂边界流量角色；仍待完整月度数据和真实 Agent 联调。更新日期：2026-11-03。下面明确区分已实现范围、后续目标与验收要求。 + +本次将外围 SCADA 定义为一个模拟外部数据源：本地 TSDB 已保存原始观测，通过只读 API 按选定时间区间和后台时钟分批读取 `scada.measurements.monitored_value`，将这一批直接交给清洗流水线，保存 `cleaned_value` 后执行异常检测及后续 Agent 调查。原始值、修正值、真实工况异常和处理结论分别保留。浏览器负责控制和展示，关闭页面不停止后台回放。 + +演示主路径为“TSDB 已存 raw → 用户点击下一步并监测 → 顺序读取一个步长内的观测 → 清洗并更新修正值 → 在步长边界执行多指标监测 → 满足条件时进入爆管调查与定位”。真实外部系统新增观测时，仍先通过入库接口保存 raw，再进入相同的清洗检测流程。演示读取已有数据不再次调用原始值插入接口，也不把同一批数据从一个内部服务读出后再写回原表。 + +演示操作以用户选定的时间区间为中心：一个月是可选择的数据范围，单次演示可以只覆盖其中几小时或几天。时间轴采用 FM 调频盘样式，玻璃刻度窗内有固定红色指示线，时间刻度带随拨动横向移动。时间轴使用 5 分钟时间戳定位；项目监测频率只决定“下一步并监测”覆盖的业务周期，两者互不替代。 + +## 首版实现范围与限制 + +已实现只读 raw 分页、持久回放控制、按项目监测频率推进、逐时刻清洗检测、质量告警与运行异常事件、周期监测回执、组合判定及调查任务。主动单步只有组合结果为 `suspected_burst` 时才创建 `burst_report`。前端改为后端状态投影，主界面只保留选定区间、5 分钟 FM 时间定位、主动周期控制和本周期监测结论，不重复展示 SCADA 设备目录与逐测点读数表。主要代码为 [回放服务](../app/services/scada_replay.py)、[组合监测](../app/services/scada_replay_monitoring.py)、[质量与检测](../app/services/scada_replay_processing.py)、[持久状态](../app/infra/db/postgresql/scada_replays.py) 和 [前端组件](../../next-tjwater-frontend/src/features/workbench/peripheral-sample/scada-replay-section.tsx)。 + +本版采用每项目最多 500 个设备、一个完整时间戳一批、至多一个未完成批次的串行处理，不实现 20 批并发队列或大于 500 个设备的分片接收。处理变慢时逐批追赶且不跳过中间观测；后台停机期间最多补算两秒现实时间，不一次性追赶全部停机时长。数据质量以有限数值和显式仪表量程为依据，不自动插值或平滑；缺测判断使用设备档案的采样周期。分组规则的恢复需要该组设备在同一帧都有可信观测，不同采样周期或采样相位应拆成规则。 + +本版冻结设备、区间、规则及批次输出，按批次原始值摘要校验源变更。完整月度数据清单、全月源版本摘要、独立事件真值和检测效果评估仍是后续数据准备工作；能力接口仅判断回放存储是否存在与当前操作权限，不声明月度数据已经就绪。现有 14 天数据不能当作一个月验收。 + +Agent 查询本轮已检测的固定批次快照，受任务的 replay_id、knowledge_cutoff、batch_sequence 约束。开始前历史只供后台基线初始化，本版调查查询不返回该部分；爆管定位已支持受回放截止约束的冻结输入适配，其他未适配计算工具继续拒绝。新 [SCADA 调查 Skill](../resources/work-order-skills/scada-anomaly/SKILL.md) 要求据实说明这些证据限制。运行异常自动建立调查，数据质量异常仅建立告警，不触发模型；恢复时推进调查代次。真实模型推理与报告效果尚未联调。 + +控制进度和事件状态使用有界轮询，沿用告警流发布业务告警；本版未新增 scada.replay.changed SSE 主题。演示面板只显示最近一个周期的组合结论，逐测点输出和完整事件历史由分页 API 保留给诊断与调查入口，不在主面板重复铺开。 + +## 已确定的设计边界 + +- 数据集覆盖一个月，包含多次独立异常及各次事件发生、持续、恢复的过程。 +- 项目配置一个 SCADA 监测频率，只用于一次主动监测所覆盖的周期；时间轴独立使用 5 分钟时间戳，它不关联设备或设备类型，本阶段不据此创建定时任务。 +- 设备采样周期仍属于设备档案。回放实际传入时间以原始观测时间戳为准，不使用项目监测任务间隔生成、重采样或判断设备缺测。 +- 复用 `scada.measurements.monitored_value` 与 `cleaned_value`；不另建一套正式 SCADA 原始值和修正值表。 +- 回放读取 API 只返回原始观测，修正值由本地清洗流水线计算和保存；GET 不更新游标、不清洗、不创建检测任务。 +- 真实压力、流量、水质异常保留在历史序列中。事件恢复或调查结束不删除、不抹平历史异常。 +- 数据处理顺序为原始观测持久化、数据质量校验、必要的修正、业务异常检测、事件与调查。 +- 步长内的数据批次按原始时间驱动清洗和基础规则检测；到达步长边界后，用户触发的本周期监测汇总多项指标。正常周期不启动 Agent。 +- 时间范围、设备、数据集版本和检测规则在一次回放创建后固定；改变这些条件需要新建回放。 +- 页面首先选择演示起止时间，再建立回放。月度数据集、设备配置和规则作为来源与设置，不占据主播放区。 +- 本次演示对已存 raw 只读，清洗阶段仅更新同一条观测的 cleaned_value；不导出整月副本、不重复插入原始值。每次回放的质量结论、检测结果和调查证据按回放身份隔离。 + +### 项目通用配置与监测频率 + +MetaDB 的 `projects.settings` 为项目级通用 JSONB 对象，默认 `{}`，用于保存可扩展的业务参数，不存放密钥。新增参数可直接增加 JSON 键，无需为每项配置新增数据库列；后端为已知参数提供类型校验，并保留其他 JSON 键。 + +其中 `settings.scada_monitoring_interval_seconds` 表示项目的 SCADA 监测频率，单位为秒。本阶段只用于主动处理边界和监测周期标识，不控制时间轴刻度，不注册 cron、后台 schedule 或周期性 Agent 任务。它与设备目录、设备类型、设备采样周期和页面刷新周期均无关联。省略或设为 `null` 表示未配置；有效范围为 60 秒至 365 天。 + +管理员可通过 `PATCH /api/v1/admin/projects/{project_id}` 配置： + +```json +{ + "settings": { + "scada_monitoring_interval_seconds": 900 + } +} +``` + +更新时按 `settings` 顶层键合并：未传入的键保留，传入的嵌套对象整体替换，`settings: {}` 不清空已有配置，`settings: null` 会被拒绝。读取项目详情和管理员项目列表均返回 `settings`。 + +组合监测仍复用该 JSON 对象，不新增固定数据库列。后端类型化嵌套对象显式声明项目实际具备的监测角色： + +```json +{ + "settings": { + "scada_monitoring_interval_seconds": 900, + "scada_monitoring": { + "rule_version": "burst-monitor-v1", + "pressure_device_ids": ["明确的压力设备 ID"], + "outlet_flow_device_ids": ["明确的出厂边界流量设备 ID"] + } + } +} +``` + +当前代码已类型校验并在创建回放时冻结 `scada_monitoring`。项目没有出厂流量或不足 5 个完整压力点时保持缺项，能力接口说明原因，监测回执返回 `insufficient_data`；不能为了让演示通过而把其他流量点冒充出厂流量或虚构设备。`tjwater_v2` 现有目录只有通用 `pipe_flow` 类型及模型管段映射，没有足以证明“出厂边界”的角色字段，因此本轮未写入该嵌套配置。 + +部署前需在 MetaDB 执行 `resources/sql/033_metadata_project_settings.sql`。迁移会将实验版本可能存在的独立监测间隔列搬入 `settings`，核对冲突后移除旧列；重复执行不丢失已有配置。当前 `tjwater_v2` 与 `tjwater2_v2` 均配置 900 秒，即 15 分钟的主动监测周期。该参数不改变 5 分钟时间轴定位，本阶段明确不实现通用定时任务调度。 + +## 实施前基础与原设计差距 + +下表记录实施前基础，用于说明改动缘由；本次实际实现以首版范围及后文接口表为准。设备目录采样周期为本轮讨论中 2026-11-02 的只读核查快照，与项目监测任务间隔是两个概念。 + +| 位置 | 当前能力 | 本次需要补齐 | +| --- | --- | --- | +| [外围 API](../samples/peripheral/api.py)、[存储路由](../samples/peripheral/routing.py) | 挂载在 `/peripheral`，使用 Bearer 与 `X-Project-Id`，按已授权项目解析 BizDB 和 TimescaleDB | 回放控制、源批次、回放状态与能力声明 | +| [历史观测接口](../samples/peripheral/store.py) | 单点、最多 7 天、最多返回 1000 条并声明截断；当前使用 `api_query_id` 查历史 | 回放读取使用规范 `device_id`，按时间和设备联合游标分页；不能直接拼接现有单点接口来装载一个月 | +| [SCADA API](../app/api/v1/endpoints/timeseries/scada.py)、[仓储](../app/infra/db/timescaledb/repositories/scada.py) | 范围读取、最新值读取、批量插入；当前范围读取返回 raw 和 corrected，缺少回放游标及完整流水线 | 增加只读 raw 分页契约；复用存储查询，补齐处理检查点、清洗与检测调度 | +| [清洗服务](../app/services/timeseries_analysis.py) | 从原始值计算并更新 `cleaned_value`；目前支持压力和流量 | 在线质量标记、因果修正及选择值依据；水质不能直接套用现有清洗 | +| [压力清洗](../app/algorithms/scada_cleaning/pressure_series.py)、[流量清洗](../app/algorithms/scada_cleaning/flow_series.py) | 包含居中窗口、后向补值、Kalman 平滑；流量路径还会补为 1 分钟序列 | 在线路径只使用已到达的数据，保留设备频率，不整月预清洗后再模拟检测 | +| [外围工单事件同步](peripheral-work-order-events.md) | 持久事件、通知唤醒、后台同步和定期对账 | 复用可靠处理方式，SCADA 的批次有自己的持久状态 | +| [调查 worker](../scripts/run_operational_worker.py) | 可靠投递 `event_investigation`，恢复执行租约，管理 Agent 执行代次 | 接收由 SCADA 异常事件建立的调查；不承担每条观测的检测 | +| [任务告警](../app/domain/schemas/task_alert.py) | 支持 raised、acknowledged、resolved；当前来源必须携带任务和执行标识 | 增加真实的回放检测来源，不能为调查前告警伪造 `task_id`、`run_id` | +| [MCP 上下文](../app/mcp/contexts.py)、[SCADA 查询](../app/mcp/query_service.py) | 校验任务、项目、执行代次和会话，保存查询证据 | 在所有回放调查读取入口执行统一的时间与数据版本约束 | +| [业务 Skill 映射](../app/mcp/work_order_skills.py) | 目前映射爆管上报、水质投诉、预约跟进 | 增加 SCADA 回放调查流程，不将检测信号直接等同于已确认爆管或污染 | +| [前端 SCADA 区域](../../next-tjwater-frontend/src/features/workbench/peripheral-sample/scada-system-section.tsx) | 旧版同时展示设备表、逐点读数、重复概览和手工合成读数 | 改为服务端状态投影；主界面只承担时间定位、主动单步和周期结论，详细数据留在诊断与调查接口 | + +实施前只读核查中，`tjwater_v2` 有压力、流量和水质监测类型，其他项目的类型集合与历史周期并不相同。项目通用配置保存一个监测任务周期，不与设备类型绑定，也不把 5/15 分钟写成枚举；设备原有采样信息仍用于缺测判断。 + +现有 [演示记录](../../present.md) 描述的是 14 天合成观测，不能作为一个月数据已经准备好的证明。一个月数据准备与本设计实现分别验收。 + +## 运行链路与职责 + +```text +本地 TSDB 已存月度原始观测；项目 BizDB 保存设备目录 + | + v +只读 raw API / 同一读取服务 <--- 持久回放状态 <--- 页面“下一步并监测” + | + | 返回一批原始观测，不再写入 raw + v +后台固定本批记录引用、摘要和处理身份 + | + v +后台质量校验与清洗 + | + v +本地 TSDB：仅更新 cleaned_value + | + v +用同批清洗结果执行基础规则检测 + | + v +到达步长边界，汇总爆管侦测、压力下降、出厂流量变化 + | + v +疑似爆管且定位输入具备 ---> burst_report 任务与 MCP Skill + | + v +工作台告警、调查进度与结果 +``` + +SCADA 单步命令在当前 API 请求内按原始时间顺序完成 raw 读取、清洗、基础规则检测和周期组合监测。每个阶段仍写入持久批次，网络中断后的同一幂等请求可以继续未完成步骤，但页面关闭后不会自动推进时间，也不需要单独的 SCADA 回放进程。 + +现有调查 worker 只处理已经创建的调查执行，计算 worker 继续处理长计算。当前阶段不增加消息中间件，不扩展通用 schedule API，也不注册 SCADA 定时任务。页面的明确单步命令写入持久 `step_target`，API 只完成这次已授权的周期处理；不能用“每隔几分钟创建一条 Agent 任务”替代。 + +最小可交付版本必须包含主动周期处理、批次可靠保存、至少一种明确配置的检测规则和结果展示；只移动页面时间轴不能代表数据已经处理。 + +## 一个自然月的数据集 + +数据集用带时区的半开区间定义。例如北京时间 `[2026-10-01 00:00, 2026-11-01 00:00)` 是 31 天，不能固定按 30 天计算。内部统一保存带时区时间，界面按 Asia/Shanghai 展示。 + +准备数据时冻结清单：`dataset_id`、`dataset_revision`、源项目、观测区间、可用历史基线区间、设备规范 ID、类型、单位、设备采样周期、采样相位、指标含义、raw 分片摘要和规则配置版本。由服务端读取项目路由，页面不传物理库名、SQL 或连接信息。摘要只覆盖原始观测与必要元数据，不覆盖会被本轮清洗更新的 cleaned_value。月度摘要由数据集准备流程预先生成，创建回放的 HTTP 请求只校验清单，不同步扫描和加载整个月。 + +首版演示直接读取已准备在本地 TSDB 的 raw，不另导出一套完整原始数据源。月份数据如果尚未齐备，由单独授权的数据准备过程补齐并生成版本清单；创建或播放回放本身不生成、删除、搬移或重复插入 SCADA 原始观测。演示批次是对已有观测的读取与处理身份，不能在统计中称为本轮新插入条数。 + +设备档案采样周期与实际观测间隔不一致时，记录配置差异和缺测情况，仍按实际时间戳读取。不会将 15 分钟数据插值成 5 分钟观测，也不会把没有记录的时刻当作正常值。项目监测任务间隔不参与这项判断。 + +多次异常应同时覆盖：真实压力异常、真实流量变化、真实水质变化、仪表或通信故障、正常调度引起的变化，以及恢复后的再次异常。数据集真值清单独立保存，用于事后计算命中、误报、漏报、重复事件数和发现延迟。真值不得出现在检测输入、Agent 上下文、预告型时间轴标记或普通业务报告中。 + +水质必须声明具体指标。当前演示中的化学示踪物浓度不能直接解释为余氯、浊度，也不能只凭 `mg/L` 选择检测阈值。流量保留方向，压力确认 m、kPa、MPa 等单位后再应用规则。 + +数据集准备负责人需要在运行前提交版本化设备清单、规则参数和独立真值。这是新数据集的输入要求，不表示本次已选定新的 5 分钟设备分组或生成了观测。 + +## 时间轴和读取语义 + +### 时间与进度 + +| 名称 | 含义 | +| --- | --- | +| `observed_at` | 原始观测时间，不随回放改写 | +| `virtual_now` | 已完成主动监测的周期边界时间 | +| `delivered_until` | 本轮源区间已读完，且所有分片的记录引用已登记为待处理的时间上限；表示模拟传入进度，不是原始值新插入时间 | +| `processed_until` | 质量校验和检测已完整提交的时间上限 | +| `read_at`、`cleaned_at`、`processed_at` | 实际批次读取、清洗提交和检测完成时间，用于延迟、故障与审计 | + +始终满足 `processed_until <= delivered_until <= virtual_now <= end_at`。还未处理任何批次时，读取和处理进度为空，不能伪装成起点已经处理。GET 返回数据不自动推进任何进度；后台确认本批处理身份后才更新读取进度。表格可显示本轮已读取但仍在清洗的 raw 并标明状态，检测只能消费已完成清洗阶段的批次。主界面的次级进度使用“已读取”和“已检测”。 + +后台仅在播放状态累计模拟时间。暂停期间不推进；恢复时重新锚定现实时间。进程重启从持久进度重新锚定，不把停机时长一次性折算成需要跳过的数据。 + +### 顺序读取 + +1. 整次回放选取 `[start_at, end_at)`；首批包含 `start_at` 的观测。 +2. 时间推进到 T 时，候选记录必须不晚于 T、早于 `end_at`，且在已成功接收的联合游标 `(observed_at, device_id)` 之后。 +3. 所有选中设备合并后按 `observed_at, device_id` 排序；优先按同一观测时刻成批，不按设备分别跑独立时钟。 +4. 每个传输片段最多 500 条、序列化正文最多 2 MiB；超限继续分片。记录片段序号与该时刻是否完整，不在半个时刻接收成功时提前推进时间上限；同一时刻的设备分组检测等待该时刻所有分片收齐。 +5. 后台将本批记录引用和原始摘要登记成功后，才推进读取游标。重试同一批使用相同编号和摘要；清洗或检测失败从该阶段恢复，不重新选择一批、不重新插入 raw。正常路径复用本次读取的 raw 对象计算清洗结果，清洗提交后直接将同批结果用于检测，不再重新查询该批 raw 或 corrected。 +6. 没有观测的时间段按设备档案采样周期与采样相位识别应到时刻。后台提交记录数为零的区间检查批次，推进覆盖进度并触发缺测判断；它不生成伪造观测。不能仅依赖“下一条数据到来”唤醒，否则连续断流永远不会被发现，终点也无法排空完成。 + +例如设备档案中的压力采样周期为 5 分钟、流量采样周期为 15 分钟。时间轴到 09:10 时，压力设备可传入 09:00、09:05、09:10，流量设备只有 09:00；到 09:15 时，再传入两类设备对应的 09:15 记录。 + +首版至多保留一个未完成批次，清洗检测提交后才选择下一批。单次请求不跳过周期内的中间观测、质量检查、异常开始和恢复，也不提供播放倍率。 + +### 页面操作 + +| 操作 | 约定 | +| --- | --- | +| 创建 | 固定数据集、范围、设备和规则，进入 ready | +| 播放 | ready 或 paused 进入 running，由后台推进 | +| 暂停 | 停止释放新批次；已持久接收的批次继续处理。返回实际生效进度，在途接收与暂停命令通过同一回放版本串行裁决 | +| 下一步并监测 | 在 ready 或 paused 状态推进到下一个项目监测周期边界；周期内的所有观测按原始时间顺序处理，到边界后提交一次本周期监测回执并保持暂停，尾步可短于完整周期 | +| 调速 | 原子更新速度和时间锚点，不改变观测顺序 | +| 拖动时间轴 | 只查看已释放的数据与已产生的事件；未来区间不预览读数，不触发事件 | +| 从指定时刻重播 | 新建 `replay_id`，独立检测与调查；新起点之前仅作为有声明的历史基线，跳过段不声称已经检测 | +| 到达终点 | 停止新批次，进入 draining；待已接收批次完成后 completed。Agent 调查是否完成另行展示 | +| 停止 | 终止继续传入，保留已接收批次、事件和证据；不自动撤销已经建立的调查 | +| 重试故障 | 修复依赖后校验固定配置，failed 进入 paused，重试原未完成批次；源版本或批次内容冲突需要新建回放，不能靠重试忽略 | + +生命周期为 ready、running、paused、draining、completed、stopped、failed。队列等待、基线不足和依赖暂不可用是处理状态或原因字段，不再增加一套并行生命周期。 + +首次运行遇到可重试投递错误，按现实时间退避重试同一批次；权限错误、数据集版本变化、同批编号内容冲突和结构缺失进入 failed，明确原因。用户恢复前必须解决原因；原批次事实继续保留。 + +### 选定演示区间 + +用户先在月度数据可用范围内选择起止时间。创建回放后，将该区间固定为当前播放范围；进度比例与剩余时长都相对该区间计算。例如选择 10 月 8 日 08:00 至 14:00，时间轴只覆盖这 6 小时，不能将月内 8 日的位置作为本轮进度。 + +主要界面常驻显示选定区间和当前监测时间。月份、数据版本、设备范围及规则收进来源与设置区域；需要校验的配置差异仍在开始前明确显示。不存在上一次选择时不自动处理整个月，等待用户确定区间。 + +演示建立后改变时间区间需要停止并创建新回放；保留旧回放的事件和证据，不把旧回放时间游标直接重设。已完成区间需要再次演示时，新建独立回放；不自动循环处理和重复触发调查。 + +选定区间之前的可用历史仅用于已声明的基线初始化，不占用主时间轴、不作为新到达批次、不预触发此前事件。基线不足时在当前区间显示具体可用性，不能悄悄扩大演示范围或读取区间之后的数据。 + +处理到选定终点后停在终点，显示数据处理是否完成及剩余调查数量；不会自动继续处理该月剩余数据。空区间明确显示没有观测，不能通过控件动画假装已经收到 SCADA 数据。 + +### 5 分钟时间定位与项目监测周期 + +时间轴以 5 分钟时间戳组织刻度、标签和选择值，只负责定位和回看，不读取 `projects.settings.scada_monitoring_interval_seconds`。方向键按 5 分钟移动；鼠标和触摸的拖动轨迹保持连续调频手感，选定值仍按最接近的 5 分钟时间戳归档。标签密度可以随面板宽度合并，不会改变拖动手感或时间选择精度。 + +能力接口另行返回当前项目监测周期；创建回放时由服务端固定到回放文档及快照的 `monitoring_interval_seconds`，页面不能提交不同周期。项目配置后续调整只影响新回放，当前演示继续使用固定值；未配置时提示设置。 + +| 交互 | 行为 | +| --- | --- | +| 方向键 | 前后移动一个 5 分钟时间戳 | +| PageUp / PageDown | 移动一个当前可见的主刻度，主刻度始终为 5 分钟的整数倍 | +| 拖动回看 | 拖动轨迹连续无缝，选择值按 5 分钟量化；只读定位不推进后台 | +| 主刻度 | 宽屏可每 5 分钟标注，窄屏按 10、15 分钟等密度显示，细刻线保留 5 分钟定位 | +| 下一步并监测 | 不读取时间轴位置；如后台已监测至 08:07，15 分钟周期处理到 08:22 | + +“下一步并监测”通过持久化 `step_target` 与 `step_pending` 管理。单步开始后，API 按原观测时刻读取、清洗和检测到目标边界，完成前禁用重复单步。若周期内有 5 分钟观测，15 分钟监测周期仍处理其中每个时刻；没有观测则提交空区间检查，不补造数据。尾步到达 end_time 时，仍遵守原始区间 `[start_time, end_time)`。 + +时间轴以 5 分钟时间戳定位，不能控制或触发周期处理；“下一步并监测”按项目监测频率推进后台周期，不将原始观测重采样。旧回放没有固定周期时，页面要求停止后重新创建，不能把后来修改的项目周期追补为历史事实。 + +### 步长边界主动监测与爆管流程 + +一次主动监测的身份为 `(replay_id, period_end, monitor_rule_version)`。同一步骤重试复用原回执，不能重复创建告警、调查或计算。监测回执保存在该步最后一个回放批次的 `monitoring` 中，并将最近一次结果投影到回放快照；内容包含周期起止、知识截止、输入批次上限、数据完整性、各指标状态、组合结论和关联任务。它不是 `scheduled_flow`，也不需要通用定时器。 + +步长边界按以下顺序执行： + +1. 确认本周期内所有已到达观测均已完成质量校验和清洗；缺测、无效值和单位冲突进入数据完整性结果，不能按正常值处理。 +2. 汇总压力下降幅度。只使用映射明确、单位一致、质量合格的压力测点，并同时保留异常测点数、覆盖测点数、基线窗口和最大或中位下降幅度。 +3. 汇总出厂流量变化。设备必须在项目配置或设备档案中明确标注为“出厂边界流量”，不能把任意 `pipe_flow` 测点当作出厂流量；保留方向、单位、基线和增量。 +4. 条件满足时运行爆管侦测。现有侦测要求至少 5 个完整压力点和目标时刻之前 15 天同周期数据；不满足时返回 `baseline_not_ready` 或 `sensor_coverage_insufficient`，不能将“未运行”解释为“无异常”。 +5. 组合上述证据形成监测结论；只有疑似爆管才创建 `event_investigation`，工作流使用现有 `burst_report`，并绑定本轮 `replay_context`。正常周期只保存回执，不启动 Agent；数据不足周期保存缺口并在界面展示。 +6. `burst_report` Skill 核对证据后决定是否调用爆管定位。定位必须有 m³/s 的有依据漏量估计或明确情景假设，以及模型节点映射、异常窗和正常对照窗；输入不具备时保持“疑似爆管、定位待补充”，不能填默认漏量。 + +组合判定采用可版本化策略，首版演示建议如下： + +| 监测结果 | 判定条件 | 后续动作 | +| --- | --- | --- | +| `insufficient_data` | 必需点位缺测、质量无效、单位或角色未配置，或爆管侦测基线不满足 | 保存缺口，不声明正常，不启动定位 | +| `normal` | 数据完整，爆管侦测未命中，压力与出厂流量规则均未命中 | 保存周期回执，不创建 Agent 任务 | +| `attention` | 只有单项指标命中，或指标间方向不一致 | 保存观察结果；下一步继续监测，必要时由人发起调查 | +| `suspected_burst` | 爆管侦测命中，且压力下降与出厂流量增加在同一业务时间窗内相互支持 | 创建一次 `burst_report` 调查；定位输入具备后进入计算 | + +爆管侦测和压力下降存在相关性，因此回执不能把两者描述成完全独立的两票。出厂流量也会受泵站调度、储水设施和正常用水变化影响。组合结论只是调查触发条件，不是现场确诊。 + +回放中的 MCP 定位已走 Server 持有的安全适配层：按 `replay_id`、`knowledge_cutoff` 和 `batch_sequence` 从已固化清洗输出组装压力与可选流量序列，再调用现有计算队列。回放只允许 monitoring 模式，并要求显式异常窗、正常对照窗和有依据的 `burst_leakage`；不能直接查询 TSDB 中尚未回放的未来记录。步长爆管侦测同样只读取最近 15 天已固化输出。计算结果保存输入、算法与证据引用，任务恢复或重试仍使用相同知识上限。 + +### FM 调频盘样式 + +用户明确选择电台调频盘的刻度、红色指示线和玻璃质感。这里的 FM 表示调谐外观，不增加电台、音频、频段切换或拟物旋钮功能。 + +视觉方向：工作台内一块冷白半透明的玻璃刻度窗，红色细线作为唯一醒目的时间定位标记；文字、控件与下方数据沿用现有工作台主题。内容顺序为选定区间、当前时间、刻度窗、控制栏、传入与检测进度。交互特征为固定指示线与横向移动的刻度带，手动拨动直接跟手。 + +![FM 调频式时间轴静态样式示意](assets/scada-fm-timeline.svg) + +图中时间和状态仅为静态样式示意，不是运行结果。玻璃下的底色用于说明透光与边缘高光，运行时仍需验证实际工作台背景下的对比度。 + +| 元素 | 设计约定 | +| --- | --- | +| 演示区间 | 常驻显示完整起止日期和时间；更换区间是次要入口，月度来源收在设置中 | +| 当前时间 | 刻度窗上方突出显示红线对应的时间，数字等宽；跨日时同时显示日期 | +| 时间刻度 | 主刻度有时间标签，次刻度细密且高低有别；实际密度按区间与容器宽度计算，窄屏优先减少标签而非缩小文字 | +| 红色指示线 | 固定在刻度窗水平中央，1 至 2px 宽，带小型顶端指示；这是时间定位色，不代表告警严重度 | +| 玻璃刻度窗 | 局部半透明、背景柔化、细边缘高光和轻微内阴影，玻璃层不影响数字、刻度和红线的清晰度 | +| 控制栏 | 播放或暂停是唯一主操作，下一批与倍速为辅助；回看时才出现回到当前 | +| 次级进度 | 单行显示已传入与已检测时间，积压时显示等待处理；不增加三块同等醒目的仪表 | + +玻璃效果是本次按用户要求对 SCADA 调频刻度窗的局部材质例外，外围面板外壳、表格、表单、菜单和 Agent 面板仍沿用现有 shadcn/ui 实体设计。新增时间指示颜色使用独立语义变量，不能让正常播放状态套用危险告警颜色。关闭透明效果或不支持背景模糊时，回退到清晰的实体刻度窗。 + +固定红线表示当前定位时间，刻度带按 5 分钟时间戳移动；只在选定区间内标注时间，两端之外留白，不带出整个月的刻度。该窗是选定区间的只读定位视窗，完整起止范围始终由上方文字明确表达；拨动不改变传入或检测进度。 + +手动拖动和触屏横拨改变查看位置，不立即改写后台时间。拨动到已释放时间时显示历史快照并标明回看；拨动到尚未释放时间时只显示待传入时间，不展示未来读数，若需从该位置开始则明确新建回放。回到当前只恢复跟随，不补发旧数据、告警或调查。 + +时间拨动按 slider 语义提供中文名称、时间范围、当前时间文本和键盘操作：方向键按 5 分钟刻度调整，PageUp/PageDown 调整一个主刻度，Home/End 到选定范围两端。键盘操作即时生效；鼠标和触摸拖动轨迹连续无缝、不带惯性，选定时间按 5 分钟量化。减少动态效果时关闭装饰性过渡，保留真实时间定位。状态不用颜色单独表达。 + +## 数据质量与业务异常 + +真实外部系统新增观测时，先通过现有入库边界将 raw 保存到 TSDB。本次演示使用已经入库的原始值，读取 API 校验身份、项目、设备与时间范围后返回 raw，直接进入质量校验和清洗；不为了模拟传入而重复执行原始值 INSERT。清洗只写回同一条观测的 cleaned_value，monitored_value 始终保留。 + +清洗完成后才启动业务异常检测。valid 可将 cleaned_value 设为原始值，corrected 保存有依据的修正值，无法可靠修正的 invalid 或 missing 保持空值并保存质量结论。suspect 保留 raw 与不确定性，不能自动替换为正常值。是否已经清洗以阶段回执和质量结论为准，不能仅用 cleaned_value 是否为空判断。 + +| 质量结论 | 选择值与后续处理 | +| --- | --- | +| valid | 使用 raw，包括可靠采集的突降、突升、低压和浓度变化 | +| corrected | 使用有依据的修正值，同时保留 raw、修正值、规则版本、原因和可用时间 | +| suspect | 保留 raw 并声明不确定性，结合其他测点和设备状态复核；不自动改成正常值 | +| invalid 或 missing | 保留原始记录或缺测事实，退出要求可信观测的规则输入，形成数据质量事件 | + +质量校验依据结构、有限数值、单位、设备身份、观测顺序、重复记录、量程与设备状态；仅偏离日常水平不能直接判 invalid。负流量、零浓度、低压均需按设备和业务含义解释。缺少仪表状态时明确证据不足，不能凭一次尖峰确认仪表损坏。 + +当前正式表中的 `cleaned_value` 可以保留，但不等于本轮已经完成清洗。读取 raw 时不清空已有 corrected，也不将旧 corrected 当作清洗输入。清洗阶段使用本批 raw 和本轮已知历史重新判断，提交本轮清洗版本并保存处理结果；检测只能使用该版本已提交的结果，不能直接取一个来源和时间不明的旧 corrected 当作已校验输入。 + +在线修正只使用当前已到达数据。当前居中窗口、后向补值、整段 Kalman 平滑和整段统计阈值保留为离线分析能力；不能一次性清洗整个月后将结果送入时间轴。在线清洗更新目标 TSDB 的修正值投影，原始值不变；原修正版本、当前修正版本和依据保留在处理证据中，旧回放仍读取自己的固定版本。 + +每条参与判断的数据记录其 `value_source`、质量结论、选用值、原始记录引用、规则版本和计算时间。回放过程不为未知错误强制生成一个修正数值。 + +这一区分对应 [EPA 在线水质监测指南第 6.1、6.2 节](https://nepis.epa.gov/Exe/ZyPURL.cgi?Dockey=P100UJYF.txt) 的“先校验、再异常检测，并保留原始值”原则；历史修订留痕可参考 [OPC UA Historical Access 第 4.4 节](https://reference.opcfoundation.org/specs/OPC-10000-11/4)。本项目的具体质量判定仍由指标和设备证据决定。 + +## 检测规则和多次事件 + +首版实时检测使用有版本的类型化规则：压力变化与上下限、流量变化与方向、水质具体指标的上下限或变化，以及数据质量规则。规则输入包含质量结论，输出只表示异常信号及依据，不确认爆管位置或污染原因。 + +每项启用的规则必须声明：`rule_id`、指标、方向、单位、阈值、使用的历史窗口、最少有效样本、连续命中点数、恢复阈值、连续恢复点数，以及事件组和最少同时命中的设备数。参数在创建回放前校验并冻结，至少需要一种已配置规则才可创建检测回放。缺失参数时该规则为 not_configured，不能借用统一 30% 或把零到非零统一解释为 100% 污染风险。 + +同一个事件组由明确的设备集合组成，不使用预设异常真值自动分组。同组、同规则、同一持续异常归并为一个事件；不同事件组可以同时产生不同事件。恢复需要可信的新观测满足恢复条件，缺测不构成恢复。恢复后的再次命中建立新事件。 + +事件保存首次异常、检测触发、最后异常、恢复时刻、证据引用和调查结论。传感器恢复正常只表示观测恢复，不代表现场维修、污染处置或业务调查已经完成。 + +事件身份由 `replay_id + rule_id + event_group_id + first_trigger_at` 确定；同一事件更新不创建多条调查。批次重试不重复创建事件，新一次回放的事件与旧回放独立。 + +现有 [爆管侦测服务](../app/services/burst_detection.py) 可作为满足条件后的进一步分析工具，不能直接等同于逐批在线检测器。当前 target 模式需要 15 天数据及至少 5 个完整压力测点。回放前没有足够历史时，展示 baseline_not_ready；支持短历史的规则仍可运行，不能用未来几天补足参考窗口。 + +正常基线只使用当时已知、质量合格且适用于该规则的历史。已确认异常可以从正常基线中排除，原始记录仍保留。事后确认或重新标注产生新版本，不改写此前检测“当时知道了什么”。 + +## 持久化和事务 + +观测数据以项目目标 TSDB 的 `scada.measurements` 为权威存储，沿用 time、device_id、monitored_value 和 cleaned_value。业务库不替代 TSDB 保存正式观测。 + +建议在 BizDB 增加以下处理对象： + +| 对象 | 权威信息 | +| --- | --- | +| `peripheral_sample.scada_replays` | 回放身份、项目、创建者、冻结配置、生命周期、版本、模拟时间锚点、读取与处理进度、工作租约和恢复检查点 | +| `peripheral_sample.scada_replay_batches` | 本批 TSDB 记录键、raw 摘要、固定规则版本、清洗输出、质量结论、检测结果、处理阶段和重试信息 | + +两个对象分别具有控制生命周期与批次处理生命周期。现有 test_data 是手工单条演练 CRUD，缺少顺序、批次和消费语义；任务表表示业务调查，也不能存正常观测批次。这里保存记录引用和处理结果，不复制一整套 raw；需要固化异常证据或修正依据时，只保存必要的原始片段。由于本次不重新摄入 raw，不新增 TSDB 原始值收件表。表和列同时写清数据库注释。 + +事件及当前告警复用 `workbench.alert_events`、`workbench.alerts`，调查及其证据复用现有 operational 表。告警来源扩展为判别联合：保留现有任务来源，新增 `kind=scada_replay`、`replay_id`、`dataset_revision`、规则组与触发批次，避免伪造调查身份。质量故障也用同一事件存储，按类别区分。 + +### 批次处理与提交 + +不使用一个长期跨库事务。批次按以下阶段单调推进，重试保持原批次身份： + +1. `selected`：后台读取 raw 后，BizDB 固定记录键、输入摘要、规则与知识截止时间，并在同一事务登记处理身份和读取检查点。GET 本身不做此写入。 +2. `clean_prepared`：直接对已读入的这一批 raw 和已加载的历史状态计算质量与修正结果，先将确定的输出及输入摘要保存在 BizDB。正常路径不调用会重新查询同批 raw 的整段清洗入口。 +3. `cleaned`：TSDB 用一个有界事务按记录键更新 cleaned_value，并校验 raw 仍与本批输入一致；不更新 monitored_value。只有 TSDB 提交成功且 BizDB 记录已提交阶段后，才允许检测。单行结果相同时不必重写字段。 +4. 若 TSDB 已提交但 BizDB 阶段尚未更新，恢复时核验预存的输入与输出；已经匹配则补记 cleaned，未写入则幂等应用预存结果，raw 已变则停止该批次。只在这种恢复或校验冲突路径增加读取，不为每批正常处理增加一次重复查询。 +5. `detected`：直接使用本批已提交的清洗结果做检测，BizDB 在同一事务保存检测结果、事件状态、调查意图、检查点和 processed_until，然后发布业务事件。检测失败只重试本阶段，不重新清洗或重写 raw。 + +以上阶段保存在批次中。页面继续突出模拟时间、已读取进度和已检测进度，清洗中或失败作为当前批次状态展示。正常路径对新批次读取一次 raw、必要时批量更新一次修正值，再复用这批处理结果完成检测；界面后续查看与故障恢复属于独立的必要读取。 + +BizDB 批次状态是持久待处理队列，通知只负责唤醒。后台启动、通知中断与定期对账均扫描未结束批次，按已提交阶段恢复；同库状态与事件在同一事务保存，不能假定两个数据库天然原子。[PostgreSQL NOTIFY](https://www.postgresql.org/docs/15/sql-notify.html) 和 [SELECT 锁定语义](https://www.postgresql.org/docs/15/sql-select.html) 提供通知及领取基础。 + +调用慢算法和外部服务前结束数据库事务，不持有事务锁等待 HTTP。清洗结果未提交，不执行异常检测；中断后从已有检查点恢复，不丢弃批次身份重新处理一遍。 + +### 重复读取与重复回放 + +读取 API 的同一次游标请求可以安全重试，但相同数据不能被当作新的处理批次。后台以项目、回放、批次序号和分片序号登记幂等身份，记录级使用规范设备 ID 与原始时间；同键同摘要复用现有处理状态，不同摘要返回冲突。 + +新一轮演示按新的 replay_id 建立独立处理状态，可以重新读取和分析选定区间，但仍不插入 raw。界面统计已读取条数和已处理条数，不能称为新写入 SCADA 条数。 + +首版同一项目只允许一条活动的读取清洗流水线，避免不同回放同时改写同一 cleaned_value。历史结果仍可同时查看。修正字段表示最近一次已发布清洗结果，旧版结果和依据保留在批次输出及必要的原始证据中；旧 Agent 执行和历史回看不直接读取被后续处理替换的修正投影。 + +工作台事件流只发布回放控制状态变化、故障与业务告警；不为每个测点值发布 SSE，也不把全部原始数据放入事件日志。未来若新增 `scada.replay.changed` 主题，只用于控制状态，并需同步主题模型、约束和消费契约；当前控制进度由轮询读取。 + +未读取数据所在 raw 分片摘要变化时停止该回放，要求新数据集版本；本轮 cleaned_value 更新不计作源版本漂移。空检查批次也需要保存处理阶段，记录数为零不代表可以绕过缺测判断。 + +每个回放同一时刻只有一个有效处理租约，采用数据库会话锁阻止两个请求重复推进同一周期。每个批次提交后再读取下一原始时间戳，不持久保存进程计时器值。 + +源数据和相关回放证据在本次演示中不自动过期；一个月是数据覆盖范围，不是保存期限。事件结束、暂停或重播不清理已有证据。生产保留策略单独制定。 + +## Agent 调查与时间隔离 + +主动单步在边界形成组合监测回执：一般关注和数据不足只保存回执，只有 `suspected_burst` 进入 `burst_report`。SCADA 演示请求只负责建立持久调查任务，不要求先生成一条合成 test_data，也不在请求内执行 Agent。 + +告警事实与“待建立调查”意图在事件事务中保存。创建调查失败时后台对账重试；组合监测调查的来源稳定标识使用 `scada_replay::monitor::`，一般事件继续使用 `scada_replay::event:`，均沿用现有外部来源幂等约束。事件不因 Agent 不可用而丢失。 + +已新增 `scada_anomaly` 工作流与对应 Skill，要求先核验数据质量、点位和单位，再读取事件窗口及正常参考。疑似爆管复用现有 `burst_report` Skill；它会区分侦测、定位候选和现场确诊，并要求漏量依据。`run_burst_location` 在回放任务中通过冻结输出适配执行，其他未适配计算继续拒绝；不能仅修改 Skill 文本或 MCP allowlist 绕过知识截止。重计算沿用 [后台计算队列](calculation-queue.md)，保留外层 HTTP 限制。 + +新增 Server 持有的回放查询上下文：`replay_id`、数据集版本、知识截止时刻 `knowledge_cutoff`、已接收批次上限、处理版本和来源事件。它与任务及执行代次绑定，不能由模型或客户端自行扩大。 + +每代调查开始时冻结其可用批次与时间上限；同一执行中后续回放推进不会偷偷改变证据。确有新的重要证据时，按现有 [调查轮次规则](operational-tasks.md) 创建或合并后续调查轮次，而不是每批次追加一次模型执行。 + +时间隔离必须覆盖 MCP 的 SCADA 查询、要素历史、任务数据快照、外围观测，以及检测算法内部读取。raw 源读取 API 只允许读取不晚于模拟时钟的观测；检测只使用已完成本轮清洗的批次结果，原始表格可以显示本轮已读取且尚在清洗的 raw。历史回看和旧代调查读取对应固定版本或原始记录引用，异常证据使用已经固化的必要片段。即使 TSDB 存有整个月记录，也必须按本轮读取检查点、设备集合和知识截止时间过滤，不能只在提示词中要求遵守。 + +所有回放调查证据保留真实来源：合成数据仍为 synthetic，历史回放有明确标识，不能因为读取的是正式表就改称现场实时观测。普通历史数据管理仍可按原权限查看数据集，但不能作为回放会话绕过截止时间的途径。 + +回放调查只生成演示分析、计划和报告；真实外围工单和现场控制不属于自动回放动作。Server 动作执行边界必须识别回放上下文，不能仅靠 Agent 提示阻止将演示事件发成真实业务工单。 + +## 接口与界面 + +以下为已经加入 Server OpenAPI 的路径，统一使用 Bearer 与 X-Project-Id。保留原有 `/peripheral/api/sample/v1` 工单与手工演练接口，回放不增加一层外围代理。 + +| 接口 | 语义 | +| --- | --- | +| `GET /api/v1/scada-replays/capabilities` | 回放表可用性、写权限、监测频率、爆管组合监测角色是否就绪及最近一轮状态；不代表完整月度数据就绪 | +| `POST /api/v1/scada-replays` | 固定区间、设备、规则和质量量程，返回 ready；携带稳定 request_id | +| `GET /api/v1/scada-replays/{replay_id}` | 权威控制、时间、计数、设备与固定规则、当前输出及故障 | +| `POST /api/v1/scada-replays/{replay_id}/commands` | play、pause、step、set_speed、stop、retry；request_id + expected_revision | +| `GET /api/v1/scada-replays/{replay_id}/observations?through=...` | 仅本轮已读取时间范围内的历史快照，每设备最近一条、最多 500 个 | +| `GET /api/v1/timeseries/scada-readings/raw` | 当前项目 TSDB 原始值联合游标分页；不插入、不隐式清洗 | +| `GET /api/v1/scada-replays/{replay_id}/events?offset=0&limit=100` | 检测变化与调查关联，返回 items、next_offset | + +raw API 允许参数为 start_time、end_time、device_ids、cursor、limit；回放请求携带经服务端验证的 replay_id，任务请求由已有绑定解析回放身份。有效范围为所选区间与模拟时间上限的交集，不能因省略 replay_id 绕过已绑定的任务限制。返回 items（time、device_id、monitored_value、quality）、next_cursor 和 scanned_through；本版不返回尚未准备的 dataset_revision，不返回 corrected 作为源输入。 + +next_cursor 绑定项目、设备、查询区间并固定本轮分页时间上限；本版不为整个查询提供 MVCC 源版本快照，按 time、device_id 排序，每页最多 500 条。读取窗口未结束但当前页为空时，scanned_through 表示已经确认没有观测的区间上限，供后台登记空检查批次。分页、刷新和网络重试均是无副作用读取。 + +查询和检查点更新使用类型化服务方法及已验证项目上下文。清洗函数接受已经读入的批次，不再调用当前会重新查询 raw 的整段清洗入口；检测函数接受这批已提交清洗结果。页面的单步 POST 命令同步完成一个监测周期，GET 预览和回看不启动任务。 + +回放读取要求 scada.view，创建及控制要求 scada.clean；普通 raw API 沿用 timeseries SCADA 路由的读权限。API 不自动创建表，不回退默认项目，也不隐式枚举全部活跃项目。 + +前端 SCADA 页签按以下顺序组织： + +1. 选定演示区间：开始时间、结束时间和项目监测步长；设备范围、规则及基线收纳在折叠的检测参数区域。 +2. FM 调频式时间轴：局部玻璃刻度窗、固定红色指示线、两侧渐暗及中央放大；按 5 分钟时间戳定位，不读取逐测点列表，也不触发监测。 +3. 主动监测操作：以“开始并监测”或“下一步并监测”为唯一主操作，按项目频率同步处理一个周期。 +4. 本周期结论：显示正常、关注、数据不足或疑似爆管，以及是否已经进入爆管调查与定位流程。 + +设备目录、raw 与修正值明细、完整事件列表和手工单点异常录入不在演示主面板展示。底层数据与 API 继续保留，供诊断、追溯和调查任务按权限读取。 + +沿用现有 workbench-shadcn-surface、日期控件和实体表面，调频刻度窗采用上文明确的局部玻璃例外；时间轴支持键盘操作和中文可访问名称。拖动回看不触发后台写操作。 + +页面可见时每 5 秒读取一次权威控制快照；单步请求执行期间由操作本身返回最终快照。页面隐藏时停止界面采样。演示面板不轮询完整事件列表，调查入口需要时再按分页 API 查询。 + +## 实施范围与验证 + +这是跨 Server、供水前端和 Agent 契约的变更，不新增后台进程。数据、存储、身份及网络复用现有部署。 + +| 仓库 | 主要落点 | +| --- | --- | +| Server | `app/api/v1/endpoints/scada_replay.py` 的回放控制与能力；`app/services/` 和 `app/infra/db/` 的批次、质量和事件；现有 task_alert、operational_task、MCP 上下文、查询和 Skill | +| Server 契约与存储 | `resources/sql/` 的 BizDB 两张回放表迁移；`contracts/` 的 raw 读取、Server OpenAPI 和清单；外围协议不变；TSDB 沿用已有观测表 | +| 前端 | `src/features/workbench/peripheral-sample/` 的回放控制与状态 hook；SCADA 客户端与生成契约；告警来源和事件解析 | +| Agent | `src/product/` 的生成契约及任务上下文适配，保留已有原生 MCP 与执行权限边界 | + +验收用例必须覆盖： + +- 5/15 分钟设备混合、乱序源记录、同一时间戳跨页、空片段、结束边界和多次异常。 +- 重复批次、相同编号不同内容、响应丢失、请求中断和恢复重试；不漏读、不重复事件。 +- GET 重试、分页重复请求、清洗输出预存后中断、TSDB 更新提交而 BizDB 阶段未更新、检测失败；逐阶段恢复，不提前检测。 +- 正常处理每批只读一次 raw，清洗后检测复用结果；回放不执行 raw INSERT，不在读取时清空旧 corrected，空数据不伪造观测。 +- 周期内多批次单步、重复单步拒绝、并发请求互斥和尾步完成状态。 +- 5 分钟时间定位与 15 分钟监测周期相互独立、非整点起点、周期配置变更不影响当前回放、尾步不足完整周期和未配置时禁止开始。 +- 每个步长边界只产生一份幂等监测回执；正常、数据不足、单指标关注和疑似爆管分别处理，不为正常周期创建 Agent。 +- 爆管侦测基线不足、压力下降、出厂流量变化、设备角色错误、指标方向冲突和多指标同时命中；只有疑似爆管建立 `burst_report`。 +- 回放 MCP 侦测与定位只能读取固定批次和知识截止前数据；定位缺少漏量依据时停止在待补充，不调用算法。 +- 选定小时区间、跨日区间和再次演示；刻度与进度只对应选定区间，基线初始化不释放旧事件，到终点不自动播放整月。 +- FM 红线固定、刻度带移动、拖动回看不重复投递、未来位置不泄露读数；桌面与 375px、320px 窄屏下的长日期、键盘、减少动态效果及玻璃回退。 +- 真实突降不被当作采集误差修掉；负流量、零水质值、空值、错误单位和缺测恢复分别处理。 +- 首个异常、持续异常、可信恢复、再次异常及两个事件组并发；不会逐点创建调查。 +- 缺少基线、缺少水质指标配置、历史 corrected 无可用时间，以及 source_revision 变化。 +- MCP、任务数据接口及算法内部的未来查询均被拒绝；下一代调查才能使用新增证据。 +- 合成来源、历史回放来源、事件真值隔离；回放不发布真实现场动作。 +- 合成测试链路、真实项目 API、常驻后台、真实 Agent 执行和浏览器分别记录验证结果。 + +后续实现的验证命令从各仓库执行：Server 运行新增回放、批次、质量、事件与查询边界的聚焦 pytest，并保留现有 SCADA/告警/调查回归；前端运行 `pnpm lint`、`pnpm typecheck`、`pnpm test` 和聚焦 `pnpm test:browser`；Agent 修改后运行 `bun run check` 及对应聚焦测试。具体已执行结果见下方，不把待部署链路计为通过。 + +数据库应用遵守 [工作台数据库变更门禁](workbench-database-rollout.md)：先核对目标项目，备份和回滚演练，再按明确范围应用及验收；不得默认向全部项目写入。本地 MetaDB 已对明确的演示项目应用并复验 `033_metadata_project_settings.sql`，其他项目保持原值；`tjwater2_v2` BizDB 已应用 `031_scada_replays.sql`，不据此推断其他项目已迁移。本次没有生成一个月观测或启动常驻进程。 + +回退应用前暂停新批次读取，核对 clean_prepared、cleaned 与 detected 阶段，保留原始观测、已提交修正值、回放表、告警与调查证据；升级恢复后从原阶段继续。BizDB 流程记录和 TSDB 修正值不在同一事务，回滚应用不能假定另一库结果也被撤回。不能通过删除事件、重置时间游标或清空 SCADA 原始表来伪装成从未处理。若需要撤回数据库变更，另按备份与新增业务写入情况制定恢复步骤，不在仍有写入时覆盖数据库。 + +本方案依赖的关键前提是源历史能提供稳定设备身份、可信时间戳和固定数据版本。缺少完整月度观测时，能力接口只能声明实际可用范围；缺少修正来源时采用 raw；缺少检测参数时该类检测不启用。任何这些缺口都不能由时间轴、插值或预设事件答案掩盖。 + +## 部署准备与验证记录 + +迁移脚本为 [031_scada_replays.sql](../resources/sql/031_scada_replays.sql)，显式限制物理数据库 `tjwater_v2` 与 `tjwater2_v2`，依赖既有 peripheral_sample schema、告警及调查结构。仅新增两张 BizDB 表，不新增 TSDB 观测表。`tjwater2_v2` 已应用该迁移并配置 `settings.scada_monitoring_interval_seconds=900`。 + +真实数据库验证使用 `SCADA_REPLAY_STORAGE_TEST=1 conda run -n server python -m pytest tests/integration/test_scada_replay_storage.py -q`。该测试在项目路由选出的 BizDB 与 TSDB 双回滚事务中临时建表、写测试观测，覆盖真实 SQL 的 cleaned 更新、事件持久化和主动周期推进，退出全部回滚；它不证明身份认证 HTTP 和真实 Agent 推理已经验收。浏览器测试以 mock Server 验证交互与窄屏布局，不能作为现场检测效果证明。 + +本轮验证结果（2026-11-03，当前工作区）： + +| 层级 | 结果与边界 | +| --- | --- | +| Server | 主动监测、条件式爆管调查、冻结定位适配、MCP、项目配置、raw、处理及 OpenAPI 聚焦回归共 120 项通过 | +| 数据库与契约 | 本地 MetaDB 的 033 迁移已单项目应用并重复执行验证；OpenAPI 279 条路径与清单摘要一致；本轮未运行 031 的真实双库集成测试 | +| 前端 | lint、typecheck、全量 148 文件共 884 项 Vitest 通过；设备列表、手工单点演练及对应触发适配已移除 | +| 浏览器 | 1 项 mock Server 场景通过，包含 5 分钟刻度、选择值与键盘步进、连续拖动轨迹、15 分钟主动监测、中央放大、两侧渐暗、固定红针及 1280/375/320px 布局；未作为真实检测效果证明 | +| Agent | 同步 Server OpenAPI 并生成 server-api.d.ts，bun run check 通过；无真实模型推理 | +| 仓库 | Server、供水前端、Agent 的 git diff --check 通过；未提交、推送或部署 | + +生产联调仍需核验真实 HTTP 接口、调查执行链路和 Agent 报告，并单独准备完整月度数据及事件真值。不能将以上局部测试解释为这些环节已经完成。 diff --git a/resources/sql/005_metadata_project_configuration.sql b/resources/sql/005_metadata_project_configuration.sql index 0751487..54218c1 100644 --- a/resources/sql/005_metadata_project_configuration.sql +++ b/resources/sql/005_metadata_project_configuration.sql @@ -19,13 +19,18 @@ CREATE TABLE IF NOT EXISTS projects ( gs_workspace VARCHAR(100) UNIQUE NOT NULL, map_extent JSONB, map_config JSONB NOT NULL DEFAULT '{}'::jsonb, + settings JSONB NOT NULL DEFAULT '{}'::jsonb, status VARCHAR(20) DEFAULT 'active' NOT NULL, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL, - CONSTRAINT projects_status_check CHECK (status IN ('active', 'inactive', 'archived')) + CONSTRAINT projects_status_check CHECK (status IN ('active', 'inactive', 'archived')), + CONSTRAINT projects_settings_object_check CHECK (jsonb_typeof(settings) = 'object') ); ALTER TABLE projects ADD COLUMN IF NOT EXISTS map_config JSONB NOT NULL DEFAULT '{}'::jsonb; +ALTER TABLE projects ADD COLUMN IF NOT EXISTS settings JSONB NOT NULL DEFAULT '{}'::jsonb; +COMMENT ON COLUMN projects.settings IS + '项目级通用配置（JSON 对象，不存放密钥）；scada_monitoring_interval_seconds 为监测任务间隔秒数。'; COMMENT ON COLUMN projects.map_config IS '项目级地图配置（JSON）；place_search 保存地名搜索启用状态、服务商、行政区编码及区域名称；保留既有图例与坐标参考配置。'; diff --git a/resources/sql/031_scada_replays.sql b/resources/sql/031_scada_replays.sql new file mode 100644 index 0000000..e5883f4 --- /dev/null +++ b/resources/sql/031_scada_replays.sql @@ -0,0 +1,46 @@ +BEGIN; +DO $$ BEGIN + IF current_database() NOT IN ('tjwater_v2', 'tjwater2_v2') THEN + RAISE EXCEPTION '031_scada_replays requires an explicitly verified SCADA demo project'; + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS peripheral_sample.scada_replays ( + project_id uuid NOT NULL, + replay_id uuid NOT NULL, + request_id uuid NOT NULL, + status text NOT NULL CHECK (status IN ('ready','paused','completed','stopped','failed')), + document jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (project_id,replay_id), + UNIQUE (project_id,request_id) +); +CREATE INDEX IF NOT EXISTS scada_replays_status_idx + ON peripheral_sample.scada_replays(project_id,status,created_at DESC); +CREATE TABLE IF NOT EXISTS peripheral_sample.scada_replay_batches ( + project_id uuid NOT NULL, + replay_id uuid NOT NULL, + sequence bigint NOT NULL, + stage text NOT NULL CHECK(stage IN ('selected','clean_prepared','cleaned','detected')), + document jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY(project_id,replay_id,sequence), + FOREIGN KEY(project_id,replay_id) REFERENCES peripheral_sample.scada_replays(project_id,replay_id) +); +COMMENT ON TABLE peripheral_sample.scada_replays IS 'SCADA 演示回放的权威控制状态与处理检查点，不保存正式原始观测'; +COMMENT ON COLUMN peripheral_sample.scada_replays.project_id IS '经认证与路由校验的所属项目'; +COMMENT ON COLUMN peripheral_sample.scada_replays.replay_id IS '独立回放身份，不等同 Agent 执行编号'; +COMMENT ON COLUMN peripheral_sample.scada_replays.request_id IS '创建请求的稳定幂等编号'; +COMMENT ON COLUMN peripheral_sample.scada_replays.status IS '供领取和约束使用的 document.status 查询投影'; +COMMENT ON COLUMN peripheral_sample.scada_replays.document IS '冻结配置、设备快照、时钟、检查点、规则状态和命令回执'; +COMMENT ON COLUMN peripheral_sample.scada_replays.created_at IS '实际创建时间'; +COMMENT ON COLUMN peripheral_sample.scada_replays.updated_at IS '实际控制或进度更新时间'; +COMMENT ON TABLE peripheral_sample.scada_replay_batches IS 'SCADA raw 引用、清洗输出及检测阶段的恢复凭证，禁止重写 raw'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.project_id IS '所属项目'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.replay_id IS '所属回放'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.sequence IS '回放内单调批次序号'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.stage IS '已提交处理阶段'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.document IS '观测键、raw 摘要、确定的清洗结果与检测事件'; +COMMENT ON COLUMN peripheral_sample.scada_replay_batches.created_at IS '实际登记批次时间'; +COMMIT; diff --git a/resources/sql/033_metadata_project_settings.sql b/resources/sql/033_metadata_project_settings.sql new file mode 100644 index 0000000..8de4fca --- /dev/null +++ b/resources/sql/033_metadata_project_settings.sql @@ -0,0 +1,54 @@ +BEGIN; + +DO $$ +BEGIN + IF to_regclass('public.project_databases') IS NULL + OR to_regclass('public.user_project_membership') IS NULL THEN + RAISE EXCEPTION '033_metadata_project_settings must run in the project metadata database'; + END IF; +END $$; + +ALTER TABLE public.projects + ADD COLUMN IF NOT EXISTS settings jsonb NOT NULL DEFAULT '{}'::jsonb; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'public.projects'::regclass + AND conname = 'projects_settings_object_check' + ) THEN + ALTER TABLE public.projects + ADD CONSTRAINT projects_settings_object_check + CHECK (jsonb_typeof(settings) = 'object'); + END IF; + + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'projects' + AND column_name = 'scada_monitoring_interval_seconds' + ) THEN + IF EXISTS ( + SELECT 1 FROM public.projects + WHERE scada_monitoring_interval_seconds IS NOT NULL + AND settings ? 'scada_monitoring_interval_seconds' + AND settings->'scada_monitoring_interval_seconds' + IS DISTINCT FROM to_jsonb(scada_monitoring_interval_seconds) + ) THEN + RAISE EXCEPTION 'Conflicting SCADA monitoring intervals; resolve before migration'; + END IF; + + UPDATE public.projects + SET settings = settings || jsonb_build_object( + 'scada_monitoring_interval_seconds', scada_monitoring_interval_seconds + ) + WHERE scada_monitoring_interval_seconds IS NOT NULL; + + ALTER TABLE public.projects DROP COLUMN scada_monitoring_interval_seconds; + END IF; +END $$; + +COMMENT ON COLUMN public.projects.settings IS + '项目级通用配置（JSON 对象，不存放密钥）；scada_monitoring_interval_seconds 为监测任务间隔秒数。'; + +COMMIT; diff --git a/resources/work-order-skills/burst-report/SKILL.md b/resources/work-order-skills/burst-report/SKILL.md index 207085f..6947dec 100644 --- a/resources/work-order-skills/burst-report/SKILL.md +++ b/resources/work-order-skills/burst-report/SKILL.md @@ -2,7 +2,7 @@ name: burst-report description: 调查供水爆管上报，形成有依据的候选位置、隔离阀门、影响范围和后续处置建议。 metadata: - version: "2.0.6" + version: "2.0.8" --- # 爆管上报处置 @@ -13,6 +13,8 @@ metadata: 读取当前任务、事件时刻、地点、来源、已有报告、证据、动作和反馈。新一轮优先检查新增事实和上轮未决项，按时间、来源、对象及输入比较新旧结果，不按 ID 推断先后，也不因原工单未更新就断言没有新证据。保留本流程名称和版本。 +若任务来源带有 `replay_context`，这是 SCADA 历史回放中的疑似爆管演示。先读取任务摘要中的步长监测结论，再用 `query_scada_readings` 核对冻结批次；不得查询知识截止之后的数据。回放中的 `run_burst_location` 只接受 monitoring 模式，并要求显式提供本轮已处理范围内的异常窗和正常对照窗；Server 会从冻结清洗输出构造计算输入。缺少有依据的 `burst_leakage` 时保留“定位待补充”，不得用出厂流量增量、压力下降或默认值替代。回放结果保持 synthetic，不发布真实工单或设备动作。 + ## 定位与影响分析 根据现有线索自主选择必要步骤： @@ -26,9 +28,19 @@ metadata: 查询错误、权限拒绝或接口故障不是缺数证据。必要分析无法执行时，说明失败步骤和已经完成的部分，不借有限摘要宣布调查完成。 +## 事件前后影响说明 + +每轮按实际证据说明以下阶段的影响，给出结果或具体限制；尚未发生的处置列为预测和待验证项： + +1. **事件前与爆管后、隔离前**：核对正常基线和异常窗口，说明压力、流量变化及已覆盖的异常点位、资产和区域。候选管段及破口面积有依据时，可用 `run_burst_simulation` 补充情景水力影响，标明假设。漏量估计须说明边界流量、正常用水及储量变化等依据；有可靠漏量序列和时间覆盖时可累计漏水量，保留单位、方法和不确定性。定位工具的 `burst_leakage` 是输入，不能当成反演所得漏量。 +2. **关阀隔离方案**：说明止漏目标、直接隔离节点和关阀后新增失去水源连接的节点，分别保留 `affected_nodes` 与 `supply_affected_nodes` 及其计数。结合关阀水力结果说明预计停水、低压或供水不足的范围和覆盖限制；有用户关联资料时说明受影响用户、重要设施及保障需求。拓扑失源、模型预测和现场实际停水分别标注，不把节点数当户数。 +3. **实际隔离与恢复供水后**：后续轮次按实际关阀、抢修和恢复时间读取前后监测与现场反馈，对照预测核查止漏效果、实际影响范围和恢复情况，说明持续低压、异常流量、再次异常及尚待验证的影响。核对正常基线、观察窗口和判定依据，工单状态或单个正常读数不足以证明影响消除。 + +比较须核对对象、单位、时间对齐、模型版本、需水与水源边界、泵阀状态及计算设置；区分实测变化与模型方案差异。关阀工具不会自动继承爆管情景，条件不同的独立计算不能直接作为同一事故前后效果。可比时给出变化量和方案差异，缺少可比基准时说明当前结果及缺口，不补造改善比例。必要的紧急核验、通知和供水保障不等待可选的完整模拟。 + ## 处置与报告 -报告分别给出位置候选及依据、隔离阀门及可用性、替代方案、拓扑与水力影响、用户影响边界，以及处置和闭环缺口。只有初步结果时明确范围，不宣称已经定位、确诊或闭环。位置、候选、监测、影响和阀门方案按实际调查步骤同步展示，遵循后台展示规则，不用报告定位链接代替过程展示。覆盖面保留分组、缺失几何和示意边界，不能当作实际停水、水力影响或用户服务边界。无数据或展示失败时说明具体限制；展示不替代证据，也不要求等待浏览器。 +报告分别给出位置候选及依据、隔离阀门及可用性、替代方案、事件前后分阶段的拓扑与水力影响、用户影响边界，以及处置和闭环缺口。影响说明保留时间窗、对象、证据、实测或预测属性及限制。只有初步结果时明确范围，不宣称已经定位、确诊或闭环。位置、候选、监测、影响和阀门方案按实际调查步骤同步展示，遵循后台展示规则，不用报告定位链接代替过程展示。覆盖面保留分组、缺失几何和示意边界，不能当作实际停水、水力影响或用户服务边界。无数据或展示失败时说明具体限制；展示不替代证据，也不要求等待浏览器。 形成具体建议后，先保存阶段结论，再提议必要动作。人工核验、通知、定位隔离、抢修、适用的清洗复测和恢复回访使用关联的 `work_order_type=general` 草稿；原爆管事件保持独立。草稿写明位置、任务、依赖、执行前核验和回执要求。没有通知适配器时明确由人工实施。设备命令仅用于真实可控目标，审批和执行由 Server 控制。 diff --git a/resources/work-order-skills/scada-anomaly/SKILL.md b/resources/work-order-skills/scada-anomaly/SKILL.md new file mode 100644 index 0000000..91fa192 --- /dev/null +++ b/resources/work-order-skills/scada-anomaly/SKILL.md @@ -0,0 +1,18 @@ +--- +name: scada-anomaly +description: 调查 SCADA 回放中的异常信号，区分读数质量与运行异常，形成可追溯的演示报告。 +metadata: + version: "1.0.1" +--- + +# SCADA 回放异常调查 + +先读取任务上下文和来源中的 replay_context，核对规则、设备、单位、异常时刻及知识截止时间。数据来自已存 SCADA 的演示回放，不代表新采集的现场故障。 + +通过 query_scada_readings 读取事件前后本轮已处理的观测；查询结束时间不得晚于 knowledge_cutoff。结合 monitored_value 与 cleaned_value 判断数值变化，不将被可靠采集的低压或水质变化当作数据错误。缺测、无效值和未完成清洗应明确记录。 + +报告说明事件前后影响：在已开放且已处理的观测范围内，核对异常前参考、异常期间及已有恢复观测，列出压力、流量或水质变化、持续时间、涉及测点和已核实的资产关联。只有对象、单位、时间与运行条件可比时才给出变化量；说明参考窗口、证据和覆盖限制。尚无恢复观测时列为待验证，读数恢复不能证明现场修复或用户影响消除。停水范围、关阀影响及泄漏量需要相应拓扑、水力、流量平衡或现场依据；当前证据无法支持时明确待分析，不由单点低压推定停水区、受影响户数或漏量。 + +当前回放调查仅开放冻结 SCADA 证据、设备与模型目录；不支持原生计算或绕过回放上限的其他数据入口。回放前历史用于后台基线初始化，当前证据接口不返回该部分。需要更多历史或进一步计算时，报告具体缺口，不能读取未来或虚构分析结果。 + +报告列出观测事实、规则信号、质量问题、尚未确认的原因和建议核查。压力异常不等于已确认爆管，浓度变化不等于已确认污染。回放只生成演示分析与报告，不创建真实现场工单、不下发设备命令。 diff --git a/tests/api/test_admin_metadata_endpoints.py b/tests/api/test_admin_metadata_endpoints.py index 6f23dcf..a0febbf 100644 --- a/tests/api/test_admin_metadata_endpoints.py +++ b/tests/api/test_admin_metadata_endpoints.py @@ -616,3 +616,52 @@ def test_project_map_config_accepts_search_and_preserves_extra_options(): AdminProjectUpdateRequest(map_config=None) with pytest.raises(ValidationError): AdminProjectUpdateRequest(map_config={'place_search': {'enabled': True, 'admin_code': 'wrong', 'region': '凤阳县'}}) + + +def test_project_scada_monitoring_interval_is_a_strict_schedule_parameter(): + from pydantic import ValidationError + from app.domain.schemas.admin_metadata import AdminProjectUpdateRequest + + config = { + "scada_monitoring_interval_seconds": 900, + "future_option": {"enabled": True, "values": [1, "demo", None]}, + } + request = AdminProjectUpdateRequest(settings=config) + assert request.model_dump(exclude_unset=True) == {"settings": config} + for value in (0, 59, True, 900.0, "900", 31_536_001): + with pytest.raises(ValidationError): + AdminProjectUpdateRequest(settings={"scada_monitoring_interval_seconds": value}) + for value in (None, [], "settings", {"future_option": object()}): + with pytest.raises(ValidationError): + AdminProjectUpdateRequest(settings=value) + assert AdminProjectUpdateRequest().model_dump(exclude_unset=True) == {} + assert AdminProjectUpdateRequest(settings={}).model_dump(exclude_unset=True) == {"settings": {}} + assert AdminProjectUpdateRequest( + settings={"scada_monitoring_interval_seconds": None} + ).model_dump(exclude_unset=True) == {"settings": {"scada_monitoring_interval_seconds": None}} + + +@pytest.mark.anyio +async def test_update_project_passes_only_requested_settings_and_returns_extra_keys(monkeypatch): + from app.domain.schemas.admin_metadata import AdminProjectUpdateRequest + + now = datetime.now(timezone.utc) + project = _project( + code="demo", description=None, gs_workspace="demo", map_extent=None, + settings={"scada_monitoring_interval_seconds": 900, "future_option": {"enabled": True}}, + status="active", created_at=now, updated_at=now, + ) + repo = SimpleNamespace(session=object(), update_project=AsyncMock(return_value=project)) + monkeypatch.setattr(admin_metadata, "log_audit_event", AsyncMock()) + + response = await admin_metadata.update_admin_project( + AdminProjectUpdateRequest(settings={"future_option": {"enabled": True}}), + project_id=project.id, + current_user=_user(role="admin", is_superuser=True), + metadata_repo=repo, + ) + + repo.update_project.assert_awaited_once_with( + project.id, updates={"settings": {"future_option": {"enabled": True}}} + ) + assert response.settings.model_dump() == {**project.settings, "scada_monitoring": None} diff --git a/tests/api/test_operational_task_dynamic_flow.py b/tests/api/test_operational_task_dynamic_flow.py index eb4d071..6803441 100644 --- a/tests/api/test_operational_task_dynamic_flow.py +++ b/tests/api/test_operational_task_dynamic_flow.py @@ -86,6 +86,7 @@ async def test_preconfigured_service_access_avoids_server_delegation(monkeypatch source_ref=snapshot.source_ref, observed_at=now, recorded_at=now, recorded_by="service", has_snapshot=True) service = SimpleNamespace(can_read_source=AsyncMock(return_value=True), + replay_context=AsyncMock(return_value=None), record_data_snapshot=AsyncMock(return_value=(evidence, snapshot))) source = AsyncMock(return_value=snapshot) monkeypatch.setattr(endpoints, "_service", lambda _ctx, _conn: service) diff --git a/tests/api/test_scada_raw_api.py b/tests/api/test_scada_raw_api.py new file mode 100644 index 0000000..5b7e18e --- /dev/null +++ b/tests/api/test_scada_raw_api.py @@ -0,0 +1,60 @@ +"""HTTP boundary checks without project database access.""" +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.v1.endpoints.timeseries import scada +from app.domain.schemas.scada import ScadaRawPage + + +@pytest.fixture +def raw_api(monkeypatch): + app = FastAPI() + app.include_router(scada.router) + project = uuid4() + app.dependency_overrides[scada.get_timescale_connection] = lambda: object() + app.dependency_overrides[scada.get_postgres_connection] = lambda: object() + app.dependency_overrides[scada.get_project_context] = lambda: SimpleNamespace(project_id=project) + devices = AsyncMock(return_value={"P1"}) + monkeypatch.setattr(scada.ScadaInfoRepository, "get_existing_device_ids", devices) + read = AsyncMock(return_value=ScadaRawPage(items=[], next_cursor=None, scanned_through=datetime(2026, 10, 2, tzinfo=timezone.utc))) + monkeypatch.setattr(scada, "read_raw_page", read) + with TestClient(app) as client: + yield client, devices, read, project + + +def test_raw_http_returns_only_read_projection_and_binds_project(raw_api): + client, devices, read, project = raw_api + response = client.get("/timeseries/scada-readings/raw", params={ + "start_time": "2026-10-01T00:00:00Z", "end_time": "2026-10-02T00:00:00Z", "device_ids": "P1,P1", + }) + assert response.status_code == 200 + assert response.json()["items"] == [] + assert read.await_args.kwargs["project_id"] == str(project) + assert read.await_args.kwargs["device_ids"] == ["P1"] + devices.assert_awaited_once() + + +def test_raw_http_rejects_naive_time_before_replay_lookup(raw_api): + client, devices, read, _ = raw_api + response = client.get("/timeseries/scada-readings/raw", params={ + "start_time": "2026-10-01T00:00:00", "end_time": "2026-10-02T00:00:00Z", + "device_ids": "P1", "replay_id": str(uuid4()), + }) + assert response.status_code == 422 + devices.assert_not_awaited() + read.assert_not_awaited() + + +def test_raw_http_rejects_cross_project_device_before_tsdb(raw_api): + client, _, read, _ = raw_api + response = client.get("/timeseries/scada-readings/raw", params={ + "start_time": "2026-10-01T00:00:00Z", "end_time": "2026-10-02T00:00:00Z", "device_ids": "other", + }) + assert response.status_code == 422 + read.assert_not_awaited() diff --git a/tests/api/test_scada_replay_api.py b/tests/api/test_scada_replay_api.py new file mode 100644 index 0000000..8e6f4e5 --- /dev/null +++ b/tests/api/test_scada_replay_api.py @@ -0,0 +1,113 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.v1.endpoints import scada_replay +from app.auth.project_dependencies import ProjectContext, get_project_context + + +@pytest.fixture +def replay_api(monkeypatch): + context = ProjectContext(uuid4(), "demo", uuid4(), "member") + project = SimpleNamespace(settings={"scada_monitoring_interval_seconds": 900, "other": True}) + metadata = SimpleNamespace(get_project_by_id=AsyncMock(return_value=project)) + repo = SimpleNamespace(available=AsyncMock(return_value=True), current=AsyncMock(return_value=None)) + create = AsyncMock(side_effect=RuntimeError("creation mock must be configured")) + monkeypatch.setattr(scada_replay.ScadaReplayService, "create", create) + app = FastAPI() + app.include_router(scada_replay.router) + app.dependency_overrides[get_project_context] = lambda: context + app.dependency_overrides[scada_replay.repository] = lambda: repo + app.dependency_overrides[scada_replay.get_metadata_repository] = lambda: metadata + with TestClient(app) as client: + yield client, project, metadata, context, create + + +def test_capabilities_reads_frequency_from_current_project_settings(replay_api): + client, project, metadata, context, _ = replay_api + response = client.get("/scada-replays/capabilities") + assert response.status_code == 200 + assert response.json()["monitoring_interval_seconds"] == 900 + assert response.json()["burst_monitoring_ready"] is False + assert "尚未配置" in response.json()["burst_monitoring_reason"] + metadata.get_project_by_id.assert_awaited_once_with(context.project_id) + project.settings = {} + assert client.get("/scada-replays/capabilities").json()["monitoring_interval_seconds"] is None + + +def test_capabilities_reports_ready_only_with_five_pressures_and_outlet_flow(replay_api): + client, project, *_ = replay_api + project.settings = { + "scada_monitoring_interval_seconds": 900, + "scada_monitoring": { + "pressure_device_ids": ["P1", "P2", "P3", "P4", "P5"], + "outlet_flow_device_ids": ["F1"], + }, + } + + response = client.get("/scada-replays/capabilities") + + assert response.status_code == 200 + assert response.json()["burst_monitoring_ready"] is True + assert response.json()["burst_monitoring_reason"] is None + + +def test_capabilities_reports_only_missing_storage_not_worker(): + repo = SimpleNamespace(available=AsyncMock(return_value=False)) + context = ProjectContext(uuid4(), "demo", uuid4(), "member") + + result = asyncio.run(scada_replay.replay_capabilities( + repo=repo, + ctx=context, + settings=scada_replay.ProjectSettings(scada_monitoring_interval_seconds=900), + )) + + assert result.available is False + assert result.reason == "需先部署 SCADA 回放存储结构" + + +def test_step_command_processes_pending_period_in_same_request(monkeypatch): + calls = [] + + class Service: + def __init__(self, repo, context, timescale): + calls.append((repo, context, timescale)) + + async def command(self, replay_id, command): + calls.append((replay_id, command.action)) + return SimpleNamespace(step_pending=True) + + async def process_pending_step(self, replay_id): + calls.append((replay_id, "processed")) + return "completed-period" + + monkeypatch.setattr(scada_replay, "ScadaReplayService", Service) + replay_id = uuid4() + context = ProjectContext(uuid4(), "demo", uuid4(), "member") + command = scada_replay.ScadaReplayCommand( + request_id=uuid4(), expected_revision=1, action="step" + ) + + result = asyncio.run(scada_replay.replay_command( + replay_id, command, repo="repo", ctx=context, timescale="timescale" + )) + + assert result == "completed-period" + assert calls[-2:] == [(replay_id, "step"), (replay_id, "processed")] + + +def test_client_cannot_override_project_frequency_on_creation(replay_api): + client, _, _, _, create = replay_api + response = client.post("/scada-replays", json={ + "request_id": str(uuid4()), "start_time": "2026-10-08T00:00:00Z", + "end_time": "2026-10-08T01:00:00Z", "device_ids": ["P1"], + "rules": [{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}], + "monitoring_interval_seconds": 300, + }) + assert response.status_code == 422 + create.assert_not_awaited() diff --git a/tests/integration/test_scada_replay_storage.py b/tests/integration/test_scada_replay_storage.py new file mode 100644 index 0000000..39803e6 --- /dev/null +++ b/tests/integration/test_scada_replay_storage.py @@ -0,0 +1,82 @@ +"""Real project SQL, with all schema, observations, events and tasks rolled back.""" +import os +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import psycopg +from psycopg.rows import dict_row +import pytest + +from app.auth.project_dependencies import ProjectContext +from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand +from app.infra.db.metadb.database import SessionLocal, engine +from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository +from app.infra.db.postgresql.scada_replays import ScadaReplayRepository +from app.infra.db.postgresql.scada import ScadaInfoRepository +from app.services.scada_replay import ScadaReplayService + +pytestmark = pytest.mark.skipif(os.getenv("SCADA_REPLAY_STORAGE_TEST") != "1", reason="explicit rollback-only SQL test required") + + +@pytest.fixture +def anyio_backend(): return "asyncio" + + +@pytest.mark.anyio +async def test_real_manual_steps_clean_raw_and_persist_events_roll_back(monkeypatch): + actual_project = UUID("871d7538-589c-47de-a6d5-3185ccb9f0e1") + async with SessionLocal() as session: + metadata = MetadataRepository(session) + business = await metadata.get_project_db_routing(actual_project, "biz_data") + timeseries = await metadata.get_project_db_routing(actual_project, "iot_data") + await engine.dispose() + pg = await psycopg.AsyncConnection.connect(business.dsn, row_factory=dict_row, connect_timeout=5) + ts = await psycopg.AsyncConnection.connect(timeseries.dsn, row_factory=dict_row, connect_timeout=5) + try: + async with pg.transaction(force_rollback=True), ts.transaction(force_rollback=True): + actual = await (await pg.execute("SELECT current_database() AS db")).fetchone() + assert actual["db"] == "tjwater2_v2" + migration = (Path(__file__).parents[2] / "resources/sql/031_scada_replays.sql").read_text() + migration = migration.replace("BEGIN;\n", "", 1).removesuffix("COMMIT;\n") + await pg.execute(migration) + device = "SCADA_TEST_" + uuid4().hex[:12] + start = datetime(2026, 10, 8, tzinfo=timezone.utc) + for minute, value in ((0, 10), (15, 20), (30, 20), (45, 10), (60, 10)): + await ts.execute("INSERT INTO scada.measurements(time,device_id,monitored_value) VALUES(%s,%s,%s)", + (start + timedelta(minutes=minute), device, value)) + monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{ + "device_id": device, "device_type": "pressure", "measurement_unit": "m", + "transmission_frequency": "0:15:00", "transmission_mode": "non_realtime" + }])) + context = ProjectContext( + project_id=uuid4(), + project_code="tjwater2_v2", + user_id=uuid4(), + project_role="member", + ) + repo = ScadaReplayRepository(pg, context.project_id) + wall = [start] + service = ScadaReplayService(repo, context, ts, clock=lambda: wall[0]) + config = ScadaReplayCreate(request_id=uuid4(), start_time=start, end_time=start + timedelta(minutes=75), + device_ids=[device], + rules=[{"rule_id": "pressure", "device_ids": [device], "kind": "bounds", "upper": 15, "unit": "m"}]) + result = await service.create(config, monitoring_interval_seconds=900) + while result.status != "completed": + await service.command(result.replay_id, ScadaReplayCommand( + request_id=uuid4(), expected_revision=result.revision, action="step" + )) + result = await service.process_pending_step(result.replay_id) + assert result.status == "completed" + assert result.processed_count == 5 + assert result.anomaly_count == 1 + events = await repo.events(result.replay_id) + assert {event["change"] for event in events} == {"raised", "resolved"} + assert all(event["task_id"] is None for event in events) + raw = await (await ts.execute("SELECT monitored_value,cleaned_value FROM scada.measurements WHERE device_id=%s ORDER BY time", (device,))).fetchall() + assert [row["monitored_value"] for row in raw] == [10, 20, 20, 10, 10] + assert [row["cleaned_value"] for row in raw] == [10, 20, 20, 10, 10] + finally: + await pg.close(); await ts.close() diff --git a/tests/unit/test_mcp_query.py b/tests/unit/test_mcp_query.py index 9463d8a..129c3ba 100644 --- a/tests/unit/test_mcp_query.py +++ b/tests/unit/test_mcp_query.py @@ -300,6 +300,27 @@ def binding(task=False): return value +@pytest.mark.parametrize("tool_name", ["run_burst_detection", "query_element_history"]) +def test_replay_binding_denies_unscoped_computations_and_history(tool_name): + bound = binding(task=True) + bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00+00:00", "batch_sequence": 1} + with pytest.raises(HTTPException) as error: + asyncio.run(query_service.authorize_query(SimpleNamespace(), bound, TOOLS[tool_name], SimpleNamespace())) + assert error.value.status_code == 403 + + +def test_replay_binding_denies_future_and_allows_device_metadata(monkeypatch): + bound = binding(task=True) + bound.replay_context = {"replay_id": str(uuid4()), "knowledge_cutoff": "2026-10-08T01:00:00+00:00", "batch_sequence": 1} + args = SimpleNamespace(start=datetime(2026, 10, 8, tzinfo=timezone.utc), + end=datetime(2026, 10, 8, 2, tzinfo=timezone.utc)) + with pytest.raises(HTTPException) as error: + asyncio.run(query_service.authorize_query(SimpleNamespace(), bound, TOOLS["query_scada_readings"], args)) + assert error.value.status_code == 422 + monkeypatch.setattr(query_service, "permissions_for_context", lambda _: PERMISSIONS) + asyncio.run(query_service.authorize_query(SimpleNamespace(project=object()), bound, TOOLS["list_scada_devices"], SimpleNamespace())) + + @pytest.mark.parametrize("seconds", [30, 120, 240]) def test_all_tools_advertise_the_effective_host_request_budget(runtime, seconds): app, client, _, _ = runtime diff --git a/tests/unit/test_metadata_repository_projects.py b/tests/unit/test_metadata_repository_projects.py index c07d82f..8d8db03 100644 --- a/tests/unit/test_metadata_repository_projects.py +++ b/tests/unit/test_metadata_repository_projects.py @@ -7,6 +7,7 @@ from sqlalchemy.dialects import postgresql from app.infra.db.metadb.repositories import metadata_repository from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository +from app.domain.schemas.project_settings import ProjectSettings @pytest.fixture @@ -99,3 +100,60 @@ async def test_update_map_config_preserves_legacy_settings_and_other_projects(): repository.get_project_by_id.assert_awaited_once_with(project_id) assert project.map_config == {'legend': 'original', 'coordinate_reference': {'source_srid': 990001}, 'place_search': search} session.commit.assert_awaited_once() + + +@pytest.mark.anyio +async def test_update_project_settings_preserves_unmentioned_parameters(): + original = {"scada_monitoring_interval_seconds": 300, "future_option": {"enabled": True}} + project = SimpleNamespace(settings=original) + session = SimpleNamespace(commit=AsyncMock(), refresh=AsyncMock()) + repository = MetadataRepository(session) + repository.get_project_by_id = AsyncMock(return_value=project) + project_id = uuid4() + + await repository.update_project( + project_id, + updates={"settings": {"scada_monitoring_interval_seconds": 900}}, + ) + + assert project.settings == {"scada_monitoring_interval_seconds": 900, "future_option": {"enabled": True}} + assert original["scada_monitoring_interval_seconds"] == 300 + session.commit.assert_awaited_once() + + +@pytest.mark.anyio +async def test_empty_settings_update_preserves_values_and_explicit_null_clears_interval(): + project = SimpleNamespace(settings={"scada_monitoring_interval_seconds": 900, "other": True}) + session = SimpleNamespace(commit=AsyncMock(), refresh=AsyncMock()) + repository = MetadataRepository(session) + repository.get_project_by_id = AsyncMock(return_value=project) + project_id = uuid4() + + await repository.update_project(project_id, updates={"settings": {}}) + assert project.settings == {"scada_monitoring_interval_seconds": 900, "other": True} + await repository.update_project( + project_id, updates={"settings": {"scada_monitoring_interval_seconds": None}} + ) + assert project.settings == {"scada_monitoring_interval_seconds": None, "other": True} + + +def test_scada_monitoring_roles_are_typed_inside_extensible_project_settings(): + settings = ProjectSettings.model_validate({ + "scada_monitoring_interval_seconds": 900, + "scada_monitoring": { + "rule_version": "burst-monitor-v1", + "pressure_device_ids": ["P1", "P2"], + "outlet_flow_device_ids": ["F1"], + }, + "future_option": True, + }) + + assert settings.scada_monitoring.pressure_device_ids == ["P1", "P2"] + assert settings.model_dump()["future_option"] is True + with pytest.raises(ValueError): + ProjectSettings.model_validate({ + "scada_monitoring": { + "pressure_device_ids": ["D1"], + "outlet_flow_device_ids": ["D1"], + } + }) diff --git a/tests/unit/test_scada_raw.py b/tests/unit/test_scada_raw.py new file mode 100644 index 0000000..fdcd280 --- /dev/null +++ b/tests/unit/test_scada_raw.py @@ -0,0 +1,65 @@ +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from app.infra.db.timescaledb.repositories.scada import ScadaRepository +from app.services.scada_raw import read_raw_page + +START = datetime(2026, 10, 8, tzinfo=timezone.utc) + + +def reading(minutes, device="P1", value=1): + return {"time": START + timedelta(minutes=minutes), "device_id": device, + "monitored_value": value, "source_hash": "hash"} + + +def test_pagination_keeps_same_timestamp_devices_and_freezes_clock(monkeypatch): + fetch = AsyncMock(side_effect=[[reading(0), reading(0, "P2")], [reading(0, "P2")]]) + monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch) + async def run(): + args = dict(project_id="project", device_ids=["P1", "P2"], start=START, + end=START + timedelta(hours=1), limit=1) + first = await read_raw_page(None, **args, through=START + timedelta(minutes=15)) + second = await read_raw_page(None, **args, cursor=first.next_cursor, through=START + timedelta(minutes=30)) + assert first.items[0].device_id == "P1" + assert second.items[0].device_id == "P2" + assert second.scanned_through == START + timedelta(minutes=15) + assert fetch.await_args.args[-1] == (START, "P1") + asyncio.run(run()) + + +def test_cursor_cannot_expand_query_or_move_into_future(monkeypatch): + fetch = AsyncMock(return_value=[reading(0), reading(5)]) + monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch) + async def run(): + args = dict(project_id="project", device_ids=["P1"], start=START, + end=START + timedelta(hours=1), limit=1) + first = await read_raw_page(None, **args, through=START + timedelta(minutes=15)) + for change in ({"project_id": "other"}, {"device_ids": ["P2"]}, {"through": START}): + with pytest.raises(ValueError, match="游标"): + await read_raw_page(None, **(args | change), cursor=first.next_cursor) + asyncio.run(run()) + assert fetch.await_count == 1 + + +def test_raw_response_does_not_expose_corrected_values_and_marks_bad_numbers(monkeypatch): + monkeypatch.setattr(ScadaRepository, "get_raw_page", AsyncMock(return_value=[ + reading(0, value=float("nan")), reading(5, value=None), reading(10, value=0), + ])) + page = asyncio.run(read_raw_page(None, project_id="project", device_ids=["P1"], start=START, + end=START + timedelta(hours=1))) + assert [item.quality for item in page.items] == ["invalid", "missing", "unvalidated"] + assert page.items[2].monitored_value == 0 + assert "cleaned_value" not in page.model_dump_json() + + +@pytest.mark.parametrize("cursor", ["not-base64", "e30=", "W10="]) +def test_malformed_cursor_is_rejected_without_query(monkeypatch, cursor): + fetch = AsyncMock() + monkeypatch.setattr(ScadaRepository, "get_raw_page", fetch) + with pytest.raises(ValueError): + asyncio.run(read_raw_page(None, project_id="project", device_ids=["P1"], start=START, + end=START + timedelta(hours=1), cursor=cursor)) + fetch.assert_not_awaited() diff --git a/tests/unit/test_scada_replay_monitoring.py b/tests/unit/test_scada_replay_monitoring.py new file mode 100644 index 0000000..49ea09e --- /dev/null +++ b/tests/unit/test_scada_replay_monitoring.py @@ -0,0 +1,108 @@ +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +from app.domain.schemas.scada_replay import ScadaReplayCreate +from app.services import scada_replay_monitoring + + +TARGET = datetime(2026, 10, 16, tzinfo=timezone.utc) +PRESSURE_IDS = [f"P{index}" for index in range(1, 6)] +FLOW_IDS = ["F1"] + + +def _document(): + config = ScadaReplayCreate.model_validate({ + "request_id": str(uuid4()), + "start_time": TARGET - timedelta(days=15), + "end_time": TARGET + timedelta(hours=1), + "device_ids": PRESSURE_IDS + FLOW_IDS, + "rules": [ + {"rule_id": "pressure_drop", "device_ids": PRESSURE_IDS, + "kind": "relative_change", "direction": "decrease", "threshold": 20, "unit": "m"}, + {"rule_id": "outlet_flow", "device_ids": FLOW_IDS, + "kind": "relative_change", "direction": "increase", "threshold": 20, "unit": "m3/h"}, + ], + }) + return { + "replay_id": str(uuid4()), + "config": config.model_dump(mode="json"), + "monitoring_interval_seconds": 900, + "monitoring_config": { + "rule_version": "burst-monitor-v1", + "pressure_device_ids": PRESSURE_IDS, + "outlet_flow_device_ids": FLOW_IDS, + }, + } + + +def _rows(): + rows = [] + count = 15 * 96 + start = TARGET - timedelta(minutes=15 * (count - 1)) + for index in range(count): + at = start + timedelta(minutes=15 * index) + for device_id in PRESSURE_IDS: + rows.append({"time": at.isoformat(), "device_id": device_id, + "cleaned_value": 70 if at == TARGET else 100, + "quality": "valid"}) + rows.append({"time": at.isoformat(), "device_id": "F1", + "cleaned_value": 150 if at == TARGET else 100, + "quality": "valid"}) + return rows + + +def test_multiple_indicators_create_suspected_burst_result(monkeypatch): + monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: { + "rows": [{"Score": -0.05}], + "summary": {"latest_sensor_rankings": []}, + }) + + result = scada_replay_monitoring.evaluate_monitoring_period( + _document(), _rows(), + period_start=TARGET - timedelta(minutes=15), + period_end=TARGET, + batch_sequence=1440, + ) + + assert result["status"] == "suspected_burst" + assert result["pressure"]["hit_device_ids"] == PRESSURE_IDS + assert result["outlet_flow"]["hit_device_ids"] == FLOW_IDS + assert result["burst_detection"]["hit"] is True + + +def test_single_indicator_stays_attention(monkeypatch): + monkeypatch.setattr(scada_replay_monitoring, "run_burst_detection", lambda **_: { + "rows": [{"Score": 0.1}], + "summary": {"latest_sensor_rankings": []}, + }) + rows = _rows() + for row in rows: + if row["device_id"] == "F1" and row["time"] == TARGET.isoformat(): + row["cleaned_value"] = 100 + + result = scada_replay_monitoring.evaluate_monitoring_period( + _document(), rows, + period_start=TARGET - timedelta(minutes=15), + period_end=TARGET, + batch_sequence=1440, + ) + + assert result["status"] == "attention" + assert result["pressure"]["hit"] is True + assert result["outlet_flow"]["hit"] is False + assert result["burst_detection"]["hit"] is False + + +def test_missing_project_roles_is_data_gap_not_normal(): + document = _document() + document["monitoring_config"] = None + + result = scada_replay_monitoring.evaluate_monitoring_period( + document, [], + period_start=TARGET - timedelta(minutes=15), + period_end=TARGET, + batch_sequence=1, + ) + + assert result["status"] == "insufficient_data" + assert result["burst_detection"]["status"] == "not_run" diff --git a/tests/unit/test_scada_replay_processing.py b/tests/unit/test_scada_replay_processing.py new file mode 100644 index 0000000..085dd02 --- /dev/null +++ b/tests/unit/test_scada_replay_processing.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +from app.domain.schemas.scada_replay import ScadaReplayCreate +from app.services.scada_replay_processing import prepare_cleaning, detect_frame + +START = datetime(2026, 10, 8, tzinfo=timezone.utc) + + +def config(**changes): + return ScadaReplayCreate.model_validate({ + "request_id": str(uuid4()), "start_time": START, "end_time": START + timedelta(hours=2), + "device_ids": ["P1"], "rules": [{"rule_id": "pressure", "device_ids": ["P1"], + "kind": "relative_change", "direction": "decrease", "threshold": 30, "unit": "m"}], + **changes, + }) + + +def rows(value, minute=0): + return [{"time": START + timedelta(minutes=minute), "device_id": "P1", "monitored_value": value, "source_hash": "h"}] + + +def test_quality_does_not_repair_a_real_pressure_drop_or_reverse_flow(): + for value in (0, -5, 1): + result = prepare_cleaning(rows(value), config()) + assert result[0]["quality"] == "valid" + assert result[0]["cleaned_value"] == value + invalid = prepare_cleaning(rows(-5), config(quality_ranges={"P1": {"lower": 0}})) + assert invalid[0]["quality"] == "invalid" + assert invalid[0]["monitored_value"] == -5 + assert invalid[0]["cleaned_value"] is None + + +def test_persistent_drop_is_one_episode_and_recovery_then_drop_is_new_episode(): + cfg = config() + doc = {"replay_id": "run", "histories": {"P1": [10, 10, 10, 10]}} + all_events = [] + for minute, value in enumerate((1, 1, None, 10, 10, 1)): + doc, events = detect_frame(doc, prepare_cleaning(rows(value, minute), cfg), cfg, START + timedelta(minutes=minute)) + all_events.extend(events) + business = [event for event in all_events if event.get("category") != "data_quality"] + quality = [event for event in all_events if event.get("category") == "data_quality"] + assert [event["change"] for event in business] == ["raised", "resolved", "raised"] + assert [event["change"] for event in quality] == ["raised", "resolved"] + assert business[0]["event_id"] != business[2]["event_id"] + assert doc["anomaly_count"] == 3 + assert doc["processed_count"] == 6 + + +def test_water_quality_zero_baseline_uses_explicit_bounds(): + cfg = config(rules=[{"rule_id": "tracer", "device_ids": ["P1"], "kind": "bounds", "upper": 0.1, "unit": "m"}]) + doc = {"replay_id": "run"} + doc, events = detect_frame(doc, prepare_cleaning(rows(0), cfg), cfg, START) + assert events == [] + _, events = detect_frame(doc, prepare_cleaning(rows(0.4, 15), cfg), cfg, START + timedelta(minutes=15)) + assert events[0]["change"] == "raised" + + +def test_empty_clock_ticks_do_not_break_consecutive_observations(): + cfg = config(rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, + "unit": "m", "consecutive_samples": 2}]) + doc, events = detect_frame({"replay_id": "run"}, prepare_cleaning(rows(20), cfg), cfg, START) + assert events == [] + doc, _ = detect_frame(doc, [], cfg, START + timedelta(minutes=1)) + _, events = detect_frame(doc, prepare_cleaning(rows(20, 15), cfg), cfg, START + timedelta(minutes=15)) + assert events[0]["change"] == "raised" diff --git a/tests/unit/test_scada_replay_queries.py b/tests/unit/test_scada_replay_queries.py new file mode 100644 index 0000000..781b3b7 --- /dev/null +++ b/tests/unit/test_scada_replay_queries.py @@ -0,0 +1,145 @@ +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import HTTPException + +from app.mcp.analysis_models import BurstLocationInput +from app.mcp.catalog import TOOLS +from app.mcp.models import BoundContext +from app.mcp import query_service +from app.services.scada_replay_queries import replay_burst_location_inputs + + +START = datetime(2026, 10, 8, tzinfo=timezone.utc) + + +class Cursor: + def __init__(self, rows): self.rows = rows + async def fetchone(self): return self.rows[0] if self.rows else None + async def fetchall(self): return self.rows + + +class Connection: + def __init__(self, results): + self.results = list(results) + self.statements = [] + + @asynccontextmanager + async def transaction(self): yield + + async def execute(self, statement, params): + self.statements.append((statement, params)) + return Cursor(self.results.pop(0)) + + async def commit(self): pass + + +def _args(**changes): + return BurstLocationInput.model_validate({ + "context_handle": "a" * 43, + "burst_leakage": 0.02, + "pressure_scada_ids": ["J1"], + "flow_scada_ids": ["L1"], + "scada_burst_start": START + timedelta(minutes=15), + "scada_burst_end": START + timedelta(minutes=30), + "scada_normal_start": START, + "scada_normal_end": START + timedelta(minutes=14), + **changes, + }) + + +def test_replay_location_builds_series_only_from_frozen_outputs(): + replay_id = str(uuid4()) + run = { + "start_time": START.isoformat(), + "devices": [ + {"device_id": "P1", "node_id": "J1", "link_id": None}, + {"device_id": "F1", "node_id": None, "link_id": "L1"}, + ], + "monitoring_config": { + "pressure_device_ids": ["P1"], + "outlet_flow_device_ids": ["F1"], + }, + } + readings = [ + {"reading": {"device_id": "P1", "time": START.isoformat(), "cleaned_value": 100, "quality": "valid"}}, + {"reading": {"device_id": "F1", "time": START.isoformat(), "cleaned_value": 50, "quality": "valid"}}, + {"reading": {"device_id": "P1", "time": (START + timedelta(minutes=15)).isoformat(), "cleaned_value": 70, "quality": "valid"}}, + {"reading": {"device_id": "F1", "time": (START + timedelta(minutes=15)).isoformat(), "cleaned_value": 80, "quality": "valid"}}, + ] + connection = Connection([[{"document": run}], readings]) + context = {"replay_id": replay_id, "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(), + "batch_sequence": 9} + + result = asyncio.run(replay_burst_location_inputs(connection, uuid4(), context, _args())) + + assert result["burst_pressure"] == {"J1": 70} + assert result["normal_pressure"] == {"J1": 100} + assert result["burst_flow"] == {"L1": 80} + assert result["normal_flow"] == {"L1": 50} + assert connection.statements[1][1][2] == 9 + + +def test_replay_location_authorization_requires_explicit_bounded_normal_window(monkeypatch): + binding = BoundContext( + context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host", + tool_cap=["run_burst_location"], permission_cap=["burst.run", "scada.view"], + native_session_id="s", purpose="task_execution", task_id=uuid4(), + expires_at=START + timedelta(days=1), + replay_context={"replay_id": str(uuid4()), + "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(), + "batch_sequence": 9}, + ) + monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"burst.run", "scada.view"}) + scope = SimpleNamespace(project=SimpleNamespace(project_role="member")) + + asyncio.run(query_service.authorize_query(scope, binding, TOOLS["run_burst_location"], _args())) + + missing_normal = _args().model_copy(update={"scada_normal_start": None, "scada_normal_end": None}) + with pytest.raises(HTTPException) as caught: + asyncio.run(query_service.authorize_query(scope, binding, TOOLS["run_burst_location"], missing_normal)) + assert caught.value.status_code == 422 + + +def test_replay_location_execution_uses_frozen_adapter_instead_of_live_scada(monkeypatch): + from app.mcp import analysis_service + from app.services import scada_replay_queries + + binding = BoundContext( + context_handle="a" * 43, project_id=uuid4(), subject="u", client_id="host", + tool_cap=["run_burst_location"], permission_cap=["burst.run", "scada.view"], + native_session_id="s", purpose="task_execution", task_id=uuid4(), + expires_at=START + timedelta(days=1), + replay_context={"replay_id": str(uuid4()), + "knowledge_cutoff": (START + timedelta(hours=1)).isoformat(), + "batch_sequence": 9}, + ) + monkeypatch.setattr(query_service, "permissions_for_context", lambda _: {"burst.run", "scada.view"}) + frozen = {"burst_leakage": 0.02, "burst_pressure": {"J1": 70}, "normal_pressure": {"J1": 100}} + build = AsyncMock(return_value=frozen) + calculate = AsyncMock(return_value={"candidates": [{"id": "PIPE-1"}]}) + monkeypatch.setattr(scada_replay_queries, "replay_burst_location_inputs", build) + monkeypatch.setattr(analysis_service, "calculate_replay_burst_location", calculate) + + @asynccontextmanager + async def routing(): yield + + scope = SimpleNamespace( + connection=Connection([]), + project=SimpleNamespace(project_id=binding.project_id, project_code="demo", project_role="member"), + user=SimpleNamespace(username="tester"), + simulation_routing=routing, + ) + result = asyncio.run(query_service.execute_query( + scope, binding, TOOLS["run_burst_location"], _args(), + )) + + build.assert_awaited_once() + calculate.assert_awaited_once_with(frozen, network="demo", username="tester") + assert result.origin == "synthetic" + assert result.data["result"]["candidates"][0]["id"] == "PIPE-1" diff --git a/tests/unit/test_scada_replay_service.py b/tests/unit/test_scada_replay_service.py new file mode 100644 index 0000000..c9bb435 --- /dev/null +++ b/tests/unit/test_scada_replay_service.py @@ -0,0 +1,242 @@ +import asyncio +from contextlib import asynccontextmanager +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import HTTPException + +from app.domain.schemas.scada_replay import ScadaReplayCreate, ScadaReplayCommand +from app.infra.db.postgresql.scada import ScadaInfoRepository +from app.infra.db.timescaledb.repositories.scada import ScadaRepository +from app.services.scada_replay import ScadaReplayService + +START = datetime(2026, 10, 8, tzinfo=timezone.utc) + + +class Connection: + @asynccontextmanager + async def transaction(self): + yield + + +class Repository: + def __init__(self): + self.project_id = str(uuid4()) + self.connection = Connection() + self.document = None + self.batches = {} + + async def available(self): return True + @asynccontextmanager + async def control(self): yield + @asynccontextmanager + async def processing(self): yield True + async def by_request(self, key): + return deepcopy(self.document) if self.document and self.document["config"]["request_id"] == str(key) else None + async def active_other(self, key): return False + async def insert(self, document): self.document = deepcopy(document) + async def save(self, document): self.document = deepcopy(document) + async def get(self, _): return deepcopy(self.document) + async def current(self): return deepcopy(self.document) + async def pending_batch(self, _): + return next((deepcopy(batch) for batch in self.batches.values() if batch["stage"] != "detected"), None) + async def save_batch(self, _, batch): self.batches[batch["sequence"]] = deepcopy(batch) + async def monitoring_outputs(self, *_args): return [] + + +def setup(monkeypatch): + repo = Repository() + context = SimpleNamespace( + project_id=repo.project_id, + user_id=uuid4(), + ) + service = ScadaReplayService(repo, context, Connection(), clock=lambda: START + timedelta(seconds=2)) + monkeypatch.setattr(ScadaInfoRepository, "get_scadas", AsyncMock(return_value=[{ + "device_id": "P1", "device_type": "pressure", "measurement_unit": "m", "transmission_frequency": "0:15:00" + }])) + async def raw_page(_connection, _devices, _start, _end, _through, _limit, after=None, **_): + return ([{"time": START, "device_id": "P1", "monitored_value": 20., "source_hash": "h"}] + if after is None else []) + query = AsyncMock(side_effect=raw_page) + cleaning = AsyncMock() + monkeypatch.setattr(ScadaRepository, "get_raw_page", query) + monkeypatch.setattr(ScadaRepository, "apply_replay_cleaning", cleaning) + monkeypatch.setattr(service, "_create_investigations", AsyncMock()) + create = ScadaReplayCreate(request_id=uuid4(), start_time=START, end_time=START + timedelta(hours=1), + device_ids=["P1"], rules=[{"rule_id": "p", "device_ids": ["P1"], "kind": "bounds", "upper": 10, "unit": "m"}]) + return repo, service, query, cleaning, create + + +def test_step_command_is_revisioned_idempotent_and_does_not_process_until_requested(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + restored = await service.create(create, monitoring_interval_seconds=300) + assert restored.replay_id == first.replay_id + assert restored.monitoring_interval_seconds == 900 + command = ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step") + played = await service.command(first.replay_id, command) + assert played.status == "paused" and played.step_pending and played.revision == 2 + assert (await service.command(first.replay_id, command)).revision == 2 + with pytest.raises(HTTPException) as conflict: + await service.command(first.replay_id, command.model_copy(update={"request_id": uuid4()})) + assert conflict.value.status_code == 409 + asyncio.run(run()) + query.assert_not_awaited(); cleaning.assert_not_awaited() + + +def test_one_raw_read_then_correction_then_detection_without_reread(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + repo.document["warmed"] = True + await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")) + result = await service.process_pending_step(first.replay_id) + assert result.processed_count == 1 + assert result.anomaly_count == 1 + assert repo.batches[1]["stage"] == "detected" + asyncio.run(run()) + assert query.await_count == 2 + assert cleaning.await_count == 2 + assert cleaning.await_args_list[0].args[1][0]["cleaned_value"] == 20 + + +def test_detection_failure_resumes_cleaned_batch_without_duplicate_reads_or_writes(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + publish = AsyncMock(side_effect=[RuntimeError("Agent storage unavailable"), None, None]) + monkeypatch.setattr(service, "_create_investigations", publish) + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + repo.document["warmed"] = True + await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")) + with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id) + assert repo.document["status"] == "failed" + assert repo.batches[1]["stage"] == "cleaned" + assert repo.document["processed_count"] == 0 + await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=2, action="retry")) + result = await service.process_pending_step(first.replay_id) + assert result.processed_count == 1 and result.anomaly_count == 1 + asyncio.run(run()) + assert query.await_count == 2 + assert cleaning.await_count == 2 + + +def test_cleaning_failure_never_runs_detection_and_preserves_prepared_result(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + cleaning.side_effect = RuntimeError("Timescale unavailable") + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + repo.document["warmed"] = True + await service.command(first.replay_id, ScadaReplayCommand(request_id=uuid4(), expected_revision=1, action="step")) + with pytest.raises(RuntimeError): await service.process_pending_step(first.replay_id) + assert repo.batches[1]["stage"] == "clean_prepared" + assert repo.document["processed_count"] == 0 + asyncio.run(run()) + service._create_investigations.assert_not_awaited() + + +def test_creation_requires_explicit_project_monitoring_interval(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + + async def run(): + with pytest.raises(HTTPException) as missing: + await service.create(create, monitoring_interval_seconds=None) + assert missing.value.status_code == 422 + assert repo.document is None + + asyncio.run(run()) + query.assert_not_awaited() + cleaning.assert_not_awaited() + + +def test_step_processes_every_observation_within_one_monitoring_interval(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + rows = [{"time": START + timedelta(minutes=minute), "device_id": "P1", + "monitored_value": 20., "source_hash": f"h{minute}"} for minute in (0, 5, 10, 15, 20)] + + async def raw_page(_connection, _devices, _start, _end, through, _limit, after=None, **_): + return next(([row] for row in rows if row["time"] <= through + and (after is None or (row["time"], row["device_id"]) > after)), []) + + query.side_effect = raw_page + + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + repo.document["warmed"] = True + stepping = await service.command(first.replay_id, ScadaReplayCommand( + request_id=uuid4(), expected_revision=1, action="step" + )) + assert stepping.status == "paused" and stepping.step_pending + with pytest.raises(HTTPException): + await service.command(first.replay_id, ScadaReplayCommand( + request_id=uuid4(), expected_revision=2, action="step" + )) + result = await service.process_pending_step(first.replay_id) + assert result.processed_count == 4 + assert result.virtual_now == START + timedelta(minutes=15) + assert result.status == "paused" + assert not result.step_pending + assert result.processed_until == START + timedelta(minutes=15) + assert result.monitoring_count == 1 + assert result.last_monitoring.status == "insufficient_data" + assert "step_target" not in repo.document + assert all(call.args[4] == START + timedelta(minutes=15) for call in query.await_args_list) + + asyncio.run(run()) + assert cleaning.await_count == 4 + + +def test_partial_final_step_checks_empty_interval_and_completes_without_fabricated_rows(monkeypatch): + repo, service, query, cleaning, create = setup(monkeypatch) + create = create.model_copy(update={"end_time": START + timedelta(minutes=22)}) + query.side_effect = None + query.return_value = [] + + async def run(): + first = await service.create(create, monitoring_interval_seconds=900) + repo.document.update(warmed=True, virtual_now=(START + timedelta(minutes=15)).isoformat()) + await service.command(first.replay_id, ScadaReplayCommand( + request_id=uuid4(), expected_revision=1, action="step" + )) + result = await service.process_pending_step(first.replay_id) + assert result.virtual_now == create.end_time + assert result.processed_until == create.end_time + assert result.status == "completed" and not result.step_pending + assert result.processed_count == 0 + + asyncio.run(run()) + assert query.await_args.args[4] == create.end_time + + +def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(monkeypatch): + repo, service, *_ = setup(monkeypatch) + create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4())) + monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"}) + monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object()) + monkeypatch.setattr( + "app.services.operational_tasks.OperationalTaskService", + lambda *_: SimpleNamespace(create=create_task), + ) + monitoring = { + "status": "suspected_burst", + "period_end": (START + timedelta(minutes=15)).isoformat(), + "knowledge_cutoff": (START + timedelta(minutes=15)).isoformat(), + "rule_version": "burst-monitor-v1", + "pressure": {"hit_device_ids": ["P1", "P2"]}, + "outlet_flow": {"hit_device_ids": ["F1"]}, + "burst_detection": {"target_score": -0.05}, + "task_id": None, + } + document = {"replay_id": str(uuid4())} + + asyncio.run(service._create_monitoring_task(document, monitoring, {"sequence": 8})) + + command = create_task.await_args.args[0] + assert command.workflow == "burst_report" + assert command.source.replay_context.batch_sequence == 8 + assert command.source.replay_context.knowledge_cutoff == START + timedelta(minutes=15) + assert monitoring["task_id"] == str(create_task.return_value.task_id)