feat(scada): add project monitoring replay

app/infra/db/metadb/models.py

@@ -44,6 +44,7 @@ class Project(Base):
     gs_workspace: Mapped[str] = mapped_column(String(100), unique=True)
     map_extent: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
     map_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
+    settings: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict, server_default="{}")
     status: Mapped[str] = mapped_column(String(20), default="active")
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), default=datetime.utcnow

app/infra/db/metadb/repositories/metadata_repository.py

@@ -67,6 +67,7 @@ class ProjectDetail:
     map_extent: Optional[dict]
     status: str
     map_config: dict = field(default_factory=dict)
+    settings: dict = field(default_factory=dict)
 
 
 @dataclass(frozen=True)
@@ -223,6 +224,7 @@ class MetadataRepository:
         status: str,
         creator_user_id: UUID | None = None,
         map_config: dict | None = None,
+        settings: dict | None = None,
     ) -> models.Project:
         project = models.Project(
             id=uuid4(),
@@ -233,6 +235,7 @@ class MetadataRepository:
             map_extent=map_extent,
             status=status,
             map_config=map_config or {},
+            settings=settings or {},
             created_at=_utcnow(),
             updated_at=_utcnow(),
         )
@@ -263,6 +266,7 @@ class MetadataRepository:
         timescale_dsn: str,
         pool_min_size: int = 1,
         pool_max_size: int = 4,
+        settings: dict | None = None,
     ) -> models.Project:
         """Atomically expose a fully provisioned project and both DB routes."""
         business_secret = _encrypt_database_secret(business_dsn)
@@ -276,6 +280,7 @@ class MetadataRepository:
             map_extent=map_extent,
             status="active",
             map_config={},
+            settings=settings or {},
             created_at=_utcnow(),
             updated_at=_utcnow(),
         )
@@ -333,6 +338,8 @@ class MetadataRepository:
                 setattr(project, field, updates[field])
         if "map_config" in updates:
             project.map_config = {**(project.map_config or {}), **updates["map_config"]}
+        if "settings" in updates:
+            project.settings = {**(project.settings or {}), **updates["settings"]}
         project.updated_at = _utcnow()
         await self.session.commit()
         await self.session.refresh(project)
@@ -350,6 +357,7 @@ class MetadataRepository:
             gs_workspace=project.gs_workspace,
             map_extent=project.map_extent,
             map_config=project.map_config or {},
+            settings=project.settings or {},
             status=project.status,
         )
 

app/infra/db/postgresql/scada_replays.py

@@ -0,0 +1,188 @@
+"""Short-transaction storage for replay control and resumable batches."""
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+
+from fastapi import HTTPException
+from psycopg.types.json import Jsonb
+
+
+class ScadaReplayRepository:
+    def __init__(self, connection, project_id):
+        self.connection, self.project_id = connection, str(project_id)
+        self.lock_key = f"scada-replay:{self.project_id}"
+
+    async def available(self) -> bool:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT to_regclass('peripheral_sample.scada_replays') IS NOT NULL "
+                "AND to_regclass('peripheral_sample.scada_replay_batches') IS NOT NULL AS ready"
+            )
+            return (await cur.fetchone())["ready"]
+
+    @asynccontextmanager
+    async def control(self):
+        async with self.connection.transaction():
+            await self.connection.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (self.lock_key,))
+            yield
+
+    @asynccontextmanager
+    async def processing(self):
+        # Session lock outlives short transactions, never an open SQL transaction
+        # across Timescale IO. A dead connection automatically releases ownership.
+        async with self.connection.transaction():
+            cur = await self.connection.execute("SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS owned", (self.lock_key,))
+            owned = (await cur.fetchone())["owned"]
+        try:
+            yield owned
+        finally:
+            if owned:
+                async with self.connection.transaction():
+                    await self.connection.execute("SELECT pg_advisory_unlock(hashtextextended(%s,0))", (self.lock_key,))
+
+    async def get(self, replay_id) -> dict:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id=%s",
+                (self.project_id, str(replay_id)),
+            )
+            row = await cur.fetchone()
+        if not row:
+            raise HTTPException(404, "当前项目不存在此 SCADA 回放")
+        return row["document"]
+
+    async def current(self) -> dict | None:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s"
+                " ORDER BY created_at DESC LIMIT 1", (self.project_id,),
+            )
+            row = await cur.fetchone()
+        return row["document"] if row else None
+
+    async def by_request(self, request_id) -> dict | None:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT document FROM peripheral_sample.scada_replays WHERE project_id=%s AND request_id=%s",
+                (self.project_id, str(request_id)),
+            )
+            row = await cur.fetchone()
+        return row["document"] if row else None
+
+    async def active_other(self, replay_id) -> bool:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT 1 FROM peripheral_sample.scada_replays WHERE project_id=%s AND replay_id<>%s "
+                "AND status IN ('ready','paused') LIMIT 1",
+                (self.project_id, str(replay_id)),
+            )
+            return await cur.fetchone() is not None
+
+    async def insert(self, document: dict):
+        await self.connection.execute(
+            "INSERT INTO peripheral_sample.scada_replays(project_id,replay_id,request_id,status,document) "
+            "VALUES(%s,%s,%s,%s,%s)",
+            (self.project_id, document["replay_id"], document["config"]["request_id"], document["status"], Jsonb(document)),
+        )
+
+    async def save(self, document: dict):
+        document["updated_at"] = datetime.now(timezone.utc).isoformat()
+        await self.connection.execute(
+            "UPDATE peripheral_sample.scada_replays SET status=%s,document=%s,updated_at=now() "
+            "WHERE project_id=%s AND replay_id=%s",
+            (document["status"], Jsonb(document), self.project_id, document["replay_id"]),
+        )
+
+    async def pending_batch(self, replay_id) -> dict | None:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT document FROM peripheral_sample.scada_replay_batches "
+                "WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1",
+                (self.project_id, str(replay_id)),
+            )
+            row = await cur.fetchone()
+        return row["document"] if row else None
+
+    async def save_batch(self, replay_id, batch: dict):
+        await self.connection.execute(
+            "INSERT INTO peripheral_sample.scada_replay_batches(project_id,replay_id,sequence,stage,document) "
+            "VALUES(%s,%s,%s,%s,%s) ON CONFLICT(project_id,replay_id,sequence) "
+            "DO UPDATE SET stage=excluded.stage,document=excluded.document",
+            (self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)),
+        )
+
+    async def events(self, replay_id, offset=0, limit=100) -> list[dict]:
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT e.value AS event FROM peripheral_sample.scada_replay_batches b "
+                "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'events','[]'::jsonb)) e "
+                "WHERE b.project_id=%s AND b.replay_id=%s ORDER BY b.sequence DESC LIMIT %s OFFSET %s",
+                (self.project_id, str(replay_id), limit, offset),
+            )
+            return [r["event"] for r in await cur.fetchall()]
+
+    async def monitoring_outputs(
+        self,
+        replay_id,
+        device_ids: list[str],
+        start: datetime,
+        end: datetime,
+        before_sequence: int,
+    ) -> list[dict]:
+        """Read only committed replay outputs available before the current batch."""
+        if not device_ids:
+            return []
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
+                "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
+                "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage='detected' AND b.sequence<%s "
+                "AND r.value->>'device_id'=ANY(%s) "
+                "AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
+                "ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'",
+                (self.project_id, str(replay_id), before_sequence, device_ids, start, end),
+            )
+            return [row["reading"] for row in await cur.fetchall()]
+
+    async def detected_outputs(
+        self,
+        replay_id,
+        device_ids: list[str],
+        start: datetime,
+        end: datetime,
+        before_sequence: int,
+        *,
+        limit: int | None = None,
+        offset: int = 0,
+    ) -> list[dict]:
+        """Read immutable detector outputs available to a bounded replay investigation."""
+        statement = (
+            "SELECT r.value AS reading FROM peripheral_sample.scada_replay_batches b "
+            "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
+            "WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence<=%s AND b.stage='detected' "
+            "AND r.value->>'device_id'=ANY(%s) "
+            "AND (r.value->>'time')::timestamptz>=%s AND (r.value->>'time')::timestamptz<=%s "
+            "ORDER BY (r.value->>'time')::timestamptz,r.value->>'device_id'"
+        )
+        params: tuple = (
+            self.project_id, str(replay_id), before_sequence, device_ids, start, end,
+        )
+        if limit is not None:
+            statement += " LIMIT %s OFFSET %s"
+            params += (limit, offset)
+        async with self.connection.transaction():
+            cur = await self.connection.execute(statement, params)
+            return [row["reading"] for row in await cur.fetchall()]
+
+    async def latest_outputs(self, replay_id, through: datetime, limit: int = 500) -> list[dict]:
+        """Return the latest committed cleaned output for each replay device."""
+        async with self.connection.transaction():
+            cur = await self.connection.execute(
+                "SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading "
+                "FROM peripheral_sample.scada_replay_batches b "
+                "CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
+                "WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') "
+                "AND (r.value->>'time')::timestamptz<=%s "
+                "ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s",
+                (self.project_id, str(replay_id), through, limit),
+            )
+            return [row["reading"] for row in await cur.fetchall()]

app/infra/db/timescaledb/repositories/scada.py

@@ -9,6 +9,86 @@ class ScadaRepository:
     VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
     RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
 
+    @staticmethod
+    async def get_warmup_values(
+        conn: AsyncConnection,
+        device_ids: list[str],
+        before: datetime,
+        per_device_limit: int,
+    ) -> list[dict]:
+        """Return the latest raw values before a replay starts for detector warmup."""
+        async with conn.cursor() as cur:
+            await cur.execute(
+                "SELECT d.device_id,h.monitored_value FROM unnest(%s::text[]) d(device_id) "
+                "CROSS JOIN LATERAL (SELECT monitored_value,time FROM scada.measurements "
+                "WHERE device_id=d.device_id AND time<%s ORDER BY time DESC LIMIT %s) h "
+                "ORDER BY d.device_id,h.time",
+                (device_ids, before, per_device_limit),
+            )
+            return [dict(row) for row in await cur.fetchall()]
+
+    @staticmethod
+    async def get_raw_page(
+        conn: AsyncConnection, device_ids: list[str], start: datetime,
+        end: datetime, through: datetime, limit: int,
+        after: tuple[datetime, str] | None = None, *, frame: bool = False,
+    ) -> list[dict]:
+        """Read raw only, with a stable composite key and a bounded frame/page."""
+        predicate = "device_id=ANY(%s) AND time>=%s AND time<%s AND time<=%s"
+        params: tuple = (device_ids, start, end, through)
+        if after is not None:
+            predicate += " AND (time,device_id)>(%s,%s)"
+            params += after
+        if frame:
+            statement = (
+                "WITH next_time AS (SELECT min(time) AS time FROM scada.measurements WHERE "
+                + predicate + ") SELECT m.time,m.device_id,m.monitored_value,"
+                "md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
+                "FROM scada.measurements m WHERE m.device_id=ANY(%s) "
+                "AND m.time=(SELECT time FROM next_time) ORDER BY m.time,m.device_id LIMIT %s"
+            )
+            params += (device_ids, limit)
+        else:
+            statement = (
+                "SELECT time,device_id,monitored_value,"
+                "md5(COALESCE(monitored_value::text,'NULL')) AS source_hash "
+                "FROM scada.measurements WHERE " + predicate
+                + " ORDER BY time,device_id LIMIT %s"
+            )
+            params += (limit,)
+        async with conn.cursor() as cur:
+            await cur.execute(statement, params)
+            return [dict(row) for row in await cur.fetchall()]
+
+    @staticmethod
+    async def apply_replay_cleaning(conn: AsyncConnection, rows: list[dict]) -> None:
+        """Guard every raw version; update corrected values without inserting raw."""
+        if not rows:
+            return
+        async with conn.transaction():
+            async with conn.cursor() as cur:
+                await cur.execute(
+                    "SELECT m.time,m.device_id,md5(COALESCE(m.monitored_value::text,'NULL')) AS source_hash "
+                    "FROM scada.measurements m JOIN unnest(%s::timestamptz[],%s::text[]) "
+                    "b(time,device_id) USING(time,device_id) FOR UPDATE OF m",
+                    ([r["time"] for r in rows], [r["device_id"] for r in rows]),
+                )
+                existing = {(r["time"].isoformat(), r["device_id"]): r["source_hash"]
+                            for r in await cur.fetchall()}
+                if len(existing) != len(rows) or any(
+                    existing.get((datetime.fromisoformat(r["time"]).isoformat(), r["device_id"]))
+                    != r["source_hash"] for r in rows
+                ):
+                    raise ValueError("SOURCE_CHANGED: SCADA 原始值在处理期间发生变化")
+                await cur.execute(
+                    "UPDATE scada.measurements m SET cleaned_value=b.value "
+                    "FROM unnest(%s::timestamptz[],%s::text[],%s::double precision[]) "
+                    "b(time,device_id,value) WHERE m.time=b.time AND m.device_id=b.device_id "
+                    "AND m.cleaned_value IS DISTINCT FROM b.value",
+                    ([r["time"] for r in rows], [r["device_id"] for r in rows],
+                     [r["cleaned_value"] for r in rows]),
+                )
+
     @staticmethod
     async def insert_scada_batch(conn: AsyncConnection, data: List[dict]):
         if not data: