feat(work-orders): synchronize peripheral changes through events

scripts/sync_peripheral_work_orders.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import json
 import logging
 import os
 import sys
@@ -15,6 +16,7 @@ import httpx
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 from samples.peripheral.seed_identity import SEED_ACTOR_ID
+from app.domain.schemas.workbench_events import WorkbenchEvent
 
 try:
     from scripts.run_operational_worker import CredentialProvider, WorkerSettings
@@ -211,30 +213,125 @@ async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, periphe
     return synchronized
 
 
-async def run(once: bool):
+class PeripheralWorkOrderSynchronizer:
+    """Wake on committed source events; periodically reconcile missed changes."""
+
+    def __init__(self, client: httpx.AsyncClient, settings: WorkerSettings,
+                 peripheral_url: str, credentials: CredentialProvider) -> None:
+        self.client = client
+        self.settings = settings
+        self.peripheral_url = peripheral_url
+        self.credentials = credentials
+        self.wake = asyncio.Event()
+        self.cursor: int | None = None
+
+    async def _reset_cursor(self, headers: dict[str, str]) -> None:
+        response = await self.client.get(
+            f"{self.settings.server_url.rstrip('/')}/api/v1/workbench/sync-snapshot",
+            headers=headers,
+        )
+        response.raise_for_status()
+        cursor = response.json()["event_cursor"]
+        if not isinstance(cursor, int) or isinstance(cursor, bool) or cursor < 0:
+            raise ValueError("invalid workbench event cursor")
+        self.cursor = cursor
+        # Capture the cursor before scanning sources, so updates during the
+        # initial/reset scan are replayed by the following SSE connection.
+        self.wake.set()
+
+    async def listen(self) -> None:
+        while True:
+            try:
+                headers = await self.credentials.headers(self.client)
+                if self.cursor is None:
+                    await self._reset_cursor(headers)
+                async with self.client.stream(
+                    "GET", f"{self.settings.server_url.rstrip('/')}/api/v1/workbench/events",
+                    headers={**headers, "Accept": "text/event-stream"},
+                    params={"after": self.cursor}, timeout=None,
+                ) as response:
+                    response.raise_for_status()
+                    event_name = ""
+                    async for line in response.aiter_lines():
+                        if line.startswith("event:"):
+                            event_name = line[6:].strip()
+                        elif line.startswith("data:") and event_name == "reset_required":
+                            await self._reset_cursor(headers)
+                            break
+                        elif line.startswith("data:") and event_name == "workbench_event":
+                            event = WorkbenchEvent.model_validate(json.loads(line[5:].strip()))
+                            if str(event.project_id) != self.settings.project_id:
+                                raise ValueError("workbench event project mismatch")
+                            if event.event_seq <= self.cursor:
+                                continue
+                            self.cursor = event.event_seq
+                            if (event.topic == "workbench.work_order.changed"
+                                    and event.aggregate_type == "peripheral_work_order"
+                                    and event.metadata.get("source") == "peripheral_sample"):
+                                self.wake.set()
+                        elif not line:
+                            event_name = ""
+                await asyncio.sleep(1)
+            except (httpx.HTTPError, ValueError, KeyError):
+                logger.warning("Work-order event stream disconnected; reconciliation remains active")
+                await asyncio.sleep(5)
+
+    async def serve(self, reconcile_seconds: int = 300) -> None:
+        if reconcile_seconds < 1:
+            raise ValueError("reconciliation interval must be at least one second")
+        listener = asyncio.create_task(self.listen())
+        wait_seconds = reconcile_seconds
+        retry_seconds = 5
+        try:
+            while True:
+                try:
+                    await asyncio.wait_for(self.wake.wait(), timeout=wait_seconds)
+                except TimeoutError:
+                    pass
+                # Clear before work, so arrivals during a scan request another
+                # pass. A failed scan gets a retry even after its cursor advances.
+                self.wake.clear()
+                try:
+                    count = await sync_once(
+                        self.client, self.settings, self.peripheral_url, self.credentials,
+                    )
+                    logger.info("Peripheral progress synchronization completed: %d revisions", count)
+                    wait_seconds, retry_seconds = reconcile_seconds, 5
+                except (httpx.HTTPError, ValueError, KeyError):
+                    logger.error("Peripheral synchronization failed; persisted revisions will be retried")
+                    wait_seconds = retry_seconds
+                    retry_seconds = min(retry_seconds * 2, 60)
+        finally:
+            listener.cancel()
+            await asyncio.gather(listener, return_exceptions=True)
+
+
+async def run(once: bool, reconcile_seconds: int = 300) -> None:
     settings = WorkerSettings.from_environment()
     peripheral_url = os.environ["OPERATIONAL_ACTION_PERIPHERAL_URL"]
     credentials = CredentialProvider(settings)
     async with httpx.AsyncClient(timeout=30, follow_redirects=False, trust_env=False) as client:
-        while True:
+        if once:
             try:
                 count = await sync_once(client, settings, peripheral_url, credentials)
                 logger.info("Peripheral progress synchronization completed: %d revisions", count)
             except (httpx.HTTPError, ValueError, KeyError):
-                logger.error("Peripheral synchronization failed; next tick will reconcile persisted revisions")
-                if once:
-                    raise SystemExit(1) from None
-            if once:
-                return
-            await asyncio.sleep(settings.interval_seconds)
+                logger.error("Peripheral synchronization failed; persisted revisions remain available")
+                raise SystemExit(1) from None
+            return
+        await PeripheralWorkOrderSynchronizer(client, settings, peripheral_url, credentials).serve(reconcile_seconds)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--once", action="store_true")
+    parser.add_argument("--reconcile-seconds", type=int, default=300,
+                        help="missed-event reconciliation interval (default: 300 seconds)")
     args = parser.parse_args()
+    if args.reconcile_seconds < 1:
+        parser.error("--reconcile-seconds must be at least 1")
     logging.basicConfig(level=logging.INFO)
     try:
-        asyncio.run(run(args.once))
+        asyncio.run(run(args.once, args.reconcile_seconds))
     except KeyboardInterrupt:
         pass