fix(workbench): serialize events and target work order sync

2026-10-08 15:41:49 +08:00
parent b70283c2ff
commit 4a63fcd7ff
20 changed files with 673 additions and 90 deletions
@@ -28,6 +28,7 @@ logger = logging.getLogger("peripheral-task-sync")
 STATUS = {"draft": "scheduled", "open": "assigned", "in_progress": "in_progress",
          "awaiting_reply": "awaiting_feedback", "completed": "completed", "cancelled": "cancelled"}
 EVENT_WORK_ORDER_TYPES = frozenset({"water_quality_complaint", "burst_report"})
+MAX_TARGETED_ORDERS = 200


 def peripheral_order_id(external_id: str | None) -> str | None:
@@ -69,31 +70,66 @@ def work_order_snapshot(order: dict) -> dict:
    return snapshot


+async def work_order_pages(client, url, headers, work_order_ids, deleted_ids):
+    if work_order_ids is not None:
+        for order_id in sorted(work_order_ids):
+            response = await client.get(f"{url}/{order_id}", headers=headers)
+            if response.status_code == 404:
+                if response.json().get("code") != "WORK_ORDER_NOT_FOUND":
+                    raise ValueError("peripheral work order lookup did not confirm deletion")
+                deleted_ids.add(order_id)
+                continue
+            response.raise_for_status()
+            order = response.json()
+            if order["work_order_id"] != order_id:
+                raise ValueError("peripheral work order identity mismatch")
+            yield {"items": [order]}
+        return
+    cursor = None
+    while True:
+        params = {"limit": 100}
+        if cursor:
+            params["cursor"] = cursor
+        response = await client.get(url, headers=headers, params=params)
+        response.raise_for_status()
+        page = response.json()
+        yield page
+        cursor = page.get("next_cursor")
+        if not cursor:
+            return
+
+
 async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, peripheral_url: str,
-                    credentials: CredentialProvider) -> int:
+                    credentials: CredentialProvider, *, work_order_ids: set[str] | None = None) -> int:
+    if work_order_ids is not None:
+        work_order_ids = {str(UUID(order_id)) for order_id in work_order_ids}
    headers = await credentials.headers(client)
    tasks_url = f"{settings.server_url.rstrip('/')}/api/v1/operational-tasks"
    existing: dict[str, dict] = {}
    event_tasks: dict[str | None, dict] = {}
-    investigations: list[dict] = []
-    offset = 0
-    while True:
-        response = await client.get(tasks_url, headers=headers, params={"limit": 100, "offset": offset})
-        response.raise_for_status()
-        page = response.json()
-        for task in page["items"]:
-            if task["task_type"] == "external_work_order":
-                existing[task["source"].get("external_id")] = task
-            elif task["task_type"] == "event_investigation":
-                event_tasks[task.get("source", {}).get("external_id")] = task
-                if task["status"] in {"queued", "waiting_human", "running"}:
-                    investigations.append(task)
-        if page.get("next_offset") is None:
-            break
-        offset = page["next_offset"]
+    investigations: dict[str, dict] = {}
+    for order_id in sorted(work_order_ids) if work_order_ids is not None else [None]:
+        offset = 0
+        while True:
+            params = {"limit": 100, "offset": offset}
+            if order_id is not None:
+                params["peripheral_work_order_id"] = order_id
+            response = await client.get(tasks_url, headers=headers, params=params)
+            response.raise_for_status()
+            page = response.json()
+            for task in page["items"]:
+                if task["task_type"] == "external_work_order":
+                    existing[task["source"].get("external_id")] = task
+                elif task["task_type"] == "event_investigation":
+                    event_tasks[task.get("source", {}).get("external_id")] = task
+                    if task["status"] in {"queued", "waiting_human", "running"}:
+                        investigations[task["task_id"]] = task
+            if page.get("next_offset") is None:
+                break
+            offset = page["next_offset"]

    linked: dict[str, list[str]] = {}
-    for task in investigations:
+    for task in investigations.values():
        response = await client.get(f"{tasks_url}/{task['task_id']}", headers=headers)
        response.raise_for_status()
        for action in response.json().get("actions", []):
@@ -102,16 +138,13 @@ async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, periphe
                linked.setdefault(receipt["receipt_id"], []).append(task["task_id"])
    linked_order_ids = set(linked)

-    cursor, synchronized = None, 0
+    synchronized = 0
+    deleted_ids: set[str] = set()
    seen_order_ids: set[str] = set()
-    while True:
-        params = {"limit": 100}
-        if cursor:
-            params["cursor"] = cursor
-        response = await client.get(f"{peripheral_url.rstrip('/')}/api/sample/v1/work-orders",
-                                    headers=headers, params=params)
-        response.raise_for_status()
-        page = response.json()
+    async for page in work_order_pages(
+        client, f"{peripheral_url.rstrip('/')}/api/sample/v1/work-orders", headers,
+        work_order_ids, deleted_ids,
+    ):
        for order in page["items"]:
            if order["project_id"] != settings.project_id or order["origin"] != "synthetic":
                raise ValueError("peripheral work order scope or origin mismatch")
@@ -178,9 +211,6 @@ async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, periphe
                result = await client.post(f"{tasks_url}/{target}/external-feedback", headers=headers, json=payload)
                result.raise_for_status()
                synchronized += 1
-        cursor = page.get("next_cursor")
-        if not cursor:
-            break

    for external_id, task in existing.items():
        order_id = peripheral_order_id(external_id)
@@ -188,18 +218,21 @@ async def sync_once(client: httpx.AsyncClient, settings: WorkerSettings, periphe
            continue
        if order_id in seen_order_ids:
            continue
+        if work_order_ids is not None and order_id not in work_order_ids:
+            continue
        # The list may change during pagination. Only a scoped 404 proves that
        # the source order is now absent; transient errors must stop this tick.
-        response = await client.get(
-            f"{peripheral_url.rstrip('/')}/api/sample/v1/work-orders/{order_id}",
-            headers=headers,
-        )
-        if response.status_code == 200:
-            continue
-        if response.status_code != 404:
-            response.raise_for_status()
-        if response.json().get("code") != "WORK_ORDER_NOT_FOUND":
-            raise ValueError("peripheral work order lookup did not confirm deletion")
+        if order_id not in deleted_ids:
+            response = await client.get(
+                f"{peripheral_url.rstrip('/')}/api/sample/v1/work-orders/{order_id}",
+                headers=headers,
+            )
+            if response.status_code == 200:
+                continue
+            if response.status_code != 404:
+                response.raise_for_status()
+            if response.json().get("code") != "WORK_ORDER_NOT_FOUND":
+                raise ValueError("peripheral work order lookup did not confirm deletion")
        result = await client.post(
            f"{tasks_url}/{task['task_id']}/withdraw-external-source",
            headers=headers,
@@ -224,6 +257,16 @@ class PeripheralWorkOrderSynchronizer:
        self.credentials = credentials
        self.wake = asyncio.Event()
        self.cursor: int | None = None
+        self.pending_order_ids: set[str] = set()
+        self.full_reconcile = False
+
+    def _queue_order(self, order_id: str) -> None:
+        if not self.full_reconcile:
+            self.pending_order_ids.add(order_id)
+            if len(self.pending_order_ids) > MAX_TARGETED_ORDERS:
+                self.pending_order_ids.clear()
+                self.full_reconcile = True
+        self.wake.set()

    async def _reset_cursor(self, headers: dict[str, str]) -> None:
        response = await self.client.get(
@@ -235,6 +278,7 @@ class PeripheralWorkOrderSynchronizer:
        if not isinstance(cursor, int) or isinstance(cursor, bool) or cursor < 0:
            raise ValueError("invalid workbench event cursor")
        self.cursor = cursor
+        self.full_reconcile = True
        # Capture the cursor before scanning sources, so updates during the
        # initial/reset scan are replayed by the following SSE connection.
        self.wake.set()
@@ -264,11 +308,25 @@ class PeripheralWorkOrderSynchronizer:
                                raise ValueError("workbench event project mismatch")
                            if event.event_seq <= self.cursor:
                                continue
-                            self.cursor = event.event_seq
                            if (event.topic == "workbench.work_order.changed"
                                    and event.aggregate_type == "peripheral_work_order"
                                    and event.metadata.get("source") == "peripheral_sample"):
-                                self.wake.set()
+                                self._queue_order(str(UUID(event.aggregate_id)))
+                            elif event.topic == "operational.action.changed":
+                                # The source create can commit before its action
+                                # receipt is saved. Revisit just those receipt IDs.
+                                task_id = str(UUID(event.metadata["task_id"]))
+                                task = await self.client.get(
+                                    f"{self.settings.server_url.rstrip('/')}/api/v1/operational-tasks/{task_id}",
+                                    headers=headers,
+                                )
+                                task.raise_for_status()
+                                for action in task.json().get("actions", []):
+                                    receipt = action.get("receipt")
+                                    if (action["kind"] == "create_work_order" and receipt
+                                            and receipt["adapter"] == "peripheral_sample"):
+                                        self._queue_order(str(UUID(receipt["receipt_id"])))
+                            self.cursor = event.event_seq
                        elif not line:
                            event_name = ""
                await asyncio.sleep(1)
@@ -280,26 +338,37 @@ class PeripheralWorkOrderSynchronizer:
        if reconcile_seconds < 1:
            raise ValueError("reconciliation interval must be at least one second")
        listener = asyncio.create_task(self.listen())
-        wait_seconds = reconcile_seconds
        retry_seconds = 5
+        next_reconcile = asyncio.get_running_loop().time() + reconcile_seconds
        try:
            while True:
                try:
-                    await asyncio.wait_for(self.wake.wait(), timeout=wait_seconds)
+                    remaining = max(0, next_reconcile - asyncio.get_running_loop().time())
+                    await asyncio.wait_for(self.wake.wait(), timeout=remaining)
                except TimeoutError:
-                    pass
+                    self.full_reconcile = True
                # Clear before work, so arrivals during a scan request another
                # pass. A failed scan gets a retry even after its cursor advances.
                self.wake.clear()
+                full_reconcile = self.full_reconcile
+                self.full_reconcile = False
+                order_ids, self.pending_order_ids = self.pending_order_ids, set()
                try:
                    count = await sync_once(
                        self.client, self.settings, self.peripheral_url, self.credentials,
+                        **({"work_order_ids": order_ids} if not full_reconcile else {}),
                    )
                    logger.info("Peripheral progress synchronization completed: %d revisions", count)
-                    wait_seconds, retry_seconds = reconcile_seconds, 5
+                    retry_seconds = 5
+                    if full_reconcile:
+                        next_reconcile = asyncio.get_running_loop().time() + reconcile_seconds
                except (httpx.HTTPError, ValueError, KeyError):
                    logger.error("Peripheral synchronization failed; persisted revisions will be retried")
-                    wait_seconds = retry_seconds
+                    self.full_reconcile |= full_reconcile
+                    for order_id in order_ids:
+                        self._queue_order(order_id)
+                    await asyncio.sleep(retry_seconds)
+                    self.wake.set()
                    retry_seconds = min(retry_seconds * 2, 60)
        finally:
            listener.cancel()