feat: add project-scoped operations

scripts/run_operational_worker.py

@@ -12,7 +12,7 @@ import json
 import logging
 import os
 import time
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
 from uuid import UUID, uuid4
@@ -37,11 +37,21 @@ class WorkerSettings:
     client_secret: str = ""
     token_refresh_seconds: float = 300
     task_created_from: str | None = None
+    project_ids: tuple[str, ...] = ()
+
+    def project_scopes(self) -> tuple["WorkerSettings", ...]:
+        """Keep credentials, sessions and cursors separate for each explicit project."""
+        return tuple(replace(self, project_id=project_id, project_ids=())
+                     for project_id in self.project_ids or (self.project_id,))
 
     @classmethod
     def from_environment(cls) -> "WorkerSettings":
         load_dotenv(Path(__file__).resolve().parents[1] / ".env", override=False)
-        project_id = str(UUID(os.environ["OPERATIONAL_WORKER_PROJECT_ID"]))
+        configured_projects = os.environ.get("OPERATIONAL_WORKER_PROJECT_IDS", "").strip()
+        project_ids = tuple(dict.fromkeys(
+            str(UUID(value.strip())) for value in configured_projects.split(",")
+        )) if configured_projects else ()
+        project_id = project_ids[0] if project_ids else str(UUID(os.environ["OPERATIONAL_WORKER_PROJECT_ID"]))
         interval = float(os.environ.get("OPERATIONAL_WORKER_RECONCILE_SECONDS", "30"))
         if interval < 1:
             raise ValueError("worker interval must be at least one second")
@@ -60,6 +70,7 @@ class WorkerSettings:
             client_secret=os.environ.get("OPERATIONAL_WORKER_CLIENT_SECRET", ""),
             token_refresh_seconds=float(os.environ.get("OPERATIONAL_WORKER_TOKEN_REFRESH_SECONDS", "300")),
             task_created_from=os.environ.get("OPERATIONAL_WORKER_TASK_CREATED_FROM") or None,
+            project_ids=project_ids,
         )
         if settings.task_created_from:
             boundary = datetime.fromisoformat(settings.task_created_from.replace("Z", "+00:00"))
@@ -305,37 +316,45 @@ class OperationalWorker:
                 await asyncio.sleep(5)
 
 
-async def run(once: bool) -> None:
-    settings = WorkerSettings.from_environment()
+async def run_project(settings: WorkerSettings, client: httpx.AsyncClient, once: bool) -> None:
     logger.info(
         "Investigation worker scope: project=%s task_created_from=%s concurrency=%d",
         settings.project_id, settings.task_created_from or "all", settings.max_concurrency,
     )
-    async with httpx.AsyncClient(timeout=30, follow_redirects=False, trust_env=False) as client:
-        worker = OperationalWorker(settings, client)
-        wake = asyncio.Event()
-        listener = None if once else asyncio.create_task(worker.listen(wake))
-        try:
-            while True:
-                try:
-                    accepted = await worker.tick()
-                    logger.info("Task dispatch tick completed: %d accepted", accepted)
-                except (httpx.HTTPError, ValueError, KeyError):
-                    # Do not log request headers, response bodies or provider secrets.
-                    logger.error("Task dispatch tick failed; persisted tasks remain available for retry")
-                    if once:
-                        raise SystemExit(1) from None
+    worker = OperationalWorker(settings, client)
+    wake = asyncio.Event()
+    listener = None if once else asyncio.create_task(worker.listen(wake))
+    try:
+        while True:
+            try:
+                accepted = await worker.tick()
+                logger.info("Task dispatch tick completed: project=%s accepted=%d", settings.project_id, accepted)
+            except (httpx.HTTPError, ValueError, KeyError):
+                logger.error("Task dispatch tick failed: project=%s; persisted tasks remain available for retry", settings.project_id)
                 if once:
-                    return
-                try:
-                    await asyncio.wait_for(wake.wait(), timeout=settings.interval_seconds)
-                except TimeoutError:
-                    pass
-                wake.clear()
-        finally:
-            if listener is not None:
-                listener.cancel()
-                await asyncio.gather(listener, return_exceptions=True)
+                    raise
+            if once:
+                return
+            try:
+                await asyncio.wait_for(wake.wait(), timeout=settings.interval_seconds)
+            except TimeoutError:
+                pass
+            wake.clear()
+    finally:
+        if listener is not None:
+            listener.cancel()
+            await asyncio.gather(listener, return_exceptions=True)
+
+
+async def run(once: bool) -> None:
+    settings = WorkerSettings.from_environment()
+    async with httpx.AsyncClient(timeout=30, follow_redirects=False, trust_env=False) as client:
+        results = await asyncio.gather(
+            *(run_project(scope, client, once) for scope in settings.project_scopes()),
+            return_exceptions=once,
+        )
+        if any(isinstance(result, Exception) for result in results):
+            raise SystemExit(1)
 
 
 if __name__ == "__main__":

scripts/sync_peripheral_work_orders.py

@@ -372,7 +372,7 @@ class PeripheralWorkOrderSynchronizer:
                             event_name = ""
                 await asyncio.sleep(1)
             except (httpx.HTTPError, ValueError, KeyError):
-                logger.warning("Work-order event stream disconnected; reconciliation remains active")
+                logger.warning("Work-order event stream disconnected: project=%s; reconciliation remains active", self.settings.project_id)
                 await asyncio.sleep(5)
 
     async def serve(self, reconcile_seconds: int = 300) -> None:
@@ -399,12 +399,12 @@ class PeripheralWorkOrderSynchronizer:
                         self.client, self.settings, self.peripheral_url, self.credentials,
                         **({"work_order_ids": order_ids} if not full_reconcile else {}),
                     )
-                    logger.info("Peripheral progress synchronization completed: %d revisions", count)
+                    logger.info("Peripheral progress synchronization completed: project=%s revisions=%d", self.settings.project_id, count)
                     retry_seconds = 5
                     if full_reconcile:
                         next_reconcile = asyncio.get_running_loop().time() + reconcile_seconds
                 except (httpx.HTTPError, ValueError, KeyError):
-                    logger.error("Peripheral synchronization failed; persisted revisions will be retried")
+                    logger.error("Peripheral synchronization failed: project=%s; persisted revisions will be retried", self.settings.project_id)
                     self.full_reconcile |= full_reconcile
                     for order_id in order_ids:
                         self._queue_order(order_id)
@@ -419,17 +419,21 @@ class PeripheralWorkOrderSynchronizer:
 async def run(once: bool, reconcile_seconds: int = 300) -> None:
     settings = WorkerSettings.from_environment()
     peripheral_url = os.environ["OPERATIONAL_ACTION_PERIPHERAL_URL"]
-    credentials = CredentialProvider(settings)
     async with httpx.AsyncClient(timeout=30, follow_redirects=False, trust_env=False) as client:
         if once:
-            try:
-                count = await sync_once(client, settings, peripheral_url, credentials)
-                logger.info("Peripheral progress synchronization completed: %d revisions", count)
-            except (httpx.HTTPError, ValueError, KeyError):
+            results = await asyncio.gather(*(
+                sync_once(client, scope, peripheral_url, CredentialProvider(scope))
+                for scope in settings.project_scopes()
+            ), return_exceptions=True)
+            if any(isinstance(result, Exception) for result in results):
                 logger.error("Peripheral synchronization failed; persisted revisions remain available")
                 raise SystemExit(1) from None
+            logger.info("Peripheral progress synchronization completed: %d revisions", sum(results))
             return
-        await PeripheralWorkOrderSynchronizer(client, settings, peripheral_url, credentials).serve(reconcile_seconds)
+        await asyncio.gather(*(
+            PeripheralWorkOrderSynchronizer(client, scope, peripheral_url, CredentialProvider(scope)).serve(reconcile_seconds)
+            for scope in settings.project_scopes()
+        ))
 
 
 if __name__ == "__main__":