feat: extend operational investigation workflows

scripts/run_operational_worker.py

@@ -124,6 +124,20 @@ class OperationalWorker:
     async def tick(self) -> int:
         headers = await self.credentials.headers(self.client)
         server = self.settings.server_url.rstrip("/")
+        agent = self.settings.agent_url.rstrip("/")
+        target_agent_instance_id: str | None = None
+
+        async def target_instance_id() -> str:
+            nonlocal target_agent_instance_id
+            if target_agent_instance_id is None:
+                response = await self.client.get(f"{agent}/health")
+                response.raise_for_status()
+                value = response.json().get("agent_instance_id")
+                if not isinstance(value, str) or not value.strip():
+                    raise ValueError("Agent health response has no instance identity")
+                target_agent_instance_id = value.strip()
+            return target_agent_instance_id
+
         scope = {"task_type": "event_investigation"}
         if self.settings.task_created_from:
             scope["task_created_from"] = self.settings.task_created_from
@@ -136,10 +150,13 @@ class OperationalWorker:
         for status_value in ("claimed", "running"):
             response = await self.client.get(
                 f"{server}/api/v1/operational-task-runs", headers=headers,
-                params={"status": status_value, "limit": self.settings.max_concurrency, **scope},
+                params={"status": status_value, "limit": 100, **scope},
             )
             response.raise_for_status()
             for item in response.json()["items"]:
+                owner = item.get("agent_instance_id")
+                if not isinstance(owner, str) or owner != await target_instance_id():
+                    continue
                 run_id = str(item["run_id"])
                 active_run_ids.add(run_id)
                 session_id = item.get("agent_session_id")
@@ -152,7 +169,7 @@ class OperationalWorker:
             # Refresh the Agent vault with the worker's existing OAuth provider.
             # Background runs never depend on an interactive browser refresh.
             renewed = await self.client.post(
-                f"{self.settings.agent_url.rstrip('/')}/api/v1/session/{session_id}/credentials",
+                f"{agent}/api/v1/session/{session_id}/credentials",
                 headers=headers, json={},
             )
             if renewed.status_code not in (202, 404, 409):
@@ -167,6 +184,7 @@ class OperationalWorker:
         )
         response.raise_for_status()
         queued = response.json()["items"]
+        dispatch_instance_id = await target_instance_id() if queued else None
 
         async def dispatch(run: dict) -> bool:
             if run.get("status") != "queued":
@@ -176,18 +194,21 @@ class OperationalWorker:
             request_id = str(UUID(run["execution_request_id"]))
             try:
                 result = await self.client.post(
-                    f"{self.settings.agent_url.rstrip('/')}/api/v1/task-executions",
+                    f"{agent}/api/v1/task-executions",
                     headers=headers,
                     json={
                         "task_id": task_id,
                         "run_id": run_id,
                         "execution_request_id": request_id,
+                        "agent_instance_id": dispatch_instance_id,
                     },
                 )
                 if result.is_success:
                     payload = result.json()
                     if payload.get("task_id") != task_id or payload.get("run_id") != run_id:
                         raise ValueError("Agent returned a different task run")
+                    if payload.get("agent_instance_id") != dispatch_instance_id:
+                        raise ValueError("Agent returned a different instance identity")
                     session_id = payload.get("session_id")
                     if isinstance(session_id, str) and session_id:
                         self._agent_sessions[run_id] = session_id
@@ -221,7 +242,12 @@ class OperationalWorker:
                         f"{self.settings.agent_url.rstrip('/')}/api/v1/task-messages/deliver",
                         headers=headers, json={"message_id": message_id},
                     )
-                    if delivered.status_code in {400, 403, 404, 410, 422}:
+                    if delivered.status_code == 404:
+                        # A different Agent instance may own the target session.
+                        # Keep the durable message pending instead of turning a
+                        # routing miss into a terminal delivery failure.
+                        logger.info("Agent message target is not local; delivery remains pending: %s", message_id)
+                    elif delivered.status_code in {400, 403, 410, 422}:
                         failed = await self.client.post(
                             f"{self.settings.server_url.rstrip('/')}/api/v1/agent-messages/{message_id}/delivery",
                             headers=headers, json={