fix(tasks): isolate worker execution scopes

Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.

app/api/v1/endpoints/operational_tasks.py

@@ -9,6 +9,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
 from psycopg import AsyncConnection
 from app.auth.metadata_dependencies import get_metadata_repository
 
+from app.core.config import settings
 from app.auth.permissions import (
     OPERATIONAL_TASK_EXECUTE,
     OPERATIONAL_TASK_EXTERNAL_SYNC,
@@ -122,6 +123,7 @@ def _service(ctx: ProjectContext, conn: AsyncConnection) -> OperationalTaskServi
         OperationalTaskRepository(conn, ctx.project_id),
         ctx,
         _task_permissions(ctx),
+        execution_scope=settings.operational_task_execution_scope,
     )
 
 

app/api/v1/endpoints/workbench_events.py

@@ -36,6 +36,7 @@ def _service(ctx: ProjectContext, conn: AsyncConnection) -> OperationalTaskServi
         OperationalTaskRepository(conn, ctx.project_id),
         ctx,
         permissions_for_context(ctx),
+        execution_scope=settings.operational_task_execution_scope,
     )
 
 

app/core/config.py

@@ -8,6 +8,12 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 class Settings(BaseSettings):
     PROJECT_NAME: str = "TJWater Server"
     ENVIRONMENT: str = "production"
+    OPERATIONAL_TASK_EXECUTION_SCOPE: str = ""
+
+    @property
+    def operational_task_execution_scope(self) -> str:
+        return self.OPERATIONAL_TASK_EXECUTION_SCOPE.strip() or self.ENVIRONMENT
+
     API_V1_STR: str = "/api/v1"
     NETWORK_NAME: str = "default_network"
     # 敏感配置加密密钥 (Fernet)

app/domain/schemas/operational_task.py

@@ -574,6 +574,7 @@ class OperationalTaskRun(StrictModel):
 
 
 class OperationalTaskSummary(StrictModel):
+    execution_scope: NonEmptyStr
     project_id: UUID
     task_id: UUID
     task_type: TaskType

app/infra/db/postgresql/operational_tasks.py

@@ -264,6 +264,7 @@ class OperationalTaskRepository:
     async def list_runs(
         self, *, status: str | None, limit: int, cursor_created_at=None, cursor_run_id=None,
         task_type: str | None = None, task_created_from: datetime | None = None,
+        execution_scope: str = "production",
     ) -> list[dict]:
         result = await self.connection.execute(
             """SELECT run_id,task_id,generation,execution_request_id,status,revision,
@@ -282,18 +283,20 @@ class OperationalTaskRepository:
                        AND t.task_id=operational_task_runs.task_id
                        AND (%s::text IS NULL OR t.task_type=%s::text)
                        AND (%s::timestamptz IS NULL OR t.created_at>=%s::timestamptz)
+                       AND t.document->>'execution_scope'=%s
                  )
                ORDER BY created_at,run_id LIMIT %s""",
             (
                 self.project_id, status, status, cursor_created_at,
                 cursor_created_at, cursor_run_id,
-                task_type, task_type, task_created_from, task_created_from, limit,
+                task_type, task_type, task_created_from, task_created_from, execution_scope, limit,
             ),
         )
         return [dict(row) for row in await result.fetchall()]
 
     async def claim_expired_runs(
         self, limit: int, *, task_type: str | None = None, task_created_from: datetime | None = None,
+        execution_scope: str = "production",
     ) -> list[dict]:
         cursor = await self.connection.execute(
             """SELECT run_id,task_id,generation,execution_request_id,status,revision,
@@ -310,9 +313,10 @@ class OperationalTaskRepository:
                        AND t.task_id=operational_task_runs.task_id
                        AND (%s::text IS NULL OR t.task_type=%s::text)
                        AND (%s::timestamptz IS NULL OR t.created_at>=%s::timestamptz)
+                       AND t.document->>'execution_scope'=%s
                  )
                ORDER BY lease_expires_at,run_id FOR UPDATE SKIP LOCKED LIMIT %s""",
-            (self.project_id, task_type, task_type, task_created_from, task_created_from, limit),
+            (self.project_id, task_type, task_type, task_created_from, task_created_from, execution_scope, limit),
         )
         return [dict(row) for row in await cursor.fetchall()]
 
@@ -711,15 +715,16 @@ class OperationalTaskRepository:
         row = await cursor.fetchone()
         return dict(row) if row else None
 
-    async def recovery_candidates(self, limit: int, created_from=None) -> list[dict]:
+    async def recovery_candidates(self, limit: int, created_from=None, execution_scope: str = "production") -> list[dict]:
         cursor = await self.connection.execute(
             """SELECT document FROM workbench.operational_tasks
                WHERE project_id=%s AND task_type='event_investigation'
                  AND status NOT IN ('cancelled','failed')
                  AND document#>>'{handling_context,phase}'='waiting_scada'
+                 AND document->>'execution_scope'=%s
                  AND (%s::timestamptz IS NULL OR created_at>=%s)
                ORDER BY COALESCE((document#>>'{handling_context,last_checked_at}')::timestamptz,created_at),task_id LIMIT %s""",
-            (self.project_id, created_from, created_from, limit),
+            (self.project_id, execution_scope, created_from, created_from, limit),
         )
         return [row["document"] for row in await cursor.fetchall()]
 

app/mcp/query_service.py

@@ -12,6 +12,7 @@ from datetime import datetime, timezone
 from uuid import uuid4
 from fastapi import HTTPException
 from fastapi.encoders import jsonable_encoder
+from app.core.config import settings
 from app.auth.permissions import permissions_for_context
 from app.infra.db.postgresql import read_queries as reads
 from app.infra.db.postgresql.spatial_queries import nearby_elements
@@ -52,6 +53,7 @@ def task_service(scope: QueryScope) -> OperationalTaskService:
         OperationalTaskRepository(scope.connection, scope.project.project_id),
         scope.project,
         set(permissions_for_context(scope.project)),
+        execution_scope=settings.operational_task_execution_scope,
     )
 
 

app/services/operational_recovery.py

@@ -131,9 +131,11 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
     from app.infra.db.timescaledb.repositories.scada import ScadaRepository
 
     async with service.repo.transaction():
-        candidates = await service.repo.recovery_candidates(limit, created_from)
+        candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope)
     resumed = 0
     for task in candidates:
+        if task.get("execution_scope") != service.execution_scope:
+            continue
         context = task["handling_context"]
         returned_at = datetime.fromisoformat(context["returned_at"])
         incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])

app/services/operational_tasks.py

@@ -88,11 +88,13 @@ def _task_json(model: OperationalTaskSummary) -> dict:
 
 
 class OperationalTaskService:
-    def __init__(self, repository, context, permissions: Iterable[str], *, clock=None):
+    def __init__(self, repository, context, permissions: Iterable[str], *, clock=None,
+                 execution_scope: str = "production"):
         self.repo = repository
         self.context = context
         self.permissions = frozenset(permissions)
         self.clock = clock or (lambda: datetime.now(timezone.utc))
+        self.execution_scope = execution_scope
 
     def _now(self) -> datetime:
         now = self.clock()
@@ -195,6 +197,7 @@ class OperationalTaskService:
             now = self._now()
             task_id = uuid4()
             task = OperationalTaskSummary(
+                execution_scope=self.execution_scope,
                 project_id=self.context.project_id,
                 task_id=task_id,
                 task_type=command.task_type,
@@ -489,6 +492,7 @@ class OperationalTaskService:
         task_type: str | None = None, task_created_from: datetime | None = None,
     ) -> tuple[list[OperationalTaskRun], str | None]:
         rows = await self.repo.list_runs(
+            execution_scope=self.execution_scope,
             status=status,
             limit=limit + 1,
             cursor_created_at=cursor_created_at,
@@ -510,6 +514,9 @@ class OperationalTaskService:
         scope = f"run:{run_id}:claim"
         async with self.repo.transaction():
             await self.repo.require_execution_storage()
+            task = await self._task(task_id)
+            require(task.get("execution_scope") == self.execution_scope,
+                    "EXECUTION_SCOPE_MISMATCH", "当前环境不能领取其他执行范围的任务", status=403)
             replay = await self._request_replay(scope, command.request_id, body)
             if replay:
                 return OperationalTaskRun.model_validate(replay["run"])
@@ -759,6 +766,7 @@ class OperationalTaskService:
         async with self.repo.transaction():
             for run in await self.repo.claim_expired_runs(
                 limit, task_type=task_type, task_created_from=task_created_from,
+                execution_scope=self.execution_scope,
             ):
                 # Lease recovery already holds a row lock. Try rather than wait
                 # for the shared run lock to avoid reversing the lock order of
@@ -2516,6 +2524,7 @@ class OperationalTaskService:
         due_at: datetime, now: datetime,
     ) -> OperationalTaskSummary:
         return OperationalTaskSummary(
+            execution_scope=self.execution_scope,
             project_id=self.context.project_id, task_id=task_id,
             task_type="scheduled_flow", workflow=schedule.schedule_key,
             lifecycle_status="open", status="queued", priority=schedule.priority, revision=1,

app/services/scada_replay.py

@@ -6,6 +6,7 @@ import json
 from uuid import NAMESPACE_URL, uuid4, uuid5
 
 from fastapi import HTTPException
+from app.core.config import settings
 from app.auth.permissions import permissions_for_context
 
 from app.domain.schemas.project_settings import ScadaMonitoringSettings
@@ -439,7 +440,8 @@ class ScadaReplayService:
         alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
                                   self.context, internal_producer_id=producer)
         tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
-                                       self.context, permissions_for_context(self.context))
+                                       self.context, permissions_for_context(self.context),
+                                       execution_scope=settings.operational_task_execution_scope)
         for event in changes:
             # Period diagnosis owns measurement notifications during manual steps.
             # Retain rule events as evidence; quality alerts remain independent.
@@ -503,6 +505,7 @@ class ScadaReplayService:
                 tasks = OperationalTaskService(
                     OperationalTaskRepository(self.repo.connection, self.context.project_id),
                     self.context, permissions_for_context(self.context),
+                    execution_scope=settings.operational_task_execution_scope,
                 )
                 await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
                     "replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
@@ -520,6 +523,7 @@ class ScadaReplayService:
             OperationalTaskRepository(self.repo.connection, self.context.project_id),
             self.context,
             permissions_for_context(self.context),
+            execution_scope=settings.operational_task_execution_scope,
         )
         external = incident["external_id"]
         pressure = monitoring["pressure"]