fix(tasks): isolate worker execution scopes

Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
This commit is contained in:
2026-11-08 01:54:48 +08:00
parent 2a7b96f813
commit 33df25f22b
14 changed files with 156 additions and 16 deletions
+3 -1
View File
@@ -131,9 +131,11 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
async with service.repo.transaction():
candidates = await service.repo.recovery_candidates(limit, created_from)
candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope)
resumed = 0
for task in candidates:
if task.get("execution_scope") != service.execution_scope:
continue
context = task["handling_context"]
returned_at = datetime.fromisoformat(context["returned_at"])
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
+10 -1
View File
@@ -88,11 +88,13 @@ def _task_json(model: OperationalTaskSummary) -> dict:
class OperationalTaskService:
def __init__(self, repository, context, permissions: Iterable[str], *, clock=None):
def __init__(self, repository, context, permissions: Iterable[str], *, clock=None,
execution_scope: str = "production"):
self.repo = repository
self.context = context
self.permissions = frozenset(permissions)
self.clock = clock or (lambda: datetime.now(timezone.utc))
self.execution_scope = execution_scope
def _now(self) -> datetime:
now = self.clock()
@@ -195,6 +197,7 @@ class OperationalTaskService:
now = self._now()
task_id = uuid4()
task = OperationalTaskSummary(
execution_scope=self.execution_scope,
project_id=self.context.project_id,
task_id=task_id,
task_type=command.task_type,
@@ -489,6 +492,7 @@ class OperationalTaskService:
task_type: str | None = None, task_created_from: datetime | None = None,
) -> tuple[list[OperationalTaskRun], str | None]:
rows = await self.repo.list_runs(
execution_scope=self.execution_scope,
status=status,
limit=limit + 1,
cursor_created_at=cursor_created_at,
@@ -510,6 +514,9 @@ class OperationalTaskService:
scope = f"run:{run_id}:claim"
async with self.repo.transaction():
await self.repo.require_execution_storage()
task = await self._task(task_id)
require(task.get("execution_scope") == self.execution_scope,
"EXECUTION_SCOPE_MISMATCH", "当前环境不能领取其他执行范围的任务", status=403)
replay = await self._request_replay(scope, command.request_id, body)
if replay:
return OperationalTaskRun.model_validate(replay["run"])
@@ -759,6 +766,7 @@ class OperationalTaskService:
async with self.repo.transaction():
for run in await self.repo.claim_expired_runs(
limit, task_type=task_type, task_created_from=task_created_from,
execution_scope=self.execution_scope,
):
# Lease recovery already holds a row lock. Try rather than wait
# for the shared run lock to avoid reversing the lock order of
@@ -2516,6 +2524,7 @@ class OperationalTaskService:
due_at: datetime, now: datetime,
) -> OperationalTaskSummary:
return OperationalTaskSummary(
execution_scope=self.execution_scope,
project_id=self.context.project_id, task_id=task_id,
task_type="scheduled_flow", workflow=schedule.schedule_key,
lifecycle_status="open", status="queued", priority=schedule.priority, revision=1,
+5 -1
View File
@@ -6,6 +6,7 @@ import json
from uuid import NAMESPACE_URL, uuid4, uuid5
from fastapi import HTTPException
from app.core.config import settings
from app.auth.permissions import permissions_for_context
from app.domain.schemas.project_settings import ScadaMonitoringSettings
@@ -439,7 +440,8 @@ class ScadaReplayService:
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
self.context, internal_producer_id=producer)
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context))
self.context, permissions_for_context(self.context),
execution_scope=settings.operational_task_execution_scope)
for event in changes:
# Period diagnosis owns measurement notifications during manual steps.
# Retain rule events as evidence; quality alerts remain independent.
@@ -503,6 +505,7 @@ class ScadaReplayService:
tasks = OperationalTaskService(
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context, permissions_for_context(self.context),
execution_scope=settings.operational_task_execution_scope,
)
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
@@ -520,6 +523,7 @@ class ScadaReplayService:
OperationalTaskRepository(self.repo.connection, self.context.project_id),
self.context,
permissions_for_context(self.context),
execution_scope=settings.operational_task_execution_scope,
)
external = incident["external_id"]
pressure = monitoring["pressure"]