fix(tasks): isolate worker execution scopes
Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
This commit is contained in:
@@ -131,9 +131,11 @@ async def reconcile_recoveries(service: OperationalTaskService, timescale: Async
|
||||
from app.infra.db.timescaledb.repositories.scada import ScadaRepository
|
||||
|
||||
async with service.repo.transaction():
|
||||
candidates = await service.repo.recovery_candidates(limit, created_from)
|
||||
candidates = await service.repo.recovery_candidates(limit, created_from, service.execution_scope)
|
||||
resumed = 0
|
||||
for task in candidates:
|
||||
if task.get("execution_scope") != service.execution_scope:
|
||||
continue
|
||||
context = task["handling_context"]
|
||||
returned_at = datetime.fromisoformat(context["returned_at"])
|
||||
incident_at = datetime.fromisoformat(task["trigger"]["occurred_at"])
|
||||
|
||||
@@ -88,11 +88,13 @@ def _task_json(model: OperationalTaskSummary) -> dict:
|
||||
|
||||
|
||||
class OperationalTaskService:
|
||||
def __init__(self, repository, context, permissions: Iterable[str], *, clock=None):
|
||||
def __init__(self, repository, context, permissions: Iterable[str], *, clock=None,
|
||||
execution_scope: str = "production"):
|
||||
self.repo = repository
|
||||
self.context = context
|
||||
self.permissions = frozenset(permissions)
|
||||
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
||||
self.execution_scope = execution_scope
|
||||
|
||||
def _now(self) -> datetime:
|
||||
now = self.clock()
|
||||
@@ -195,6 +197,7 @@ class OperationalTaskService:
|
||||
now = self._now()
|
||||
task_id = uuid4()
|
||||
task = OperationalTaskSummary(
|
||||
execution_scope=self.execution_scope,
|
||||
project_id=self.context.project_id,
|
||||
task_id=task_id,
|
||||
task_type=command.task_type,
|
||||
@@ -489,6 +492,7 @@ class OperationalTaskService:
|
||||
task_type: str | None = None, task_created_from: datetime | None = None,
|
||||
) -> tuple[list[OperationalTaskRun], str | None]:
|
||||
rows = await self.repo.list_runs(
|
||||
execution_scope=self.execution_scope,
|
||||
status=status,
|
||||
limit=limit + 1,
|
||||
cursor_created_at=cursor_created_at,
|
||||
@@ -510,6 +514,9 @@ class OperationalTaskService:
|
||||
scope = f"run:{run_id}:claim"
|
||||
async with self.repo.transaction():
|
||||
await self.repo.require_execution_storage()
|
||||
task = await self._task(task_id)
|
||||
require(task.get("execution_scope") == self.execution_scope,
|
||||
"EXECUTION_SCOPE_MISMATCH", "当前环境不能领取其他执行范围的任务", status=403)
|
||||
replay = await self._request_replay(scope, command.request_id, body)
|
||||
if replay:
|
||||
return OperationalTaskRun.model_validate(replay["run"])
|
||||
@@ -759,6 +766,7 @@ class OperationalTaskService:
|
||||
async with self.repo.transaction():
|
||||
for run in await self.repo.claim_expired_runs(
|
||||
limit, task_type=task_type, task_created_from=task_created_from,
|
||||
execution_scope=self.execution_scope,
|
||||
):
|
||||
# Lease recovery already holds a row lock. Try rather than wait
|
||||
# for the shared run lock to avoid reversing the lock order of
|
||||
@@ -2516,6 +2524,7 @@ class OperationalTaskService:
|
||||
due_at: datetime, now: datetime,
|
||||
) -> OperationalTaskSummary:
|
||||
return OperationalTaskSummary(
|
||||
execution_scope=self.execution_scope,
|
||||
project_id=self.context.project_id, task_id=task_id,
|
||||
task_type="scheduled_flow", workflow=schedule.schedule_key,
|
||||
lifecycle_status="open", status="queued", priority=schedule.priority, revision=1,
|
||||
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||
|
||||
from fastapi import HTTPException
|
||||
from app.core.config import settings
|
||||
from app.auth.permissions import permissions_for_context
|
||||
|
||||
from app.domain.schemas.project_settings import ScadaMonitoringSettings
|
||||
@@ -439,7 +440,8 @@ class ScadaReplayService:
|
||||
alerts = TaskAlertService(TaskAlertRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, internal_producer_id=producer)
|
||||
tasks = OperationalTaskService(OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context))
|
||||
self.context, permissions_for_context(self.context),
|
||||
execution_scope=settings.operational_task_execution_scope)
|
||||
for event in changes:
|
||||
# Period diagnosis owns measurement notifications during manual steps.
|
||||
# Retain rule events as evidence; quality alerts remain independent.
|
||||
@@ -503,6 +505,7 @@ class ScadaReplayService:
|
||||
tasks = OperationalTaskService(
|
||||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context, permissions_for_context(self.context),
|
||||
execution_scope=settings.operational_task_execution_scope,
|
||||
)
|
||||
await tasks.advance_replay_investigation(UUID(incident["task_id"]), {
|
||||
"replay_id": document["replay_id"], "knowledge_cutoff": monitoring["knowledge_cutoff"],
|
||||
@@ -520,6 +523,7 @@ class ScadaReplayService:
|
||||
OperationalTaskRepository(self.repo.connection, self.context.project_id),
|
||||
self.context,
|
||||
permissions_for_context(self.context),
|
||||
execution_scope=settings.operational_task_execution_scope,
|
||||
)
|
||||
external = incident["external_id"]
|
||||
pressure = monitoring["pressure"]
|
||||
|
||||
Reference in New Issue
Block a user