OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tasks): isolate worker execution scopes

Browse Source 
Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
This commit is contained in:
jiang
2026-11-08 01:54:48 +08:00
parent 2a7b96f813
commit 33df25f22b
14 changed files with 156 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_operational_tasks.py
+58 -2
View File
@@ -235,6 +235,7 @@ async def test_targeted_notification_cannot_be_acknowledged_without_permission()
def _task_document(context, now, *, lease=None):
task_id = uuid4()
return OperationalTaskSummary(
execution_scope="production",
project_id=context.project_id,
task_id=task_id,
task_type="event_investigation",
@@ -436,16 +437,39 @@ def _active_run(task, now):
@pytest.mark.anyio
async def test_run_claim_persists_agent_instance_ownership():
@pytest.mark.parametrize("task_scope,worker_scope,allowed", [
(None, "production", False),
(None, "development", False),
("production", "production", True),
("production", "development", False),
("development", "production", False),
("development", "development", True),
])
async def test_run_claim_persists_agent_instance_ownership(task_scope, worker_scope, allowed):
now = datetime(2026, 9, 23, 8, tzinfo=timezone.utc)
context = _context()
task = _task_document(context, now, lease=None)
task.update(status="queued", agent_session_id=None, agent_lease_expires_at=None)
if task_scope is None:
task.pop("execution_scope", None)
else:
task["execution_scope"] = task_scope
run = _active_run(task, now)
run.update(status="queued", revision=1, agent_session_id=None,
lease_expires_at=None, heartbeat_at=None, started_at=None)
repository = RunTaskRepository(task, run)
service = OperationalTaskService(repository, context, set(), clock=lambda: now)
service = OperationalTaskService(repository, context, set(), clock=lambda: now,
execution_scope=worker_scope)
if not allowed:
with pytest.raises(DomainError) as failure:
await service.claim_run(task["task_id"], run["run_id"], AgentRunCommand(
request_id=run["execution_request_id"], expected_revision=1,
source_session_id="session-1", agent_instance_id="agent-a"))
assert failure.value.code == "EXECUTION_SCOPE_MISMATCH"
assert repository.run["status"] == "queued"
assert repository.task["status"] == "queued"
return
claimed = await service.claim_run(
task["task_id"], run["run_id"],
@@ -2371,3 +2395,35 @@ async def test_recovery_resolves_alerts_with_original_facts_and_stable_event_id(
assert command.event_id == uuid5(NAMESPACE_URL, f"recovery:{task_id}:recovery:evidence-1:incident-1")
assert command.source.model_dump(mode="json") == alert["source"]
assert command.title == alert["title"] and command.description == alert["description"]
@pytest.mark.anyio
@pytest.mark.parametrize("scope", ["production", "development"])
async def test_task_creation_persists_server_execution_scope(monkeypatch, scope):
from app.domain.schemas.operational_task import OperationalTaskCreate
now = datetime(2026, 11, 8, tzinfo=timezone.utc)
context = _context()
document = _task_document(context, now)
repository = TaskRepository(document)
repository.insert_task = AsyncMock()
repository.insert_run = AsyncMock()
service = OperationalTaskService(repository, context, set(), clock=lambda: now,
execution_scope=scope)
monkeypatch.setattr(service, "_task_event", AsyncMock())
monkeypatch.setattr(service, "_run_event", AsyncMock())
monkeypatch.setattr(service, "detail", AsyncMock())
command = OperationalTaskCreate(request_id=uuid4(), **{
key: document[key] for key in ("task_type", "workflow", "title", "summary", "source", "trigger")
})
await service.create(command)
assert repository.insert_task.await_args.args[0]["execution_scope"] == scope
def test_task_scope_is_required_in_persisted_contract():
task = _task_document(_context(), datetime(2026, 11, 8, tzinfo=timezone.utc))
task.pop("execution_scope")
with pytest.raises(ValidationError) as error:
OperationalTaskSummary.model_validate(task)
assert any(item["loc"] == ("execution_scope",) and item["type"] == "missing"
for item in error.value.errors())
tests/unit/test_scada_replay_service.py
+7 -7
View File
@@ -579,7 +579,7 @@ def test_suspected_monitoring_creates_burst_report_with_frozen_replay_context(mo
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr(
"app.services.operational_tasks.OperationalTaskService",
lambda *_: SimpleNamespace(create=create_task),
lambda *_, **_kwargs: SimpleNamespace(create=create_task),
)
monitoring = {
"status": "suspected_burst",
@@ -618,7 +618,7 @@ def test_monitoring_summary_counts_signals_without_copying_reading_details(monke
_, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_, **_kwargs: SimpleNamespace(create=create_task))
monitoring = incident_monitoring(0)
monitoring["anomaly_signals"] = [
{"device_ids": [f"P{i}"], "detail": f"P{i}=28.14402136077437 m，超出冻结正常同时段范围 [28.3253, 28.8685] m"}
@@ -641,7 +641,7 @@ def test_continuing_burst_reuses_task_across_persisted_periods(monkeypatch):
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
lambda *_: SimpleNamespace(create=create_task, advance_replay_investigation=advance_task))
lambda *_, **_kwargs: SimpleNamespace(create=create_task, advance_replay_investigation=advance_task))
document = {"replay_id": str(uuid4())}
async def run():
@@ -683,7 +683,7 @@ def test_incident_recovers_once_then_allows_new_accident(monkeypatch):
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService",
lambda *_: SimpleNamespace(create=create_task))
lambda *_, **_kwargs: SimpleNamespace(create=create_task))
document = {"replay_id": str(uuid4())}
async def run():
@@ -784,7 +784,7 @@ def test_combined_monitoring_keeps_rule_evidence_and_quality_notifications(monke
monkeypatch.setattr("app.infra.db.postgresql.task_alerts.TaskAlertRepository", lambda *_: object())
monkeypatch.setattr("app.services.task_alerts.TaskAlertService", lambda *_args, **_kwargs: SimpleNamespace(push=push))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=AsyncMock()))
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_, **_kwargs: SimpleNamespace(create=AsyncMock()))
def event(rule, category=None):
return {"event_id": str(uuid4()), "rule_id": rule, "change": "raised", "occurred_at": START.isoformat(),
"detail": "检测信号", "category": category, "task_id": None}
@@ -812,7 +812,7 @@ def test_failed_incident_publication_rolls_back_state_and_retries_same_task_requ
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.services.scada_replay.permissions_for_context", lambda _: {"task.write"})
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_, **_kwargs: SimpleNamespace(create=create_task))
def evaluate(_doc, _rows, *, period_start, period_end, batch_sequence):
return {**incident_monitoring(0), "period_start": period_start.isoformat(), "period_end": period_end.isoformat(),
"knowledge_cutoff": period_end.isoformat(), "batch_sequence": batch_sequence}
@@ -843,7 +843,7 @@ def test_any_single_measurement_anomaly_enters_diagnosis_below_burst_threshold(m
_, service, *_ = setup(monkeypatch)
create_task = AsyncMock(return_value=SimpleNamespace(task_id=uuid4()))
monkeypatch.setattr("app.infra.db.postgresql.operational_tasks.OperationalTaskRepository", lambda *_: object())
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_: SimpleNamespace(create=create_task))
monkeypatch.setattr("app.services.operational_tasks.OperationalTaskService", lambda *_, **_kwargs: SimpleNamespace(create=create_task))
monitoring = incident_monitoring(0, status="attention")
for key in ("pressure", "outlet_flow", "burst_detection"):
monitoring[key]["hit"] = key == indicator