Persist a required Server-owned execution scope and enforce it for listing, claiming, lease recovery and SCADA verification. Backfill existing tasks explicitly through migration 034 so local development workers no longer claim production tasks from the shared database.
2552 lines
130 KiB
Python
2552 lines
130 KiB
Python
"""State machines for durable operational tasks and approved external actions."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Iterable
|
|
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
|
|
|
|
from app.auth.permissions import (
|
|
OPERATIONAL_ACTION_COMMAND_APPROVE,
|
|
OPERATIONAL_ACTION_COMMAND_EXECUTE,
|
|
OPERATIONAL_ACTION_WORK_ORDER_APPROVE,
|
|
OPERATIONAL_ACTION_WORK_ORDER_EXECUTE,
|
|
OPERATIONAL_TASK_EXECUTE,
|
|
OPERATIONAL_TASK_ACCESS_APPROVE,
|
|
OPERATIONAL_TASK_EXTERNAL_SYNC,
|
|
OPERATIONAL_TASK_WRITE,
|
|
SCADA_VIEW,
|
|
SIMULATION_VIEW,
|
|
)
|
|
from app.domain.domain_errors import DomainError, require
|
|
from app.services import task_questions
|
|
from app.domain.schemas.operational_task import (
|
|
ActionDecisionCommand,
|
|
ActionExecuteCommand,
|
|
ActionProposalCommand,
|
|
ActionReconcileCommand,
|
|
WorkOrderDraftUpdateCommand,
|
|
WorkOrderConfirmDispatchCommand,
|
|
TaskAccessRequest,
|
|
TaskAccessRequestCreate,
|
|
TaskAccessRequestDecision,
|
|
AgentArtifactsCommand,
|
|
AgentArtifactsResult,
|
|
AgentTaskContext,
|
|
AgentRunCancelCommand,
|
|
AgentRunCommand,
|
|
AgentRunDispatchFailureCommand,
|
|
AgentRunFailureCommand,
|
|
CreateWorkOrderPayload,
|
|
DispatchDueSchedulesResult,
|
|
EvidenceRecord,
|
|
ExecutionPlanSnapshot,
|
|
ExternalFeedbackCommand,
|
|
ExternalFeedbackResult,
|
|
ExternalSourceWithdrawalCommand,
|
|
HumanApprovalAction,
|
|
HumanEvidenceCommand,
|
|
InvestigationReportRevision,
|
|
InvestigationReportReference,
|
|
InvestigationProgressSnapshot,
|
|
IssueCommandPayload,
|
|
NotificationAcknowledgeCommand,
|
|
OperationalTaskCreate,
|
|
OperationalTaskDetail,
|
|
OperationalTaskNotification,
|
|
OperationalTaskPermissions,
|
|
OperationalTaskSchedule,
|
|
OperationalTaskScheduleCreate,
|
|
OperationalTaskScheduleUpdate,
|
|
OperationalTaskSummary,
|
|
OperationalTaskRetryCommand,
|
|
OperationalTaskRun,
|
|
RecoverExpiredLeasesResult,
|
|
StoredEvidenceRecord,
|
|
TaskPresentationEventCommand,
|
|
TaskPresentationEventResult,
|
|
TaskPresentationCursor,
|
|
TaskPresentationHistoryItem,
|
|
TaskPresentationHistoryPage,
|
|
)
|
|
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
|
|
from app.services.content_digest import digest
|
|
from app.services.operational_presentation import resolve_overlay
|
|
from app.services.operational_recovery import accept_completion, track_work_order
|
|
from app.services.operational_scada_coverage import require_complete_scada_evidence
|
|
|
|
|
|
LEASE_DURATION = timedelta(seconds=60)
|
|
|
|
|
|
def _json(model) -> dict:
|
|
return model.model_dump(mode="json", by_alias=True)
|
|
|
|
|
|
def _task_json(model: OperationalTaskSummary) -> dict:
|
|
return model.model_dump(mode="json", by_alias=True, exclude={"active_run", "latest_run"})
|
|
|
|
|
|
class OperationalTaskService:
|
|
def __init__(self, repository, context, permissions: Iterable[str], *, clock=None,
|
|
execution_scope: str = "production"):
|
|
self.repo = repository
|
|
self.context = context
|
|
self.permissions = frozenset(permissions)
|
|
self.clock = clock or (lambda: datetime.now(timezone.utc))
|
|
self.execution_scope = execution_scope
|
|
|
|
def _now(self) -> datetime:
|
|
now = self.clock()
|
|
require(now.tzinfo is not None, "SCHEMA_INVALID", "Clock must be timezone-aware")
|
|
return now
|
|
|
|
def _actor(self) -> str:
|
|
return f"principal:{self.context.user_id}"
|
|
|
|
@staticmethod
|
|
def _require_run_owner(run: dict, command: AgentRunCommand) -> None:
|
|
require(run.get("agent_session_id") == command.source_session_id,
|
|
"SESSION_MISMATCH", "Agent session does not own this run", status=409)
|
|
owner = run.get("agent_instance_id")
|
|
if owner is not None:
|
|
require(owner == command.agent_instance_id,
|
|
"AGENT_INSTANCE_MISMATCH", "Agent instance does not own this run", status=409)
|
|
|
|
@property
|
|
def _supports_runs(self) -> bool:
|
|
return hasattr(self.repo, "get_active_run") and hasattr(self.repo, "update_run")
|
|
|
|
async def _active_run(self, task: dict) -> dict | None:
|
|
if self._supports_runs:
|
|
return await self.repo.get_active_run(str(task["task_id"]))
|
|
if task.get("status") not in {"queued", "running"}:
|
|
return None
|
|
now = self._now()
|
|
return _json(OperationalTaskRun(
|
|
run_id=uuid5(NAMESPACE_URL, f"legacy-run:{task['task_id']}:{task['execution_generation']}"),
|
|
task_id=task["task_id"],
|
|
generation=task["execution_generation"],
|
|
execution_request_id=task["execution_request_id"],
|
|
status="running" if task["status"] == "running" else "queued",
|
|
revision=task["revision"],
|
|
agent_session_id=task.get("agent_session_id"),
|
|
lease_expires_at=task.get("agent_lease_expires_at"),
|
|
heartbeat_at=task.get("updated_at") if task["status"] == "running" else None,
|
|
started_at=task.get("created_at") if task["status"] == "running" else None,
|
|
dispatch_attempts=0,
|
|
next_dispatch_at=task.get("created_at") or now,
|
|
created_at=task.get("created_at") or now,
|
|
updated_at=task.get("updated_at") or now,
|
|
))
|
|
|
|
async def _append_event(self, **kwargs) -> int | None:
|
|
append_event = getattr(self.repo, "append_event", None)
|
|
return await append_event(**kwargs) if append_event is not None else None
|
|
|
|
async def _list_access_requests(self, task_id: UUID | str) -> list[dict]:
|
|
list_access_requests = getattr(self.repo, "list_access_requests", None)
|
|
if list_access_requests is None:
|
|
return []
|
|
return await list_access_requests(str(task_id))
|
|
|
|
def _permission_view(self) -> OperationalTaskPermissions:
|
|
permissions = self.permissions
|
|
return OperationalTaskPermissions(
|
|
can_publish_artifacts=OPERATIONAL_TASK_EXECUTE in permissions,
|
|
can_answer_questions=OPERATIONAL_TASK_WRITE in permissions,
|
|
can_propose_actions=OPERATIONAL_TASK_WRITE in permissions,
|
|
can_sync_external_feedback=OPERATIONAL_TASK_EXTERNAL_SYNC in permissions,
|
|
can_decide_work_orders=OPERATIONAL_ACTION_WORK_ORDER_APPROVE in permissions,
|
|
can_execute_work_orders=OPERATIONAL_ACTION_WORK_ORDER_EXECUTE in permissions,
|
|
can_decide_commands=OPERATIONAL_ACTION_COMMAND_APPROVE in permissions,
|
|
can_execute_commands=OPERATIONAL_ACTION_COMMAND_EXECUTE in permissions,
|
|
can_decide_access=OPERATIONAL_TASK_ACCESS_APPROVE in permissions,
|
|
)
|
|
|
|
async def create(self, command: OperationalTaskCreate) -> OperationalTaskDetail:
|
|
if command.source.replay_context:
|
|
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
|
replay_context = command.source.replay_context
|
|
replay_doc = await ScadaReplayRepository(self.repo.connection, self.context.project_id).get(replay_context.replay_id)
|
|
require(replay_doc.get("delivered_until") is not None
|
|
and replay_context.knowledge_cutoff <= datetime.fromisoformat(replay_doc["delivered_until"])
|
|
and replay_context.batch_sequence <= replay_doc["batch_sequence"],
|
|
"REPLAY_SCOPE_INVALID", "回放调查不能使用尚未读取的批次", status=422)
|
|
body = _json(command)
|
|
request_id = str(command.request_id)
|
|
scope = "task:create"
|
|
async with self.repo.transaction():
|
|
await self.repo.lock(f"request:{scope}:{request_id}")
|
|
prior = await self.repo.get_request(scope, request_id)
|
|
if prior:
|
|
self._require_same_request(prior, body)
|
|
return await self.detail(UUID(prior["response"]["task_id"]))
|
|
if command.source.external_id:
|
|
await self.repo.lock(
|
|
f"external-source:{command.source.kind}:{command.source.external_id}"
|
|
)
|
|
existing = await self.repo.get_task_by_external_source(
|
|
command.source.kind, command.source.external_id
|
|
)
|
|
if existing:
|
|
await self.repo.save_request(
|
|
scope, request_id, digest(body), {"task_id": existing["task_id"]}
|
|
)
|
|
return await self.detail(UUID(existing["task_id"]))
|
|
now = self._now()
|
|
task_id = uuid4()
|
|
task = OperationalTaskSummary(
|
|
execution_scope=self.execution_scope,
|
|
project_id=self.context.project_id,
|
|
task_id=task_id,
|
|
task_type=command.task_type,
|
|
workflow=command.workflow,
|
|
lifecycle_status="open",
|
|
status="queued",
|
|
priority=command.priority,
|
|
revision=1,
|
|
execution_generation=1,
|
|
execution_request_id=self._execution_request_id(task_id, 1),
|
|
title=command.title,
|
|
summary=command.summary,
|
|
source=command.source,
|
|
trigger=command.trigger,
|
|
scheduled_for=command.scheduled_for,
|
|
agent_session_id=command.agent_session_id,
|
|
agent_lease_expires_at=None,
|
|
current_plan_revision=None,
|
|
current_report_revision=None,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
task_document = _task_json(task)
|
|
await self.repo.insert_task(task_document, request_id)
|
|
if command.task_type != "external_work_order":
|
|
run = self._new_run(task_id, 1, now)
|
|
await self.repo.insert_run(_json(run))
|
|
await self._run_event(run, now, queued=True)
|
|
await self.repo.save_request(
|
|
scope, request_id, digest(body), {"task_id": str(task_id)}
|
|
)
|
|
await self._task_event(task_document, now)
|
|
return await self.detail(task_id)
|
|
|
|
async def advance_replay_investigation(self, task_id: UUID, replay_context: dict, request_id: UUID) -> None:
|
|
"""A recovery is new evidence, coalesced through the existing run lifecycle."""
|
|
scope = f"task:{task_id}:replay-evidence"
|
|
async with self.repo.transaction():
|
|
if await self._request_replay(scope, request_id, replay_context):
|
|
return
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
prior = task["source"].get("replay_context")
|
|
require(prior and prior["replay_id"] == replay_context["replay_id"], "REPLAY_SCOPE_INVALID",
|
|
"Replay evidence belongs to another investigation", status=409)
|
|
require(datetime.fromisoformat(prior["knowledge_cutoff"]) <= datetime.fromisoformat(replay_context["knowledge_cutoff"]),
|
|
"REPLAY_SCOPE_INVALID", "Replay evidence cannot move backwards", status=409)
|
|
task["source"]["replay_context"] = replay_context
|
|
task["source"]["external_revision"] = task["source"].get("external_revision", 1) + 1
|
|
now = self._now()
|
|
await self._requeue_task(task, now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
await self._save_request(scope, request_id, replay_context, {"task_id": str(task_id)})
|
|
|
|
async def list(
|
|
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
|
|
peripheral_work_order_id: UUID | None = None,
|
|
) -> tuple[list[OperationalTaskSummary], int | None]:
|
|
rows = await self.repo.list_tasks(
|
|
task_type=task_type, status=status, limit=limit + 1, offset=offset,
|
|
**({"peripheral_work_order_id": str(peripheral_work_order_id)}
|
|
if peripheral_work_order_id is not None else {}),
|
|
)
|
|
items: list[OperationalTaskSummary] = []
|
|
for row in rows[:limit]:
|
|
active_run = row.get("active_run") if "active_run" in row else await self._active_run(row)
|
|
items.append(OperationalTaskSummary.model_validate({
|
|
**row,
|
|
"active_run": active_run,
|
|
}))
|
|
return items, offset + limit if len(rows) > limit else None
|
|
|
|
async def detail(self, task_id: UUID) -> OperationalTaskDetail:
|
|
task = await self.repo.get_task(str(task_id))
|
|
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
|
|
plan = await self.repo.get_latest_plan(str(task_id))
|
|
report = await self.repo.get_latest_report(str(task_id))
|
|
if report is not None:
|
|
report = await self._report_with_progress_diagram(task_id, report)
|
|
evidence = [
|
|
self._public_evidence(document)
|
|
for document in await self.repo.list_evidence(str(task_id))
|
|
]
|
|
actions = await self.repo.list_actions(str(task_id))
|
|
access_requests = await self._list_access_requests(task_id)
|
|
active_run = await self._active_run(task)
|
|
latest_run = (await self.repo.get_latest_run(str(task_id))
|
|
if hasattr(self.repo, "get_latest_run") else active_run)
|
|
progress = await self.repo.get_latest_progress(str(task_id), task["execution_generation"])
|
|
work_order_snapshot = None
|
|
if task["task_type"] == "external_work_order" and task["source"].get("external_id"):
|
|
feedback = await self.repo.get_feedback(
|
|
str(task_id), task["source"]["external_id"]
|
|
)
|
|
if feedback:
|
|
work_order_snapshot = feedback["document"].get("work_order_snapshot")
|
|
return OperationalTaskDetail(
|
|
**{key: value for key, value in task.items() if key not in {"active_run", "latest_run"}},
|
|
active_run=active_run,
|
|
latest_run=latest_run,
|
|
latest_plan=plan,
|
|
latest_report=report,
|
|
latest_progress=progress,
|
|
evidence=evidence,
|
|
actions=actions,
|
|
access_requests=access_requests,
|
|
questions=await task_questions.list_questions(self, task_id),
|
|
permissions=self._permission_view(),
|
|
work_order_snapshot=work_order_snapshot,
|
|
)
|
|
|
|
async def _report_with_progress_diagram(
|
|
self, task_id: UUID, value: dict | InvestigationReportRevision
|
|
) -> InvestigationReportRevision:
|
|
"""Archive an eligible recorded graph, without inventing completion states.
|
|
|
|
Also projects older reports that predate graph archival. Read paths never
|
|
rewrite their stored document or borrow a newer round's progress.
|
|
"""
|
|
snapshot = InvestigationReportRevision.model_validate(value)
|
|
if (snapshot.report.diagram is not None or snapshot.report.stage != "final"
|
|
or snapshot.execution_generation is None):
|
|
return snapshot
|
|
stored = await self.repo.get_latest_progress(str(task_id), snapshot.execution_generation)
|
|
if stored is None:
|
|
return snapshot
|
|
progress = InvestigationProgressSnapshot.model_validate(stored)
|
|
diagram = progress.progress.diagram
|
|
if (diagram is None or progress.execution_generation != snapshot.execution_generation
|
|
or progress.source_session_id != snapshot.source_session_id
|
|
or progress.published_at > snapshot.published_at
|
|
or any(not set(node.evidence_refs) <= set(snapshot.report.evidence_refs)
|
|
for node in diagram.node_refs)):
|
|
return snapshot
|
|
return snapshot.model_copy(update={
|
|
"report": snapshot.report.model_copy(update={"diagram": diagram}),
|
|
})
|
|
|
|
async def report(self, task_id: UUID, revision: int) -> InvestigationReportRevision:
|
|
await self._task(task_id)
|
|
value = await self.repo.get_report(str(task_id), revision)
|
|
require(value is not None, "REPORT_NOT_FOUND", "Report revision not found", status=404)
|
|
return await self._report_with_progress_diagram(task_id, value)
|
|
|
|
async def list_reports(
|
|
self, task_id: UUID, limit: int, offset: int
|
|
) -> tuple[list[InvestigationReportRevision], int | None]:
|
|
await self._task(task_id)
|
|
rows = await self.repo.list_reports(str(task_id), limit + 1, offset)
|
|
return [await self._report_with_progress_diagram(task_id, row) for row in rows[:limit]], (
|
|
offset + limit if len(rows) > limit else None
|
|
)
|
|
|
|
async def agent_context(self, task_id: UUID, report_revision: int | None = None) -> AgentTaskContext:
|
|
scopes: list[str] = []
|
|
if SIMULATION_VIEW in self.permissions:
|
|
scopes.extend([
|
|
"workbench_runtime", "peripheral_points", "peripheral_work_orders",
|
|
"peripheral_observations", "peripheral_test_data",
|
|
])
|
|
if SCADA_VIEW in self.permissions:
|
|
scopes.extend(["scada_devices", "scada_readings"])
|
|
for request in await self._list_access_requests(task_id):
|
|
if request["status"] == "approved":
|
|
scopes.extend(request["approved_scopes"])
|
|
return AgentTaskContext(
|
|
task=await self.detail(task_id),
|
|
approved_data_scopes=list(dict.fromkeys(scopes)),
|
|
selected_report=await self.report(task_id, report_revision) if report_revision is not None else None,
|
|
)
|
|
|
|
async def replay_context(self, task_id: UUID) -> dict | None:
|
|
return (await self._task(task_id)).get("source", {}).get("replay_context")
|
|
|
|
async def can_read_source(self, task_id: UUID, source: TaskDataSource) -> bool:
|
|
task = await self._task(task_id)
|
|
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
|
|
if permission in self.permissions:
|
|
return True
|
|
active_run = await self._active_run(task)
|
|
if active_run is None or active_run.get("status") not in {"claimed", "running"}:
|
|
return False
|
|
if not active_run.get("agent_session_id"):
|
|
return False
|
|
self._require_active_run_lease(active_run, self._now())
|
|
return any(
|
|
row["status"] == "approved" and source in row["approved_scopes"]
|
|
for row in await self._list_access_requests(task_id)
|
|
)
|
|
|
|
async def request_access(self, task_id: UUID, command: TaskAccessRequestCreate) -> TaskAccessRequest:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:access:request"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return TaskAccessRequest.model_validate(replay["access_request"])
|
|
availability = getattr(self.repo, "access_requests_available", None)
|
|
require(
|
|
availability is None or await availability(),
|
|
"DEPENDENCY_UNAVAILABLE",
|
|
"Task data access requests are not enabled for this project database",
|
|
status=503,
|
|
)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
active_run = await self._active_run(task)
|
|
require(active_run is not None and active_run.get("agent_session_id") == command.source_session_id,
|
|
"SESSION_MISMATCH", "Current Agent session must own the task run", status=409)
|
|
self._require_active_run_lease(active_run, self._now())
|
|
pending = [
|
|
item for item in await self._list_access_requests(task_id)
|
|
if item["status"] == "pending"
|
|
]
|
|
require(
|
|
not pending,
|
|
"ACCESS_REQUEST_PENDING",
|
|
"Resolve the current task data access request before creating another",
|
|
refs=[item["access_request_id"] for item in pending],
|
|
status=409,
|
|
)
|
|
now = self._now()
|
|
request = TaskAccessRequest(
|
|
access_request_id=uuid4(), task_id=task_id, node_id=command.node_id,
|
|
scopes=list(dict.fromkeys(command.scopes)), reason=command.reason,
|
|
status="pending", approved_scopes=[], version=1,
|
|
requested_by=self._actor(), requested_at=now,
|
|
)
|
|
await self.repo.insert_access_request(str(task_id), _json(request))
|
|
task.update(revision=task["revision"] + 1, updated_at=now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
await self._notify(task_id, "access_approval_pending", task["title"], command.reason, now,
|
|
required_permission=OPERATIONAL_TASK_ACCESS_APPROVE,
|
|
subject_type="access_request", subject_id=request.access_request_id)
|
|
await self._save_request(scope, command.request_id, body, {"access_request": _json(request)})
|
|
return request
|
|
|
|
async def decide_access(self, task_id: UUID, access_request_id: UUID,
|
|
command: TaskAccessRequestDecision) -> TaskAccessRequest:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:access:{access_request_id}:decision"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return TaskAccessRequest.model_validate(replay["access_request"])
|
|
require(OPERATIONAL_TASK_ACCESS_APPROVE in self.permissions, "PERMISSION_DENIED",
|
|
"Current user cannot approve task data access", status=403)
|
|
await self.repo.lock(f"access:{access_request_id}")
|
|
raw = await self.repo.get_access_request(str(task_id), str(access_request_id))
|
|
require(raw is not None, "ACCESS_REQUEST_NOT_FOUND", "Access request not found", status=404)
|
|
request = TaskAccessRequest.model_validate(raw)
|
|
require(request.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Access request version changed", status=409)
|
|
require(request.status == "pending", "INVALID_TRANSITION",
|
|
"Access request has already been decided", status=409)
|
|
approved = set(command.approved_scopes)
|
|
require(approved <= set(request.scopes), "ACCESS_SCOPE_INVALID",
|
|
"Approved scopes must be requested", status=422)
|
|
require((command.decision == "approved") == bool(approved), "ACCESS_SCOPE_INVALID",
|
|
"Approval needs at least one scope; rejection grants none", status=422)
|
|
for source in approved:
|
|
permission = SCADA_VIEW if source.startswith("scada_") else SIMULATION_VIEW
|
|
require(permission in self.permissions, "PERMISSION_DENIED",
|
|
"Approver cannot read a requested source", status=403)
|
|
now = self._now()
|
|
updated = request.model_copy(update={
|
|
"version": request.version + 1, "status": command.decision,
|
|
"approved_scopes": sorted(approved), "decided_by": self._actor(),
|
|
"decided_at": now, "comment": command.comment,
|
|
})
|
|
await self.repo.update_access_request(str(task_id), _json(updated), request.version)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["access_approval_pending"],
|
|
subject_type="access_request",
|
|
subject_id=access_request_id,
|
|
now=now,
|
|
reason=f"access_request_{command.decision}",
|
|
)
|
|
task = await self._task(task_id)
|
|
await self._requeue_task(task, now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
await self._save_request(scope, command.request_id, body, {"access_request": _json(updated)})
|
|
return updated
|
|
|
|
async def list_runs(
|
|
self, *, status: str | None, limit: int,
|
|
cursor_created_at=None, cursor_run_id=None,
|
|
task_type: str | None = None, task_created_from: datetime | None = None,
|
|
) -> tuple[list[OperationalTaskRun], str | None]:
|
|
rows = await self.repo.list_runs(
|
|
execution_scope=self.execution_scope,
|
|
status=status,
|
|
limit=limit + 1,
|
|
cursor_created_at=cursor_created_at,
|
|
cursor_run_id=cursor_run_id,
|
|
task_type=task_type,
|
|
task_created_from=task_created_from,
|
|
)
|
|
items = [OperationalTaskRun.model_validate(row) for row in rows[:limit]]
|
|
next_cursor = None
|
|
if len(rows) > limit and items:
|
|
last = items[-1]
|
|
next_cursor = f"{last.created_at.isoformat()}|{last.run_id}"
|
|
return items, next_cursor
|
|
|
|
async def claim_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunCommand
|
|
) -> OperationalTaskRun:
|
|
body = _json(command)
|
|
scope = f"run:{run_id}:claim"
|
|
async with self.repo.transaction():
|
|
await self.repo.require_execution_storage()
|
|
task = await self._task(task_id)
|
|
require(task.get("execution_scope") == self.execution_scope,
|
|
"EXECUTION_SCOPE_MISMATCH", "当前环境不能领取其他执行范围的任务", status=403)
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return OperationalTaskRun.model_validate(replay["run"])
|
|
await self.repo.lock(f"run:{run_id}")
|
|
run = await self._run(task_id, run_id)
|
|
require(
|
|
str(command.request_id) == str(run["execution_request_id"]),
|
|
"EXECUTION_REQUEST_MISMATCH",
|
|
"Claim request does not match this run",
|
|
status=409,
|
|
)
|
|
require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
|
|
"Run revision changed", status=409)
|
|
require(run["status"] == "queued", "INVALID_TRANSITION",
|
|
"Only queued runs can be claimed", status=409)
|
|
now = self._now()
|
|
expected = run["revision"]
|
|
run.update(
|
|
status="running",
|
|
revision=expected + 1,
|
|
agent_session_id=command.source_session_id,
|
|
agent_instance_id=command.agent_instance_id,
|
|
lease_expires_at=now + LEASE_DURATION,
|
|
heartbeat_at=now,
|
|
started_at=run.get("started_at") or now,
|
|
updated_at=now,
|
|
)
|
|
run_model = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(run_model), expected)
|
|
task = await self._task(task_id)
|
|
task.update(
|
|
status="running",
|
|
agent_session_id=command.source_session_id,
|
|
agent_lease_expires_at=now + LEASE_DURATION,
|
|
revision=task["revision"] + 1,
|
|
updated_at=now,
|
|
)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._run_event(run_model, now)
|
|
await self._task_event(task, now)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["run_delivery_degraded"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="run_claimed",
|
|
)
|
|
await self._save_request(scope, command.request_id, body, {"run": _json(run_model)})
|
|
return run_model
|
|
|
|
async def heartbeat_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunCommand
|
|
) -> OperationalTaskRun:
|
|
body = _json(command)
|
|
scope = f"run:{run_id}:heartbeat"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return OperationalTaskRun.model_validate(replay["run"])
|
|
await self.repo.lock(f"run:{run_id}")
|
|
run = await self._run(task_id, run_id)
|
|
require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
|
|
"Run revision changed", status=409)
|
|
require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
|
|
"Only an active run has a lease", status=409)
|
|
self._require_run_owner(run, command)
|
|
now = self._now()
|
|
self._require_active_run_lease(run, now)
|
|
expected = run["revision"]
|
|
run.update(
|
|
status="running",
|
|
revision=expected + 1,
|
|
heartbeat_at=now,
|
|
lease_expires_at=now + LEASE_DURATION,
|
|
updated_at=now,
|
|
)
|
|
run_model = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(run_model), expected)
|
|
# Deliberately no task update and no browser event: heartbeats are
|
|
# lease maintenance, not business-state changes.
|
|
await self._save_request(scope, command.request_id, body, {"run": _json(run_model)})
|
|
return run_model
|
|
|
|
async def complete_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunCommand
|
|
) -> OperationalTaskDetail:
|
|
return await self._finish_run(task_id, run_id, command, outcome="succeeded")
|
|
|
|
async def task_runs(self, task_id: UUID, limit: int, offset: int) -> list[OperationalTaskRun]:
|
|
await self._task(task_id)
|
|
return [OperationalTaskRun.model_validate(row)
|
|
for row in await self.repo.list_task_runs(str(task_id), limit, offset)]
|
|
|
|
async def task_run(self, task_id: UUID, run_id: UUID) -> OperationalTaskRun:
|
|
return OperationalTaskRun.model_validate(await self._run(task_id, run_id))
|
|
|
|
async def _fail_run_messages(self, run_id, reason: str, now: datetime) -> None:
|
|
if hasattr(self.repo, "fail_run_messages"):
|
|
await self.repo.fail_run_messages(str(run_id), reason, now)
|
|
|
|
async def fail_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunFailureCommand
|
|
) -> OperationalTaskDetail:
|
|
return await self._finish_run(
|
|
task_id, run_id, command, outcome="failed",
|
|
failure_code=command.failure_code, failure_detail=command.failure_detail,
|
|
)
|
|
|
|
async def cancel_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunCancelCommand
|
|
) -> OperationalTaskDetail:
|
|
body = _json(command)
|
|
scope = f"run:{run_id}:cancel"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return await self.detail(task_id)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"run:{run_id}")
|
|
run = await self._run(task_id, run_id)
|
|
require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
|
|
"Run revision changed", status=409)
|
|
require(run["status"] in {"queued", "claimed", "running"},
|
|
"INVALID_TRANSITION", "Run is already terminal", status=409)
|
|
now = self._now()
|
|
await task_questions.cancel_pending(self, task_id, now)
|
|
expected = run["revision"]
|
|
run.update(
|
|
status="cancelled", revision=expected + 1, finished_at=now,
|
|
lease_expires_at=None, failure_code="CANCELLED",
|
|
failure_detail=command.reason, updated_at=now,
|
|
)
|
|
run_model = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(run_model), expected)
|
|
await self._fail_run_messages(run_id, "Run cancelled", now)
|
|
task = await self._task(task_id)
|
|
task.update(
|
|
lifecycle_status="cancelled", status="cancelled",
|
|
revision=task["revision"] + 1, agent_session_id=None,
|
|
agent_lease_expires_at=None, updated_at=now,
|
|
)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._run_event(run_model, now)
|
|
await self._task_event(task, now)
|
|
await self._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
|
|
return await self.detail(task_id)
|
|
|
|
async def retry_task(
|
|
self, task_id: UUID, command: OperationalTaskRetryCommand
|
|
) -> OperationalTaskDetail:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:retry"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return await self.detail(task_id)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
require(task["task_type"] != "external_work_order", "INVALID_TRANSITION",
|
|
"External work orders do not have Agent runs", status=409)
|
|
require(await self.repo.get_active_run(str(task_id)) is None,
|
|
"ACTIVE_RUN_EXISTS", "Task already has an active run", status=409)
|
|
now = self._now()
|
|
await task_questions.cancel_pending(self, task_id, now)
|
|
generation = task["execution_generation"] + 1
|
|
run = self._new_run(task_id, generation, now)
|
|
await self.repo.insert_run(_json(run))
|
|
task.update(
|
|
lifecycle_status="open", status="queued",
|
|
revision=task["revision"] + 1,
|
|
execution_generation=generation,
|
|
execution_request_id=str(run.execution_request_id),
|
|
agent_session_id=None, agent_lease_expires_at=None, updated_at=now,
|
|
)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._run_event(run, now, queued=True)
|
|
await self._task_event(task, now)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["task_failed", "run_delivery_degraded"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="task_retried",
|
|
)
|
|
await self._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
|
|
return await self.detail(task_id)
|
|
|
|
async def record_dispatch_failure(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunDispatchFailureCommand
|
|
) -> OperationalTaskRun:
|
|
body = _json(command)
|
|
scope = f"run:{run_id}:dispatch-failure"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return OperationalTaskRun.model_validate(replay["run"])
|
|
await self.repo.lock(f"run:{run_id}")
|
|
run = await self._run(task_id, run_id)
|
|
require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
|
|
"Run revision changed", status=409)
|
|
require(run["status"] == "queued", "INVALID_TRANSITION",
|
|
"Only queued runs accept dispatch failures", status=409)
|
|
now = self._now()
|
|
expected = run["revision"]
|
|
attempts = run["dispatch_attempts"] + 1
|
|
retry_delay = min(300, 5 * (2 ** min(attempts - 1, 6)))
|
|
run.update(
|
|
revision=expected + 1,
|
|
dispatch_attempts=attempts,
|
|
failure_code=command.failure_code,
|
|
failure_detail=command.failure_detail,
|
|
next_dispatch_at=now + timedelta(seconds=retry_delay),
|
|
updated_at=now,
|
|
)
|
|
if not command.retryable:
|
|
run.update(status="failed", finished_at=now, next_dispatch_at=now)
|
|
run_model = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(run_model), expected)
|
|
await self._run_event(run_model, now, queued=command.retryable)
|
|
if attempts == 10:
|
|
task = await self._task(task_id)
|
|
await self._notify(
|
|
task_id, "run_delivery_degraded", task["title"],
|
|
"Agent 执行投递连续失败 10 次,请检查服务连接", now,
|
|
)
|
|
if not command.retryable:
|
|
await self._fail_run_messages(run_id, "Run dispatch failed", now)
|
|
task = await self._task(task_id)
|
|
task.update(
|
|
lifecycle_status="failed", status="failed",
|
|
revision=task["revision"] + 1, updated_at=now,
|
|
)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task))
|
|
)
|
|
await self._task_event(task, now)
|
|
await self._save_request(scope, command.request_id, body, {"run": _json(run_model)})
|
|
return run_model
|
|
|
|
async def recover_expired_leases(
|
|
self, limit: int, *, task_type: str | None = None, task_created_from: datetime | None = None,
|
|
) -> RecoverExpiredLeasesResult:
|
|
now = self._now()
|
|
recovered: list[UUID] = []
|
|
async with self.repo.transaction():
|
|
for run in await self.repo.claim_expired_runs(
|
|
limit, task_type=task_type, task_created_from=task_created_from,
|
|
execution_scope=self.execution_scope,
|
|
):
|
|
# Lease recovery already holds a row lock. Try rather than wait
|
|
# for the shared run lock to avoid reversing the lock order of
|
|
# message/finish transactions; a busy run is checked next tick.
|
|
if hasattr(self.repo, "try_lock") and not await self.repo.try_lock(f"run:{run['run_id']}"):
|
|
continue
|
|
task_id = UUID(str(run["task_id"]))
|
|
expected = run["revision"]
|
|
run.update(
|
|
status="timed_out", revision=expected + 1, finished_at=now,
|
|
lease_expires_at=None, failure_code="LEASE_TIMEOUT",
|
|
failure_detail="Agent run lease expired", updated_at=now,
|
|
)
|
|
timed_out = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(timed_out), expected)
|
|
await self._fail_run_messages(run["run_id"], "Run lease expired", now)
|
|
await self._run_event(timed_out, now)
|
|
task = await self._task(task_id)
|
|
timeout_count = await self.repo.consecutive_timed_out_runs(str(task_id))
|
|
task.update(
|
|
revision=task["revision"] + 1, agent_session_id=None,
|
|
agent_lease_expires_at=None, updated_at=now,
|
|
)
|
|
if timeout_count >= 3:
|
|
task.update(lifecycle_status="failed", status="failed")
|
|
await self._notify(
|
|
task_id, "task_failed", task["title"],
|
|
"Agent 连续三次执行超时,任务已停止自动重试", now,
|
|
)
|
|
else:
|
|
generation = task["execution_generation"] + 1
|
|
next_run = self._new_run(task_id, generation, now)
|
|
await self.repo.insert_run(_json(next_run))
|
|
task.update(
|
|
lifecycle_status="open", status="queued",
|
|
execution_generation=generation,
|
|
execution_request_id=str(next_run.execution_request_id),
|
|
)
|
|
await self._run_event(next_run, now, queued=True)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
recovered.append(task_id)
|
|
return RecoverExpiredLeasesResult(task_ids=recovered, recovered_at=now)
|
|
|
|
async def publish_artifacts(
|
|
self, task_id: UUID, command: AgentArtifactsCommand
|
|
) -> AgentArtifactsResult:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:artifacts"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
detail = await self.detail(task_id)
|
|
return AgentArtifactsResult(task=detail, **replay)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
active_run = await self._active_run(task)
|
|
require(active_run is not None, "RUN_NOT_FOUND",
|
|
"Task has no active Agent run", status=409)
|
|
if command.run_id is not None:
|
|
require(str(command.run_id) == str(active_run["run_id"]),
|
|
"EXECUTION_GENERATION_MISMATCH",
|
|
"Artifacts target a stale run", status=409)
|
|
require(active_run.get("agent_session_id") == command.source_session_id,
|
|
"SESSION_MISMATCH", "Artifacts must come from the claimed Agent session",
|
|
status=409)
|
|
require(active_run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
|
|
"Only a running run accepts Agent artifacts", status=409)
|
|
now = self._now()
|
|
self._require_active_run_lease(active_run, now)
|
|
if command.report and (
|
|
command.report.stage == "final" or command.report.outcome == "insufficient_data"
|
|
):
|
|
await self._require_no_pending_calculations(task_id, active_run)
|
|
previous_report = await self.repo.get_latest_report(str(task_id))
|
|
require(not previous_report or previous_report.get("execution_generation") != task["execution_generation"]
|
|
or previous_report["report"].get("outcome") != "insufficient_data",
|
|
"INVESTIGATION_ENDED", "Insufficient-data investigation requires a new run", status=409)
|
|
known_evidence = {item["evidence_id"] for item in await self.repo.list_evidence(str(task_id))}
|
|
for item in command.evidence:
|
|
document = _json(EvidenceRecord(
|
|
**_json(item), recorded_at=now, recorded_by=self._actor()
|
|
))
|
|
prior = await self.repo.get_evidence(str(task_id), item.evidence_id)
|
|
if prior:
|
|
require(
|
|
digest(self._evidence_facts(prior)) == digest(_json(item)),
|
|
"EVIDENCE_CONFLICT",
|
|
"Evidence ids are immutable", status=409)
|
|
else:
|
|
require(item.kind == "synthetic", "EVIDENCE_ORIGIN_INVALID",
|
|
"Agent may only add explicitly synthetic evidence; system, external, and human evidence must use their trusted ingestion endpoints",
|
|
status=422)
|
|
await self.repo.insert_evidence(str(task_id), document)
|
|
known_evidence.add(item.evidence_id)
|
|
plan_revision = task.get("current_plan_revision")
|
|
report_revision = task.get("current_report_revision")
|
|
current_plan = None
|
|
actions = {
|
|
item["action_id"]: HumanApprovalAction.model_validate(item)
|
|
for item in await self.repo.list_actions(str(task_id))
|
|
}
|
|
progress_revision = None
|
|
if command.progress is not None:
|
|
require(command.run_id is not None, "RUN_REQUIRED",
|
|
"Progress must identify its current execution run", status=422)
|
|
if command.progress.diagram:
|
|
for node in command.progress.diagram.node_refs:
|
|
self._require_evidence_refs(node.evidence_refs, known_evidence)
|
|
require(all(str(ref) in actions for ref in node.action_refs),
|
|
"ACTION_NOT_FOUND", "Progress references an unknown action", status=422)
|
|
previous = await self.repo.get_latest_progress(str(task_id), task["execution_generation"])
|
|
progress_revision = (previous["revision"] if previous else 0) + 1
|
|
snapshot = InvestigationProgressSnapshot(
|
|
revision=progress_revision, run_id=active_run["run_id"],
|
|
execution_generation=task["execution_generation"],
|
|
source_session_id=command.source_session_id, published_at=now, progress=command.progress,
|
|
)
|
|
await self.repo.save_progress(str(task_id), _json(snapshot))
|
|
await self._append_event(
|
|
topic="operational.progress.changed", aggregate_type="operational_task_progress",
|
|
aggregate_id=str(task_id), aggregate_revision=progress_revision, occurred_at=now,
|
|
metadata={"task_id": str(task_id), "run_id": str(active_run["run_id"]),
|
|
"execution_generation": task["execution_generation"]},
|
|
)
|
|
if command.plan:
|
|
self._require_evidence_refs(
|
|
(ref for node in command.plan.nodes for ref in node.evidence_refs),
|
|
known_evidence,
|
|
)
|
|
self._validate_plan_state(command.plan, known_evidence, actions)
|
|
if command.plan.schema_name == "execution-plan@2":
|
|
requests = {
|
|
row["access_request_id"]: TaskAccessRequest.model_validate(row)
|
|
for row in await self._list_access_requests(task_id)
|
|
}
|
|
for node in command.plan.nodes:
|
|
if node.kind == "gate":
|
|
gate_ref = str(node.gate_ref)
|
|
access_request = requests.get(gate_ref)
|
|
action = actions.get(gate_ref)
|
|
require(
|
|
access_request is not None or action is not None,
|
|
"GATE_REFERENCE_NOT_FOUND",
|
|
"Gate must reference a persisted request or action",
|
|
refs=[node.id], status=409,
|
|
)
|
|
if node.status == "completed":
|
|
require(
|
|
(
|
|
access_request is not None
|
|
and access_request.status != "pending"
|
|
) or (
|
|
action is not None
|
|
and action.approval_status != "pending"
|
|
),
|
|
"GATE_NOT_RESOLVED",
|
|
"A completed gate requires a completed human decision",
|
|
refs=[node.id], status=409,
|
|
)
|
|
plan_revision = (plan_revision or 0) + 1
|
|
snapshot = ExecutionPlanSnapshot(
|
|
revision=plan_revision,
|
|
execution_generation=task["execution_generation"],
|
|
source_session_id=command.source_session_id,
|
|
published_at=now, published_by=self._actor(), plan=command.plan,
|
|
)
|
|
await self.repo.insert_plan(str(task_id), _json(snapshot))
|
|
current_plan = snapshot
|
|
if command.report:
|
|
self._require_evidence_refs(command.report.evidence_refs, known_evidence)
|
|
if command.report.stage == "final":
|
|
require_complete_scada_evidence(
|
|
task, await self.repo.list_evidence(str(task_id)), command.report.evidence_refs,
|
|
)
|
|
for gap in command.report.data_gaps:
|
|
require(set(gap.evidence_refs) <= set(command.report.evidence_refs),
|
|
"REPORT_GAP_EVIDENCE_INVALID",
|
|
"Gap evidence must be included in report evidence", status=422)
|
|
if command.report.diagram:
|
|
for node in command.report.diagram.node_refs:
|
|
self._require_evidence_refs(node.evidence_refs, set(command.report.evidence_refs))
|
|
require(all(str(ref) in actions for ref in node.action_refs),
|
|
"ACTION_NOT_FOUND", "Diagram references an unknown action", status=422)
|
|
if command.report.outcome == "insufficient_data":
|
|
require(not any(action.approval_status == "pending" or (
|
|
action.approval_status == "approved" and action.execution_status in {"not_submitted", "submitted", "unknown"}
|
|
) for action in actions.values()), "REPORT_ACTION_PENDING",
|
|
"Resolve existing actions before ending for insufficient data", status=409)
|
|
report_revision = (report_revision or 0) + 1
|
|
report = InvestigationReportRevision(
|
|
revision=report_revision,
|
|
execution_generation=task["execution_generation"],
|
|
source_session_id=command.source_session_id,
|
|
published_at=now, published_by=self._actor(), report=command.report,
|
|
)
|
|
report = await self._report_with_progress_diagram(task_id, report)
|
|
await self.repo.insert_report(str(task_id), _json(report))
|
|
await self._append_event(
|
|
topic="operational.report.published",
|
|
aggregate_type="operational_report",
|
|
aggregate_id=str(task_id),
|
|
aggregate_revision=report_revision,
|
|
occurred_at=now,
|
|
metadata={"task_id": str(task_id), "run_id": str(active_run["run_id"])},
|
|
)
|
|
previous_result = previous_report["report"] if previous_report else None
|
|
if command.report.stage == "final" and (
|
|
previous_report is None
|
|
or previous_report.get("execution_generation") != task["execution_generation"]
|
|
or previous_result.get("stage") != "final"
|
|
or any(previous_result.get(key, [] if key == "ui_blocks" else None) != _json(command.report).get(key)
|
|
for key in ("current_conclusion", "conclusion_boundary", "outcome", "stop_reason",
|
|
"evidence_refs", "recommendations", "pending_items", "data_gaps", "ui_blocks"))
|
|
):
|
|
await self._resolve_notifications(
|
|
task_id, kinds=["report_updated"], subject_type="task", subject_id=task_id,
|
|
now=now, reason="report_superseded",
|
|
)
|
|
await self._notify(
|
|
task_id, "report_updated", task["title"], command.report.current_conclusion, now,
|
|
report_reference=InvestigationReportReference(
|
|
execution_generation=task["execution_generation"], revision=report_revision,
|
|
),
|
|
)
|
|
if current_plan is None:
|
|
stored_plan = await self.repo.get_latest_plan(str(task_id))
|
|
if stored_plan is not None:
|
|
current_plan = ExecutionPlanSnapshot.model_validate(stored_plan)
|
|
task.update(
|
|
revision=task["revision"] + 1,
|
|
current_plan_revision=plan_revision,
|
|
current_report_revision=report_revision,
|
|
agent_lease_expires_at=now + LEASE_DURATION,
|
|
updated_at=now,
|
|
)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
run_expected = active_run["revision"]
|
|
active_run.update(
|
|
revision=run_expected + 1,
|
|
heartbeat_at=now,
|
|
lease_expires_at=now + LEASE_DURATION,
|
|
updated_at=now,
|
|
)
|
|
if self._supports_runs:
|
|
await self.repo.update_run(
|
|
_json(OperationalTaskRun.model_validate(active_run)), run_expected
|
|
)
|
|
else:
|
|
# Compatibility for pre-015 repository adapters only.
|
|
if command.report:
|
|
if task["task_type"] == "event_investigation":
|
|
task["status"] = self._investigation_status(actions.values(), task) if command.report.stage == "final" else "running"
|
|
else:
|
|
task["status"] = self._status_from_plan(current_plan.plan.nodes, actions.values()) if current_plan else "running"
|
|
if task["status"] in {"waiting_human", "completed", "failed", "cancelled"}:
|
|
task["lifecycle_status"] = task["status"]
|
|
task["agent_lease_expires_at"] = (
|
|
now + LEASE_DURATION if task["status"] == "running" else None
|
|
)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task))
|
|
)
|
|
await self._task_event(task, now)
|
|
response = {"plan_revision": plan_revision, "report_revision": report_revision}
|
|
if progress_revision is not None:
|
|
response["progress_revision"] = progress_revision
|
|
await self._save_request(scope, command.request_id, body, response)
|
|
return AgentArtifactsResult(task=await self.detail(task_id), **response)
|
|
|
|
async def presentation_cursor(self, task_id: UUID) -> TaskPresentationCursor:
|
|
# The shared event-order lock gives the run and cursor one committed boundary.
|
|
async with self.repo.transaction():
|
|
task = await self._task(task_id)
|
|
run = await self._active_run(task)
|
|
cursor = await self.repo.current_event_cursor()
|
|
return TaskPresentationCursor(
|
|
task_id=task_id, execution_generation=task["execution_generation"],
|
|
active_run=({key: run[key] for key in ("run_id", "generation", "status")} if run else None),
|
|
event_cursor=cursor,
|
|
)
|
|
|
|
async def presentation_history(
|
|
self, task_id: UUID, *, after: int = 0, through: int | None = None, limit: int = 100,
|
|
) -> TaskPresentationHistoryPage:
|
|
require(0 <= after and (through is None or through >= 0) and 1 <= limit <= 200,
|
|
"INVALID_ARGUMENT", "Invalid presentation history page", status=422)
|
|
async with self.repo.transaction():
|
|
await self._task(task_id)
|
|
cursor = await self.repo.current_event_cursor()
|
|
boundary = min(through, cursor) if through is not None else cursor
|
|
rows = await self.repo.list_presentation_events(str(task_id), after, boundary, limit + 1)
|
|
items = [TaskPresentationHistoryItem(
|
|
event_seq=row["event_seq"], occurred_at=row["occurred_at"],
|
|
run_id=row["metadata"]["run_id"],
|
|
execution_generation=row["metadata"]["execution_generation"],
|
|
command=row["metadata"]["command"],
|
|
) for row in rows[:limit]]
|
|
return TaskPresentationHistoryPage(
|
|
task_id=task_id, event_cursor=boundary, items=items,
|
|
next_after=items[-1].event_seq if len(rows) > limit else None,
|
|
)
|
|
|
|
async def publish_presentation_event(
|
|
self, task_id: UUID, command: TaskPresentationEventCommand
|
|
) -> TaskPresentationEventResult:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:presentation"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return TaskPresentationEventResult.model_validate(replay)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"run:{command.run_id}")
|
|
task = await self._task(task_id)
|
|
self._require_actionable_task(task)
|
|
run = await self._active_run(task)
|
|
require(run is not None and str(run["run_id"]) == str(command.run_id)
|
|
and run["generation"] == task["execution_generation"],
|
|
"EXECUTION_GENERATION_MISMATCH", "Presentation targets a stale execution run", status=409)
|
|
require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
|
|
"Only a running run can publish presentation", status=409)
|
|
require(run.get("agent_session_id") == command.source_session_id,
|
|
"SESSION_MISMATCH", "Presentation must come from the claimed Agent session", status=409)
|
|
now = self._now()
|
|
self._require_active_run_lease(run, now)
|
|
payload = _json(command.command)
|
|
if command.command.action == "open_chart":
|
|
known = {item["evidence_id"] for item in await self.repo.list_evidence(str(task_id))}
|
|
self._require_evidence_refs(command.command.evidence_refs, known)
|
|
elif command.command.action == "set_overlay":
|
|
stored = await self.repo.get_evidence(str(task_id), command.command.evidence_id)
|
|
require(stored is not None, "EVIDENCE_NOT_FOUND", "Overlay evidence does not belong to this task", status=422)
|
|
evidence = StoredEvidenceRecord.model_validate(stored)
|
|
require(evidence.data_snapshot is not None, "SNAPSHOT_NOT_FOUND",
|
|
"Overlay requires a Server-ingested immutable data snapshot", status=422)
|
|
resolved = resolve_overlay(evidence.data_snapshot, command.command.mode)
|
|
if command.command.data is not None:
|
|
supplied = resolve_overlay(
|
|
evidence.data_snapshot.model_copy(update={
|
|
"data": command.command.data.model_dump(mode="json", exclude_unset=True),
|
|
}), command.command.mode,
|
|
)
|
|
require(supplied == resolved, "OVERLAY_DATA_MISMATCH",
|
|
"Overlay data must match the immutable evidence snapshot", status=422)
|
|
payload["data"] = resolved
|
|
metadata = {"task_id": str(task_id), "run_id": str(run["run_id"]),
|
|
"execution_generation": run["generation"], "command": payload}
|
|
try:
|
|
size = len(json.dumps(metadata, ensure_ascii=False, allow_nan=False).encode("utf-8"))
|
|
except (ValueError, TypeError) as exc:
|
|
raise DomainError("PRESENTATION_DATA_INVALID", "Presentation data must be finite JSON", status=422) from exc
|
|
require(size <= 512 * 1024, "PRESENTATION_TOO_LARGE",
|
|
"Presentation event exceeds the 512 KiB limit", status=422)
|
|
event_seq = await self._append_event(
|
|
topic="operational.presentation.requested", aggregate_type="operational_task_presentation",
|
|
aggregate_id=str(task_id), aggregate_revision=run["revision"], occurred_at=now,
|
|
metadata=metadata,
|
|
)
|
|
require(event_seq is not None, "DEPENDENCY_UNAVAILABLE", "Presentation event storage is unavailable", status=503)
|
|
result = TaskPresentationEventResult(event_seq=event_seq, run_id=run["run_id"],
|
|
execution_generation=run["generation"])
|
|
await self._save_request(scope, command.request_id, body, _json(result))
|
|
return result
|
|
|
|
async def propose_action(
|
|
self, task_id: UUID, command: ActionProposalCommand
|
|
) -> HumanApprovalAction:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:action:propose"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
action = await self.repo.get_action(str(task_id), replay["action_id"])
|
|
return HumanApprovalAction.model_validate(action)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
self._require_actionable_task(task)
|
|
self._require_real_action(task, command.kind)
|
|
self._validate_action_payload(command)
|
|
if task["task_type"] == "event_investigation":
|
|
report = await self.repo.get_latest_report(str(task_id))
|
|
require(report is not None
|
|
and report.get("execution_generation") == task["execution_generation"]
|
|
and report["report"].get("outcome") == "concluded",
|
|
"ACTION_CONCLUSION_REQUIRED",
|
|
"Publish a current concluded report before proposing an action", status=409)
|
|
if command.node_id:
|
|
if task["task_type"] != "event_investigation":
|
|
latest_plan = await self.repo.get_latest_plan(str(task_id))
|
|
require(latest_plan is not None, "PLAN_NODE_NOT_FOUND",
|
|
"Action cannot be attached before a plan is published", status=409)
|
|
node_ids = {node["id"] for node in latest_plan["plan"]["nodes"]}
|
|
require(command.node_id in node_ids, "PLAN_NODE_NOT_FOUND",
|
|
"Action references an unknown plan node", refs=[command.node_id], status=409)
|
|
known = {item["evidence_id"] for item in await self.repo.list_evidence(str(task_id))}
|
|
self._require_evidence_refs(command.evidence_refs, known)
|
|
now = self._now()
|
|
if task["status"] == "running":
|
|
active_run = await self._active_run(task)
|
|
require(active_run is not None, "RUN_NOT_FOUND",
|
|
"Task has no active Agent run", status=409)
|
|
self._require_active_run_lease(active_run, now)
|
|
approval_permission, execution_permission = self._action_permissions(command.kind)
|
|
action = HumanApprovalAction(
|
|
action_id=uuid4(), task_id=task_id, version=1, kind=command.kind,
|
|
reason=command.reason, payload=command.payload,
|
|
evidence_refs=command.evidence_refs, node_id=command.node_id,
|
|
required_approval_permission=approval_permission,
|
|
required_execution_permission=execution_permission,
|
|
approval_status="pending", execution_status="not_submitted",
|
|
execution_request_id=None,
|
|
proposed_by=self._actor(), proposed_at=now,
|
|
)
|
|
await self.repo.insert_action(str(task_id), _json(action))
|
|
await self._action_event(action, now)
|
|
if task["status"] != "running":
|
|
task.update(
|
|
lifecycle_status="waiting_human", status="waiting_human",
|
|
revision=task["revision"] + 1,
|
|
agent_lease_expires_at=None, updated_at=now,
|
|
)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task))
|
|
)
|
|
await self._task_event(task, now)
|
|
await self._notify(
|
|
task_id, "action_approval_pending", task["title"], command.reason, now,
|
|
required_permission=approval_permission,
|
|
subject_type="action", subject_id=action.action_id,
|
|
)
|
|
await self._save_request(
|
|
scope, command.request_id, body, {"action_id": str(action.action_id)}
|
|
)
|
|
return action
|
|
|
|
async def update_work_order_draft(
|
|
self, task_id: UUID, action_id: UUID, command: WorkOrderDraftUpdateCommand
|
|
) -> HumanApprovalAction:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:action:{action_id}:draft"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return HumanApprovalAction.model_validate(replay["action"])
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"action:{action_id}")
|
|
task = await self._task(task_id)
|
|
self._require_actionable_task(task)
|
|
action = await self._action(task_id, action_id)
|
|
self._require_real_action(task, action.kind)
|
|
require(action.kind == "create_work_order", "INVALID_ACTION_KIND",
|
|
"Only work-order drafts can be edited", status=422)
|
|
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Action version changed", status=409)
|
|
require(action.approval_status == "pending" and action.execution_status == "not_submitted",
|
|
"INVALID_TRANSITION", "Submitted actions cannot be edited", status=409)
|
|
require(OPERATIONAL_ACTION_WORK_ORDER_APPROVE in self.permissions,
|
|
"PERMISSION_DENIED", "Current user cannot edit this work-order draft", status=403)
|
|
now = self._now()
|
|
updated = action.model_copy(update={
|
|
"version": action.version + 1, "payload": command.payload,
|
|
"edited_by": self._actor(), "edited_at": now,
|
|
})
|
|
await self.repo.update_action(str(task_id), _json(updated))
|
|
await self._action_event(updated, now)
|
|
await self._save_request(scope, command.request_id, body, {"action": _json(updated)})
|
|
return updated
|
|
|
|
async def confirm_work_order_dispatch(
|
|
self, task_id: UUID, action_id: UUID, command: WorkOrderConfirmDispatchCommand,
|
|
dispatcher, credential: str | None,
|
|
) -> HumanApprovalAction:
|
|
require(OPERATIONAL_ACTION_WORK_ORDER_APPROVE in self.permissions
|
|
and OPERATIONAL_ACTION_WORK_ORDER_EXECUTE in self.permissions,
|
|
"PERMISSION_DENIED", "Current user cannot confirm and dispatch work orders", status=403)
|
|
# A SELECT on the default non-autocommit connection starts an outer
|
|
# transaction. End it explicitly, or approval/submit become savepoints
|
|
# whose event lock survives into the peripheral HTTP call.
|
|
async with self.repo.transaction():
|
|
current = await self._action(task_id, action_id)
|
|
require(current.kind == "create_work_order", "INVALID_ACTION_KIND",
|
|
"Only work orders support one-step confirmation", status=422)
|
|
approval = await self.decide_action(task_id, action_id, ActionDecisionCommand(
|
|
request_id=uuid5(NAMESPACE_URL, f"confirm:{command.request_id}:approve"),
|
|
expected_version=command.expected_version, decision="approved",
|
|
))
|
|
return await self.submit_action(task_id, action_id, ActionExecuteCommand(
|
|
request_id=uuid5(NAMESPACE_URL, f"confirm:{command.request_id}:dispatch"),
|
|
expected_version=approval.version,
|
|
), dispatcher, credential)
|
|
|
|
async def decide_action(
|
|
self, task_id: UUID, action_id: UUID, command: ActionDecisionCommand
|
|
) -> HumanApprovalAction:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:action:{action_id}:decision"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return HumanApprovalAction.model_validate(replay["action"])
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"action:{action_id}")
|
|
task = await self._task(task_id)
|
|
self._require_actionable_task(task)
|
|
action = await self._action(task_id, action_id)
|
|
self._require_real_action(task, action.kind)
|
|
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Action version changed", status=409)
|
|
require(action.approval_status == "pending", "INVALID_TRANSITION",
|
|
"Action has already been decided", status=409)
|
|
require(action.required_approval_permission in self.permissions,
|
|
"PERMISSION_DENIED", "Current user cannot decide this action", status=403)
|
|
now = self._now()
|
|
action = action.model_copy(update={
|
|
"version": action.version + 1, "approval_status": command.decision,
|
|
"decided_by": self._actor(), "decided_at": now,
|
|
"decision_comment": command.comment,
|
|
})
|
|
await self.repo.update_action(str(task_id), _json(action))
|
|
await self._action_event(action, now)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["action_pending", "action_approval_pending"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=now,
|
|
reason=f"action_{command.decision}",
|
|
)
|
|
if command.decision == "rejected":
|
|
task = await self._task(task_id)
|
|
now = self._now()
|
|
await self._requeue_task(task, now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
await self._save_request(scope, command.request_id, body, {"action": _json(action)})
|
|
return action
|
|
|
|
async def submit_action(
|
|
self, task_id: UUID, action_id: UUID, command: ActionExecuteCommand,
|
|
dispatcher, credential: str | None,
|
|
) -> HumanApprovalAction:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:action:{action_id}:submit"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return HumanApprovalAction.model_validate(replay["action"])
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"action:{action_id}")
|
|
task = await self._task(task_id)
|
|
self._require_actionable_task(task)
|
|
action = await self._action(task_id, action_id)
|
|
self._require_real_action(task, action.kind)
|
|
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Action version changed", status=409)
|
|
require(action.approval_status == "approved", "INVALID_TRANSITION",
|
|
"Only an approved action can be submitted", status=409)
|
|
require(action.execution_status in {"not_submitted", "failed", "unsupported"},
|
|
"INVALID_TRANSITION", "Action cannot be submitted in its current state", status=409)
|
|
require(action.required_execution_permission in self.permissions,
|
|
"PERMISSION_DENIED", "Current user cannot submit this action", status=403)
|
|
now = self._now()
|
|
action = action.model_copy(update={
|
|
"version": action.version + 1, "execution_status": "unknown",
|
|
"execution_request_id": uuid5(
|
|
NAMESPACE_URL,
|
|
f"operational-action:{self.context.project_id}:{task_id}:{action_id}:{action.version}",
|
|
),
|
|
"executed_by": self._actor(), "executed_at": now,
|
|
"execution_detail": "外部动作已开始提交,结果待回执确认",
|
|
})
|
|
await self.repo.update_action(str(task_id), _json(action))
|
|
await self._action_event(action, now)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["external_coordination_failed"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=now,
|
|
reason="external_coordination_retried",
|
|
)
|
|
await self._save_request(
|
|
scope, command.request_id, body, {"action": _json(action)}
|
|
)
|
|
|
|
async with self.repo.transaction():
|
|
task = await self.detail(task_id)
|
|
try:
|
|
outcome = await dispatcher.dispatch(
|
|
context=self.context, task=task, action=action, credential=credential
|
|
)
|
|
except Exception:
|
|
async with self.repo.transaction():
|
|
await self.repo.lock(f"action:{action_id}")
|
|
current = await self._action(task_id, action_id)
|
|
if (
|
|
current.version == action.version
|
|
and current.execution_status == "unknown"
|
|
):
|
|
await self._notify(
|
|
task_id,
|
|
"external_coordination_failed",
|
|
task.title,
|
|
"外部系统提交中断,执行结果未知,请核对回执",
|
|
self._now(),
|
|
required_permission=action.required_execution_permission,
|
|
subject_type="action", subject_id=action_id,
|
|
)
|
|
raise
|
|
|
|
async with self.repo.transaction():
|
|
await self.repo.lock(f"request:{scope}:{command.request_id}")
|
|
await self.repo.lock(f"action:{action_id}")
|
|
current = await self._action(task_id, action_id)
|
|
require(current.version == action.version and current.execution_status == "unknown",
|
|
"VERSION_CONFLICT", "Action changed while awaiting external receipt", status=409)
|
|
self._validate_dispatch_outcome(outcome)
|
|
current = current.model_copy(update={
|
|
"version": current.version + 1,
|
|
"execution_status": outcome.status,
|
|
"receipt": outcome.receipt,
|
|
"execution_detail": outcome.detail,
|
|
})
|
|
await self.repo.update_action(str(task_id), _json(current))
|
|
outcome_recorded_at = self._now()
|
|
await self._action_event(current, outcome_recorded_at)
|
|
if outcome.status in {"unknown", "failed", "unsupported"}:
|
|
await self._notify(
|
|
task_id,
|
|
"external_coordination_failed",
|
|
task.title,
|
|
outcome.detail,
|
|
outcome_recorded_at,
|
|
required_permission=action.required_execution_permission,
|
|
subject_type="action", subject_id=action_id,
|
|
)
|
|
elif outcome.status == "succeeded":
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["external_coordination_failed"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=outcome_recorded_at,
|
|
reason="external_coordination_succeeded",
|
|
)
|
|
if outcome.status in {"succeeded", "failed", "unsupported"}:
|
|
task_document = await self._task(task_id)
|
|
track_work_order(task_document, _json(current))
|
|
await self._requeue_task(task_document, outcome_recorded_at)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task_document))
|
|
)
|
|
await self._task_event(task_document, outcome_recorded_at)
|
|
await self._save_request(scope, command.request_id, body, {"action": _json(current)})
|
|
return current
|
|
|
|
async def reconcile_action(
|
|
self, task_id: UUID, action_id: UUID, command: ActionReconcileCommand,
|
|
dispatcher, credential: str | None
|
|
) -> HumanApprovalAction:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:action:{action_id}:reconcile"
|
|
async with self.repo.transaction():
|
|
replay = await self.repo.get_request(scope, str(command.request_id))
|
|
if replay:
|
|
self._require_same_request(replay, body)
|
|
return HumanApprovalAction.model_validate(replay["response"]["action"])
|
|
action = await self._action(task_id, action_id)
|
|
require(action.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Action version changed", status=409)
|
|
require(action.execution_status in {"submitted", "unknown"}, "INVALID_TRANSITION",
|
|
"Only submitted or unknown actions can be reconciled", status=409)
|
|
require(action.required_execution_permission in self.permissions,
|
|
"PERMISSION_DENIED", "Current user cannot reconcile this action", status=403)
|
|
task = await self.detail(task_id)
|
|
outcome = await dispatcher.reconcile(
|
|
context=self.context, task=task, action=action,
|
|
credential=credential,
|
|
)
|
|
self._validate_dispatch_outcome(outcome)
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return HumanApprovalAction.model_validate(replay["action"])
|
|
await self.repo.lock(f"action:{action_id}")
|
|
current = await self._action(task_id, action_id)
|
|
require(current.version == action.version, "VERSION_CONFLICT",
|
|
"Action changed while reconciling", status=409)
|
|
previous_execution_status = current.execution_status
|
|
current = current.model_copy(update={
|
|
"version": current.version + 1, "execution_status": outcome.status,
|
|
"receipt": outcome.receipt, "execution_detail": outcome.detail,
|
|
})
|
|
await self.repo.update_action(str(task_id), _json(current))
|
|
outcome_recorded_at = self._now()
|
|
await self._action_event(current, outcome_recorded_at)
|
|
if (
|
|
outcome.status in {"unknown", "failed", "unsupported"}
|
|
and outcome.status != previous_execution_status
|
|
):
|
|
task = await self._task(task_id)
|
|
await self._notify(
|
|
task_id,
|
|
"external_coordination_failed",
|
|
task["title"],
|
|
outcome.detail,
|
|
outcome_recorded_at,
|
|
required_permission=action.required_execution_permission,
|
|
subject_type="action", subject_id=action_id,
|
|
)
|
|
elif outcome.status == "succeeded":
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["external_coordination_failed"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=outcome_recorded_at,
|
|
reason="external_coordination_succeeded",
|
|
)
|
|
if outcome.status in {"succeeded", "failed", "unsupported"}:
|
|
task_document = await self._task(task_id)
|
|
track_work_order(task_document, _json(current))
|
|
await self._requeue_task(task_document, outcome_recorded_at)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task_document))
|
|
)
|
|
await self._task_event(task_document, outcome_recorded_at)
|
|
await self._save_request(
|
|
scope, command.request_id, body, {"action": _json(current)}
|
|
)
|
|
return current
|
|
|
|
async def external_feedback(
|
|
self, task_id: UUID, command: ExternalFeedbackCommand
|
|
) -> ExternalFeedbackResult:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:external-feedback"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return ExternalFeedbackResult(task=await self.detail(task_id), **replay)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
if task["task_type"] == "event_investigation":
|
|
self._require_actionable_task(task)
|
|
own_source = task["source"].get("external_id")
|
|
matching_source = own_source is not None and command.source_event_id in {
|
|
own_source, f"peripheral_sample:{own_source}",
|
|
}
|
|
if task["status"] == "completed" and not matching_source:
|
|
actions = await self.repo.list_actions(str(task_id))
|
|
require(any(
|
|
item["kind"] == "create_work_order" and item.get("receipt")
|
|
and item["receipt"].get("adapter") == "peripheral_sample"
|
|
and command.source_event_id == f"peripheral_sample:{item['receipt']['receipt_id']}"
|
|
for item in actions
|
|
), "EXTERNAL_SOURCE_MISMATCH", "Feedback does not match an associated child work order", status=409)
|
|
require(command.source_revision > 1, "INITIAL_CHILD_FEEDBACK",
|
|
"The child creation receipt is not new field feedback", status=409)
|
|
if task["task_type"] == "external_work_order":
|
|
require(
|
|
task["source"].get("external_id") == command.source_event_id,
|
|
"EXTERNAL_SOURCE_MISMATCH",
|
|
"External feedback does not match the task source",
|
|
status=409,
|
|
)
|
|
if command.work_order_snapshot is not None:
|
|
require(
|
|
task["task_type"] == "external_work_order",
|
|
"INVALID_FEEDBACK_SNAPSHOT",
|
|
"Work order snapshots only apply to external work order tasks",
|
|
status=422,
|
|
)
|
|
existing = await self.repo.get_feedback(str(task_id), command.source_event_id)
|
|
current_source_revision = (
|
|
task.get("source", {}).get("external_revision")
|
|
if task["task_type"] == "external_work_order" or (
|
|
task["task_type"] == "event_investigation" and matching_source
|
|
)
|
|
else None
|
|
)
|
|
if current_source_revision is not None:
|
|
require(
|
|
command.source_revision >= current_source_revision,
|
|
"STALE_EXTERNAL_REVISION",
|
|
"External feedback revision is stale",
|
|
status=409,
|
|
)
|
|
if existing:
|
|
prior = existing["document"]
|
|
if command.source_revision == existing["source_revision"]:
|
|
require(digest({key: value for key, value in prior.items() if key != "request_id"})
|
|
== digest({key: value for key, value in body.items() if key != "request_id"}), "IDEMPOTENCY_CONFLICT",
|
|
"External feedback revision was reused with different content", status=409)
|
|
response = {
|
|
"source_event_id": command.source_event_id,
|
|
"source_revision": command.source_revision,
|
|
"accepted_at": existing["accepted_at"],
|
|
}
|
|
await self._save_request(scope, command.request_id, body, response)
|
|
return ExternalFeedbackResult(task=await self.detail(task_id), **response)
|
|
require(command.source_revision > existing["source_revision"],
|
|
"STALE_EXTERNAL_REVISION", "External feedback revision is stale", status=409)
|
|
if (task["task_type"] == "event_investigation" and matching_source
|
|
and current_source_revision is not None):
|
|
require(command.source_revision > current_source_revision, "STALE_EXTERNAL_REVISION",
|
|
"Original event feedback must contain a new source version", status=409)
|
|
now = self._now()
|
|
await self.repo.upsert_feedback(str(task_id), body, now)
|
|
if command.evidence:
|
|
evidence = EvidenceRecord(
|
|
**_json(command.evidence),
|
|
recorded_at=now, recorded_by=self._actor(),
|
|
)
|
|
prior_evidence = await self.repo.get_evidence(
|
|
str(task_id), command.evidence.evidence_id
|
|
)
|
|
if prior_evidence:
|
|
require(
|
|
digest(self._evidence_facts(prior_evidence))
|
|
== digest(self._evidence_facts(_json(evidence))),
|
|
"EVIDENCE_CONFLICT", "Evidence ids are immutable", status=409)
|
|
else:
|
|
await self.repo.insert_evidence(str(task_id), _json(evidence))
|
|
self._apply_feedback_status(task, command.status)
|
|
if task["task_type"] == "external_work_order":
|
|
task["source"] = {
|
|
**task["source"],
|
|
"external_revision": command.source_revision,
|
|
}
|
|
if command.work_order_snapshot is not None:
|
|
task.update(
|
|
title=command.work_order_snapshot.title,
|
|
summary=command.work_order_snapshot.summary,
|
|
scheduled_for=command.work_order_snapshot.scheduled_at,
|
|
priority=(
|
|
"urgent"
|
|
if command.work_order_snapshot.priority in {"high", "urgent"}
|
|
else "normal"
|
|
),
|
|
)
|
|
if task["task_type"] == "event_investigation":
|
|
# Recover older tasks whose successful dispatch predates tracking.
|
|
for item in await self.repo.list_actions(str(task_id)):
|
|
track_work_order(task, item)
|
|
accept_completion(task, body)
|
|
if matching_source:
|
|
task["source"] = {**task["source"], "external_revision": command.source_revision}
|
|
await self._requeue_task(task, now)
|
|
else:
|
|
task.update(revision=task["revision"] + 1, updated_at=now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
if task["status"] != "failed":
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["task_failed"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="external_status_recovered",
|
|
)
|
|
if task["status"] != "waiting_human":
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["human_input_required"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="external_status_progressed",
|
|
)
|
|
if task["task_type"] == "external_work_order":
|
|
await self._append_event(
|
|
topic="workbench.work_order.changed",
|
|
aggregate_type="operational_task",
|
|
aggregate_id=str(task_id),
|
|
aggregate_revision=task["revision"],
|
|
occurred_at=now,
|
|
metadata={"task_id": str(task_id), "namespace": "operational-task"},
|
|
)
|
|
if task["status"] == "failed":
|
|
await self._notify(
|
|
task_id, "task_failed",
|
|
task["title"], command.detail, now,
|
|
)
|
|
response = {
|
|
"source_event_id": command.source_event_id,
|
|
"source_revision": command.source_revision,
|
|
"accepted_at": now,
|
|
}
|
|
await self._save_request(scope, command.request_id, body, _json_dict(response))
|
|
return ExternalFeedbackResult(task=await self.detail(task_id), **response)
|
|
|
|
async def withdraw_external_source(
|
|
self, task_id: UUID, command: ExternalSourceWithdrawalCommand
|
|
) -> OperationalTaskDetail:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:withdraw-external-source"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return await self.detail(task_id)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
source = task["source"]
|
|
require(
|
|
task["task_type"] == "external_work_order"
|
|
and source.get("kind") == "external_system"
|
|
and source.get("external_id") == command.source_event_id
|
|
and command.source_event_id.startswith("peripheral_sample:"),
|
|
"EXTERNAL_SOURCE_MISMATCH",
|
|
"Only matching peripheral work orders can be withdrawn",
|
|
status=409,
|
|
)
|
|
if source.get("deleted_at") is None:
|
|
now = self._now()
|
|
task["source"] = {**source, "deleted_at": now}
|
|
task.update(
|
|
status="cancelled",
|
|
lifecycle_status="cancelled",
|
|
revision=task["revision"] + 1,
|
|
updated_at=now,
|
|
)
|
|
await self.repo.update_task(
|
|
_task_json(OperationalTaskSummary.model_validate(task))
|
|
)
|
|
await self._task_event(task, now)
|
|
await self._append_event(
|
|
topic="workbench.work_order.changed",
|
|
aggregate_type="operational_task",
|
|
aggregate_id=str(task_id),
|
|
aggregate_revision=task["revision"],
|
|
occurred_at=now,
|
|
metadata={"task_id": str(task_id), "namespace": "operational-task"},
|
|
)
|
|
for action in await self.repo.list_actions(str(task_id)):
|
|
action_id = UUID(str(action["action_id"]))
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["action_pending", "action_approval_pending"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=now,
|
|
reason="external_source_withdrawn",
|
|
)
|
|
if action["execution_status"] == "not_submitted":
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["external_coordination_failed"],
|
|
subject_type="action",
|
|
subject_id=action_id,
|
|
now=now,
|
|
reason="external_source_withdrawn",
|
|
)
|
|
for access_request in await self._list_access_requests(task_id):
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["access_approval_pending"],
|
|
subject_type="access_request",
|
|
subject_id=UUID(str(access_request["access_request_id"])),
|
|
now=now,
|
|
reason="external_source_withdrawn",
|
|
)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["human_input_required", "run_delivery_degraded", "task_failed"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="external_source_withdrawn",
|
|
)
|
|
await self._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
|
|
return await self.detail(task_id)
|
|
|
|
async def acknowledge_notification(
|
|
self, notification_id: UUID, command: NotificationAcknowledgeCommand
|
|
) -> OperationalTaskNotification:
|
|
body = _json(command)
|
|
scope = (
|
|
f"notification:{notification_id}:user:{self.context.user_id}:ack"
|
|
)
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return OperationalTaskNotification.model_validate(replay["notification"])
|
|
await self.repo.lock(f"notification:{notification_id}")
|
|
value = await self.repo.get_notification(
|
|
str(notification_id), str(self.context.user_id)
|
|
)
|
|
require(value is not None, "NOTIFICATION_NOT_FOUND", "Notification not found", status=404)
|
|
notification = OperationalTaskNotification.model_validate(value)
|
|
require(
|
|
notification.required_permission is None
|
|
or notification.required_permission in self.permissions,
|
|
"PERMISSION_DENIED",
|
|
"Current user is not a recipient of this notification",
|
|
status=403,
|
|
)
|
|
require(notification.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Notification version changed", status=409)
|
|
if notification.status == "unread":
|
|
notification = notification.model_copy(update={
|
|
"status": "read", "version": notification.version + 1,
|
|
"read_at": self._now(), "read_by": self._actor(),
|
|
})
|
|
await self.repo.mark_notification_read(
|
|
str(notification_id), str(self.context.user_id),
|
|
notification.version, notification.read_at,
|
|
)
|
|
await self._append_event(
|
|
topic="operational.notification.changed",
|
|
aggregate_type="operational_notification_read",
|
|
aggregate_id=f"{notification_id}:{self.context.user_id}",
|
|
aggregate_revision=notification.version,
|
|
occurred_at=notification.read_at,
|
|
metadata={"task_id": str(notification.task_id)},
|
|
)
|
|
await self._save_request(
|
|
scope, command.request_id, body, {"notification": _json(notification)}
|
|
)
|
|
return notification
|
|
|
|
async def record_data_snapshot(
|
|
self, task_id: UUID, snapshot: TaskDataSnapshot
|
|
) -> tuple[EvidenceRecord, TaskDataSnapshot]:
|
|
"""Persist the bounded source response before an Agent may cite it."""
|
|
await self._task(task_id)
|
|
now = self._now()
|
|
evidence = StoredEvidenceRecord(
|
|
evidence_id=snapshot.source_ref,
|
|
kind=snapshot.origin,
|
|
summary=f"{snapshot.source} 数据快照",
|
|
source_label=snapshot.source,
|
|
source_ref=snapshot.source_ref,
|
|
observed_at=snapshot.queried_at,
|
|
recorded_at=now,
|
|
recorded_by=self._actor(),
|
|
has_snapshot=True,
|
|
data_snapshot=snapshot,
|
|
)
|
|
async with self.repo.transaction():
|
|
await self.repo.lock(f"task:{task_id}:evidence:{snapshot.source_ref}")
|
|
prior = await self.repo.get_evidence(str(task_id), snapshot.source_ref)
|
|
if prior:
|
|
stored = StoredEvidenceRecord.model_validate(prior)
|
|
require(
|
|
stored.data_snapshot is not None
|
|
and self._snapshot_facts(stored.data_snapshot)
|
|
== self._snapshot_facts(snapshot),
|
|
"EVIDENCE_CONFLICT",
|
|
"Evidence ids are immutable",
|
|
status=409,
|
|
)
|
|
return self._public_evidence(prior), stored.data_snapshot
|
|
await self.repo.insert_evidence(str(task_id), _json(evidence))
|
|
return self._public_evidence(_json(evidence)), snapshot
|
|
|
|
async def add_human_evidence(
|
|
self, task_id: UUID, command: HumanEvidenceCommand
|
|
) -> OperationalTaskDetail:
|
|
body = _json(command)
|
|
scope = f"task:{task_id}:human-evidence"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return await self.detail(task_id)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
task = await self._task(task_id)
|
|
require(command.evidence.kind == "human_input", "EVIDENCE_KIND_INVALID",
|
|
"This endpoint only accepts human_input evidence", status=422)
|
|
now = self._now()
|
|
record = EvidenceRecord(
|
|
**_json(command.evidence), recorded_at=now, recorded_by=self._actor()
|
|
)
|
|
prior = await self.repo.get_evidence(str(task_id), command.evidence.evidence_id)
|
|
if prior:
|
|
require(
|
|
digest(self._evidence_facts(prior)) == digest(_json(command.evidence)),
|
|
"EVIDENCE_CONFLICT", "Evidence ids are immutable", status=409,
|
|
)
|
|
else:
|
|
await self.repo.insert_evidence(str(task_id), _json(record))
|
|
await self._requeue_task(task, now)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._task_event(task, now)
|
|
await self._resolve_notifications(
|
|
task_id,
|
|
kinds=["human_input_required"],
|
|
subject_type="task",
|
|
subject_id=task_id,
|
|
now=now,
|
|
reason="human_evidence_added",
|
|
)
|
|
await self._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
|
|
return await self.detail(task_id)
|
|
|
|
async def data_snapshot(self, task_id: UUID, evidence_id: str) -> TaskDataSnapshot:
|
|
await self._task(task_id)
|
|
value = await self.repo.get_evidence(str(task_id), evidence_id)
|
|
require(value is not None, "EVIDENCE_NOT_FOUND", "Evidence not found", status=404)
|
|
stored = StoredEvidenceRecord.model_validate(value)
|
|
require(stored.data_snapshot is not None, "SNAPSHOT_NOT_FOUND",
|
|
"Evidence has no stored data snapshot", status=404)
|
|
return stored.data_snapshot
|
|
|
|
async def create_schedule(
|
|
self, command: OperationalTaskScheduleCreate
|
|
) -> OperationalTaskSchedule:
|
|
body = _json(command)
|
|
scope = "schedule:create"
|
|
async with self.repo.transaction():
|
|
await self.repo.lock(f"schedule-request:{command.request_id}")
|
|
request = await self.repo.get_request(scope, str(command.request_id))
|
|
if request:
|
|
self._require_same_request(request, body)
|
|
prior = await self.repo.get_schedule_by_request(str(command.request_id))
|
|
require(prior is not None, "STATE_CORRUPT",
|
|
"Schedule idempotency record has no schedule", status=500)
|
|
return OperationalTaskSchedule.model_validate(prior)
|
|
prior = await self.repo.get_schedule_by_request(str(command.request_id))
|
|
if prior:
|
|
return OperationalTaskSchedule.model_validate(prior)
|
|
now = self._now()
|
|
schedule = OperationalTaskSchedule(
|
|
schedule_id=uuid4(), schedule_key=command.schedule_key,
|
|
title=command.title, summary=command.summary,
|
|
interval_seconds=command.interval_seconds,
|
|
next_run_at=command.next_run_at, priority=command.priority,
|
|
enabled=True, version=1, created_at=now, updated_at=now,
|
|
)
|
|
await self.repo.insert_schedule(_json(schedule), str(command.request_id))
|
|
await self._save_request(
|
|
scope, command.request_id, body,
|
|
{"schedule_id": str(schedule.schedule_id)},
|
|
)
|
|
return schedule
|
|
|
|
async def list_schedules(self) -> list[OperationalTaskSchedule]:
|
|
return [
|
|
OperationalTaskSchedule.model_validate(item)
|
|
for item in await self.repo.list_schedules()
|
|
]
|
|
|
|
async def update_schedule(
|
|
self, schedule_id: UUID, command: OperationalTaskScheduleUpdate
|
|
) -> OperationalTaskSchedule:
|
|
body = _json(command)
|
|
scope = f"schedule:{schedule_id}:update"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return OperationalTaskSchedule.model_validate(replay["schedule"])
|
|
await self.repo.lock(f"schedule:{schedule_id}")
|
|
value = await self.repo.get_schedule(str(schedule_id))
|
|
require(value is not None, "SCHEDULE_NOT_FOUND", "Schedule not found", status=404)
|
|
schedule = OperationalTaskSchedule.model_validate(value)
|
|
require(schedule.version == command.expected_version, "VERSION_CONFLICT",
|
|
"Schedule version changed", status=409)
|
|
schedule = schedule.model_copy(update={
|
|
"enabled": command.enabled,
|
|
"version": schedule.version + 1,
|
|
"updated_at": self._now(),
|
|
})
|
|
await self.repo.update_schedule(_json(schedule))
|
|
await self._save_request(
|
|
scope, command.request_id, body, {"schedule": _json(schedule)}
|
|
)
|
|
return schedule
|
|
|
|
async def list_notifications(
|
|
self, *, unread_only: bool, limit: int, offset: int
|
|
) -> tuple[list[OperationalTaskNotification], int | None]:
|
|
rows = await self.repo.list_notifications(
|
|
user_id=str(self.context.user_id),
|
|
allowed_permissions=sorted(self.permissions),
|
|
unread_only=unread_only, limit=limit + 1, offset=offset
|
|
)
|
|
return (
|
|
[OperationalTaskNotification.model_validate(item) for item in rows[:limit]],
|
|
offset + limit if len(rows) > limit else None,
|
|
)
|
|
|
|
async def dispatch_due_schedules(self, limit: int) -> DispatchDueSchedulesResult:
|
|
now = self._now()
|
|
task_ids: list[UUID] = []
|
|
async with self.repo.transaction():
|
|
for schedule_document in await self.repo.claim_due_schedules(limit):
|
|
schedule = OperationalTaskSchedule.model_validate(schedule_document)
|
|
due_at = schedule.next_run_at
|
|
task_id = uuid5(schedule.schedule_id, due_at.isoformat())
|
|
request_id = uuid5(NAMESPACE_URL, f"schedule:{schedule.schedule_id}:{due_at.isoformat()}")
|
|
task = self._scheduled_task(schedule, task_id, due_at, now)
|
|
await self.repo.insert_task(_task_json(task), str(request_id))
|
|
run = self._new_run(task_id, 1, now)
|
|
await self.repo.insert_run(_json(run))
|
|
await self.repo.insert_occurrence(
|
|
str(schedule.schedule_id), due_at, str(task_id), now
|
|
)
|
|
next_schedule = schedule.model_copy(update={
|
|
"next_run_at": due_at + timedelta(seconds=schedule.interval_seconds),
|
|
"version": schedule.version + 1, "updated_at": now,
|
|
})
|
|
await self.repo.advance_schedule(_json(next_schedule))
|
|
await self._run_event(run, now, queued=True)
|
|
await self._task_event(_task_json(task), now)
|
|
task_ids.append(task_id)
|
|
return DispatchDueSchedulesResult(task_ids=task_ids, dispatched_at=now)
|
|
|
|
async def _require_no_pending_calculations(self, task_id: UUID, run: dict) -> None:
|
|
if hasattr(self.repo, "pending_calculation_jobs"):
|
|
pending = await self.repo.pending_calculation_jobs(
|
|
str(task_id), str(run["run_id"]), str(run["execution_request_id"]),
|
|
)
|
|
require(not pending, "PENDING_CALCULATIONS",
|
|
"Wait for submitted calculations before publishing a final report or completing the run",
|
|
refs=pending, status=409, recovery="retry_same_request")
|
|
|
|
async def _finish_run(
|
|
self, task_id: UUID, run_id: UUID, command: AgentRunCommand,
|
|
*, outcome: str, failure_code: str | None = None,
|
|
failure_detail: str | None = None,
|
|
) -> OperationalTaskDetail:
|
|
body = _json(command)
|
|
scope = f"run:{run_id}:{outcome}"
|
|
async with self.repo.transaction():
|
|
replay = await self._request_replay(scope, command.request_id, body)
|
|
if replay:
|
|
return await self.detail(task_id)
|
|
await self.repo.lock(f"task:{task_id}")
|
|
await self.repo.lock(f"run:{run_id}")
|
|
run = await self._run(task_id, run_id)
|
|
require(run["revision"] == command.expected_revision, "VERSION_CONFLICT",
|
|
"Run revision changed", status=409)
|
|
require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
|
|
"Only an active run can finish", status=409)
|
|
self._require_run_owner(run, command)
|
|
now = self._now()
|
|
self._require_active_run_lease(run, now)
|
|
task = await self._task(task_id)
|
|
final_outcome = outcome
|
|
pending_question = any(q["status"] == "pending" and str(q["run_id"]) == str(run_id)
|
|
for q in await task_questions.list_questions(self, task_id))
|
|
actions = []
|
|
if outcome == "succeeded":
|
|
await self._require_no_pending_calculations(task_id, run)
|
|
if hasattr(self.repo, "pending_run_messages"):
|
|
pending_messages = await self.repo.pending_run_messages(str(run_id))
|
|
require(not pending_messages, "PENDING_AGENT_MESSAGES",
|
|
"Run has messages not yet entered into Agent context",
|
|
status=409, recovery="retry_same_request", refs=pending_messages)
|
|
plan_document = await self.repo.get_latest_plan(str(task_id))
|
|
report_document = await self.repo.get_latest_report(str(task_id))
|
|
plan = (
|
|
ExecutionPlanSnapshot.model_validate(plan_document)
|
|
if plan_document is not None else None
|
|
)
|
|
report = (
|
|
InvestigationReportRevision.model_validate(report_document)
|
|
if report_document is not None else None
|
|
)
|
|
same_generation = (
|
|
report is not None
|
|
and report.execution_generation == run["generation"]
|
|
and report.source_session_id == command.source_session_id
|
|
and (task["task_type"] == "event_investigation" or (
|
|
plan is not None
|
|
and plan.execution_generation == run["generation"]
|
|
and plan.source_session_id == command.source_session_id
|
|
))
|
|
)
|
|
if pending_question:
|
|
lifecycle = "waiting_human"
|
|
elif not same_generation or report.report.stage != "final":
|
|
final_outcome = "failed"
|
|
lifecycle = "failed"
|
|
failure_code = "ARTIFACT_INCOMPLETE"
|
|
failure_detail = (
|
|
"Run requires a final report from the same generation"
|
|
)
|
|
else:
|
|
actions = [
|
|
HumanApprovalAction.model_validate(item)
|
|
for item in await self.repo.list_actions(str(task_id))
|
|
]
|
|
access_requests = [
|
|
TaskAccessRequest.model_validate(item)
|
|
for item in await self._list_access_requests(task_id)
|
|
]
|
|
if task["task_type"] == "event_investigation":
|
|
lifecycle = self._investigation_status(actions, task)
|
|
recovery = task.get("handling_context") or {}
|
|
if lifecycle == "completed" and recovery:
|
|
require(recovery.get("verification_evidence_id") in report.report.evidence_refs,
|
|
"RECOVERY_EVIDENCE_REQUIRED", "最终复核报告必须引用恢复监测证据", status=409)
|
|
require(report.report.outcome == "concluded", "RECOVERY_EVIDENCE_REQUIRED",
|
|
"恢复复核通过后才能结束事件", status=409)
|
|
recovery.update(phase="completed", title="恢复供水复核成功", completed_at=now.isoformat())
|
|
await self._resolve_investigation_alerts(str(task_id), recovery["verification_evidence_id"])
|
|
elif report.report.outcome == "insufficient_data":
|
|
terminal = all(node.status in {"completed", "skipped", "cancelled"}
|
|
for node in plan.plan.nodes)
|
|
pending_action = any(action.approval_status == "pending" or (
|
|
action.approval_status == "approved" and action.execution_status in {"not_submitted", "submitted", "unknown"}
|
|
) for action in actions)
|
|
lifecycle = "completed" if terminal and not pending_action else "running"
|
|
else:
|
|
lifecycle = self._status_from_plan(
|
|
plan.plan.nodes, actions, () if task["task_type"] == "event_investigation" else access_requests
|
|
)
|
|
if lifecycle == "running":
|
|
final_outcome = "failed"
|
|
lifecycle = "failed"
|
|
failure_code = "ARTIFACT_INCOMPLETE"
|
|
failure_detail = "Final plan still contains running work"
|
|
else:
|
|
lifecycle = "failed"
|
|
expected = run["revision"]
|
|
run.update(
|
|
status=final_outcome, revision=expected + 1, finished_at=now,
|
|
lease_expires_at=None, heartbeat_at=now,
|
|
failure_code=failure_code, failure_detail=failure_detail, updated_at=now,
|
|
)
|
|
run_model = OperationalTaskRun.model_validate(run)
|
|
await self.repo.update_run(_json(run_model), expected)
|
|
if final_outcome != "succeeded":
|
|
await task_questions.cancel_pending(self, task_id, now)
|
|
await self._fail_run_messages(run_id, failure_code or "Run failed", now)
|
|
task.update(
|
|
lifecycle_status=lifecycle,
|
|
status=lifecycle,
|
|
revision=task["revision"] + 1,
|
|
agent_session_id=None,
|
|
agent_lease_expires_at=None,
|
|
updated_at=now,
|
|
)
|
|
await self.repo.update_task(_task_json(OperationalTaskSummary.model_validate(task)))
|
|
await self._run_event(run_model, now)
|
|
await self._task_event(task, now)
|
|
if lifecycle == "failed":
|
|
await self._notify(
|
|
task_id,
|
|
"task_failed",
|
|
task["title"],
|
|
failure_detail or "工况任务执行失败",
|
|
now,
|
|
)
|
|
elif (lifecycle == "waiting_human" and not pending_question
|
|
and (task.get("handling_context") or {}).get("phase") != "waiting_scada"):
|
|
pending_actions = any(
|
|
action.approval_status == "pending" for action in actions
|
|
)
|
|
pending_access = any(
|
|
request.get("status") == "pending"
|
|
for request in await self._list_access_requests(task_id)
|
|
)
|
|
if not pending_actions and not pending_access:
|
|
await self._notify(
|
|
task_id,
|
|
"human_input_required",
|
|
task["title"],
|
|
"任务等待人工补充信息或业务确认",
|
|
now,
|
|
)
|
|
await self._save_request(scope, command.request_id, body, {"task_id": str(task_id)})
|
|
return await self.detail(task_id)
|
|
|
|
async def _run(self, task_id: UUID, run_id: UUID) -> dict:
|
|
run = await self.repo.get_run(str(run_id))
|
|
require(
|
|
run is not None and str(run["task_id"]) == str(task_id),
|
|
"RUN_NOT_FOUND", "Operational task run not found", status=404,
|
|
)
|
|
return run
|
|
|
|
async def _resolve_investigation_alerts(self, task_id: str, evidence_id: str) -> None:
|
|
from app.domain.schemas.task_alert import TaskAlertEvent
|
|
from app.infra.db.postgresql.task_alerts import TaskAlertRepository
|
|
from app.services.task_alerts import TaskAlertService
|
|
|
|
if not hasattr(self.repo, "list_investigation_alerts"):
|
|
return
|
|
for alert in await self.repo.list_investigation_alerts(task_id):
|
|
command = TaskAlertEvent.model_validate({
|
|
**{key: value for key, value in alert.items() if key not in {"status", "updated_at", "revision"}},
|
|
"event_id": uuid5(NAMESPACE_URL, f"recovery:{task_id}:{evidence_id}:{alert['alert_id']}"),
|
|
"event_type": "resolved",
|
|
})
|
|
await TaskAlertService(TaskAlertRepository(self.repo.connection, self.repo.project_id), None,
|
|
internal_producer_id=command.producer_id).push(command)
|
|
|
|
async def _task(self, task_id: UUID) -> dict:
|
|
task = await self.repo.get_task(str(task_id))
|
|
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
|
|
return task
|
|
|
|
@staticmethod
|
|
def _require_real_action(task: dict, kind: str) -> None:
|
|
require(kind == "create_work_order" or not (task.get("source") or {}).get("replay_context"),
|
|
"REPLAY_ACTION_DISABLED",
|
|
"回放任务不支持设备命令;可生成工单草稿,由有权限的用户确认发布", status=409)
|
|
|
|
@staticmethod
|
|
def _require_actionable_task(task: dict) -> None:
|
|
source = task.get("source") or {}
|
|
require(
|
|
task.get("lifecycle_status") != "cancelled" and task.get("status") != "cancelled"
|
|
and source.get("deleted_at") is None,
|
|
"TASK_NOT_ACTIONABLE",
|
|
"Cancelled or withdrawn tasks do not accept action changes",
|
|
status=409,
|
|
)
|
|
|
|
async def _action(self, task_id: UUID, action_id: UUID) -> HumanApprovalAction:
|
|
value = await self.repo.get_action(str(task_id), str(action_id))
|
|
require(value is not None, "ACTION_NOT_FOUND", "Action not found", status=404)
|
|
return HumanApprovalAction.model_validate(value)
|
|
|
|
async def _request_replay(self, scope: str, request_id: UUID, body: dict) -> dict | None:
|
|
await self.repo.lock(f"request:{scope}:{request_id}")
|
|
prior = await self.repo.get_request(scope, str(request_id))
|
|
if prior:
|
|
self._require_same_request(prior, body)
|
|
return prior["response"]
|
|
return None
|
|
|
|
async def _save_request(self, scope: str, request_id: UUID, body: dict, response: dict) -> None:
|
|
await self.repo.save_request(scope, str(request_id), digest(body), response)
|
|
|
|
@staticmethod
|
|
def _require_same_request(prior: dict, body: dict) -> None:
|
|
require(prior["request_digest"] == digest(body), "IDEMPOTENCY_CONFLICT",
|
|
"request_id was used with different content", status=409)
|
|
|
|
@staticmethod
|
|
def _execution_request_id(task_id: UUID, generation: int) -> UUID:
|
|
return uuid5(NAMESPACE_URL, f"operational-task:{task_id}:generation:{generation}")
|
|
|
|
@staticmethod
|
|
def _status_from_plan(
|
|
nodes,
|
|
actions: Iterable[HumanApprovalAction],
|
|
access_requests: Iterable[TaskAccessRequest] = (),
|
|
) -> str:
|
|
statuses = {node.status for node in nodes}
|
|
if "running" in statuses:
|
|
return "running"
|
|
if any(
|
|
node.status == "failed" and node.id.startswith("AGENT_RUN_FAILURE")
|
|
for node in nodes
|
|
):
|
|
return "failed"
|
|
if any(action.approval_status == "pending" for action in actions):
|
|
return "waiting_human"
|
|
if any(request.status == "pending" for request in access_requests):
|
|
return "waiting_human"
|
|
if statuses <= {"completed", "skipped", "cancelled"}:
|
|
return "completed"
|
|
if "failed" in statuses and statuses <= {"completed", "failed", "skipped", "cancelled"}:
|
|
return "failed"
|
|
if any(node.owner == "human" and node.status in {"pending", "waiting"} for node in nodes):
|
|
return "waiting_human"
|
|
return "running"
|
|
|
|
@staticmethod
|
|
def _investigation_status(actions: Iterable[HumanApprovalAction], task: dict | None = None) -> str:
|
|
"""Field completion and recovery evidence precede incident closure."""
|
|
if (task or {}).get("handling_context", {}).get("phase") in {"waiting_work_order", "waiting_scada"}:
|
|
return "waiting_human"
|
|
return "waiting_human" if any(
|
|
action.approval_status == "pending" or (
|
|
action.approval_status == "approved"
|
|
and action.execution_status in {"not_submitted", "submitted", "unknown"}
|
|
) for action in actions
|
|
) else "completed"
|
|
|
|
@staticmethod
|
|
def _validate_plan_state(plan, known_evidence: set[str], actions: dict[str, HumanApprovalAction]) -> None:
|
|
nodes = {node.id: node for node in plan.nodes}
|
|
predecessors: dict[str, set[str]] = {node.id: set() for node in plan.nodes}
|
|
for edge in plan.edges:
|
|
predecessors[edge.to_node].add(edge.from_node)
|
|
for node in plan.nodes:
|
|
if node.status in {"running", "completed"}:
|
|
incomplete = [
|
|
predecessor for predecessor in predecessors[node.id]
|
|
if nodes[predecessor].status not in {"completed", "skipped"}
|
|
]
|
|
require(not incomplete, "PLAN_DEPENDENCY_INCOMPLETE",
|
|
"A running or completed node has incomplete predecessors",
|
|
refs=incomplete, status=409)
|
|
if node.status == "completed":
|
|
structural = plan.schema_name == "execution-plan@2" and node.kind in {
|
|
"parallel_split", "parallel_join", "gate"
|
|
}
|
|
require(structural or bool(node.evidence_refs or node.action_refs),
|
|
"PLAN_NODE_UNSUPPORTED",
|
|
"Completed nodes require evidence or an executed action receipt",
|
|
refs=[node.id], status=409)
|
|
for action_id in node.action_refs:
|
|
action = actions.get(str(action_id))
|
|
require(action is not None, "ACTION_NOT_FOUND",
|
|
"Plan node references an unknown action", refs=[str(action_id)], status=409)
|
|
if node.status == "completed":
|
|
require(action.execution_status == "succeeded" and action.receipt is not None,
|
|
"ACTION_RECEIPT_REQUIRED",
|
|
"A completed action node requires a successful external receipt",
|
|
refs=[str(action_id)], status=409)
|
|
|
|
@staticmethod
|
|
def _evidence_facts(document: dict) -> dict:
|
|
return {
|
|
key: document.get(key)
|
|
for key in (
|
|
"evidence_id", "kind", "summary", "source_label", "source_ref", "observed_at"
|
|
)
|
|
}
|
|
|
|
@staticmethod
|
|
def _snapshot_facts(snapshot: TaskDataSnapshot) -> dict:
|
|
return snapshot.model_dump(
|
|
mode="json",
|
|
exclude={"queried_at"},
|
|
)
|
|
|
|
@staticmethod
|
|
def _require_active_run_lease(run: dict, now: datetime) -> None:
|
|
lease = OperationalTaskRun.model_validate(run).lease_expires_at
|
|
require(
|
|
lease is not None and lease > now,
|
|
"LEASE_EXPIRED",
|
|
"Agent run lease has expired",
|
|
status=409,
|
|
)
|
|
|
|
@staticmethod
|
|
def _public_evidence(document: dict) -> EvidenceRecord:
|
|
return EvidenceRecord.model_validate(
|
|
{key: value for key, value in document.items() if key != "data_snapshot"}
|
|
)
|
|
|
|
@staticmethod
|
|
def _require_evidence_refs(refs: Iterable[str], known: set[str]) -> None:
|
|
missing = sorted(set(refs) - known)
|
|
require(not missing, "EVIDENCE_NOT_FOUND", "Artifact references unknown evidence",
|
|
refs=missing, status=409)
|
|
|
|
@staticmethod
|
|
def _validate_action_payload(command: ActionProposalCommand) -> None:
|
|
model = CreateWorkOrderPayload if command.kind == "create_work_order" else IssueCommandPayload
|
|
try:
|
|
model.model_validate(command.payload)
|
|
except Exception as exc:
|
|
raise DomainError("ACTION_PAYLOAD_INVALID", "Action payload does not match its kind") from exc
|
|
|
|
@staticmethod
|
|
def _action_permissions(kind: str) -> tuple[str, str]:
|
|
if kind == "create_work_order":
|
|
return OPERATIONAL_ACTION_WORK_ORDER_APPROVE, OPERATIONAL_ACTION_WORK_ORDER_EXECUTE
|
|
return OPERATIONAL_ACTION_COMMAND_APPROVE, OPERATIONAL_ACTION_COMMAND_EXECUTE
|
|
|
|
@staticmethod
|
|
def _validate_dispatch_outcome(outcome) -> None:
|
|
if outcome.status in {"submitted", "succeeded"}:
|
|
require(outcome.receipt is not None, "RECEIPT_REQUIRED",
|
|
"Submitted actions require an external receipt", status=502)
|
|
if outcome.status == "succeeded":
|
|
require(outcome.receipt.completed_at is not None, "RECEIPT_REQUIRED",
|
|
"Successful actions require a completion receipt", status=502)
|
|
|
|
@staticmethod
|
|
def _apply_feedback_status(task: dict, status: str) -> None:
|
|
task["status"] = {
|
|
"scheduled": "queued", "assigned": "running", "in_progress": "running",
|
|
"awaiting_feedback": "waiting_human", "completed": "completed",
|
|
"cancelled": "cancelled", "failed": "failed",
|
|
}[status]
|
|
task["lifecycle_status"] = {
|
|
"scheduled": "open", "assigned": "open", "in_progress": "open",
|
|
"awaiting_feedback": "waiting_human", "completed": "completed",
|
|
"cancelled": "cancelled", "failed": "failed",
|
|
}[status]
|
|
task["agent_lease_expires_at"] = None
|
|
|
|
async def _requeue_task(self, task: dict, now: datetime) -> None:
|
|
await task_questions.cancel_pending(self, UUID(str(task["task_id"])), now)
|
|
if self._supports_runs:
|
|
active_run = await self.repo.get_active_run(str(task["task_id"]))
|
|
while active_run is not None:
|
|
await self.repo.lock(f"run:{active_run['run_id']}")
|
|
# Completion may have won the run lock after our first read.
|
|
refreshed_run = await self.repo.get_active_run(str(task["task_id"]))
|
|
if refreshed_run is None or str(refreshed_run["run_id"]) == str(active_run["run_id"]):
|
|
active_run = refreshed_run
|
|
break
|
|
active_run = refreshed_run
|
|
if active_run is not None and active_run["status"] == "queued":
|
|
# Multiple inputs can arrive before the worker claims the next
|
|
# generation. They are all visible when that queued run starts,
|
|
# so coalesce them instead of violating the one-active-run rule.
|
|
task.update(
|
|
lifecycle_status="open", status="queued",
|
|
revision=task["revision"] + 1,
|
|
execution_generation=active_run["generation"],
|
|
execution_request_id=str(active_run["execution_request_id"]),
|
|
agent_session_id=None, agent_lease_expires_at=None,
|
|
updated_at=now,
|
|
)
|
|
return
|
|
if active_run is not None:
|
|
expected = active_run["revision"]
|
|
active_run.update(
|
|
status="cancelled", revision=expected + 1,
|
|
lease_expires_at=None, finished_at=now,
|
|
failure_code="SUPERSEDED_BY_REQUEUE",
|
|
failure_detail="New task input superseded the active run",
|
|
updated_at=now,
|
|
)
|
|
cancelled = OperationalTaskRun.model_validate(active_run)
|
|
await self.repo.update_run(_json(cancelled), expected)
|
|
await self._fail_run_messages(active_run["run_id"], "Run superseded by requeue", now)
|
|
await self._run_event(cancelled, now)
|
|
generation = task["execution_generation"] + 1
|
|
task_id = UUID(task["task_id"])
|
|
task.update(
|
|
lifecycle_status="open", status="queued", revision=task["revision"] + 1,
|
|
execution_generation=generation,
|
|
execution_request_id=str(self._execution_request_id(task_id, generation)),
|
|
agent_session_id=None, agent_lease_expires_at=None, updated_at=now,
|
|
)
|
|
run = self._new_run(task_id, generation, now)
|
|
if self._supports_runs:
|
|
await self.repo.insert_run(_json(run))
|
|
await self._run_event(run, now, queued=True)
|
|
|
|
async def _notify(
|
|
self, task_id: UUID, kind: str, title: str, description: str, now: datetime,
|
|
*, required_permission: str | None = None,
|
|
subject_type: str = "task", subject_id: UUID | None = None,
|
|
report_reference: InvestigationReportReference | None = None,
|
|
) -> None:
|
|
notification = OperationalTaskNotification(
|
|
notification_id=uuid4(), task_id=task_id, kind=kind,
|
|
title=title, description=description, status="unread", version=1,
|
|
created_at=now, required_permission=required_permission,
|
|
lifecycle_status="active", subject_type=subject_type,
|
|
subject_id=subject_id or task_id,
|
|
report_reference=report_reference,
|
|
)
|
|
await self.repo.insert_notification(_json(notification))
|
|
await self._append_event(
|
|
topic="operational.notification.changed",
|
|
aggregate_type="operational_notification",
|
|
aggregate_id=str(notification.notification_id),
|
|
aggregate_revision=notification.version,
|
|
occurred_at=now,
|
|
metadata={"task_id": str(task_id), "kind": kind},
|
|
)
|
|
|
|
async def _resolve_notifications(
|
|
self,
|
|
task_id: UUID,
|
|
*,
|
|
kinds: list[str],
|
|
subject_type: str,
|
|
subject_id: UUID,
|
|
now: datetime,
|
|
reason: str,
|
|
) -> None:
|
|
resolve = getattr(self.repo, "resolve_notifications", None)
|
|
if resolve is None:
|
|
return
|
|
notifications = await resolve(
|
|
str(task_id),
|
|
kinds=kinds,
|
|
subject_type=subject_type,
|
|
subject_id=str(subject_id),
|
|
resolved_at=now,
|
|
resolution_reason=reason,
|
|
)
|
|
for notification in notifications:
|
|
await self._append_event(
|
|
topic="operational.notification.changed",
|
|
aggregate_type="operational_notification",
|
|
aggregate_id=notification["notification_id"],
|
|
aggregate_revision=2,
|
|
occurred_at=now,
|
|
metadata={
|
|
"task_id": str(task_id),
|
|
"kind": notification["kind"],
|
|
"change": "resolved",
|
|
"reason": reason,
|
|
},
|
|
)
|
|
|
|
def _new_run(
|
|
self, task_id: UUID, generation: int, now: datetime
|
|
) -> OperationalTaskRun:
|
|
return OperationalTaskRun(
|
|
run_id=uuid4(),
|
|
task_id=task_id,
|
|
generation=generation,
|
|
execution_request_id=self._execution_request_id(task_id, generation),
|
|
status="queued",
|
|
revision=1,
|
|
dispatch_attempts=0,
|
|
next_dispatch_at=now,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
async def _task_event(self, task: dict, now: datetime) -> None:
|
|
await self._append_event(
|
|
topic="operational.task.changed",
|
|
aggregate_type="operational_task",
|
|
aggregate_id=str(task["task_id"]),
|
|
aggregate_revision=int(task["revision"]),
|
|
occurred_at=now,
|
|
metadata={
|
|
"task_id": str(task["task_id"]),
|
|
"lifecycle_status": task.get("lifecycle_status", "open"),
|
|
},
|
|
)
|
|
|
|
async def _run_event(
|
|
self, run: OperationalTaskRun, now: datetime, *, queued: bool = False
|
|
) -> None:
|
|
await self._append_event(
|
|
topic="operational.run.queued" if queued else "operational.run.changed",
|
|
aggregate_type="operational_task_run",
|
|
aggregate_id=str(run.run_id),
|
|
aggregate_revision=run.revision,
|
|
occurred_at=now,
|
|
metadata={
|
|
"task_id": str(run.task_id),
|
|
"run_id": str(run.run_id),
|
|
"status": run.status,
|
|
},
|
|
)
|
|
|
|
async def _action_event(
|
|
self, action: HumanApprovalAction, now: datetime
|
|
) -> None:
|
|
await self._append_event(
|
|
topic="operational.action.changed",
|
|
aggregate_type="operational_action",
|
|
aggregate_id=str(action.action_id),
|
|
aggregate_revision=action.version,
|
|
occurred_at=now,
|
|
metadata={"task_id": str(action.task_id)},
|
|
)
|
|
|
|
def _scheduled_task(
|
|
self, schedule: OperationalTaskSchedule, task_id: UUID,
|
|
due_at: datetime, now: datetime,
|
|
) -> OperationalTaskSummary:
|
|
return OperationalTaskSummary(
|
|
execution_scope=self.execution_scope,
|
|
project_id=self.context.project_id, task_id=task_id,
|
|
task_type="scheduled_flow", workflow=schedule.schedule_key,
|
|
lifecycle_status="open", status="queued", priority=schedule.priority, revision=1,
|
|
execution_generation=1,
|
|
execution_request_id=self._execution_request_id(task_id, 1),
|
|
title=schedule.title, summary=schedule.summary,
|
|
source={
|
|
"kind": "schedule",
|
|
"external_id": f"{schedule.schedule_id}:{due_at.isoformat()}",
|
|
},
|
|
trigger={"kind": "schedule", "occurred_at": due_at,
|
|
"detail": f"定时流程 {schedule.schedule_key} 到期"},
|
|
scheduled_for=due_at, agent_session_id=None,
|
|
agent_lease_expires_at=None, current_plan_revision=None,
|
|
current_report_revision=None, created_at=now, updated_at=now,
|
|
)
|
|
|
|
|
|
def _json_dict(value: dict) -> dict:
|
|
"""Convert datetime values without allowing arbitrary object serialization."""
|
|
return {
|
|
key: item.isoformat().replace("+00:00", "Z") if isinstance(item, datetime) else item
|
|
for key, item in value.items()
|
|
}
|