feat(workbench): persist operational workflows and events
This commit is contained in:
@@ -525,6 +525,39 @@ class MetadataRepository:
|
||||
pool_max_size=record.pool_max_size,
|
||||
)
|
||||
|
||||
async def list_active_project_db_routings(
|
||||
self, db_role: str
|
||||
) -> List[ProjectDbRouting]:
|
||||
result = await self.session.execute(
|
||||
select(models.ProjectDatabase)
|
||||
.join(models.Project, models.Project.id == models.ProjectDatabase.project_id)
|
||||
.where(
|
||||
models.ProjectDatabase.db_role == db_role,
|
||||
models.Project.status == "active",
|
||||
)
|
||||
.order_by(models.ProjectDatabase.project_id)
|
||||
)
|
||||
if not is_database_encryption_configured():
|
||||
raise ValueError("DATABASE_ENCRYPTION_KEY is not configured")
|
||||
encryptor = get_database_encryptor()
|
||||
try:
|
||||
return [
|
||||
ProjectDbRouting(
|
||||
project_id=record.project_id,
|
||||
db_role=record.db_role,
|
||||
db_type=record.db_type,
|
||||
dsn=_normalize_postgres_dsn(encryptor.decrypt(record.dsn_encrypted)),
|
||||
pool_min_size=record.pool_min_size,
|
||||
pool_max_size=record.pool_max_size,
|
||||
)
|
||||
for record in result.scalars().all()
|
||||
]
|
||||
except InvalidToken as exc:
|
||||
raise ValueError(
|
||||
"Failed to decrypt project DB DSN: DATABASE_ENCRYPTION_KEY mismatch "
|
||||
"or invalid dsn_encrypted value"
|
||||
) from exc
|
||||
|
||||
async def list_projects_for_user(self, user_id: UUID) -> List[ProjectSummary]:
|
||||
stmt = (
|
||||
select(models.Project, models.UserProjectMembership.project_role)
|
||||
|
||||
@@ -53,18 +53,52 @@ class AnalysisRepository:
|
||||
status: str,
|
||||
created_by: str,
|
||||
parameters: dict[str, Any],
|
||||
) -> None:
|
||||
) -> int:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE analysis.runs
|
||||
SET created_by = %s, status = %s, parameters = %s
|
||||
SET created_by = %s, status = %s, parameters = %s,
|
||||
revision = revision + 1
|
||||
WHERE run_id = %s
|
||||
RETURNING revision
|
||||
""",
|
||||
(created_by, status, Jsonb(parameters), run_id),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
raise LookupError(f"analysis run {run_id} does not exist")
|
||||
return int(row["revision"])
|
||||
|
||||
@staticmethod
|
||||
def append_run_changed_event_sync(
|
||||
conn: Connection,
|
||||
*,
|
||||
project_id: UUID,
|
||||
run_id: UUID,
|
||||
revision: int,
|
||||
status: str,
|
||||
occurred_at: datetime,
|
||||
results_committed: bool,
|
||||
) -> None:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO workbench.domain_event_log
|
||||
(project_id,event_id,topic,aggregate_type,aggregate_id,
|
||||
aggregate_revision,occurred_at,metadata)
|
||||
VALUES (%s,%s,'analysis.run.changed','analysis_run',%s,%s,%s,%s)
|
||||
RETURNING event_seq""",
|
||||
(
|
||||
project_id, uuid4(), str(run_id), revision, occurred_at,
|
||||
Jsonb({"run_id": str(run_id), "status": status,
|
||||
"results_committed": results_committed}),
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
cur.execute(
|
||||
"SELECT pg_notify('workbench_events', %s)",
|
||||
(f'{project_id}:{row["event_seq"]}',),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def insert_result_sync(
|
||||
|
||||
@@ -0,0 +1,589 @@
|
||||
"""PostgreSQL persistence for durable operational tasks."""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
from psycopg import AsyncConnection
|
||||
from psycopg.errors import UndefinedTable
|
||||
from psycopg.types.json import Jsonb
|
||||
from uuid import uuid4
|
||||
|
||||
from app.domain.domain_errors import DomainError
|
||||
|
||||
|
||||
class OperationalTaskRepository:
|
||||
def __init__(self, connection: AsyncConnection, project_id):
|
||||
self.connection = connection
|
||||
self.project_id = str(project_id)
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
try:
|
||||
async with self.connection.transaction():
|
||||
yield self
|
||||
except UndefinedTable as exc:
|
||||
raise DomainError(
|
||||
"DEPENDENCY_UNAVAILABLE",
|
||||
"Project operational-task storage is not provisioned",
|
||||
status=503,
|
||||
) from exc
|
||||
|
||||
async def lock(self, key: str) -> None:
|
||||
await self.connection.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))",
|
||||
(f"tjwater-operational:{self.project_id}:{key}",),
|
||||
)
|
||||
|
||||
async def get_request(self, scope: str, request_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT request_digest,response
|
||||
FROM workbench.operational_task_requests
|
||||
WHERE project_id=%s AND scope=%s AND request_id=%s""",
|
||||
(self.project_id, scope, request_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def save_request(
|
||||
self, scope: str, request_id: str, request_digest: str, response: dict
|
||||
) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_requests
|
||||
(project_id,scope,request_id,request_digest,response)
|
||||
VALUES (%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(project_id,scope,request_id) DO UPDATE SET
|
||||
response=excluded.response
|
||||
WHERE operational_task_requests.request_digest=excluded.request_digest""",
|
||||
(self.project_id, scope, request_id, request_digest, Jsonb(response)),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
raise DomainError(
|
||||
"IDEMPOTENCY_CONFLICT",
|
||||
"request_id was used with different content",
|
||||
status=409,
|
||||
)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_tasks
|
||||
WHERE project_id=%s AND task_id=%s""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def get_active_run(self, task_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT run_id,task_id,generation,execution_request_id,status,revision,
|
||||
agent_session_id,lease_expires_at,heartbeat_at,started_at,finished_at,
|
||||
failure_code,failure_detail,dispatch_attempts,next_dispatch_at,
|
||||
created_at,updated_at
|
||||
FROM workbench.operational_task_runs
|
||||
WHERE project_id=%s AND task_id=%s
|
||||
AND status IN ('queued','claimed','running')
|
||||
ORDER BY generation DESC LIMIT 1""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def get_run(self, run_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT run_id,task_id,generation,execution_request_id,status,revision,
|
||||
agent_session_id,lease_expires_at,heartbeat_at,started_at,finished_at,
|
||||
failure_code,failure_detail,dispatch_attempts,next_dispatch_at,
|
||||
created_at,updated_at
|
||||
FROM workbench.operational_task_runs
|
||||
WHERE project_id=%s AND run_id=%s""",
|
||||
(self.project_id, run_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def insert_run(self, run: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_runs
|
||||
(project_id,run_id,task_id,generation,execution_request_id,status,revision,
|
||||
agent_session_id,lease_expires_at,heartbeat_at,started_at,finished_at,
|
||||
failure_code,failure_detail,dispatch_attempts,next_dispatch_at,
|
||||
created_at,updated_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(
|
||||
self.project_id, run["run_id"], run["task_id"], run["generation"],
|
||||
run["execution_request_id"], run["status"], run["revision"],
|
||||
run.get("agent_session_id"), run.get("lease_expires_at"),
|
||||
run.get("heartbeat_at"), run.get("started_at"), run.get("finished_at"),
|
||||
run.get("failure_code"), run.get("failure_detail"),
|
||||
run.get("dispatch_attempts", 0), run["next_dispatch_at"],
|
||||
run["created_at"], run["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
async def update_run(self, run: dict, expected_revision: int) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"""UPDATE workbench.operational_task_runs SET
|
||||
status=%s,revision=%s,agent_session_id=%s,lease_expires_at=%s,
|
||||
heartbeat_at=%s,started_at=%s,finished_at=%s,failure_code=%s,
|
||||
failure_detail=%s,dispatch_attempts=%s,next_dispatch_at=%s,updated_at=%s
|
||||
WHERE project_id=%s AND run_id=%s AND revision=%s""",
|
||||
(
|
||||
run["status"], run["revision"], run.get("agent_session_id"),
|
||||
run.get("lease_expires_at"), run.get("heartbeat_at"),
|
||||
run.get("started_at"), run.get("finished_at"), run.get("failure_code"),
|
||||
run.get("failure_detail"), run.get("dispatch_attempts", 0),
|
||||
run["next_dispatch_at"], run["updated_at"], self.project_id,
|
||||
run["run_id"], expected_revision,
|
||||
),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
raise DomainError("VERSION_CONFLICT", "Run revision changed", status=409)
|
||||
|
||||
async def list_runs(
|
||||
self, *, status: str | None, limit: int, cursor_created_at=None, cursor_run_id=None
|
||||
) -> list[dict]:
|
||||
result = await self.connection.execute(
|
||||
"""SELECT run_id,task_id,generation,execution_request_id,status,revision,
|
||||
agent_session_id,lease_expires_at,heartbeat_at,started_at,finished_at,
|
||||
failure_code,failure_detail,dispatch_attempts,next_dispatch_at,
|
||||
created_at,updated_at
|
||||
FROM workbench.operational_task_runs
|
||||
WHERE project_id=%s
|
||||
AND (%s::text IS NULL OR status=%s::text)
|
||||
AND (%s::timestamptz IS NULL OR (created_at,run_id)>(%s::timestamptz,%s::uuid))
|
||||
AND (status<>'queued' OR next_dispatch_at<=now())
|
||||
ORDER BY created_at,run_id LIMIT %s""",
|
||||
(
|
||||
self.project_id, status, status, cursor_created_at,
|
||||
cursor_created_at, cursor_run_id, limit,
|
||||
),
|
||||
)
|
||||
return [dict(row) for row in await result.fetchall()]
|
||||
|
||||
async def claim_expired_runs(self, limit: int) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT run_id,task_id,generation,execution_request_id,status,revision,
|
||||
agent_session_id,lease_expires_at,heartbeat_at,started_at,finished_at,
|
||||
failure_code,failure_detail,dispatch_attempts,next_dispatch_at,
|
||||
created_at,updated_at
|
||||
FROM workbench.operational_task_runs
|
||||
WHERE project_id=%s AND status IN ('claimed','running')
|
||||
AND lease_expires_at IS NOT NULL AND lease_expires_at<=now()
|
||||
ORDER BY lease_expires_at,run_id FOR UPDATE SKIP LOCKED LIMIT %s""",
|
||||
(self.project_id, limit),
|
||||
)
|
||||
return [dict(row) for row in await cursor.fetchall()]
|
||||
|
||||
async def consecutive_timed_out_runs(self, task_id: str) -> int:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT status FROM workbench.operational_task_runs
|
||||
WHERE project_id=%s AND task_id=%s ORDER BY generation DESC""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
count = 0
|
||||
for row in await cursor.fetchall():
|
||||
if row["status"] != "timed_out":
|
||||
break
|
||||
count += 1
|
||||
return count
|
||||
|
||||
async def append_event(
|
||||
self, *, topic: str, aggregate_type: str, aggregate_id: str,
|
||||
aggregate_revision: int, occurred_at, metadata: dict | None = None,
|
||||
) -> int:
|
||||
event_id = uuid4()
|
||||
cursor = await self.connection.execute(
|
||||
"""INSERT INTO workbench.domain_event_log
|
||||
(project_id,event_id,topic,aggregate_type,aggregate_id,
|
||||
aggregate_revision,occurred_at,metadata)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING event_seq""",
|
||||
(
|
||||
self.project_id, event_id, topic, aggregate_type, aggregate_id,
|
||||
aggregate_revision, occurred_at, Jsonb(metadata or {}),
|
||||
),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
event_seq = int(row["event_seq"])
|
||||
await self.connection.execute(
|
||||
"SELECT pg_notify('workbench_events', %s)",
|
||||
(f'{self.project_id}:{event_seq}',),
|
||||
)
|
||||
return event_seq
|
||||
|
||||
async def get_task_by_create_request(self, request_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_tasks
|
||||
WHERE project_id=%s AND create_request_id=%s""",
|
||||
(self.project_id, request_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def get_task_by_external_source(
|
||||
self, source_kind: str, external_id: str
|
||||
) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_tasks
|
||||
WHERE project_id=%s
|
||||
AND document#>>'{source,kind}'=%s
|
||||
AND document#>>'{source,external_id}'=%s""",
|
||||
(self.project_id, source_kind, external_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def list_tasks(
|
||||
self,
|
||||
*,
|
||||
task_type: str | None,
|
||||
status: str | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT task.document,
|
||||
CASE WHEN active_run.run_id IS NULL THEN NULL
|
||||
ELSE to_jsonb(active_run) - 'project_id' END AS active_run
|
||||
FROM workbench.operational_tasks AS task
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT run.* FROM workbench.operational_task_runs AS run
|
||||
WHERE run.project_id=task.project_id AND run.task_id=task.task_id
|
||||
AND run.status IN ('queued','claimed','running')
|
||||
ORDER BY run.generation DESC LIMIT 1
|
||||
) AS active_run ON true
|
||||
WHERE task.project_id=%s
|
||||
AND (%s::text IS NULL OR task.task_type=%s::text)
|
||||
AND (%s::text IS NULL OR task.status=%s::text)
|
||||
ORDER BY task.updated_at DESC,task.task_id
|
||||
LIMIT %s OFFSET %s""",
|
||||
(
|
||||
self.project_id,
|
||||
task_type,
|
||||
task_type,
|
||||
status,
|
||||
status,
|
||||
limit,
|
||||
offset,
|
||||
),
|
||||
)
|
||||
return [
|
||||
{**row["document"], "active_run": row["active_run"]}
|
||||
for row in await cursor.fetchall()
|
||||
]
|
||||
|
||||
async def insert_task(self, task: dict, create_request_id: str) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_tasks
|
||||
(project_id,task_id,create_request_id,task_type,workflow,status,lifecycle_status,priority,
|
||||
revision,execution_generation,execution_request_id,agent_session_id,
|
||||
agent_lease_expires_at,current_plan_revision,current_report_revision,
|
||||
scheduled_for,created_at,updated_at,document)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
self._task_params(task, create_request_id),
|
||||
)
|
||||
|
||||
async def update_task(self, task: dict) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"""UPDATE workbench.operational_tasks SET
|
||||
status=%s,lifecycle_status=%s,revision=%s,execution_generation=%s,execution_request_id=%s,
|
||||
agent_session_id=%s,agent_lease_expires_at=%s,current_plan_revision=%s,
|
||||
current_report_revision=%s,updated_at=%s,document=%s
|
||||
WHERE project_id=%s AND task_id=%s AND revision=%s""",
|
||||
(
|
||||
task["status"], task["lifecycle_status"], task["revision"], task["execution_generation"],
|
||||
task["execution_request_id"], task.get("agent_session_id"),
|
||||
task.get("agent_lease_expires_at"), task.get("current_plan_revision"),
|
||||
task.get("current_report_revision"), task["updated_at"], Jsonb(task),
|
||||
self.project_id, task["task_id"], task["revision"] - 1,
|
||||
),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
raise DomainError("VERSION_CONFLICT", "Task revision changed", status=409)
|
||||
|
||||
def _task_params(self, task: dict, create_request_id: str) -> tuple:
|
||||
return (
|
||||
self.project_id, task["task_id"], create_request_id, task["task_type"],
|
||||
task["workflow"], task["status"], task["lifecycle_status"], task["priority"], task["revision"],
|
||||
task["execution_generation"], task["execution_request_id"],
|
||||
task.get("agent_session_id"), task.get("agent_lease_expires_at"),
|
||||
task.get("current_plan_revision"), task.get("current_report_revision"),
|
||||
task.get("scheduled_for"), task["created_at"], task["updated_at"], Jsonb(task),
|
||||
)
|
||||
|
||||
async def get_evidence(self, task_id: str, evidence_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_evidence
|
||||
WHERE project_id=%s AND task_id=%s AND evidence_id=%s""",
|
||||
(self.project_id, task_id, evidence_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def list_evidence(self, task_id: str) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_evidence
|
||||
WHERE project_id=%s AND task_id=%s ORDER BY created_at,evidence_id""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
return [row["document"] for row in await cursor.fetchall()]
|
||||
|
||||
async def insert_evidence(self, task_id: str, evidence: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_evidence
|
||||
(project_id,task_id,evidence_id,document,created_at)
|
||||
VALUES (%s,%s,%s,%s,%s)""",
|
||||
(self.project_id, task_id, evidence["evidence_id"], Jsonb(evidence),
|
||||
evidence["recorded_at"]),
|
||||
)
|
||||
|
||||
async def get_latest_plan(self, task_id: str) -> dict | None:
|
||||
return await self._latest_artifact("operational_task_plans", task_id)
|
||||
|
||||
async def get_latest_report(self, task_id: str) -> dict | None:
|
||||
return await self._latest_artifact("operational_task_reports", task_id)
|
||||
|
||||
async def _latest_artifact(self, table: str, task_id: str) -> dict | None:
|
||||
if table not in {"operational_task_plans", "operational_task_reports"}:
|
||||
raise ValueError("unsupported artifact table")
|
||||
cursor = await self.connection.execute(
|
||||
f"""SELECT document FROM workbench.{table}
|
||||
WHERE project_id=%s AND task_id=%s ORDER BY revision DESC LIMIT 1""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def insert_plan(self, task_id: str, snapshot: dict) -> None:
|
||||
await self._insert_artifact("operational_task_plans", task_id, snapshot)
|
||||
|
||||
async def insert_report(self, task_id: str, snapshot: dict) -> None:
|
||||
await self._insert_artifact("operational_task_reports", task_id, snapshot)
|
||||
|
||||
async def _insert_artifact(self, table: str, task_id: str, snapshot: dict) -> None:
|
||||
if table not in {"operational_task_plans", "operational_task_reports"}:
|
||||
raise ValueError("unsupported artifact table")
|
||||
await self.connection.execute(
|
||||
f"""INSERT INTO workbench.{table}
|
||||
(project_id,task_id,revision,document,created_at)
|
||||
VALUES (%s,%s,%s,%s,%s)""",
|
||||
(self.project_id, task_id, snapshot["revision"], Jsonb(snapshot),
|
||||
snapshot["published_at"]),
|
||||
)
|
||||
|
||||
async def get_action(self, task_id: str, action_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_actions
|
||||
WHERE project_id=%s AND task_id=%s AND action_id=%s""",
|
||||
(self.project_id, task_id, action_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def list_actions(self, task_id: str) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_actions
|
||||
WHERE project_id=%s AND task_id=%s ORDER BY created_at,action_id""",
|
||||
(self.project_id, task_id),
|
||||
)
|
||||
return [row["document"] for row in await cursor.fetchall()]
|
||||
|
||||
async def insert_action(self, task_id: str, action: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_actions
|
||||
(project_id,task_id,action_id,version,approval_status,execution_status,
|
||||
document,created_at,updated_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(self.project_id, task_id, action["action_id"], action["version"],
|
||||
action["approval_status"], action["execution_status"], Jsonb(action),
|
||||
action["proposed_at"], action["proposed_at"]),
|
||||
)
|
||||
await self._insert_action_revision(task_id, action)
|
||||
|
||||
async def update_action(self, task_id: str, action: dict) -> None:
|
||||
await self._insert_action_revision(task_id, action)
|
||||
await self.connection.execute(
|
||||
"""UPDATE workbench.operational_task_actions SET
|
||||
version=%s,approval_status=%s,execution_status=%s,document=%s,updated_at=%s
|
||||
WHERE project_id=%s AND task_id=%s AND action_id=%s""",
|
||||
(action["version"], action["approval_status"], action["execution_status"],
|
||||
Jsonb(action), action.get("executed_at") or action.get("decided_at") or
|
||||
action["proposed_at"], self.project_id, task_id, action["action_id"]),
|
||||
)
|
||||
|
||||
async def _insert_action_revision(self, task_id: str, action: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_action_revisions
|
||||
(project_id,task_id,action_id,version,document,recorded_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s)""",
|
||||
(
|
||||
self.project_id, task_id, action["action_id"], action["version"],
|
||||
Jsonb(action), action.get("executed_at") or action.get("decided_at")
|
||||
or action["proposed_at"],
|
||||
),
|
||||
)
|
||||
|
||||
async def get_feedback(self, task_id: str, source_event_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT source_revision,document,accepted_at
|
||||
FROM workbench.operational_task_external_feedback
|
||||
WHERE project_id=%s AND task_id=%s AND source_event_id=%s""",
|
||||
(self.project_id, task_id, source_event_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def upsert_feedback(self, task_id: str, feedback: dict, accepted_at) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_external_feedback
|
||||
(project_id,task_id,source_event_id,source_revision,document,accepted_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(project_id,task_id,source_event_id) DO UPDATE SET
|
||||
source_revision=excluded.source_revision,document=excluded.document,
|
||||
accepted_at=excluded.accepted_at""",
|
||||
(self.project_id, task_id, feedback["source_event_id"],
|
||||
feedback["source_revision"], Jsonb(feedback), accepted_at),
|
||||
)
|
||||
|
||||
async def insert_notification(self, notification: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_notifications
|
||||
(project_id,notification_id,task_id,created_at,document)
|
||||
VALUES (%s,%s,%s,%s,%s)""",
|
||||
(self.project_id, notification["notification_id"], notification["task_id"],
|
||||
notification["created_at"], Jsonb(notification)),
|
||||
)
|
||||
|
||||
async def get_notification(self, notification_id: str, user_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT n.document || jsonb_build_object(
|
||||
'status',CASE WHEN r.read_at IS NULL THEN 'unread' ELSE 'read' END,
|
||||
'version',COALESCE(r.version,1),
|
||||
'read_at',r.read_at,'read_by',
|
||||
CASE WHEN r.user_id IS NULL THEN NULL ELSE 'principal:' || r.user_id::text END
|
||||
) AS document
|
||||
FROM workbench.operational_task_notifications n
|
||||
LEFT JOIN workbench.operational_task_notification_reads r
|
||||
ON r.project_id=n.project_id AND r.notification_id=n.notification_id
|
||||
AND r.user_id=%s
|
||||
WHERE n.project_id=%s AND n.notification_id=%s""",
|
||||
(user_id, self.project_id, notification_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def mark_notification_read(
|
||||
self, notification_id: str, user_id: str, version: int, read_at
|
||||
) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_notification_reads
|
||||
(project_id,notification_id,user_id,version,read_at)
|
||||
VALUES (%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(project_id,notification_id,user_id) DO UPDATE SET
|
||||
version=excluded.version,read_at=excluded.read_at""",
|
||||
(self.project_id, notification_id, user_id, version, read_at),
|
||||
)
|
||||
|
||||
async def list_notifications(
|
||||
self, *, user_id: str, allowed_permissions: list[str],
|
||||
unread_only: bool, limit: int, offset: int
|
||||
) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT n.document || jsonb_build_object(
|
||||
'status',CASE WHEN r.read_at IS NULL THEN 'unread' ELSE 'read' END,
|
||||
'version',COALESCE(r.version,1),
|
||||
'read_at',r.read_at,'read_by',
|
||||
CASE WHEN r.user_id IS NULL THEN NULL ELSE 'principal:' || r.user_id::text END
|
||||
) AS document
|
||||
FROM workbench.operational_task_notifications n
|
||||
LEFT JOIN workbench.operational_task_notification_reads r
|
||||
ON r.project_id=n.project_id AND r.notification_id=n.notification_id
|
||||
AND r.user_id=%s
|
||||
WHERE n.project_id=%s
|
||||
AND (%s=false OR r.read_at IS NULL)
|
||||
AND (n.document->>'required_permission' IS NULL
|
||||
OR n.document->>'required_permission'=ANY(%s))
|
||||
ORDER BY n.created_at DESC,n.notification_id LIMIT %s OFFSET %s""",
|
||||
(user_id, self.project_id, unread_only, allowed_permissions, limit, offset),
|
||||
)
|
||||
return [row["document"] for row in await cursor.fetchall()]
|
||||
|
||||
async def get_schedule_by_request(self, request_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_schedules
|
||||
WHERE project_id=%s AND create_request_id=%s""",
|
||||
(self.project_id, request_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def insert_schedule(self, schedule: dict, request_id: str) -> None:
|
||||
await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_task_schedules
|
||||
(project_id,schedule_id,create_request_id,schedule_key,next_run_at,
|
||||
interval_seconds,enabled,version,document,created_at,updated_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(self.project_id, schedule["schedule_id"], request_id,
|
||||
schedule["schedule_key"], schedule["next_run_at"],
|
||||
schedule["interval_seconds"], schedule["enabled"], schedule["version"],
|
||||
Jsonb(schedule), schedule["created_at"], schedule["updated_at"]),
|
||||
)
|
||||
|
||||
async def list_schedules(self) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_schedules
|
||||
WHERE project_id=%s ORDER BY next_run_at,schedule_id""",
|
||||
(self.project_id,),
|
||||
)
|
||||
return [row["document"] for row in await cursor.fetchall()]
|
||||
|
||||
async def get_schedule(self, schedule_id: str) -> dict | None:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_schedules
|
||||
WHERE project_id=%s AND schedule_id=%s""",
|
||||
(self.project_id, schedule_id),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["document"] if row else None
|
||||
|
||||
async def update_schedule(self, schedule: dict) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"""UPDATE workbench.operational_task_schedules SET
|
||||
enabled=%s,version=%s,updated_at=%s,document=%s
|
||||
WHERE project_id=%s AND schedule_id=%s AND version=%s""",
|
||||
(
|
||||
schedule["enabled"], schedule["version"], schedule["updated_at"],
|
||||
Jsonb(schedule), self.project_id, schedule["schedule_id"],
|
||||
schedule["version"] - 1,
|
||||
),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
raise DomainError("VERSION_CONFLICT", "Schedule version changed", status=409)
|
||||
|
||||
async def claim_due_schedules(self, limit: int) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT document FROM workbench.operational_task_schedules
|
||||
WHERE project_id=%s AND enabled=true AND next_run_at <= now()
|
||||
ORDER BY next_run_at,schedule_id FOR UPDATE SKIP LOCKED LIMIT %s""",
|
||||
(self.project_id, limit),
|
||||
)
|
||||
return [row["document"] for row in await cursor.fetchall()]
|
||||
|
||||
async def advance_schedule(self, schedule: dict) -> None:
|
||||
await self.connection.execute(
|
||||
"""UPDATE workbench.operational_task_schedules SET
|
||||
next_run_at=%s,version=%s,updated_at=%s,document=%s
|
||||
WHERE project_id=%s AND schedule_id=%s""",
|
||||
(schedule["next_run_at"], schedule["version"], schedule["updated_at"],
|
||||
Jsonb(schedule), self.project_id, schedule["schedule_id"]),
|
||||
)
|
||||
|
||||
async def insert_occurrence(self, schedule_id: str, due_at, task_id: str, created_at) -> bool:
|
||||
cursor = await self.connection.execute(
|
||||
"""INSERT INTO workbench.operational_schedule_occurrences
|
||||
(project_id,schedule_id,due_at,task_id,created_at)
|
||||
VALUES (%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(project_id,schedule_id,due_at) DO NOTHING
|
||||
RETURNING task_id""",
|
||||
(self.project_id, schedule_id, due_at, task_id, created_at),
|
||||
)
|
||||
return await cursor.fetchone() is not None
|
||||
@@ -6,6 +6,7 @@ from contextlib import asynccontextmanager
|
||||
from psycopg import AsyncConnection
|
||||
from psycopg.errors import UndefinedTable
|
||||
from psycopg.types.json import Jsonb
|
||||
from uuid import uuid4
|
||||
|
||||
from app.domain.domain_errors import DomainError
|
||||
|
||||
@@ -86,3 +87,21 @@ class TaskAlertRepository:
|
||||
Jsonb({key: value for key, value in state.items() if key != "revision"}),
|
||||
),
|
||||
)
|
||||
|
||||
async def append_changed_event(self, state: dict) -> None:
|
||||
cursor = await self.connection.execute(
|
||||
"""INSERT INTO workbench.domain_event_log
|
||||
(project_id,event_id,topic,aggregate_type,aggregate_id,
|
||||
aggregate_revision,occurred_at,metadata)
|
||||
VALUES (%s,%s,'workbench.alert.changed','workbench_alert',%s,%s,%s,%s)
|
||||
RETURNING event_seq""",
|
||||
(
|
||||
self.project_id, uuid4(), state["alert_id"], state["revision"],
|
||||
state["updated_at"], Jsonb({"alert_id": state["alert_id"]}),
|
||||
),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
await self.connection.execute(
|
||||
"SELECT pg_notify('workbench_events', %s)",
|
||||
(f'{self.project_id}:{row["event_seq"]}',),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Per-process, project-scoped PostgreSQL LISTEN hubs for workbench SSE."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg import AsyncConnection
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.infra.db.postgresql.workbench_events import WorkbenchEventRepository
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WorkbenchEventHub:
|
||||
"""Own one LISTEN connection and fan wake-ups out to SSE subscribers."""
|
||||
|
||||
def __init__(self, project_id: UUID, dsn: str) -> None:
|
||||
self.project_id = project_id
|
||||
self.dsn = dsn
|
||||
self.subscribers = 0
|
||||
self._connection: AsyncConnection | None = None
|
||||
self._pool: AsyncConnectionPool | None = None
|
||||
self._connection_lock = asyncio.Lock()
|
||||
self._condition = asyncio.Condition()
|
||||
self._generation = 0
|
||||
self._ready = asyncio.Event()
|
||||
self._closed = False
|
||||
self._listener_task: asyncio.Task | None = None
|
||||
self._idle_task: asyncio.Task | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._listener_task is None or self._listener_task.done():
|
||||
self._closed = False
|
||||
self._ready.clear()
|
||||
self._listener_task = asyncio.create_task(
|
||||
self._listen_loop(), name=f"workbench-events:{self.project_id}"
|
||||
)
|
||||
await asyncio.wait_for(self._ready.wait(), timeout=10)
|
||||
|
||||
async def _connect(self) -> AsyncConnection:
|
||||
if self._pool is None:
|
||||
self._pool = AsyncConnectionPool(
|
||||
conninfo=self.dsn,
|
||||
min_size=0,
|
||||
max_size=1,
|
||||
open=False,
|
||||
kwargs={"autocommit": True, "row_factory": dict_row},
|
||||
)
|
||||
await self._pool.open()
|
||||
connection = await self._pool.getconn()
|
||||
await connection.execute("LISTEN workbench_events")
|
||||
return connection
|
||||
|
||||
async def _listen_loop(self) -> None:
|
||||
retry_seconds = 1
|
||||
while not self._closed:
|
||||
try:
|
||||
if self._connection is None or self._connection.closed:
|
||||
async with self._connection_lock:
|
||||
if self._connection is None or self._connection.closed:
|
||||
self._connection = await self._connect()
|
||||
self._ready.set()
|
||||
retry_seconds = 1
|
||||
notified = False
|
||||
async with self._connection_lock:
|
||||
connection = self._connection
|
||||
if connection is None:
|
||||
continue
|
||||
async for notification in connection.notifies(timeout=1, stop_after=1):
|
||||
if notification.payload.startswith(f"{self.project_id}:"):
|
||||
notified = True
|
||||
break
|
||||
if notified:
|
||||
await self._wake()
|
||||
await asyncio.sleep(0)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Workbench event listener failed for project %s", self.project_id)
|
||||
await self._drop_connection()
|
||||
await self._wake()
|
||||
await asyncio.sleep(retry_seconds)
|
||||
retry_seconds = min(retry_seconds * 2, 30)
|
||||
|
||||
async def _wake(self) -> None:
|
||||
async with self._condition:
|
||||
self._generation += 1
|
||||
self._condition.notify_all()
|
||||
|
||||
async def wait_for_wake(self, generation: int, timeout: float) -> int:
|
||||
async with self._condition:
|
||||
if self._generation != generation:
|
||||
return self._generation
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(
|
||||
self._condition.wait_for(lambda: self._generation != generation),
|
||||
timeout=timeout,
|
||||
)
|
||||
return self._generation
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
return self._generation
|
||||
|
||||
async def oldest_cursor(self) -> int | None:
|
||||
return await self._read(lambda repository: repository.oldest_cursor())
|
||||
|
||||
async def current_cursor(self) -> int:
|
||||
return await self._read(lambda repository: repository.current_cursor())
|
||||
|
||||
async def read_after(self, after: int) -> list[dict]:
|
||||
return await self._read(lambda repository: repository.read_after(after))
|
||||
|
||||
async def _read(self, operation):
|
||||
await self.start()
|
||||
async with self._connection_lock:
|
||||
if self._connection is None or self._connection.closed:
|
||||
self._connection = await self._connect()
|
||||
self._ready.set()
|
||||
repository = WorkbenchEventRepository(self._connection, self.project_id)
|
||||
return await operation(repository)
|
||||
|
||||
async def _drop_connection(self) -> None:
|
||||
async with self._connection_lock:
|
||||
connection, self._connection = self._connection, None
|
||||
if connection is not None and self._pool is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._pool.putconn(connection)
|
||||
|
||||
async def close(self) -> None:
|
||||
self._closed = True
|
||||
if self._idle_task is not None:
|
||||
self._idle_task.cancel()
|
||||
self._idle_task = None
|
||||
if self._listener_task is not None:
|
||||
self._listener_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await self._listener_task
|
||||
self._listener_task = None
|
||||
await self._drop_connection()
|
||||
if self._pool is not None:
|
||||
await self._pool.close()
|
||||
self._pool = None
|
||||
|
||||
|
||||
class WorkbenchEventHubManager:
|
||||
def __init__(self, idle_seconds: float = 60) -> None:
|
||||
self.idle_seconds = idle_seconds
|
||||
self._hubs: dict[tuple[UUID, str], WorkbenchEventHub] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@asynccontextmanager
|
||||
async def subscribe(self, project_id: UUID, dsn: str):
|
||||
key = (project_id, dsn)
|
||||
async with self._lock:
|
||||
hub = self._hubs.get(key)
|
||||
if hub is None:
|
||||
hub = WorkbenchEventHub(project_id, dsn)
|
||||
self._hubs[key] = hub
|
||||
if hub._idle_task is not None:
|
||||
hub._idle_task.cancel()
|
||||
hub._idle_task = None
|
||||
hub.subscribers += 1
|
||||
try:
|
||||
await hub.start()
|
||||
yield hub
|
||||
finally:
|
||||
async with self._lock:
|
||||
hub.subscribers = max(0, hub.subscribers - 1)
|
||||
if hub.subscribers == 0 and hub._idle_task is None:
|
||||
hub._idle_task = asyncio.create_task(self._close_when_idle(key, hub))
|
||||
|
||||
async def _close_when_idle(
|
||||
self, key: tuple[UUID, str], hub: WorkbenchEventHub
|
||||
) -> None:
|
||||
try:
|
||||
await asyncio.sleep(self.idle_seconds)
|
||||
async with self._lock:
|
||||
if hub.subscribers or self._hubs.get(key) is not hub:
|
||||
return
|
||||
self._hubs.pop(key, None)
|
||||
hub._idle_task = None
|
||||
await hub.close()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
async def close_all(self) -> None:
|
||||
async with self._lock:
|
||||
hubs = list(self._hubs.values())
|
||||
self._hubs.clear()
|
||||
await asyncio.gather(*(hub.close() for hub in hubs), return_exceptions=True)
|
||||
|
||||
|
||||
workbench_event_hubs = WorkbenchEventHubManager()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Cursor reads for the durable project-scoped workbench event log."""
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import AsyncConnection
|
||||
|
||||
|
||||
class WorkbenchEventRepository:
|
||||
def __init__(self, connection: AsyncConnection, project_id):
|
||||
self.connection = connection
|
||||
self.project_id = str(project_id)
|
||||
|
||||
async def current_cursor(self) -> int:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT COALESCE(max(event_seq),0) AS cursor FROM workbench.domain_event_log WHERE project_id=%s",
|
||||
(self.project_id,),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return int(row["cursor"])
|
||||
|
||||
async def oldest_cursor(self) -> int | None:
|
||||
cursor = await self.connection.execute(
|
||||
"SELECT min(event_seq) AS cursor FROM workbench.domain_event_log WHERE project_id=%s",
|
||||
(self.project_id,),
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return int(row["cursor"]) if row and row["cursor"] is not None else None
|
||||
|
||||
async def read_after(self, after: int, limit: int = 200) -> list[dict]:
|
||||
cursor = await self.connection.execute(
|
||||
"""SELECT event_seq,event_id,project_id,topic,aggregate_type,aggregate_id,
|
||||
aggregate_revision,occurred_at,metadata
|
||||
FROM workbench.domain_event_log
|
||||
WHERE project_id=%s AND event_seq>%s
|
||||
ORDER BY event_seq LIMIT %s""",
|
||||
(self.project_id, after, limit),
|
||||
)
|
||||
return [dict(row) for row in await cursor.fetchall()]
|
||||
@@ -4,6 +4,7 @@ from contextlib import contextmanager
|
||||
from contextvars import ContextVar, Token
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterator
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
||||
|
||||
@@ -15,6 +16,7 @@ class ActiveProjectRouting:
|
||||
project_code: str
|
||||
business_dsn: str
|
||||
timescale_dsn: str | None = None
|
||||
project_id: UUID | None = None
|
||||
|
||||
@property
|
||||
def business_database_name(self) -> str:
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Typed external action adapters. Credentials are request-scoped, never persisted."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Literal, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from app.auth.project_dependencies import ProjectContext
|
||||
from app.domain.schemas.operational_task import (
|
||||
ActionReceipt, CreateWorkOrderPayload, HumanApprovalAction,
|
||||
IssueCommandPayload, OperationalTaskDetail,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActionDispatchOutcome:
|
||||
status: Literal["submitted", "succeeded", "failed", "unsupported", "unknown"]
|
||||
detail: str
|
||||
receipt: ActionReceipt | None = None
|
||||
|
||||
|
||||
class OperationalActionDispatcher(Protocol):
|
||||
async def dispatch(
|
||||
self, *, context: ProjectContext, task: OperationalTaskDetail,
|
||||
action: HumanApprovalAction, credential: str | None,
|
||||
) -> ActionDispatchOutcome: ...
|
||||
|
||||
async def reconcile(
|
||||
self, *, context: ProjectContext, task: OperationalTaskDetail,
|
||||
action: HumanApprovalAction, credential: str | None,
|
||||
) -> ActionDispatchOutcome: ...
|
||||
|
||||
|
||||
class OperationalActionSettings(BaseSettings):
|
||||
peripheral_url: str = ""
|
||||
command_gateway_url: str = ""
|
||||
timeout_seconds: float = Field(default=10, ge=1, le=60)
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="OPERATIONAL_ACTION_", extra="ignore",
|
||||
env_file=Path(__file__).resolve().parents[3] / ".env",
|
||||
)
|
||||
|
||||
|
||||
class SampleWorkOrderReceipt(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
work_order_id: UUID
|
||||
project_id: UUID
|
||||
created_at: datetime
|
||||
origin: Literal["synthetic"]
|
||||
status: Literal[
|
||||
"draft", "open", "in_progress", "awaiting_reply", "completed", "cancelled"
|
||||
]
|
||||
|
||||
|
||||
class CommandGatewayReceipt(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
request_id: UUID
|
||||
project_id: UUID
|
||||
receipt_id: str = Field(min_length=1)
|
||||
status: Literal["accepted", "running", "completed", "failed"]
|
||||
submitted_at: datetime
|
||||
completed_at: datetime | None = None
|
||||
detail: str = Field(min_length=1)
|
||||
|
||||
|
||||
class HttpOperationalActionDispatcher:
|
||||
def __init__(
|
||||
self, settings: OperationalActionSettings | None = None,
|
||||
*, transport: httpx.AsyncBaseTransport | None = None,
|
||||
):
|
||||
self.settings = settings or OperationalActionSettings()
|
||||
self.transport = transport
|
||||
|
||||
async def dispatch(
|
||||
self, *, context: ProjectContext, task: OperationalTaskDetail,
|
||||
action: HumanApprovalAction, credential: str | None,
|
||||
) -> ActionDispatchOutcome:
|
||||
base = (self.settings.peripheral_url if action.kind == "create_work_order"
|
||||
else self.settings.command_gateway_url)
|
||||
if not base:
|
||||
return ActionDispatchOutcome("unsupported", "尚未配置该动作的外部执行适配器")
|
||||
if not credential or not credential.lower().startswith("bearer "):
|
||||
return ActionDispatchOutcome("failed", "缺少当前用户的产品身份，未提交外部动作")
|
||||
if action.task_id != task.task_id or task.project_id != context.project_id:
|
||||
return ActionDispatchOutcome("failed", "动作与当前项目任务不匹配，未提交")
|
||||
if action.approval_status != "approved":
|
||||
return ActionDispatchOutcome("failed", "动作尚未通过人工确认，未提交")
|
||||
request_id = action.execution_request_id
|
||||
if request_id is None:
|
||||
return ActionDispatchOutcome("failed", "缺少持久化执行编号，未提交")
|
||||
headers = {"Authorization": credential, "X-Project-Id": str(context.project_id)}
|
||||
try:
|
||||
if action.kind == "create_work_order":
|
||||
payload = CreateWorkOrderPayload.model_validate(action.payload)
|
||||
body = {
|
||||
"request_id": str(request_id), "title": payload.title,
|
||||
"summary": payload.description, "status": "open",
|
||||
"priority": payload.priority, "assignee": payload.assignee,
|
||||
"scheduled_at": payload.scheduled_at.isoformat() if payload.scheduled_at else None,
|
||||
"location": payload.location,
|
||||
# An approved action creates a child execution order, not a
|
||||
# new inbound complaint/burst event for the sync loop.
|
||||
"work_order_type": "general",
|
||||
}
|
||||
path = "/api/sample/v1/work-orders"
|
||||
recovery_path = f"/api/sample/v1/requests/{request_id}"
|
||||
else:
|
||||
payload = IssueCommandPayload.model_validate(action.payload)
|
||||
if payload.expires_at <= datetime.now(timezone.utc):
|
||||
return ActionDispatchOutcome("failed", "命令确认窗口已过期，未提交")
|
||||
body = {
|
||||
"request_id": str(request_id), "project_id": str(context.project_id),
|
||||
"task_id": str(task.task_id), "action_id": str(action.action_id),
|
||||
"action_version": action.version,
|
||||
"approved_by": action.decided_by,
|
||||
"approved_at": action.decided_at.isoformat() if action.decided_at else None,
|
||||
"command": payload.model_dump(mode="json"),
|
||||
}
|
||||
path = "/api/v1/commands"
|
||||
recovery_path = f"/api/v1/commands/requests/{request_id}"
|
||||
except (ValidationError, TypeError, ValueError):
|
||||
return ActionDispatchOutcome("failed", "动作参数不符合执行契约，未提交")
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
base_url=base.rstrip("/"), headers=headers, timeout=self.settings.timeout_seconds,
|
||||
follow_redirects=False, trust_env=False, transport=self.transport,
|
||||
) as client:
|
||||
try:
|
||||
response = await client.post(path, json=body)
|
||||
if response.status_code in {400, 401, 403, 404, 409, 422}:
|
||||
# Sample responses explicitly state whether any write occurred.
|
||||
# An untyped rejection is not proof that no effect took place.
|
||||
error = response.json()
|
||||
if isinstance(error, dict) and error.get("effect") == "none":
|
||||
return ActionDispatchOutcome("failed", "外部系统拒绝了动作，未生效")
|
||||
response.raise_for_status()
|
||||
return self._receipt(action.kind, response.json(), request_id, context.project_id)
|
||||
except (httpx.HTTPError, ValueError, KeyError, ValidationError):
|
||||
# A lost response can follow a successful write. Recover using the
|
||||
# same idempotency key, never manufacture success or blindly retry.
|
||||
try:
|
||||
recovered = await client.get(recovery_path)
|
||||
recovered.raise_for_status()
|
||||
value = recovered.json()
|
||||
if action.kind == "create_work_order":
|
||||
if not isinstance(value, dict) or value.get("request_id") != str(request_id):
|
||||
raise ValueError("request receipt mismatch")
|
||||
value = value["result"]
|
||||
return self._receipt(action.kind, value, request_id, context.project_id)
|
||||
except (httpx.HTTPError, ValueError, KeyError, ValidationError):
|
||||
return ActionDispatchOutcome("unknown", "外部执行结果待核实，请通过回执核对，勿重复下发")
|
||||
|
||||
async def reconcile(
|
||||
self, *, context: ProjectContext, task: OperationalTaskDetail,
|
||||
action: HumanApprovalAction, credential: str | None,
|
||||
) -> ActionDispatchOutcome:
|
||||
request_id = action.execution_request_id
|
||||
if not credential or request_id is None or action.task_id != task.task_id or task.project_id != context.project_id:
|
||||
return ActionDispatchOutcome("unknown", "无法核实当前动作的身份或执行编号")
|
||||
base = self.settings.peripheral_url if action.kind == "create_work_order" else self.settings.command_gateway_url
|
||||
if not base:
|
||||
return ActionDispatchOutcome("unknown", "回执查询服务尚未配置")
|
||||
path = (f"/api/sample/v1/requests/{request_id}" if action.kind == "create_work_order"
|
||||
else f"/api/v1/commands/requests/{request_id}")
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=base.rstrip("/"), timeout=self.settings.timeout_seconds,
|
||||
follow_redirects=False, trust_env=False, transport=self.transport,
|
||||
) as client:
|
||||
response = await client.get(path, headers={
|
||||
"Authorization": credential, "X-Project-Id": str(context.project_id),
|
||||
})
|
||||
response.raise_for_status()
|
||||
value = response.json()
|
||||
if action.kind == "create_work_order":
|
||||
if not isinstance(value, dict) or value.get("request_id") != str(request_id):
|
||||
raise ValueError("request receipt mismatch")
|
||||
value = value["result"]
|
||||
return self._receipt(action.kind, value, request_id, context.project_id)
|
||||
except (httpx.HTTPError, ValueError, KeyError, ValidationError):
|
||||
return ActionDispatchOutcome("unknown", "尚未取得可核实的执行回执")
|
||||
|
||||
@staticmethod
|
||||
def _receipt(kind: str, value: object, request_id: UUID, project_id: UUID) -> ActionDispatchOutcome:
|
||||
if kind == "create_work_order":
|
||||
order = SampleWorkOrderReceipt.model_validate(value)
|
||||
if order.project_id != project_id or order.created_at.tzinfo is None:
|
||||
raise ValueError("invalid external project or timestamp")
|
||||
receipt = ActionReceipt(
|
||||
adapter="peripheral_sample", receipt_id=str(order.work_order_id),
|
||||
submitted_at=order.created_at, completed_at=order.created_at,
|
||||
external_status=order.status, detail="合成外围工单已创建，现场工作进度由外围系统继续同步",
|
||||
)
|
||||
return ActionDispatchOutcome("succeeded", receipt.detail, receipt)
|
||||
result = CommandGatewayReceipt.model_validate(value)
|
||||
if result.project_id != project_id or result.request_id != request_id:
|
||||
raise ValueError("command receipt does not match submission")
|
||||
if result.submitted_at.tzinfo is None or (result.completed_at and result.completed_at.tzinfo is None):
|
||||
raise ValueError("command receipt timestamps must include timezone")
|
||||
if result.status == "completed" and result.completed_at is None:
|
||||
raise ValueError("command has no completion receipt")
|
||||
receipt = ActionReceipt(
|
||||
adapter="command_gateway", receipt_id=result.receipt_id,
|
||||
submitted_at=result.submitted_at, completed_at=result.completed_at,
|
||||
external_status=result.status, detail=result.detail,
|
||||
)
|
||||
status = "succeeded" if result.status == "completed" else "failed" if result.status == "failed" else "submitted"
|
||||
return ActionDispatchOutcome(status, result.detail, receipt)
|
||||
|
||||
|
||||
def get_operational_action_dispatcher() -> OperationalActionDispatcher:
|
||||
return HttpOperationalActionDispatcher()
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Read-only source adapters with bounded queries and explicit provenance."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from app.auth.permissions import SCADA_VIEW, SIMULATION_VIEW, permissions_for_context
|
||||
from app.auth.project_dependencies import ProjectContext
|
||||
from app.domain.schemas.operational_data import TaskDataQuery, TaskDataSnapshot
|
||||
|
||||
|
||||
class TaskDataSettings(BaseSettings):
|
||||
server_url: str = "http://127.0.0.1:8000"
|
||||
peripheral_url: str = ""
|
||||
timeout_seconds: float = Field(default=15, ge=1, le=60)
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="OPERATIONAL_DATA_", extra="ignore",
|
||||
env_file=Path(__file__).resolve().parents[3] / ".env",
|
||||
)
|
||||
|
||||
|
||||
class TaskDataError(Exception):
|
||||
def __init__(self, status: int, detail: str):
|
||||
super().__init__(detail)
|
||||
self.status = status
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def _matches_project(value: object, project_id: str) -> bool:
|
||||
if isinstance(value, list):
|
||||
return all(_matches_project(item, project_id) for item in value)
|
||||
if isinstance(value, dict):
|
||||
if "project_id" in value and value["project_id"] != project_id:
|
||||
return False
|
||||
return all(_matches_project(item, project_id) for item in value.values())
|
||||
return True
|
||||
|
||||
|
||||
def _select_demo_observations(
|
||||
payload: object, query: TaskDataQuery, project_id: str,
|
||||
) -> dict[str, object]:
|
||||
"""Project a bounded DEMO-* observation query from the stored demo dataset."""
|
||||
if not isinstance(payload, dict) or not _matches_project(payload, project_id):
|
||||
raise TaskDataError(502, "数据源返回了其他项目的数据")
|
||||
points = payload.get("points")
|
||||
observations = payload.get("observations")
|
||||
if not isinstance(points, list) or not isinstance(observations, list):
|
||||
raise TaskDataError(502, "外围合成样本格式无效")
|
||||
if not any(isinstance(point, dict) and point.get("point_id") == query.point_id for point in points):
|
||||
raise TaskDataError(404, "合成样本测点不存在")
|
||||
assert query.start is not None and query.end is not None
|
||||
items: list[object] = []
|
||||
try:
|
||||
for item in observations:
|
||||
if not isinstance(item, dict) or item.get("point_id") != query.point_id:
|
||||
continue
|
||||
observed_at = datetime.fromisoformat(str(item.get("observed_at")))
|
||||
if query.start <= observed_at < query.end:
|
||||
items.append(item)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TaskDataError(502, "外围合成样本格式无效") from exc
|
||||
return {
|
||||
"origin": "synthetic",
|
||||
"source_id": payload.get("source_id"),
|
||||
"source_revision": payload.get("source_revision"),
|
||||
"point_id": query.point_id,
|
||||
"window": {
|
||||
"start": query.start.isoformat(),
|
||||
"end": query.end.isoformat(),
|
||||
},
|
||||
"items": items,
|
||||
"truncated": False,
|
||||
}
|
||||
|
||||
|
||||
async def read_task_data(
|
||||
*, context: ProjectContext, query: TaskDataQuery, credential: str | None,
|
||||
settings: TaskDataSettings | None = None,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> TaskDataSnapshot:
|
||||
config = settings or TaskDataSettings()
|
||||
permission = SCADA_VIEW if query.source.startswith("scada_") else SIMULATION_VIEW
|
||||
if permission not in permissions_for_context(context):
|
||||
raise TaskDataError(403, "当前身份没有该数据源的读取权限")
|
||||
if not credential or not credential.lower().startswith("bearer "):
|
||||
raise TaskDataError(401, "数据查询需要当前产品身份")
|
||||
peripheral = query.source.startswith("peripheral_")
|
||||
base = config.peripheral_url if peripheral else config.server_url
|
||||
if not base:
|
||||
raise TaskDataError(503, "调查数据源尚未配置")
|
||||
paths = {
|
||||
"workbench_runtime": "/api/v1/workbench/runtime",
|
||||
"scada_devices": "/api/v1/scada-devices",
|
||||
"scada_readings": "/api/v1/timeseries/scada-readings",
|
||||
"peripheral_points": "/api/sample/v1/monitoring/points",
|
||||
"peripheral_work_orders": "/api/sample/v1/work-orders",
|
||||
"peripheral_observations": "/api/sample/v1/monitoring/observations",
|
||||
"peripheral_test_data": "/api/sample/v1/test-data",
|
||||
}
|
||||
demo_observations = (
|
||||
query.source == "peripheral_observations"
|
||||
and query.point_id is not None
|
||||
and query.point_id.startswith("DEMO-")
|
||||
)
|
||||
path = "/api/sample/v1/demo-data" if demo_observations else paths[query.source]
|
||||
params: dict[str, str | int] = {"limit": 100}
|
||||
if query.source == "scada_readings":
|
||||
params.update(device_ids=",".join(query.device_ids), start_time=query.start.isoformat(), end_time=query.end.isoformat())
|
||||
elif query.source == "peripheral_observations" and not demo_observations:
|
||||
params = {"point_id": query.point_id, "start": query.start.isoformat(), "end": query.end.isoformat()}
|
||||
elif demo_observations:
|
||||
params = {}
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=base.rstrip("/"), timeout=config.timeout_seconds, follow_redirects=False,
|
||||
trust_env=False, transport=transport,
|
||||
) as client:
|
||||
async with client.stream("GET", path, params=params, headers={
|
||||
"Authorization": credential, "X-Project-Id": str(context.project_id),
|
||||
}) as response:
|
||||
if response.status_code in {401, 403}:
|
||||
raise TaskDataError(response.status_code, "数据源拒绝当前身份或项目权限")
|
||||
response.raise_for_status()
|
||||
body = bytearray()
|
||||
async for chunk in response.aiter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > 2_000_000:
|
||||
raise TaskDataError(422, "查询结果过大，请缩短时间范围或减少设备数量")
|
||||
data = json.loads(body)
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
raise TaskDataError(502, "调查数据查询失败，不能据此形成已核实结论") from exc
|
||||
if demo_observations:
|
||||
data = _select_demo_observations(data, query, str(context.project_id))
|
||||
elif not _matches_project(data, str(context.project_id)):
|
||||
raise TaskDataError(502, "数据源返回了其他项目的数据")
|
||||
origin = "synthetic" if peripheral else "system_data"
|
||||
historical_replay = False
|
||||
if query.source in {"peripheral_points", "peripheral_observations"}:
|
||||
source_origin = data.get("origin") if isinstance(data, dict) else None
|
||||
if source_origin not in {"historical_replay", "synthetic"}:
|
||||
raise TaskDataError(502, "监测数据缺少明确来源，不能作为已核实证据")
|
||||
historical_replay = source_origin == "historical_replay"
|
||||
origin = "system_data" if historical_replay else "synthetic"
|
||||
truncated = isinstance(data, dict) and bool(data.get("truncated") or data.get("next_cursor") or data.get("next_offset"))
|
||||
if isinstance(data, list) and len(data) > 1000:
|
||||
data, truncated = data[:1000], True
|
||||
if isinstance(data, dict) and isinstance(data.get("items"), list):
|
||||
if len(data["items"]) > 1000:
|
||||
data = {**data, "items": data["items"][:1000]}
|
||||
truncated = True
|
||||
if isinstance(data.get("total"), int) and data["total"] > len(data["items"]):
|
||||
truncated = True
|
||||
# The digest identifies the returned snapshot, not merely a mutable URL.
|
||||
digest = hashlib.sha256(json.dumps(data, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
|
||||
if demo_observations:
|
||||
assert query.start is not None and query.end is not None
|
||||
source_ref = (
|
||||
f"peripheral_demo_data:{query.point_id}:"
|
||||
f"{query.start.isoformat()}:{query.end.isoformat()}:sha256:{digest}"
|
||||
)
|
||||
elif historical_replay:
|
||||
# Use a distinct identity: old snapshots may have incorrectly labeled
|
||||
# this same historical response synthetic. Immutable evidence stays intact.
|
||||
source_ref = f"peripheral_historical_replay:{query.source}:sha256:{digest}"
|
||||
else:
|
||||
source_ref = f"{query.source}:sha256:{digest}"
|
||||
return TaskDataSnapshot(
|
||||
source=query.source, source_ref=source_ref,
|
||||
queried_at=datetime.now(timezone.utc), origin=origin,
|
||||
data=data, truncated=truncated,
|
||||
)
|
||||
Reference in New Issue
Block a user