OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(workbench): serialize events and target work order sync

Browse Source
This commit is contained in:
jiang
2026-10-08 15:41:49 +08:00
parent b70283c2ff
commit 4a63fcd7ff
20 changed files with 673 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/v1/endpoints/operational_tasks.py
+6 -1
View File
@@ -122,12 +122,17 @@ async def list_operational_tasks(
task_status: TaskStatus | None = Query(default=None, alias="status"),
limit: int = Query(default=50, ge=1, le=100),
offset: int = Query(default=0, ge=0),
peripheral_work_order_id: UUID | None = Query(
default=None, description="Only tasks sourced from or linked to this peripheral work order",
),
ctx: ProjectContext = Depends(require_permission(OPERATIONAL_TASK_VIEW)),
conn: AsyncConnection = Depends(get_project_pg_connection),
):
try:
items, next_offset = await _service(ctx, conn).list(
task_type=task_type, status=task_status, limit=limit, offset=offset
task_type=task_type, status=task_status, limit=limit, offset=offset,
**({"peripheral_work_order_id": peripheral_work_order_id}
if peripheral_work_order_id is not None else {}),
)
return OperationalTaskPage(
items=items, limit=limit, offset=offset, next_offset=next_offset
app/api/v1/endpoints/workbench_events.py
+3 -3
View File
@@ -131,9 +131,9 @@ async def workbench_events(
if now - last_keepalive >= 15:
yield f": keepalive {int(now)}\n\n"
last_keepalive = now
# LISTEN is only a wake-up path. The timeout also reconciles
# against the durable cursor if a notification was missed.
wake_generation = await hub.wait_for_wake(wake_generation, timeout=1)
# The Hub invalidates shared reads on NOTIFY and every 30s.
# Per-client timeout only keeps the transport alive, not SQL.
wake_generation = await hub.wait_for_wake(wake_generation, timeout=15)
yield "event: reconnect\ndata: {}\n\n"
return StreamingResponse(
app/infra/db/postgresql/analysis.py
+3
View File
@@ -4,6 +4,7 @@ from uuid import UUID, uuid4
from psycopg import AsyncConnection, Connection
from psycopg.types.json import Jsonb
from app.infra.db.postgresql.workbench_events import lock_event_transaction_sync
class AnalysisRepository:
@@ -19,6 +20,7 @@ class AnalysisRepository:
parameters: dict[str, Any],
run_id: UUID | None = None,
) -> dict[str, Any]:
lock_event_transaction_sync(conn)
execution_id = run_id or uuid4()
with conn.cursor() as cur:
cur.execute(
@@ -54,6 +56,7 @@ class AnalysisRepository:
created_by: str,
parameters: dict[str, Any],
) -> int:
lock_event_transaction_sync(conn)
with conn.cursor() as cur:
cur.execute(
"""
app/infra/db/postgresql/operational_tasks.py
+17
View File
@@ -10,6 +10,7 @@ from psycopg.types.json import Jsonb
from uuid import uuid4
from app.domain.domain_errors import DomainError
from app.infra.db.postgresql.workbench_events import lock_event_transaction
class OperationalTaskRepository:
@@ -21,6 +22,7 @@ class OperationalTaskRepository:
async def transaction(self):
try:
async with self.connection.transaction():
await lock_event_transaction(self.connection)
yield self
except UndefinedTable as exc:
raise DomainError(
@@ -239,6 +241,7 @@ class OperationalTaskRepository:
status: str | None,
limit: int,
offset: int,
peripheral_work_order_id: str | None = None,
) -> list[dict]:
cursor = await self.connection.execute(
"""SELECT task.document,
@@ -255,6 +258,16 @@ class OperationalTaskRepository:
AND (task.document#>>'{source,deleted_at}') IS NULL
AND (%s::text IS NULL OR task.task_type=%s::text)
AND (%s::text IS NULL OR task.status=%s::text)
AND (%s::text IS NULL
OR (task.document#>>'{source,kind}'='external_system'
AND task.document#>>'{source,external_id}' IN (%s,%s))
OR EXISTS (
SELECT 1 FROM workbench.operational_task_actions AS action
WHERE action.project_id=task.project_id AND action.task_id=task.task_id
AND action.document->>'kind'='create_work_order'
AND action.document#>>'{receipt,adapter}'='peripheral_sample'
AND action.document#>>'{receipt,receipt_id}'=%s
))
ORDER BY task.updated_at DESC,task.task_id
LIMIT %s OFFSET %s""",
(
@@ -263,6 +276,10 @@ class OperationalTaskRepository:
task_type,
status,
status,
peripheral_work_order_id,
peripheral_work_order_id,
f"peripheral_sample:{peripheral_work_order_id}" if peripheral_work_order_id else None,
peripheral_work_order_id,
limit,
offset,
),
app/infra/db/postgresql/task_alerts.py
+2
View File
@@ -9,6 +9,7 @@ from psycopg.types.json import Jsonb
from uuid import uuid4
from app.domain.domain_errors import DomainError
from app.infra.db.postgresql.workbench_events import lock_event_transaction
class TaskAlertRepository:
@@ -20,6 +21,7 @@ class TaskAlertRepository:
async def transaction(self):
try:
async with self.connection.transaction():
await lock_event_transaction(self.connection)
yield self
except UndefinedTable as exc:
raise DomainError(
app/infra/db/postgresql/workbench_event_hub.py
+32 -11
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import contextlib
import logging
from collections import OrderedDict
from contextlib import asynccontextmanager
from uuid import UUID
@@ -27,6 +28,8 @@ class WorkbenchEventHub:
self._connection: AsyncConnection | None = None
self._pool: AsyncConnectionPool | None = None
self._connection_lock = asyncio.Lock()
self._read_lock = asyncio.Lock()
self._reads: OrderedDict[tuple, tuple[int, object]] = OrderedDict()
self._condition = asyncio.Condition()
self._generation = 0
self._ready = asyncio.Event()
@@ -59,6 +62,7 @@ class WorkbenchEventHub:
async def _listen_loop(self) -> None:
retry_seconds = 1
next_reconcile = asyncio.get_running_loop().time() + 30
while not self._closed:
try:
if self._connection is None or self._connection.closed:
@@ -76,8 +80,11 @@ class WorkbenchEventHub:
if notification.payload.startswith(f"{self.project_id}:"):
notified = True
break
if notified:
now = asyncio.get_running_loop().time()
if notified or now >= next_reconcile:
await self._wake()
if now >= next_reconcile:
next_reconcile = now + 30
await asyncio.sleep(0)
except asyncio.CancelledError:
raise
@@ -109,22 +116,35 @@ class WorkbenchEventHub:
return self._generation
async def oldest_cursor(self) -> int | None:
return await self._read(lambda repository: repository.oldest_cursor())
return await self._read(("oldest",), lambda repository: repository.oldest_cursor())
async def current_cursor(self) -> int:
return await self._read(lambda repository: repository.current_cursor())
return await self._read(("current",), lambda repository: repository.current_cursor())
async def read_after(self, after: int) -> list[dict]:
return await self._read(lambda repository: repository.read_after(after))
return await self._read(("after", after), lambda repository: repository.read_after(after))
async def _read(self, operation):
async def _read(self, key: tuple, operation):
await self.start()
async with self._connection_lock:
if self._connection is None or self._connection.closed:
self._connection = await self._connect()
self._ready.set()
repository = WorkbenchEventRepository(self._connection, self.project_id)
return await operation(repository)
async with self._read_lock:
cached = self._reads.get(key)
if cached is not None and cached[0] == self.generation:
self._reads.move_to_end(key)
return cached[1]
# Record the generation before the query: a concurrent NOTIFY must
# invalidate even an empty result from an older database snapshot.
generation = self.generation
async with self._connection_lock:
if self._connection is None or self._connection.closed:
self._connection = await self._connect()
self._ready.set()
repository = WorkbenchEventRepository(self._connection, self.project_id)
result = await operation(repository)
self._reads[key] = (generation, result)
self._reads.move_to_end(key)
while len(self._reads) > 32:
self._reads.popitem(last=False)
return result
async def _drop_connection(self) -> None:
async with self._connection_lock:
@@ -135,6 +155,7 @@ class WorkbenchEventHub:
async def close(self) -> None:
self._closed = True
self._reads.clear()
if self._idle_task is not None:
self._idle_task.cancel()
self._idle_task = None
app/infra/db/postgresql/workbench_events.py
+25 -1
View File
@@ -1,7 +1,31 @@
"""Cursor reads for the durable project-scoped workbench event log."""
from __future__ import annotations
from psycopg import AsyncConnection
from psycopg import AsyncConnection, Connection
EVENT_ORDER_LOCK = "tjwater-workbench:event-order"
async def lock_event_transaction(connection: AsyncConnection) -> None:
"""Serialize v2 event-producing transactions before business locks/writes.
PostgreSQL sequence allocation is not commit order. All producers must use
this same transaction lock so a committed higher cursor cannot hide an
earlier, still-uncommitted event. Other BizDBs remain rollout opt-out.
"""
if getattr(getattr(connection, "info", None), "dbname", None) == "tjwater_v2":
await connection.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,)
)
def lock_event_transaction_sync(connection: Connection) -> None:
"""Synchronous counterpart using the identical PostgreSQL lock key."""
if getattr(getattr(connection, "info", None), "dbname", None) == "tjwater_v2":
connection.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,)
)
class WorkbenchEventRepository:
app/services/operational_tasks.py
+28 -18
View File
@@ -200,10 +200,13 @@ class OperationalTaskService:
return await self.detail(task_id)
async def list(
self, *, task_type: str | None, status: str | None, limit: int, offset: int
self, *, task_type: str | None, status: str | None, limit: int, offset: int,
peripheral_work_order_id: UUID | None = None,
) -> tuple[list[OperationalTaskSummary], int | None]:
rows = await self.repo.list_tasks(
task_type=task_type, status=status, limit=limit + 1, offset=offset
task_type=task_type, status=status, limit=limit + 1, offset=offset,
**({"peripheral_work_order_id": str(peripheral_work_order_id)}
if peripheral_work_order_id is not None else {}),
)
items: list[OperationalTaskSummary] = []
for row in rows[:limit]:
@@ -922,9 +925,13 @@ class OperationalTaskService:
require(OPERATIONAL_ACTION_WORK_ORDER_APPROVE in self.permissions
and OPERATIONAL_ACTION_WORK_ORDER_EXECUTE in self.permissions,
"PERMISSION_DENIED", "Current user cannot confirm and dispatch work orders", status=403)
current = await self._action(task_id, action_id)
require(current.kind == "create_work_order", "INVALID_ACTION_KIND",
"Only work orders support one-step confirmation", status=422)
# A SELECT on the default non-autocommit connection starts an outer
# transaction. End it explicitly, or approval/submit become savepoints
# whose event lock survives into the peripheral HTTP call.
async with self.repo.transaction():
current = await self._action(task_id, action_id)
require(current.kind == "create_work_order", "INVALID_ACTION_KIND",
"Only work orders support one-step confirmation", status=422)
approval = await self.decide_action(task_id, action_id, ActionDecisionCommand(
request_id=uuid5(NAMESPACE_URL, f"confirm:{command.request_id}:approve"),
expected_version=command.expected_version, decision="approved",
@@ -1024,7 +1031,8 @@ class OperationalTaskService:
scope, command.request_id, body, {"action": _json(action)}
)
task = await self.detail(task_id)
async with self.repo.transaction():
task = await self.detail(task_id)
try:
outcome = await dispatcher.dispatch(
context=self.context, task=task, action=action, credential=credential
@@ -1099,19 +1107,21 @@ class OperationalTaskService:
) -> HumanApprovalAction:
body = _json(command)
scope = f"task:{task_id}:action:{action_id}:reconcile"
replay = await self.repo.get_request(scope, str(command.request_id))
if replay:
self._require_same_request(replay, body)
return HumanApprovalAction.model_validate(replay["response"]["action"])
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
require(action.execution_status in {"submitted", "unknown"}, "INVALID_TRANSITION",
"Only submitted or unknown actions can be reconciled", status=409)
require(action.required_execution_permission in self.permissions,
"PERMISSION_DENIED", "Current user cannot reconcile this action", status=403)
async with self.repo.transaction():
replay = await self.repo.get_request(scope, str(command.request_id))
if replay:
self._require_same_request(replay, body)
return HumanApprovalAction.model_validate(replay["response"]["action"])
action = await self._action(task_id, action_id)
require(action.version == command.expected_version, "VERSION_CONFLICT",
"Action version changed", status=409)
require(action.execution_status in {"submitted", "unknown"}, "INVALID_TRANSITION",
"Only submitted or unknown actions can be reconciled", status=409)
require(action.required_execution_permission in self.permissions,
"PERMISSION_DENIED", "Current user cannot reconcile this action", status=403)
task = await self.detail(task_id)
outcome = await dispatcher.reconcile(
context=self.context, task=await self.detail(task_id), action=action,
context=self.context, task=task, action=action,
credential=credential,
)
self._validate_dispatch_outcome(outcome)