OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(workbench): serialize events and target work order sync

Browse Source
This commit is contained in:
jiang
2026-10-08 15:41:49 +08:00
parent b70283c2ff
commit 4a63fcd7ff
20 changed files with 673 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/postgresql/analysis.py
+3
View File
@@ -4,6 +4,7 @@ from uuid import UUID, uuid4
from psycopg import AsyncConnection, Connection
from psycopg.types.json import Jsonb
from app.infra.db.postgresql.workbench_events import lock_event_transaction_sync
class AnalysisRepository:
@@ -19,6 +20,7 @@ class AnalysisRepository:
parameters: dict[str, Any],
run_id: UUID | None = None,
) -> dict[str, Any]:
lock_event_transaction_sync(conn)
execution_id = run_id or uuid4()
with conn.cursor() as cur:
cur.execute(
@@ -54,6 +56,7 @@ class AnalysisRepository:
created_by: str,
parameters: dict[str, Any],
) -> int:
lock_event_transaction_sync(conn)
with conn.cursor() as cur:
cur.execute(
"""
app/infra/db/postgresql/operational_tasks.py
+17
View File
@@ -10,6 +10,7 @@ from psycopg.types.json import Jsonb
from uuid import uuid4
from app.domain.domain_errors import DomainError
from app.infra.db.postgresql.workbench_events import lock_event_transaction
class OperationalTaskRepository:
@@ -21,6 +22,7 @@ class OperationalTaskRepository:
async def transaction(self):
try:
async with self.connection.transaction():
await lock_event_transaction(self.connection)
yield self
except UndefinedTable as exc:
raise DomainError(
@@ -239,6 +241,7 @@ class OperationalTaskRepository:
status: str | None,
limit: int,
offset: int,
peripheral_work_order_id: str | None = None,
) -> list[dict]:
cursor = await self.connection.execute(
"""SELECT task.document,
@@ -255,6 +258,16 @@ class OperationalTaskRepository:
AND (task.document#>>'{source,deleted_at}') IS NULL
AND (%s::text IS NULL OR task.task_type=%s::text)
AND (%s::text IS NULL OR task.status=%s::text)
AND (%s::text IS NULL
OR (task.document#>>'{source,kind}'='external_system'
AND task.document#>>'{source,external_id}' IN (%s,%s))
OR EXISTS (
SELECT 1 FROM workbench.operational_task_actions AS action
WHERE action.project_id=task.project_id AND action.task_id=task.task_id
AND action.document->>'kind'='create_work_order'
AND action.document#>>'{receipt,adapter}'='peripheral_sample'
AND action.document#>>'{receipt,receipt_id}'=%s
))
ORDER BY task.updated_at DESC,task.task_id
LIMIT %s OFFSET %s""",
(
@@ -263,6 +276,10 @@ class OperationalTaskRepository:
task_type,
status,
status,
peripheral_work_order_id,
peripheral_work_order_id,
f"peripheral_sample:{peripheral_work_order_id}" if peripheral_work_order_id else None,
peripheral_work_order_id,
limit,
offset,
),
app/infra/db/postgresql/task_alerts.py
+2
View File
@@ -9,6 +9,7 @@ from psycopg.types.json import Jsonb
from uuid import uuid4
from app.domain.domain_errors import DomainError
from app.infra.db.postgresql.workbench_events import lock_event_transaction
class TaskAlertRepository:
@@ -20,6 +21,7 @@ class TaskAlertRepository:
async def transaction(self):
try:
async with self.connection.transaction():
await lock_event_transaction(self.connection)
yield self
except UndefinedTable as exc:
raise DomainError(
app/infra/db/postgresql/workbench_event_hub.py
+32 -11
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import contextlib
import logging
from collections import OrderedDict
from contextlib import asynccontextmanager
from uuid import UUID
@@ -27,6 +28,8 @@ class WorkbenchEventHub:
self._connection: AsyncConnection | None = None
self._pool: AsyncConnectionPool | None = None
self._connection_lock = asyncio.Lock()
self._read_lock = asyncio.Lock()
self._reads: OrderedDict[tuple, tuple[int, object]] = OrderedDict()
self._condition = asyncio.Condition()
self._generation = 0
self._ready = asyncio.Event()
@@ -59,6 +62,7 @@ class WorkbenchEventHub:
async def _listen_loop(self) -> None:
retry_seconds = 1
next_reconcile = asyncio.get_running_loop().time() + 30
while not self._closed:
try:
if self._connection is None or self._connection.closed:
@@ -76,8 +80,11 @@ class WorkbenchEventHub:
if notification.payload.startswith(f"{self.project_id}:"):
notified = True
break
if notified:
now = asyncio.get_running_loop().time()
if notified or now >= next_reconcile:
await self._wake()
if now >= next_reconcile:
next_reconcile = now + 30
await asyncio.sleep(0)
except asyncio.CancelledError:
raise
@@ -109,22 +116,35 @@ class WorkbenchEventHub:
return self._generation
async def oldest_cursor(self) -> int | None:
return await self._read(lambda repository: repository.oldest_cursor())
return await self._read(("oldest",), lambda repository: repository.oldest_cursor())
async def current_cursor(self) -> int:
return await self._read(lambda repository: repository.current_cursor())
return await self._read(("current",), lambda repository: repository.current_cursor())
async def read_after(self, after: int) -> list[dict]:
return await self._read(lambda repository: repository.read_after(after))
return await self._read(("after", after), lambda repository: repository.read_after(after))
async def _read(self, operation):
async def _read(self, key: tuple, operation):
await self.start()
async with self._connection_lock:
if self._connection is None or self._connection.closed:
self._connection = await self._connect()
self._ready.set()
repository = WorkbenchEventRepository(self._connection, self.project_id)
return await operation(repository)
async with self._read_lock:
cached = self._reads.get(key)
if cached is not None and cached[0] == self.generation:
self._reads.move_to_end(key)
return cached[1]
# Record the generation before the query: a concurrent NOTIFY must
# invalidate even an empty result from an older database snapshot.
generation = self.generation
async with self._connection_lock:
if self._connection is None or self._connection.closed:
self._connection = await self._connect()
self._ready.set()
repository = WorkbenchEventRepository(self._connection, self.project_id)
result = await operation(repository)
self._reads[key] = (generation, result)
self._reads.move_to_end(key)
while len(self._reads) > 32:
self._reads.popitem(last=False)
return result
async def _drop_connection(self) -> None:
async with self._connection_lock:
@@ -135,6 +155,7 @@ class WorkbenchEventHub:
async def close(self) -> None:
self._closed = True
self._reads.clear()
if self._idle_task is not None:
self._idle_task.cancel()
self._idle_task = None
app/infra/db/postgresql/workbench_events.py
+25 -1
View File
@@ -1,7 +1,31 @@
"""Cursor reads for the durable project-scoped workbench event log."""
from __future__ import annotations
from psycopg import AsyncConnection
from psycopg import AsyncConnection, Connection
EVENT_ORDER_LOCK = "tjwater-workbench:event-order"
async def lock_event_transaction(connection: AsyncConnection) -> None:
"""Serialize v2 event-producing transactions before business locks/writes.
PostgreSQL sequence allocation is not commit order. All producers must use
this same transaction lock so a committed higher cursor cannot hide an
earlier, still-uncommitted event. Other BizDBs remain rollout opt-out.
"""
if getattr(getattr(connection, "info", None), "dbname", None) == "tjwater_v2":
await connection.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,)
)
def lock_event_transaction_sync(connection: Connection) -> None:
"""Synchronous counterpart using the identical PostgreSQL lock key."""
if getattr(getattr(connection, "info", None), "dbname", None) == "tjwater_v2":
connection.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,)
)
class WorkbenchEventRepository: