fix(workbench): serialize events and target work order sync

tests/integration/test_workbench_event_order.py

@@ -0,0 +1,232 @@
+"""Opt-in v2 concurrent writer regression; every event is rolled back."""
+import asyncio
+import os
+from datetime import datetime, timezone
+from uuid import uuid4
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import psycopg
+import pytest
+from psycopg.conninfo import make_conninfo
+from psycopg.rows import dict_row
+from psycopg.pq import TransactionStatus
+
+from app.core.config import get_pgconn_string
+from app.auth.permissions import OPERATIONAL_ACTION_WORK_ORDER_APPROVE, OPERATIONAL_ACTION_WORK_ORDER_EXECUTE
+from app.auth.project_dependencies import ProjectContext
+from app.domain.schemas.operational_task import (
+    ActionExecuteCommand, ActionReconcileCommand, ActionReceipt, HumanApprovalAction,
+    WorkOrderConfirmDispatchCommand,
+    OperationalTaskCreate,
+)
+from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
+from app.infra.db.postgresql.task_alerts import TaskAlertRepository
+from app.infra.db.postgresql.workbench_events import lock_event_transaction
+from app.infra.integrations.operational_actions import ActionDispatchOutcome
+from app.services.operational_tasks import OperationalTaskService
+
+
+pytestmark = pytest.mark.skipif(
+    os.getenv("WORKBENCH_EVENT_ORDER_TEST") != "1", reason="explicit v2 concurrency opt-in required",
+)
+
+
+@pytest.fixture
+def anyio_backend():
+    return "asyncio"
+
+
+@pytest.mark.anyio
+async def test_later_producer_cannot_allocate_a_cursor_before_first_transaction_finishes():
+    dsn = make_conninfo(get_pgconn_string(), dbname="tjwater_v2")
+    first_ready, release_first, second_ready = asyncio.Event(), asyncio.Event(), asyncio.Event()
+    project_id = uuid4()
+    async with await psycopg.AsyncConnection.connect(dsn, autocommit=True, row_factory=dict_row) as first:
+        async with await psycopg.AsyncConnection.connect(dsn, autocommit=True, row_factory=dict_row) as second:
+            async with await psycopg.AsyncConnection.connect(dsn, autocommit=True, row_factory=dict_row) as observer:
+                assert first.info.dbname == second.info.dbname == observer.info.dbname == "tjwater_v2"
+
+                async def write_first():
+                    async with first.transaction(force_rollback=True):
+                        repo = OperationalTaskRepository(first, project_id)
+                        async with repo.transaction():
+                            await repo.append_event(
+                                topic="operational.task.changed", aggregate_type="operational_task",
+                                aggregate_id=str(uuid4()), aggregate_revision=1,
+                                occurred_at=datetime.now(timezone.utc), metadata={},
+                            )
+                            first_ready.set()
+                            await release_first.wait()
+
+                async def write_second():
+                    async with second.transaction(force_rollback=True):
+                        async with TaskAlertRepository(second, project_id).transaction():
+                            second_ready.set()
+
+                first_task = asyncio.create_task(write_first())
+                second_task = None
+                try:
+                    await asyncio.wait_for(first_ready.wait(), 3)
+                    second_task = asyncio.create_task(write_second())
+                    async with asyncio.timeout(3):
+                        while True:
+                            row = await (await observer.execute(
+                                "SELECT wait_event_type FROM pg_stat_activity WHERE pid=%s",
+                                (second.info.backend_pid,),
+                            )).fetchone()
+                            if row and row["wait_event_type"] == "Lock":
+                                break
+                            assert not second_ready.is_set(), "later producer bypassed the event ordering lock"
+                            await asyncio.sleep(0.01)
+                    assert not second_ready.is_set()
+                    release_first.set()
+                    await asyncio.wait_for(asyncio.gather(first_task, second_task), 3)
+                    assert second_ready.is_set()
+                    row = await (await observer.execute(
+                        "SELECT count(*) AS n FROM workbench.domain_event_log WHERE project_id=%s",
+                        (project_id,),
+                    )).fetchone()
+                    assert row["n"] == 0
+                finally:
+                    release_first.set()
+                    pending = [task for task in (first_task, second_task) if task is not None]
+                    for task in pending:
+                        if not task.done():
+                            task.cancel()
+                    await asyncio.gather(*pending, return_exceptions=True)
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize("operation", ["confirm", "submit", "reconcile"])
+async def test_default_connection_ends_transactions_before_external_dispatch(monkeypatch, operation):
+    """Real psycopg reads/locks and service orchestration, without fixture writes."""
+    dsn = make_conninfo(get_pgconn_string(), dbname="tjwater_v2")
+    task_id, action_id, order_id = uuid4(), uuid4(), uuid4()
+    now = datetime.now(timezone.utc)
+    context = ProjectContext(project_id=uuid4(), project_code="event-order-test",
+                             user_id=uuid4(), project_role="member")
+    action = HumanApprovalAction(
+        action_id=action_id, task_id=task_id, version=1, kind="create_work_order",
+        reason="AUTOTEST transaction boundary", evidence_refs=[],
+        payload={"title": "AUTOTEST", "description": "not persisted", "work_order_type": "general"},
+        required_approval_permission=OPERATIONAL_ACTION_WORK_ORDER_APPROVE,
+        required_execution_permission=OPERATIONAL_ACTION_WORK_ORDER_EXECUTE,
+        approval_status="pending" if operation == "confirm" else "approved",
+        execution_status="submitted" if operation == "reconcile" else "not_submitted",
+        proposed_by="AUTOTEST", proposed_at=now,
+    )
+    async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
+        async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as external:
+            assert conn.info.dbname == external.info.dbname == "tjwater_v2"
+            repo = OperationalTaskRepository(conn, context.project_id)
+            service = OperationalTaskService(repo, context, {
+                OPERATIONAL_ACTION_WORK_ORDER_APPROVE, OPERATIONAL_ACTION_WORK_ORDER_EXECUTE,
+            })
+
+            async def get_action(*_args):
+                await conn.execute("SELECT 1")  # Same implicit BEGIN as a product read.
+                return action.model_dump(mode="json")
+
+            async def update_action(_task_id, document):
+                nonlocal action
+                await conn.execute("SELECT 1")
+                action = HumanApprovalAction.model_validate(document)
+
+            async def get_task(*_args):
+                await conn.execute("SELECT 1")
+                return {"title": "AUTOTEST", "lifecycle_status": "open", "source": {}}
+
+            async def get_request(*_args):
+                await conn.execute("SELECT 1")
+                return None
+
+            async def detail(*_args):
+                await conn.execute("SELECT 1")
+                return SimpleNamespace(title="AUTOTEST", task_id=task_id, project_id=context.project_id)
+
+            monkeypatch.setattr(repo, "get_action", get_action)
+            monkeypatch.setattr(repo, "get_task", get_task)
+            monkeypatch.setattr(repo, "get_request", get_request)
+            monkeypatch.setattr(repo, "update_action", update_action)
+            monkeypatch.setattr(repo, "save_request", AsyncMock())
+            monkeypatch.setattr(service, "detail", detail)
+            monkeypatch.setattr(service, "_action_event", AsyncMock())
+            monkeypatch.setattr(service, "_resolve_notifications", AsyncMock())
+            monkeypatch.setattr(service, "_notify", AsyncMock())
+
+            async def dispatch(**_kwargs):
+                assert conn.info.transaction_status == TransactionStatus.IDLE, \
+                    "implicit outer transaction survives into the external call"
+                async with external.transaction():
+                    await external.execute("SET LOCAL lock_timeout='1s'")
+                    await lock_event_transaction(external)
+                return ActionDispatchOutcome("submitted", "AUTOTEST receipt", ActionReceipt(
+                    adapter="peripheral_sample", receipt_id=str(order_id), submitted_at=now,
+                    external_status="open", detail="not persisted",
+                ))
+
+            dispatcher = SimpleNamespace(dispatch=dispatch, reconcile=dispatch)
+            command = dict(request_id=uuid4(), expected_version=1)
+            if operation == "confirm":
+                result = await service.confirm_work_order_dispatch(
+                    task_id, action_id, WorkOrderConfirmDispatchCommand(**command), dispatcher, None,
+                )
+            elif operation == "submit":
+                result = await service.submit_action(
+                    task_id, action_id, ActionExecuteCommand(**command), dispatcher, None,
+                )
+            else:
+                result = await service.reconcile_action(
+                    task_id, action_id, ActionReconcileCommand(**command), dispatcher, None,
+                )
+            assert result.execution_status == "submitted"
+
+
+@pytest.mark.anyio
+async def test_work_order_filter_matches_source_and_receipts_without_cross_project_rows():
+    dsn = make_conninfo(get_pgconn_string(), dbname="tjwater_v2")
+    project_id, order_id = uuid4(), uuid4()
+    context = ProjectContext(project_id=project_id, project_code="event-filter-test",
+                             user_id=uuid4(), project_role="member")
+    now = datetime.now(timezone.utc)
+    async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
+        assert conn.info.dbname == "tjwater_v2"
+        async with conn.transaction(force_rollback=True):
+            repo = OperationalTaskRepository(conn, project_id)
+            service = OperationalTaskService(repo, context, set())
+
+            async def create(source, *, task_type="external_work_order"):
+                return await service.create(OperationalTaskCreate(
+                    request_id=uuid4(), task_type=task_type, workflow="external-work-order",
+                    title="AUTOTEST filtered work order", summary="isolated rollback", source=source,
+                    trigger={"kind": "manual", "occurred_at": now, "detail": "filter regression"},
+                ))
+
+            projection = await create({"kind": "external_system", "external_id": f"peripheral_sample:{order_id}"})
+            investigation = await create({"kind": "external_system", "external_id": str(order_id)},
+                                         task_type="event_investigation")
+            linked = await create({"kind": "operator"}, task_type="event_investigation")
+            await create({"kind": "external_system", "external_id": f"peripheral_sample:{uuid4()}"})
+            action = HumanApprovalAction(
+                action_id=uuid4(), task_id=linked.task_id, version=1, kind="create_work_order",
+                reason="AUTOTEST receipt", evidence_refs=[],
+                payload={"title": "AUTOTEST", "description": "rollback", "work_order_type": "general"},
+                required_approval_permission=OPERATIONAL_ACTION_WORK_ORDER_APPROVE,
+                required_execution_permission=OPERATIONAL_ACTION_WORK_ORDER_EXECUTE,
+                approval_status="approved", execution_status="submitted", proposed_by="AUTOTEST", proposed_at=now,
+                receipt=ActionReceipt(adapter="peripheral_sample", receipt_id=str(order_id), submitted_at=now,
+                                      external_status="open", detail="rollback"),
+            )
+            await repo.insert_action(str(linked.task_id), action.model_dump(mode="json"))
+            items, cursor = await service.list(task_type=None, status=None, limit=100, offset=0,
+                                               peripheral_work_order_id=order_id)
+            assert {item.task_id for item in items} == {projection.task_id, investigation.task_id, linked.task_id}
+            assert cursor is None
+            foreign = OperationalTaskRepository(conn, uuid4())
+            assert await foreign.list_tasks(task_type=None, status=None, limit=100, offset=0,
+                                            peripheral_work_order_id=str(order_id)) == []
+        row = await (await conn.execute(
+            "SELECT count(*) AS n FROM workbench.operational_tasks WHERE project_id=%s", (project_id,),
+        )).fetchone()
+        assert row["n"] == 0

tests/unit/test_peripheral_push_sync.py

@@ -42,12 +42,25 @@ class PushStream(httpx.AsyncByteStream):
         )
 
 
+def test_large_source_burst_coalesces_to_full_reconciliation_with_bounded_ids():
+    settings = WorkerSettings("http://server.test", "http://agent.test", str(uuid4()), bearer="TEST_ONLY")
+    synchronizer = PeripheralWorkOrderSynchronizer(None, settings, "http://sample.test", CredentialProvider(settings))
+    order_id = str(uuid4())
+    for _ in range(1000):
+        synchronizer._queue_order(order_id)
+    assert synchronizer.pending_order_ids == {order_id}
+    for _ in range(1000):
+        synchronizer._queue_order(str(uuid4()))
+    assert synchronizer.full_reconcile
+    assert synchronizer.pending_order_ids == set()
+
+
 @pytest.mark.anyio
 async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_for_reconciliation():
     project_id, order_id, task_id = map(str, (uuid4(), uuid4(), uuid4()))
     settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
     stream = PushStream()
-    orders, tasks, cursors = [], [], []
+    orders, tasks, cursors, task_queries = [], [], [], []
     initial_scan, projected = asyncio.Event(), asyncio.Event()
     scan_count = 0
 
@@ -63,8 +76,11 @@ async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_f
         if request.url.host == "sample.test":
             scan_count += 1
             initial_scan.set()
+            if request.url.path.endswith(f"/work-orders/{order_id}"):
+                return httpx.Response(200, json=orders[0])
             return httpx.Response(200, json={"items": orders, "next_cursor": None})
         if request.method == "GET":
+            task_queries.append(dict(request.url.params))
             return httpx.Response(200, json={"items": tasks, "next_offset": None})
         body = json.loads(request.content)
         if request.url.path.endswith("external-feedback"):
@@ -100,6 +116,8 @@ async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_f
             assert tasks[0]["source"]["external_revision"] == 2
             assert tasks[0]["status"] == "in_progress"
             assert scan_count == 3  # Startup plus two pushed source changes.
+            assert task_queries[0] == {"limit": "100", "offset": "0"}
+            assert all(query["peripheral_work_order_id"] == order_id for query in task_queries[1:])
             assert cursors == ["20"]
         finally:
             runner.cancel()
@@ -221,7 +239,7 @@ async def test_failed_scan_retries_without_requiring_another_source_message(monk
             return httpx.Response(200, json={"event_cursor": 20})
         return httpx.Response(200, stream=stream)
 
-    async def sync_once(*_args):
+    async def sync_once(*_args, **_kwargs):
         nonlocal attempts
         attempts += 1
         if attempts == 1:
@@ -256,13 +274,14 @@ async def test_source_event_during_scan_schedules_another_pass(monkeypatch):
             return httpx.Response(200, json={"event_cursor": 20})
         return httpx.Response(200, stream=stream)
 
-    async def sync_once(*_args):
+    async def sync_once(*_args, **kwargs):
         nonlocal attempts
         attempts += 1
         if attempts == 1:
             scanning.set()
             await release.wait()
         else:
+            assert kwargs["work_order_ids"] == {order_id}
             caught_up.set()
         return 0
 

tests/unit/test_peripheral_task_sync.py

@@ -240,6 +240,7 @@ async def test_sample_sync_rejects_cross_project_data_before_writing():
 
 
 @pytest.mark.anyio
+@pytest.mark.parametrize("targeted", [False, True])
 @pytest.mark.parametrize(
     ("investigation_status", "expected_feedback_count", "routes_to_investigation"),
     [
@@ -252,19 +253,24 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
     investigation_status,
     expected_feedback_count,
     routes_to_investigation,
+    targeted,
 ):
     project_id, order_id, investigation_id, projection_id = map(str, (uuid4(), uuid4(), uuid4(), uuid4()))
     received = []
     def handler(request):
         if request.url.host == "sample.test":
-            return httpx.Response(200, json={"items": [{
+            order = {
                 "project_id": project_id, "work_order_id": order_id, "origin": "synthetic",
                 "created_by": str(uuid4()),
                 "revision": 2, "summary": "采样已完成", "status": "completed", "updated_at": "2026-09-23T09:00:00Z",
                 "title": "合成采样工单", "priority": "normal",
                 "work_order_type": "general", "created_at": "2026-09-23T08:00:00Z",
                 "scheduled_at": None,
-            }], "next_cursor": None})
+            }
+            if targeted:
+                assert request.url.path.endswith(f"/work-orders/{order_id}")
+                return httpx.Response(200, json=order)
+            return httpx.Response(200, json={"items": [order], "next_cursor": None})
         if request.method == "POST":
             assert request.url.path.endswith("external-feedback"), "Linked child orders must not start another investigation"
             received.append(request.url.path)
@@ -276,6 +282,8 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
             return httpx.Response(200, json={"actions": [{"kind": "create_work_order", "receipt": {
                 "adapter": "peripheral_sample", "receipt_id": order_id,
             }}]})
+        if targeted:
+            assert request.url.params["peripheral_work_order_id"] == order_id
         return httpx.Response(200, json={"items": [
             {"task_id": investigation_id, "task_type": "event_investigation", "status": investigation_status},
             {"task_id": projection_id, "task_type": "external_work_order", "source": {
@@ -285,7 +293,8 @@ async def test_created_work_order_receipt_preserves_projection_and_routes_active
     settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
     async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
         assert await sync_once(
-            client, settings, "http://sample.test", CredentialProvider(settings)
+            client, settings, "http://sample.test", CredentialProvider(settings),
+            **({"work_order_ids": {order_id}} if targeted else {}),
         ) == expected_feedback_count
     assert any(investigation_id in path for path in received) is routes_to_investigation
     assert any(projection_id in path for path in received)

tests/unit/test_workbench_event_hub.py

@@ -1,10 +1,12 @@
 import asyncio
+from types import SimpleNamespace
 from unittest.mock import AsyncMock
 from uuid import uuid4
 
 import pytest
 
 from app.infra.db.postgresql.workbench_event_hub import WorkbenchEventHub, WorkbenchEventHubManager
+from app.infra.db.postgresql import workbench_event_hub as hub_module
 
 
 @pytest.fixture
@@ -30,3 +32,60 @@ async def test_project_subscribers_share_one_hub_and_idle_close(monkeypatch):
     await asyncio.sleep(0.03)
     assert len(manager._hubs) == 0
     close.assert_awaited_once()
+
+
+@pytest.mark.anyio
+async def test_idle_subscribers_share_cursor_reads_and_notification_invalidates_the_cache(monkeypatch):
+    reads = 0
+
+    class Repository:
+        def __init__(self, *_args):
+            pass
+
+        async def read_after(self, _after):
+            nonlocal reads
+            reads += 1
+            return []
+
+    monkeypatch.setattr(hub_module, "WorkbenchEventRepository", Repository)
+    hub = WorkbenchEventHub(uuid4(), "postgresql://test/db")
+    hub._connection = SimpleNamespace(closed=False)
+    monkeypatch.setattr(hub, "start", AsyncMock())
+    await asyncio.gather(*(hub.read_after(0) for _ in range(20)))
+    assert reads == 1
+    await hub._wake()
+    await asyncio.gather(*(hub.read_after(0) for _ in range(20)))
+    assert reads == 2
+
+
+@pytest.mark.anyio
+async def test_notification_during_read_does_not_cache_stale_results(monkeypatch):
+    entered, release = asyncio.Event(), asyncio.Event()
+    reads = 0
+
+    class Repository:
+        def __init__(self, *_args):
+            pass
+
+        async def read_after(self, _after):
+            nonlocal reads
+            reads += 1
+            if reads == 1:
+                entered.set()
+                await release.wait()
+            return []
+
+    monkeypatch.setattr(hub_module, "WorkbenchEventRepository", Repository)
+    hub = WorkbenchEventHub(uuid4(), "postgresql://test/db")
+    hub._connection = SimpleNamespace(closed=False)
+    monkeypatch.setattr(hub, "start", AsyncMock())
+    first = asyncio.create_task(hub.read_after(0))
+    await entered.wait()
+    await hub._wake()
+    release.set()
+    await first
+    await hub.read_after(0)
+    assert reads == 2
+    for cursor in range(100):
+        await hub.read_after(cursor)
+    assert len(hub._reads) == 32

tests/unit/test_workbench_event_order.py

@@ -0,0 +1,49 @@
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+from uuid import uuid4
+
+import pytest
+
+from app.infra.db.postgresql.operational_tasks import OperationalTaskRepository
+from app.infra.db.postgresql.task_alerts import TaskAlertRepository
+from app.infra.db.postgresql.workbench_events import (
+    EVENT_ORDER_LOCK, lock_event_transaction_sync,
+)
+
+
+@pytest.fixture
+def anyio_backend():
+    return "asyncio"
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize("repository", [OperationalTaskRepository, TaskAlertRepository])
+@pytest.mark.parametrize("database", ["tjwater_v2", "other_project"])
+async def test_event_order_lock_precedes_business_writes_and_is_v2_only(repository, database):
+    @asynccontextmanager
+    async def transaction():
+        yield
+
+    connection = SimpleNamespace(
+        info=SimpleNamespace(dbname=database), execute=AsyncMock(), transaction=transaction,
+    )
+    async with repository(connection, uuid4()).transaction():
+        if database == "tjwater_v2":
+            connection.execute.assert_awaited_once_with(
+                "SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,),
+            )
+        else:
+            connection.execute.assert_not_awaited()
+
+
+def test_sync_producers_use_same_database_scoped_lock():
+    connection = SimpleNamespace(info=SimpleNamespace(dbname="tjwater_v2"), execute=Mock())
+    lock_event_transaction_sync(connection)
+    connection.execute.assert_called_once_with(
+        "SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (EVENT_ORDER_LOCK,),
+    )
+    connection.info.dbname = "other_project"
+    connection.execute.reset_mock()
+    lock_event_transaction_sync(connection)
+    connection.execute.assert_not_called()