feat(work-orders): synchronize peripheral changes through events
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""Opt-in checks against provisioned sample tables, never business-table writes."""
|
||||
"""Opt-in checks against provisioned sample tables and transactional source events."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import os
|
||||
import subprocess
|
||||
@@ -39,6 +39,10 @@ def sample():
|
||||
yield store, who
|
||||
# Only this test's unguessable project key; tables created by the authorized setup.
|
||||
with store.connection() as conn:
|
||||
if store.settings.db_name == "tjwater_v2" and conn.execute(
|
||||
"SELECT to_regclass('workbench.domain_event_log') AS name"
|
||||
).fetchone()["name"] is not None:
|
||||
conn.execute("DELETE FROM workbench.domain_event_log WHERE project_id=%s", (who.project_id,))
|
||||
for table in (
|
||||
"work_order_revisions", "test_data_record_revisions", "work_orders",
|
||||
"test_data_records", "feedback", "requests", "revisions", "tasks", "scenarios",
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""tjwater_v2 source/event atomicity checks; every write is rolled back."""
|
||||
from contextlib import contextmanager
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from samples.peripheral.auth import Identity
|
||||
from samples.peripheral.config import SampleSettings
|
||||
from samples.peripheral.errors import SampleFailure
|
||||
from samples.peripheral.schemas import CreateWorkOrder, DeleteResource, UpdateWorkOrder
|
||||
from samples.peripheral.store import PostgresStore
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
os.getenv("PERIPHERAL_SAMPLE_LIVE_TEST") != "1", reason="explicit live sample opt-in required",
|
||||
)
|
||||
|
||||
|
||||
def test_work_order_mutations_and_events_are_atomic_idempotent_and_rollback_cleanly(monkeypatch):
|
||||
settings = SampleSettings()
|
||||
assert settings.db_name == "tjwater_v2", "event verification is restricted to tjwater_v2"
|
||||
store = PostgresStore(settings)
|
||||
who = Identity(str(uuid4()), str(uuid4()), True)
|
||||
with store.connection() as conn:
|
||||
@contextmanager
|
||||
def connection(*, readonly=False, timeseries=False):
|
||||
assert timeseries is False
|
||||
with conn.transaction():
|
||||
yield conn
|
||||
|
||||
monkeypatch.setattr(store, "connection", connection)
|
||||
with conn.transaction(force_rollback=True):
|
||||
command = CreateWorkOrder(
|
||||
request_id=uuid4(), title="AUTOTEST push work order", summary="isolated transaction",
|
||||
)
|
||||
created = store.create_work_order(who, command)
|
||||
assert store.create_work_order(who, command) == created
|
||||
order_id = created["work_order_id"]
|
||||
store.update_work_order(who, order_id, UpdateWorkOrder(
|
||||
request_id=uuid4(), expected_revision=1, status="in_progress",
|
||||
))
|
||||
store.update_work_order(who, order_id, UpdateWorkOrder(
|
||||
request_id=uuid4(), expected_revision=2, progress_note="AUTOTEST field feedback",
|
||||
))
|
||||
with pytest.raises(SampleFailure) as stale:
|
||||
store.update_work_order(who, order_id, UpdateWorkOrder(
|
||||
request_id=uuid4(), expected_revision=1, status="completed",
|
||||
))
|
||||
assert stale.value.code == "REVISION_CONFLICT"
|
||||
store.delete_work_order(who, order_id, DeleteResource(
|
||||
request_id=uuid4(), expected_revision=3,
|
||||
))
|
||||
rows = conn.execute(
|
||||
"""SELECT project_id,topic,aggregate_type,aggregate_id,aggregate_revision,metadata
|
||||
FROM workbench.domain_event_log WHERE project_id=%s ORDER BY event_seq""",
|
||||
(who.project_id,),
|
||||
).fetchall()
|
||||
assert len(rows) == 4
|
||||
assert [row["aggregate_revision"] for row in rows] == [1, 2, 3, 4]
|
||||
assert [row["metadata"]["change"] for row in rows] == ["created", "updated", "updated", "deleted"]
|
||||
assert all(row["topic"] == "workbench.work_order.changed"
|
||||
and row["aggregate_type"] == "peripheral_work_order"
|
||||
and row["aggregate_id"] == order_id for row in rows)
|
||||
assert store.list_work_orders(who, 20)["items"] == []
|
||||
|
||||
original = store._work_order_event
|
||||
|
||||
def fail_after_event(*args, **kwargs):
|
||||
original(*args, **kwargs)
|
||||
raise RuntimeError("AUTOTEST transaction failure")
|
||||
|
||||
monkeypatch.setattr(store, "_work_order_event", fail_after_event)
|
||||
with pytest.raises(RuntimeError, match="transaction failure"):
|
||||
store.create_work_order(who, command.model_copy(update={"request_id": uuid4()}))
|
||||
assert store.list_work_orders(who, 20)["items"] == []
|
||||
assert conn.execute(
|
||||
"SELECT count(*) AS n FROM workbench.domain_event_log WHERE project_id=%s", (who.project_id,),
|
||||
).fetchone()["n"] == 4
|
||||
|
||||
for table in ("workbench.domain_event_log", "peripheral_sample.work_orders",
|
||||
"peripheral_sample.work_order_revisions", "peripheral_sample.requests"):
|
||||
assert conn.execute(f"SELECT count(*) AS n FROM {table} WHERE project_id=%s",
|
||||
(who.project_id,)).fetchone()["n"] == 0
|
||||
@@ -0,0 +1,284 @@
|
||||
import asyncio
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from scripts.run_operational_worker import CredentialProvider, WorkerSettings
|
||||
from scripts import sync_peripheral_work_orders as sync_module
|
||||
from scripts.sync_peripheral_work_orders import PeripheralWorkOrderSynchronizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
class PushStream(httpx.AsyncByteStream):
|
||||
def __init__(self):
|
||||
self.messages = asyncio.Queue()
|
||||
self.closed = False
|
||||
|
||||
async def __aiter__(self):
|
||||
while True:
|
||||
yield await self.messages.get()
|
||||
self.messages.task_done()
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
def publish(self, project_id, order_id, sequence, *, aggregate_type="peripheral_work_order"):
|
||||
event = {
|
||||
"schema": "workbench-event@1", "event_seq": sequence,
|
||||
"event_id": str(uuid4()), "project_id": project_id,
|
||||
"topic": "workbench.work_order.changed", "aggregate_type": aggregate_type,
|
||||
"aggregate_id": order_id, "aggregate_revision": 1,
|
||||
"occurred_at": "2026-10-08T00:00:00Z",
|
||||
"metadata": {"source": "peripheral_sample", "work_order_id": order_id},
|
||||
}
|
||||
self.messages.put_nowait(
|
||||
f"id: {sequence}\nevent: workbench_event\ndata: {json.dumps(event)}\n\n".encode()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_for_reconciliation():
|
||||
project_id, order_id, task_id = map(str, (uuid4(), uuid4(), uuid4()))
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
orders, tasks, cursors = [], [], []
|
||||
initial_scan, projected = asyncio.Event(), asyncio.Event()
|
||||
scan_count = 0
|
||||
|
||||
def handler(request):
|
||||
nonlocal scan_count
|
||||
assert request.headers["X-Project-Id"] == project_id
|
||||
assert request.headers["Authorization"] == "Bearer TEST_ONLY"
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
if request.url.path.endswith("/events"):
|
||||
cursors.append(request.url.params["after"])
|
||||
return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, stream=stream)
|
||||
if request.url.host == "sample.test":
|
||||
scan_count += 1
|
||||
initial_scan.set()
|
||||
return httpx.Response(200, json={"items": orders, "next_cursor": None})
|
||||
if request.method == "GET":
|
||||
return httpx.Response(200, json={"items": tasks, "next_offset": None})
|
||||
body = json.loads(request.content)
|
||||
if request.url.path.endswith("external-feedback"):
|
||||
tasks[0]["source"]["external_revision"] = body["source_revision"]
|
||||
tasks[0]["status"] = body["status"]
|
||||
projected.set()
|
||||
return httpx.Response(200, json={"task": tasks[0]})
|
||||
task = {**body, "task_id": task_id}
|
||||
tasks.append(task)
|
||||
return httpx.Response(201, json=task)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(initial_scan.wait(), timeout=3)
|
||||
orders.append({
|
||||
"project_id": project_id, "work_order_id": order_id, "origin": "synthetic",
|
||||
"revision": 1, "title": "预约维修", "summary": "等待现场处理", "priority": "normal",
|
||||
"status": "open", "created_at": "2026-10-08T00:00:00Z",
|
||||
"updated_at": "2026-10-08T00:00:00Z", "scheduled_at": None,
|
||||
"work_order_type": "general",
|
||||
})
|
||||
stream.publish(project_id, order_id, 21)
|
||||
await asyncio.wait_for(projected.wait(), timeout=3)
|
||||
assert tasks[0]["source"]["external_revision"] == 1
|
||||
projected.clear()
|
||||
orders[0].update(revision=2, status="in_progress")
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(projected.wait(), timeout=3)
|
||||
assert tasks[0]["source"]["external_revision"] == 2
|
||||
assert tasks[0]["status"] == "in_progress"
|
||||
assert scan_count == 3 # Startup plus two pushed source changes.
|
||||
assert cursors == ["20"]
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
assert stream.closed
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_expired_cursor_rebuilds_baseline_and_wakes_source_reconciliation():
|
||||
project_id = str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
refreshed = asyncio.Event()
|
||||
baselines = iter([10, 30])
|
||||
cursors = []
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
cursor = next(baselines)
|
||||
if cursor == 30:
|
||||
refreshed.set()
|
||||
return httpx.Response(200, json={"event_cursor": cursor})
|
||||
cursors.append(request.url.params["after"])
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
||||
synchronizer.wake.clear()
|
||||
stream.messages.put_nowait(b"event: reset_required\ndata: {}\n\n")
|
||||
await asyncio.wait_for(refreshed.wait(), timeout=3)
|
||||
assert synchronizer.wake.is_set()
|
||||
assert synchronizer.cursor == 30
|
||||
assert cursors == ["10"]
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_projection_events_and_duplicate_source_events_do_not_trigger_extra_scans():
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
||||
synchronizer.wake.clear()
|
||||
stream.publish(project_id, order_id, 21, aggregate_type="operational_action")
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert not synchronizer.wake.is_set()
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert synchronizer.wake.is_set()
|
||||
synchronizer.wake.clear()
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert not synchronizer.wake.is_set()
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_disconnect_resumes_after_last_consumed_event():
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
stream.publish(project_id, order_id, 21)
|
||||
first_message = stream.messages.get_nowait()
|
||||
stream.messages.task_done()
|
||||
reconnected, cursors = asyncio.Event(), []
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
cursors.append(request.url.params["after"])
|
||||
if len(cursors) == 1:
|
||||
return httpx.Response(200, content=first_message)
|
||||
reconnected.set()
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(reconnected.wait(), timeout=3)
|
||||
assert cursors == ["20", "21"]
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_failed_scan_retries_without_requiring_another_source_message(monkeypatch):
|
||||
project_id = str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
recovered, attempts = asyncio.Event(), 0
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async def sync_once(*_args):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise httpx.ConnectError("AUTOTEST temporary source outage")
|
||||
recovered.set()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(recovered.wait(), timeout=8)
|
||||
assert attempts == 2
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_source_event_during_scan_schedules_another_pass(monkeypatch):
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
scanning, release, caught_up = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
||||
attempts = 0
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async def sync_once(*_args):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
scanning.set()
|
||||
await release.wait()
|
||||
else:
|
||||
caught_up.set()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(scanning.wait(), timeout=3)
|
||||
stream.publish(project_id, order_id, 21)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
release.set()
|
||||
await asyncio.wait_for(caught_up.wait(), timeout=3)
|
||||
assert attempts == 2
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
@@ -0,0 +1,50 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from samples.peripheral.auth import Identity
|
||||
from samples.peripheral.config import SampleSettings
|
||||
from samples.peripheral.store import PostgresStore
|
||||
|
||||
|
||||
class EventConnection:
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
|
||||
def execute(self, statement, parameters):
|
||||
self.statements.append((statement, parameters))
|
||||
return self
|
||||
|
||||
def fetchone(self):
|
||||
return {"event_seq": 42}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("revision", "deleted", "change"), [
|
||||
(1, False, "created"), (2, False, "updated"), (3, True, "deleted"),
|
||||
])
|
||||
def test_work_order_signals_use_same_connection_and_contain_only_identity(revision, deleted, change):
|
||||
conn = EventConnection()
|
||||
who = Identity(str(uuid4()), str(uuid4()), True)
|
||||
order = {"work_order_id": str(uuid4()), "revision": revision,
|
||||
"updated_at": "2026-10-08T00:00:00Z", "title": "private title"}
|
||||
|
||||
PostgresStore(SampleSettings())._work_order_event(conn, who, order, deleted=deleted)
|
||||
|
||||
statement, parameters = conn.statements[0]
|
||||
assert "workbench.domain_event_log" in statement
|
||||
assert "'workbench.work_order.changed','peripheral_work_order'" in statement
|
||||
assert parameters[0] == who.project_id
|
||||
assert parameters[2:5] == (order["work_order_id"], revision, order["updated_at"])
|
||||
assert parameters[5].obj == {
|
||||
"source": "peripheral_sample", "work_order_id": order["work_order_id"], "change": change,
|
||||
}
|
||||
assert conn.statements[1] == (
|
||||
"SELECT pg_notify('workbench_events', %s)", (f"{who.project_id}:42",),
|
||||
)
|
||||
|
||||
|
||||
def test_event_rollout_does_not_write_to_other_databases():
|
||||
conn = EventConnection()
|
||||
store = PostgresStore(SampleSettings(db_name="another_project"))
|
||||
store._work_order_event(conn, Identity(str(uuid4()), str(uuid4()), True), {})
|
||||
assert conn.statements == []
|
||||
Reference in New Issue
Block a user