feat(work-orders): synchronize peripheral changes through events
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
import asyncio
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from scripts.run_operational_worker import CredentialProvider, WorkerSettings
|
||||
from scripts import sync_peripheral_work_orders as sync_module
|
||||
from scripts.sync_peripheral_work_orders import PeripheralWorkOrderSynchronizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
class PushStream(httpx.AsyncByteStream):
|
||||
def __init__(self):
|
||||
self.messages = asyncio.Queue()
|
||||
self.closed = False
|
||||
|
||||
async def __aiter__(self):
|
||||
while True:
|
||||
yield await self.messages.get()
|
||||
self.messages.task_done()
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
def publish(self, project_id, order_id, sequence, *, aggregate_type="peripheral_work_order"):
|
||||
event = {
|
||||
"schema": "workbench-event@1", "event_seq": sequence,
|
||||
"event_id": str(uuid4()), "project_id": project_id,
|
||||
"topic": "workbench.work_order.changed", "aggregate_type": aggregate_type,
|
||||
"aggregate_id": order_id, "aggregate_revision": 1,
|
||||
"occurred_at": "2026-10-08T00:00:00Z",
|
||||
"metadata": {"source": "peripheral_sample", "work_order_id": order_id},
|
||||
}
|
||||
self.messages.put_nowait(
|
||||
f"id: {sequence}\nevent: workbench_event\ndata: {json.dumps(event)}\n\n".encode()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_for_reconciliation():
|
||||
project_id, order_id, task_id = map(str, (uuid4(), uuid4(), uuid4()))
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
orders, tasks, cursors = [], [], []
|
||||
initial_scan, projected = asyncio.Event(), asyncio.Event()
|
||||
scan_count = 0
|
||||
|
||||
def handler(request):
|
||||
nonlocal scan_count
|
||||
assert request.headers["X-Project-Id"] == project_id
|
||||
assert request.headers["Authorization"] == "Bearer TEST_ONLY"
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
if request.url.path.endswith("/events"):
|
||||
cursors.append(request.url.params["after"])
|
||||
return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, stream=stream)
|
||||
if request.url.host == "sample.test":
|
||||
scan_count += 1
|
||||
initial_scan.set()
|
||||
return httpx.Response(200, json={"items": orders, "next_cursor": None})
|
||||
if request.method == "GET":
|
||||
return httpx.Response(200, json={"items": tasks, "next_offset": None})
|
||||
body = json.loads(request.content)
|
||||
if request.url.path.endswith("external-feedback"):
|
||||
tasks[0]["source"]["external_revision"] = body["source_revision"]
|
||||
tasks[0]["status"] = body["status"]
|
||||
projected.set()
|
||||
return httpx.Response(200, json={"task": tasks[0]})
|
||||
task = {**body, "task_id": task_id}
|
||||
tasks.append(task)
|
||||
return httpx.Response(201, json=task)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(initial_scan.wait(), timeout=3)
|
||||
orders.append({
|
||||
"project_id": project_id, "work_order_id": order_id, "origin": "synthetic",
|
||||
"revision": 1, "title": "预约维修", "summary": "等待现场处理", "priority": "normal",
|
||||
"status": "open", "created_at": "2026-10-08T00:00:00Z",
|
||||
"updated_at": "2026-10-08T00:00:00Z", "scheduled_at": None,
|
||||
"work_order_type": "general",
|
||||
})
|
||||
stream.publish(project_id, order_id, 21)
|
||||
await asyncio.wait_for(projected.wait(), timeout=3)
|
||||
assert tasks[0]["source"]["external_revision"] == 1
|
||||
projected.clear()
|
||||
orders[0].update(revision=2, status="in_progress")
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(projected.wait(), timeout=3)
|
||||
assert tasks[0]["source"]["external_revision"] == 2
|
||||
assert tasks[0]["status"] == "in_progress"
|
||||
assert scan_count == 3 # Startup plus two pushed source changes.
|
||||
assert cursors == ["20"]
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
assert stream.closed
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_expired_cursor_rebuilds_baseline_and_wakes_source_reconciliation():
|
||||
project_id = str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
refreshed = asyncio.Event()
|
||||
baselines = iter([10, 30])
|
||||
cursors = []
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
cursor = next(baselines)
|
||||
if cursor == 30:
|
||||
refreshed.set()
|
||||
return httpx.Response(200, json={"event_cursor": cursor})
|
||||
cursors.append(request.url.params["after"])
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
||||
synchronizer.wake.clear()
|
||||
stream.messages.put_nowait(b"event: reset_required\ndata: {}\n\n")
|
||||
await asyncio.wait_for(refreshed.wait(), timeout=3)
|
||||
assert synchronizer.wake.is_set()
|
||||
assert synchronizer.cursor == 30
|
||||
assert cursors == ["10"]
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_projection_events_and_duplicate_source_events_do_not_trigger_extra_scans():
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
||||
synchronizer.wake.clear()
|
||||
stream.publish(project_id, order_id, 21, aggregate_type="operational_action")
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert not synchronizer.wake.is_set()
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert synchronizer.wake.is_set()
|
||||
synchronizer.wake.clear()
|
||||
stream.publish(project_id, order_id, 22)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
assert not synchronizer.wake.is_set()
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_disconnect_resumes_after_last_consumed_event():
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
stream.publish(project_id, order_id, 21)
|
||||
first_message = stream.messages.get_nowait()
|
||||
stream.messages.task_done()
|
||||
reconnected, cursors = asyncio.Event(), []
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
cursors.append(request.url.params["after"])
|
||||
if len(cursors) == 1:
|
||||
return httpx.Response(200, content=first_message)
|
||||
reconnected.set()
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
listener = asyncio.create_task(synchronizer.listen())
|
||||
try:
|
||||
await asyncio.wait_for(reconnected.wait(), timeout=3)
|
||||
assert cursors == ["20", "21"]
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_failed_scan_retries_without_requiring_another_source_message(monkeypatch):
|
||||
project_id = str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
recovered, attempts = asyncio.Event(), 0
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async def sync_once(*_args):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise httpx.ConnectError("AUTOTEST temporary source outage")
|
||||
recovered.set()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(recovered.wait(), timeout=8)
|
||||
assert attempts == 2
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_source_event_during_scan_schedules_another_pass(monkeypatch):
|
||||
project_id, order_id = str(uuid4()), str(uuid4())
|
||||
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
||||
stream = PushStream()
|
||||
scanning, release, caught_up = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
||||
attempts = 0
|
||||
|
||||
def handler(request):
|
||||
if request.url.path.endswith("sync-snapshot"):
|
||||
return httpx.Response(200, json={"event_cursor": 20})
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
async def sync_once(*_args):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
scanning.set()
|
||||
await release.wait()
|
||||
else:
|
||||
caught_up.set()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
synchronizer = PeripheralWorkOrderSynchronizer(
|
||||
client, settings, "http://sample.test", CredentialProvider(settings),
|
||||
)
|
||||
runner = asyncio.create_task(synchronizer.serve())
|
||||
try:
|
||||
await asyncio.wait_for(scanning.wait(), timeout=3)
|
||||
stream.publish(project_id, order_id, 21)
|
||||
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
||||
release.set()
|
||||
await asyncio.wait_for(caught_up.wait(), timeout=3)
|
||||
assert attempts == 2
|
||||
finally:
|
||||
runner.cancel()
|
||||
await asyncio.gather(runner, return_exceptions=True)
|
||||
@@ -0,0 +1,50 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from samples.peripheral.auth import Identity
|
||||
from samples.peripheral.config import SampleSettings
|
||||
from samples.peripheral.store import PostgresStore
|
||||
|
||||
|
||||
class EventConnection:
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
|
||||
def execute(self, statement, parameters):
|
||||
self.statements.append((statement, parameters))
|
||||
return self
|
||||
|
||||
def fetchone(self):
|
||||
return {"event_seq": 42}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("revision", "deleted", "change"), [
|
||||
(1, False, "created"), (2, False, "updated"), (3, True, "deleted"),
|
||||
])
|
||||
def test_work_order_signals_use_same_connection_and_contain_only_identity(revision, deleted, change):
|
||||
conn = EventConnection()
|
||||
who = Identity(str(uuid4()), str(uuid4()), True)
|
||||
order = {"work_order_id": str(uuid4()), "revision": revision,
|
||||
"updated_at": "2026-10-08T00:00:00Z", "title": "private title"}
|
||||
|
||||
PostgresStore(SampleSettings())._work_order_event(conn, who, order, deleted=deleted)
|
||||
|
||||
statement, parameters = conn.statements[0]
|
||||
assert "workbench.domain_event_log" in statement
|
||||
assert "'workbench.work_order.changed','peripheral_work_order'" in statement
|
||||
assert parameters[0] == who.project_id
|
||||
assert parameters[2:5] == (order["work_order_id"], revision, order["updated_at"])
|
||||
assert parameters[5].obj == {
|
||||
"source": "peripheral_sample", "work_order_id": order["work_order_id"], "change": change,
|
||||
}
|
||||
assert conn.statements[1] == (
|
||||
"SELECT pg_notify('workbench_events', %s)", (f"{who.project_id}:42",),
|
||||
)
|
||||
|
||||
|
||||
def test_event_rollout_does_not_write_to_other_databases():
|
||||
conn = EventConnection()
|
||||
store = PostgresStore(SampleSettings(db_name="another_project"))
|
||||
store._work_order_event(conn, Identity(str(uuid4()), str(uuid4()), True), {})
|
||||
assert conn.statements == []
|
||||
Reference in New Issue
Block a user