304 lines
13 KiB
Python
304 lines
13 KiB
Python
import asyncio
|
|
import json
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from scripts.run_operational_worker import CredentialProvider, WorkerSettings
|
|
from scripts import sync_peripheral_work_orders as sync_module
|
|
from scripts.sync_peripheral_work_orders import PeripheralWorkOrderSynchronizer
|
|
|
|
|
|
@pytest.fixture
|
|
def anyio_backend():
|
|
return "asyncio"
|
|
|
|
|
|
class PushStream(httpx.AsyncByteStream):
|
|
def __init__(self):
|
|
self.messages = asyncio.Queue()
|
|
self.closed = False
|
|
|
|
async def __aiter__(self):
|
|
while True:
|
|
yield await self.messages.get()
|
|
self.messages.task_done()
|
|
|
|
async def aclose(self):
|
|
self.closed = True
|
|
|
|
def publish(self, project_id, order_id, sequence, *, aggregate_type="peripheral_work_order"):
|
|
event = {
|
|
"schema": "workbench-event@1", "event_seq": sequence,
|
|
"event_id": str(uuid4()), "project_id": project_id,
|
|
"topic": "workbench.work_order.changed", "aggregate_type": aggregate_type,
|
|
"aggregate_id": order_id, "aggregate_revision": 1,
|
|
"occurred_at": "2026-10-08T00:00:00Z",
|
|
"metadata": {"source": "peripheral_sample", "work_order_id": order_id},
|
|
}
|
|
self.messages.put_nowait(
|
|
f"id: {sequence}\nevent: workbench_event\ndata: {json.dumps(event)}\n\n".encode()
|
|
)
|
|
|
|
|
|
def test_large_source_burst_coalesces_to_full_reconciliation_with_bounded_ids():
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", str(uuid4()), bearer="TEST_ONLY")
|
|
synchronizer = PeripheralWorkOrderSynchronizer(None, settings, "http://sample.test", CredentialProvider(settings))
|
|
order_id = str(uuid4())
|
|
for _ in range(1000):
|
|
synchronizer._queue_order(order_id)
|
|
assert synchronizer.pending_order_ids == {order_id}
|
|
for _ in range(1000):
|
|
synchronizer._queue_order(str(uuid4()))
|
|
assert synchronizer.full_reconcile
|
|
assert synchronizer.pending_order_ids == set()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_source_push_synchronizes_new_and_updated_orders_without_waiting_for_reconciliation():
|
|
project_id, order_id, task_id = map(str, (uuid4(), uuid4(), uuid4()))
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
orders, tasks, cursors, task_queries = [], [], [], []
|
|
initial_scan, projected = asyncio.Event(), asyncio.Event()
|
|
scan_count = 0
|
|
|
|
def handler(request):
|
|
nonlocal scan_count
|
|
assert request.headers["X-Project-Id"] == project_id
|
|
assert request.headers["Authorization"] == "Bearer TEST_ONLY"
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
return httpx.Response(200, json={"event_cursor": 20})
|
|
if request.url.path.endswith("/events"):
|
|
cursors.append(request.url.params["after"])
|
|
return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, stream=stream)
|
|
if request.url.host == "sample.test":
|
|
scan_count += 1
|
|
initial_scan.set()
|
|
if request.url.path.endswith(f"/work-orders/{order_id}"):
|
|
return httpx.Response(200, json=orders[0])
|
|
return httpx.Response(200, json={"items": orders, "next_cursor": None})
|
|
if request.method == "GET":
|
|
task_queries.append(dict(request.url.params))
|
|
return httpx.Response(200, json={"items": tasks, "next_offset": None})
|
|
body = json.loads(request.content)
|
|
if request.url.path.endswith("external-feedback"):
|
|
tasks[0]["source"]["external_revision"] = body["source_revision"]
|
|
tasks[0]["status"] = body["status"]
|
|
projected.set()
|
|
return httpx.Response(200, json={"task": tasks[0]})
|
|
task = {**body, "task_id": task_id}
|
|
tasks.append(task)
|
|
return httpx.Response(201, json=task)
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
runner = asyncio.create_task(synchronizer.serve())
|
|
try:
|
|
await asyncio.wait_for(initial_scan.wait(), timeout=3)
|
|
orders.append({
|
|
"project_id": project_id, "work_order_id": order_id, "origin": "synthetic",
|
|
"revision": 1, "title": "预约维修", "summary": "等待现场处理", "priority": "normal",
|
|
"status": "open", "created_at": "2026-10-08T00:00:00Z",
|
|
"updated_at": "2026-10-08T00:00:00Z", "scheduled_at": None,
|
|
"work_order_type": "general",
|
|
})
|
|
stream.publish(project_id, order_id, 21)
|
|
await asyncio.wait_for(projected.wait(), timeout=3)
|
|
assert tasks[0]["source"]["external_revision"] == 1
|
|
projected.clear()
|
|
orders[0].update(revision=2, status="in_progress")
|
|
stream.publish(project_id, order_id, 22)
|
|
await asyncio.wait_for(projected.wait(), timeout=3)
|
|
assert tasks[0]["source"]["external_revision"] == 2
|
|
assert tasks[0]["status"] == "in_progress"
|
|
assert scan_count == 3 # Startup plus two pushed source changes.
|
|
assert task_queries[0] == {"limit": "100", "offset": "0"}
|
|
assert all(query["peripheral_work_order_id"] == order_id for query in task_queries[1:])
|
|
assert cursors == ["20"]
|
|
finally:
|
|
runner.cancel()
|
|
await asyncio.gather(runner, return_exceptions=True)
|
|
assert stream.closed
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_expired_cursor_rebuilds_baseline_and_wakes_source_reconciliation():
|
|
project_id = str(uuid4())
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
refreshed = asyncio.Event()
|
|
baselines = iter([10, 30])
|
|
cursors = []
|
|
|
|
def handler(request):
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
cursor = next(baselines)
|
|
if cursor == 30:
|
|
refreshed.set()
|
|
return httpx.Response(200, json={"event_cursor": cursor})
|
|
cursors.append(request.url.params["after"])
|
|
return httpx.Response(200, stream=stream)
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
listener = asyncio.create_task(synchronizer.listen())
|
|
try:
|
|
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
|
synchronizer.wake.clear()
|
|
stream.messages.put_nowait(b"event: reset_required\ndata: {}\n\n")
|
|
await asyncio.wait_for(refreshed.wait(), timeout=3)
|
|
assert synchronizer.wake.is_set()
|
|
assert synchronizer.cursor == 30
|
|
assert cursors == ["10"]
|
|
finally:
|
|
listener.cancel()
|
|
await asyncio.gather(listener, return_exceptions=True)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_projection_events_and_duplicate_source_events_do_not_trigger_extra_scans():
|
|
project_id, order_id = str(uuid4()), str(uuid4())
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
|
|
def handler(request):
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
return httpx.Response(200, json={"event_cursor": 20})
|
|
return httpx.Response(200, stream=stream)
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
listener = asyncio.create_task(synchronizer.listen())
|
|
try:
|
|
await asyncio.wait_for(synchronizer.wake.wait(), timeout=3)
|
|
synchronizer.wake.clear()
|
|
stream.publish(project_id, order_id, 21, aggregate_type="operational_action")
|
|
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
|
assert not synchronizer.wake.is_set()
|
|
stream.publish(project_id, order_id, 22)
|
|
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
|
assert synchronizer.wake.is_set()
|
|
synchronizer.wake.clear()
|
|
stream.publish(project_id, order_id, 22)
|
|
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
|
assert not synchronizer.wake.is_set()
|
|
finally:
|
|
listener.cancel()
|
|
await asyncio.gather(listener, return_exceptions=True)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_disconnect_resumes_after_last_consumed_event():
|
|
project_id, order_id = str(uuid4()), str(uuid4())
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
stream.publish(project_id, order_id, 21)
|
|
first_message = stream.messages.get_nowait()
|
|
stream.messages.task_done()
|
|
reconnected, cursors = asyncio.Event(), []
|
|
|
|
def handler(request):
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
return httpx.Response(200, json={"event_cursor": 20})
|
|
cursors.append(request.url.params["after"])
|
|
if len(cursors) == 1:
|
|
return httpx.Response(200, content=first_message)
|
|
reconnected.set()
|
|
return httpx.Response(200, stream=stream)
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
listener = asyncio.create_task(synchronizer.listen())
|
|
try:
|
|
await asyncio.wait_for(reconnected.wait(), timeout=3)
|
|
assert cursors == ["20", "21"]
|
|
finally:
|
|
listener.cancel()
|
|
await asyncio.gather(listener, return_exceptions=True)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_failed_scan_retries_without_requiring_another_source_message(monkeypatch):
|
|
project_id = str(uuid4())
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
recovered, attempts = asyncio.Event(), 0
|
|
|
|
def handler(request):
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
return httpx.Response(200, json={"event_cursor": 20})
|
|
return httpx.Response(200, stream=stream)
|
|
|
|
async def sync_once(*_args, **_kwargs):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
raise httpx.ConnectError("AUTOTEST temporary source outage")
|
|
recovered.set()
|
|
return 0
|
|
|
|
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
runner = asyncio.create_task(synchronizer.serve())
|
|
try:
|
|
await asyncio.wait_for(recovered.wait(), timeout=8)
|
|
assert attempts == 2
|
|
finally:
|
|
runner.cancel()
|
|
await asyncio.gather(runner, return_exceptions=True)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_source_event_during_scan_schedules_another_pass(monkeypatch):
|
|
project_id, order_id = str(uuid4()), str(uuid4())
|
|
settings = WorkerSettings("http://server.test", "http://agent.test", project_id, bearer="TEST_ONLY")
|
|
stream = PushStream()
|
|
scanning, release, caught_up = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
|
attempts = 0
|
|
|
|
def handler(request):
|
|
if request.url.path.endswith("sync-snapshot"):
|
|
return httpx.Response(200, json={"event_cursor": 20})
|
|
return httpx.Response(200, stream=stream)
|
|
|
|
async def sync_once(*_args, **kwargs):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
scanning.set()
|
|
await release.wait()
|
|
else:
|
|
assert kwargs["work_order_ids"] == {order_id}
|
|
caught_up.set()
|
|
return 0
|
|
|
|
monkeypatch.setattr(sync_module, "sync_once", sync_once)
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
synchronizer = PeripheralWorkOrderSynchronizer(
|
|
client, settings, "http://sample.test", CredentialProvider(settings),
|
|
)
|
|
runner = asyncio.create_task(synchronizer.serve())
|
|
try:
|
|
await asyncio.wait_for(scanning.wait(), timeout=3)
|
|
stream.publish(project_id, order_id, 21)
|
|
await asyncio.wait_for(stream.messages.join(), timeout=3)
|
|
release.set()
|
|
await asyncio.wait_for(caught_up.wait(), timeout=3)
|
|
assert attempts == 2
|
|
finally:
|
|
runner.cancel()
|
|
await asyncio.gather(runner, return_exceptions=True)
|