feat(scada): add calibrated monitoring and incident recovery

tests/unit/test_calculation_worker.py

@@ -27,8 +27,7 @@ def run_async(function):
     return wrapper
 
 
-def test_worker_settings_require_explicit_valid_scope(monkeypatch):
-    monkeypatch.setenv('CALCULATION_WORKER_PROJECT_IDS', '6b39b128-2ba8-454f-9bbe-e9553902ab53')
+def test_worker_settings_require_valid_concurrency(monkeypatch):
     settings = WorkerSettings.from_environment()
     assert settings.max_concurrency == 2
     monkeypatch.setenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '30')
@@ -85,7 +84,7 @@ async def test_cancel_and_lost_lease_stop_process_before_persisting(monkeypatch,
     @asynccontextmanager
     async def repository(project):
         yield Repository()
-    worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.05), repository=repository)
+    worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.05), repository=repository)
     await worker.run_job({'project_id': 'project', 'job_id': 'job', 'timeout_seconds': 60})
     assert finishes == ([expected] if expected else [])
     assert processes[0].returncode is not None
@@ -111,7 +110,7 @@ async def test_timeout_stops_process_before_recording(monkeypatch):
     @asynccontextmanager
     async def repository(project):
         yield Repository()
-    worker = CalculationWorker(WorkerSettings(('project',), heartbeat_seconds=.02), repository=repository)
+    worker = CalculationWorker(WorkerSettings(heartbeat_seconds=.02), repository=repository)
     await worker.run_job({'project_id': 'project', 'timeout_seconds': 0})
     assert statuses == ['timed_out']
 
@@ -134,7 +133,52 @@ async def test_duplicate_supervisor_never_claims(monkeypatch):
         pytest.fail('A duplicate supervisor must never open the queue')
         yield
     with pytest.raises(RuntimeError, match='already active'):
-        await CalculationWorker(WorkerSettings(('project',)), repository=repository).run()
+        await CalculationWorker(WorkerSettings(), repository=repository).run()
+
+
+@run_async
+async def test_unavailable_project_does_not_block_other_projects(monkeypatch):
+    from app.infra.db import calculation_worker as infrastructure
+
+    class Lease:
+        async def ping(self):
+            pass
+
+    @asynccontextmanager
+    async def supervisor_lease():
+        yield Lease()
+
+    checked = asyncio.Event()
+
+    class Repository:
+        async def recover(self):
+            pass
+
+        async def claim(self, owner, lease_seconds):
+            checked.set()
+            return None
+
+    @asynccontextmanager
+    async def repository(project):
+        if project == 'unavailable':
+            raise RuntimeError('offline')
+        yield Repository()
+
+    async def projects():
+        return ('unavailable', 'healthy')
+
+    monkeypatch.setattr(infrastructure, 'calculation_supervisor_lease', supervisor_lease)
+    worker = CalculationWorker(
+        WorkerSettings(poll_seconds=.01, lease_seconds=1, heartbeat_seconds=.1),
+        repository=repository,
+        project_provider=projects,
+    )
+    task = asyncio.create_task(worker.run())
+    try:
+        await asyncio.wait_for(checked.wait(), 2)
+    finally:
+        task.cancel()
+        await asyncio.gather(task, return_exceptions=True)
 
 
 @run_async
@@ -212,7 +256,7 @@ async def test_cleanup_failure_is_not_reported_cancelled(monkeypatch, tmp_path):
     @asynccontextmanager
     async def repository(project):
         yield Repository()
-    worker = CalculationWorker(WorkerSettings(('project',)), repository=repository)
+    worker = CalculationWorker(WorkerSettings(), repository=repository)
     await worker.run_job({'project_id': 'project', 'timeout_seconds': 60})
     assert results == [('failed', 'CLEANUP_FAILED')]
     assert len(list(tmp_path.glob('*/cleanup-failed'))) == 1