feat(scada): add calibrated monitoring and incident recovery

tests/integration/test_calculation_queue_storage.py

@@ -31,6 +31,7 @@ def test_concurrent_submission_claims_fencing_and_project_isolation():
                 (legacy_job, uuid4(), owner))
             lifecycle = (migrations / "027_calculation_job_lifecycle.sql").read_text()
             await conn.execute(lifecycle)
+            await conn.execute((migrations / "030_remove_calculation_queue_expiry.sql").read_text())
             cursor = await conn.execute("SELECT result,result_expires_at,result_expires_at>now()+interval '6 days' AS grace FROM workbench.calculation_jobs WHERE job_id=%s", (legacy_job,))
             first = await cursor.fetchone()
             assert first["result"] == {"preserved": True} and first["grace"]
@@ -90,15 +91,6 @@ def test_concurrent_submission_claims_fencing_and_project_isolation():
             assert cancelled["started_at"] is None
             assert await repo.claim("replacement", 60) is None
 
-            # A polling client expires queued jobs without any worker process.
-            waiting = await repo.enqueue(owner_user_id=owner, request_key="expired-queue",
-                tool="run_simulation", parameters={}, binding={}, timeout_seconds=30)
-            await conn.execute("UPDATE workbench.calculation_jobs SET queue_expires_at=now()-interval '1 second' WHERE job_id=%s", (waiting["job_id"],))
-            status = await repo.get(waiting["job_id"], owner)
-            assert status["status"] == "timed_out" and status["error"]["code"] == "QUEUE_TIMEOUT"
-            assert status["finished_at"] is not None
-            assert "result" not in status
-
             # Retention scrubs only payload, keeping deduplication and status.
             completed = await repo.enqueue(owner_user_id=owner, request_key="retained-identity",
                 tool="run_simulation", parameters={}, binding={}, timeout_seconds=30)