feat(scada): add calibrated monitoring and incident recovery

2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
@@ -10,6 +10,7 @@ from contextlib import asynccontextmanager, contextmanager
 from dataclasses import dataclass
 from datetime import datetime, timezone
 import json
+import logging
 import os
 from pathlib import Path
 import signal
@@ -20,9 +21,11 @@ import time
 from uuid import UUID, uuid4


+logger = logging.getLogger(__name__)
+
+
@dataclass(frozen=True)
 class WorkerSettings:
-    project_ids: tuple[str, ...]
    max_concurrency: int = 2
    poll_seconds: float = 2
    lease_seconds: int = 30
@@ -30,14 +33,12 @@ class WorkerSettings:

    @classmethod
    def from_environment(cls):
-        projects = tuple(dict.fromkeys(str(UUID(p.strip())) for p in
-            os.environ['CALCULATION_WORKER_PROJECT_IDS'].split(',')))
-        result = cls(projects, int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
+        result = cls(int(os.getenv('CALCULATION_WORKER_MAX_CONCURRENCY', '2')),
                     float(os.getenv('CALCULATION_WORKER_POLL_SECONDS', '2')),
                     int(os.getenv('CALCULATION_WORKER_LEASE_SECONDS', '30')),
                     float(os.getenv('CALCULATION_WORKER_HEARTBEAT_SECONDS', '5')))
-        if not projects or not 1 <= result.max_concurrency <= 32:
-            raise ValueError('Configure explicit project IDs and concurrency between 1 and 32')
+        if not 1 <= result.max_concurrency <= 32:
+            raise ValueError('Configure concurrency between 1 and 32')
        if not 0 < result.heartbeat_seconds < result.lease_seconds / 3 or result.poll_seconds <= 0:
            raise ValueError('Heartbeat must be positive and below one third of the lease')
        return result
@@ -173,9 +174,13 @@ async def recover_job_resources(project_ids):


 class CalculationWorker:
-    def __init__(self, settings, *, repository=queue_repository):
+    def __init__(self, settings, *, repository=queue_repository, project_provider=None):
        self.settings = settings
        self.repository = repository
+        if project_provider is None:
+            from app.infra.db.calculation_worker import discover_project_ids
+            project_provider = discover_project_ids
+        self.project_provider = project_provider
        self.owner = str(uuid4())

    async def run_job(self, row):
@@ -239,36 +244,55 @@ class CalculationWorker:
        # Dedicated session-level metadata lock: independent API workers never
        # participate; duplicate supervisors exit before claiming any jobs.
        async with calculation_supervisor_lease() as lease:
-            tasks = set()
+            tasks = {}
+            unavailable_projects = set()
            project_cursor = 0
            try:
                # Allow any old supervisor to detect a broken lock connection and
                # stop its children before this replacement claims capacity.
                await asyncio.sleep(self.settings.lease_seconds)
-                await recover_job_resources(self.settings.project_ids)
+                await recover_job_resources(await self.project_provider())
                while True:
                    async with asyncio.timeout(self.settings.heartbeat_seconds):
                        await lease.ping()
-                    for task in tuple(tasks):
+                    for task, identity in tuple(tasks.items()):
                        if task.done():
-                            tasks.remove(task)
-                            task.result()  # Fail closed if heartbeat/persistence failed.
-                    projects = self.settings.project_ids
+                            del tasks[task]
+                            try:
+                                task.result()
+                            except Exception:
+                                logger.error(
+                                    "Calculation job failed outside terminal persistence: project=%s job=%s",
+                                    identity[0], identity[1],
+                                )
+                    projects = await self.project_provider()
+                    if not projects:
+                        await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
+                        continue
+                    project_cursor %= len(projects)
                    ordered = projects[project_cursor:] + projects[:project_cursor]
                    project_cursor = (project_cursor + 1) % len(projects)
                    for project_id in ordered:
                        if len(tasks) >= self.settings.max_concurrency:
                            break
-                        async with self.repository(project_id) as repository:
-                            await repository.recover()
-                            row = await repository.claim(self.owner, self.settings.lease_seconds)
+                        try:
+                            async with self.repository(project_id) as repository:
+                                await repository.recover()
+                                row = await repository.claim(self.owner, self.settings.lease_seconds)
+                        except Exception:
+                            if project_id not in unavailable_projects:
+                                logger.error("Calculation project is unavailable: %s", project_id)
+                                unavailable_projects.add(project_id)
+                            continue
+                        unavailable_projects.discard(project_id)
                        if row:
-                            tasks.add(asyncio.create_task(self.run_job(row)))
+                            task = asyncio.create_task(self.run_job(row))
+                            tasks[task] = (project_id, str(row.get("job_id", "unknown")))
                    await asyncio.sleep(min(self.settings.poll_seconds, self.settings.heartbeat_seconds))
            finally:
                for task in tasks:
                    task.cancel()
-                await asyncio.gather(*tasks, return_exceptions=True)
+                await asyncio.gather(*tasks.keys(), return_exceptions=True)


 async def guard_child(input_path, output_path):