OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scada): add calibrated monitoring and incident recovery

Browse Source
This commit is contained in:
jiang
2026-11-07 19:44:58 +08:00
parent 26fa5d1909
commit ff8aba4964
109 changed files with 9165 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/infra/db/calculation_worker.py
+23
View File
@@ -7,6 +7,29 @@ from uuid import UUID
from sqlalchemy import text
async def discover_project_ids() -> tuple[str, ...]:
"""Return every active PostgreSQL-backed project from authoritative metadata."""
from sqlalchemy import select
from app.infra.db.metadb.database import SessionLocal
from app.infra.db.metadb import models
async with SessionLocal() as session:
rows = await session.scalars(
select(models.Project.id)
.join(
models.ProjectDatabase,
models.ProjectDatabase.project_id == models.Project.id,
)
.where(
models.Project.status == "active",
models.ProjectDatabase.db_role == "biz_data",
models.ProjectDatabase.db_type == "postgresql",
)
.order_by(models.Project.id)
)
return tuple(str(project_id) for project_id in rows.all())
@asynccontextmanager
async def queue_repository(project_id):
"""Resolve the current project route and open its calculation queue."""
app/infra/db/postgresql/calculation_jobs.py
+9 -15
View File
@@ -7,10 +7,10 @@ from app.mcp.errors import CalculationCapacityExceeded
# Explicit metadata projection prevents status polling from decoding large JSON results.
JOB_COLUMNS = "job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds,status,error,lease_generation,lease_owner,lease_expires_at,created_at,updated_at,started_at,finished_at,queue_expires_at,result_expires_at"
JOB_COLUMNS = "job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds,status,error,lease_generation,lease_owner,lease_expires_at,created_at,updated_at,started_at,finished_at,result_expires_at"
PUBLIC_JOB_COLUMNS = "job_id,status,tool,error,created_at,updated_at,started_at,finished_at,queue_expires_at,result_expires_at"
PUBLIC_JOB_COLUMNS = "job_id,status,tool,error,created_at,updated_at,started_at,finished_at,result_expires_at"
class CalculationJobRepository:
@@ -19,12 +19,6 @@ class CalculationJobRepository:
self.project_id = str(project_id)
async def maintain(self):
await self.connection.execute(
"""UPDATE workbench.calculation_jobs SET status='timed_out',
error='{"code":"QUEUE_TIMEOUT"}'::jsonb,finished_at=clock_timestamp(),updated_at=clock_timestamp()
WHERE project_id=%s AND status='queued' AND queue_expires_at<=clock_timestamp()""",
(self.project_id,),
)
await self.connection.execute(
"""UPDATE workbench.calculation_jobs SET result=NULL
WHERE project_id=%s AND result IS NOT NULL AND result_expires_at<=clock_timestamp()""",
@@ -32,14 +26,14 @@ class CalculationJobRepository:
)
async def enqueue(self, *, owner_user_id, request_key, tool, parameters, binding, timeout_seconds,
queue_wait_seconds=3600, max_active_jobs=100):
max_active_jobs=100):
async with self.connection.transaction():
return await self._enqueue(owner_user_id=owner_user_id, request_key=request_key,
tool=tool, parameters=parameters, binding=binding, timeout_seconds=timeout_seconds,
queue_wait_seconds=queue_wait_seconds, max_active_jobs=max_active_jobs)
max_active_jobs=max_active_jobs)
async def _enqueue(self, *, owner_user_id, request_key, tool, parameters, binding,
timeout_seconds, queue_wait_seconds, max_active_jobs):
timeout_seconds, max_active_jobs):
# Serialize admission within the caller's transaction, across every API process.
await self.connection.execute(
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 1748531781))", (self.project_id,))
@@ -64,11 +58,11 @@ class CalculationJobRepository:
raise CalculationCapacityExceeded()
cursor = await self.connection.execute(
f"""INSERT INTO workbench.calculation_jobs
(job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds,queue_expires_at)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,clock_timestamp()+%s*interval '1 second')
(job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
RETURNING {JOB_COLUMNS}""",
(uuid4(), self.project_id, owner_user_id, request_key, tool,
Jsonb(parameters), Jsonb(binding), timeout_seconds, queue_wait_seconds),
Jsonb(parameters), Jsonb(binding), timeout_seconds),
)
return dict(await cursor.fetchone())
@@ -118,7 +112,7 @@ class CalculationJobRepository:
lease_generation=j.lease_generation+1,
lease_expires_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp()
WHERE j.job_id=(SELECT job_id FROM workbench.calculation_jobs
WHERE project_id=%s AND status='queued' AND queue_expires_at>clock_timestamp() ORDER BY created_at,job_id
WHERE project_id=%s AND status='queued' ORDER BY created_at,job_id
FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING j.*""",
(owner, lease_seconds, self.project_id),
)
app/infra/db/postgresql/network_assets.py
+18
View File
@@ -6,6 +6,24 @@ from psycopg import AsyncConnection
class NetworkAssetRepository:
@staticmethod
async def get_isolation_model(conn: AsyncConnection) -> list[dict[str, Any]]:
"""Read source nodes, link state and pipe lengths in one model snapshot."""
from app.infra.db.postgresql.read_queries import read_rows
return await read_rows(conn, """
SELECT l.id, l.link_type::text AS kind,
l.start_node_id, l.end_node_id, p.length, p.diameter,
COALESCE(s.status::text, p.status::text, '') AS status
FROM network.links l
LEFT JOIN network.pipes p ON p.link_id = l.id
LEFT JOIN network.link_initial_settings s ON s.link_id = l.id
UNION ALL
SELECT n.id, n.node_type::text, NULL, NULL, NULL, NULL, ''
FROM network.nodes n
WHERE n.node_type::text IN ('reservoir', 'tank')
""")
@staticmethod
async def get_pipes_by_ids(
conn: AsyncConnection,
app/infra/db/postgresql/operational_tasks.py
+22
View File
@@ -711,6 +711,28 @@ class OperationalTaskRepository:
row = await cursor.fetchone()
return dict(row) if row else None
async def recovery_candidates(self, limit: int, created_from=None) -> list[dict]:
cursor = await self.connection.execute(
"""SELECT document FROM workbench.operational_tasks
WHERE project_id=%s AND task_type='event_investigation'
AND status NOT IN ('cancelled','failed')
AND document#>>'{handling_context,phase}'='waiting_scada'
AND document#>>'{source,replay_context}' IS NULL
AND (%s::timestamptz IS NULL OR created_at>=%s)
ORDER BY COALESCE((document#>>'{handling_context,last_checked_at}')::timestamptz,created_at),task_id LIMIT %s""",
(self.project_id, created_from, created_from, limit),
)
return [row["document"] for row in await cursor.fetchall()]
async def list_investigation_alerts(self, task_id: str) -> list[dict]:
cursor = await self.connection.execute(
"""SELECT document FROM workbench.alerts WHERE project_id=%s
AND status IN ('active','acknowledged')
AND document->>'condition_id'=%s ORDER BY alert_id""",
(self.project_id, f"operational-task:{task_id}"),
)
return [row["document"] for row in await cursor.fetchall()]
async def upsert_feedback(self, task_id: str, feedback: dict, accepted_at) -> None:
await self.connection.execute(
"""INSERT INTO workbench.operational_task_external_feedback
app/infra/db/postgresql/read_queries.py
+1
View File
@@ -312,6 +312,7 @@ async def task_context_summary(conn: AsyncConnection, *, project_id: UUID, task_
'current_report_revision',t.current_report_revision,
'title_preview',left(t.document->>'title',160),'preview_only',true,
'detail',jsonb_build_object('section','task')),
'handling_context',COALESCE(t.document->'handling_context','{}'::jsonb),
'latest_report',(SELECT REPORT_SUMMARY FROM workbench.operational_task_reports r
WHERE r.project_id=t.project_id AND r.task_id=t.task_id ORDER BY r.revision DESC LIMIT 1),
'selected_report',CASE WHEN %s::int IS NULL THEN NULL ELSE
app/infra/db/postgresql/scada_replays.py
+21 -2
View File
@@ -96,7 +96,7 @@ class ScadaReplayRepository:
async with self.connection.transaction():
cur = await self.connection.execute(
"SELECT document FROM peripheral_sample.scada_replay_batches "
"WHERE project_id=%s AND replay_id=%s AND stage<>'detected' ORDER BY sequence LIMIT 1",
"WHERE project_id=%s AND replay_id=%s AND sequence>0 AND stage<>'detected' ORDER BY sequence LIMIT 1",
(self.project_id, str(replay_id)),
)
row = await cur.fetchone()
@@ -110,6 +110,25 @@ class ScadaReplayRepository:
(self.project_id, str(replay_id), batch["sequence"], batch["stage"], Jsonb(batch)),
)
async def clear_unpublished_baseline(self, replay_id) -> None:
"""Discard only an incomplete attempt, retaining every published snapshot."""
await self.connection.execute(
"DELETE FROM peripheral_sample.scada_replay_batches "
"WHERE project_id=%s AND replay_id=%s AND sequence<0 "
"AND stage='clean_prepared' AND document->>'kind'='baseline'",
(self.project_id, str(replay_id)),
)
async def publish_baseline(self, replay_id) -> None:
"""Publish with the replay's baseline metadata in the caller's transaction."""
await self.connection.execute(
"UPDATE peripheral_sample.scada_replay_batches "
"SET stage='detected',document=jsonb_set(document,'{stage}','\"detected\"'::jsonb) "
"WHERE project_id=%s AND replay_id=%s AND sequence<0 "
"AND stage='clean_prepared' AND document->>'kind'='baseline'",
(self.project_id, str(replay_id)),
)
async def events(self, replay_id, offset=0, limit=100) -> list[dict]:
async with self.connection.transaction():
cur = await self.connection.execute(
@@ -180,7 +199,7 @@ class ScadaReplayRepository:
"SELECT DISTINCT ON(r.value->>'device_id') r.value AS reading "
"FROM peripheral_sample.scada_replay_batches b "
"CROSS JOIN LATERAL jsonb_array_elements(COALESCE(b.document->'outputs','[]'::jsonb)) r "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.stage IN ('cleaned','detected') "
"WHERE b.project_id=%s AND b.replay_id=%s AND b.sequence>0 AND b.stage IN ('cleaned','detected') "
"AND (r.value->>'time')::timestamptz<=%s "
"ORDER BY r.value->>'device_id',(r.value->>'time')::timestamptz DESC LIMIT %s",
(self.project_id, str(replay_id), through, limit),
app/infra/db/timescaledb/repositories/scada.py
+5
View File
@@ -9,6 +9,11 @@ class ScadaRepository:
VALUE_FIELDS = frozenset({"monitored_value", "cleaned_value"})
RESULT_COLUMNS = "time, device_id, monitored_value, cleaned_value"
@staticmethod
async def begin_replay_baseline_snapshot(conn: AsyncConnection) -> None:
"""Set the caller's source transaction to a repeatable read-only snapshot."""
await conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
@staticmethod
async def get_warmup_values(
conn: AsyncConnection,