151 lines
8.5 KiB
Python
151 lines
8.5 KiB
Python
"""Durable project calculation queue. Every worker update is generation fenced."""
|
|
from uuid import uuid4
|
|
|
|
from fastapi import HTTPException
|
|
from psycopg.types.json import Jsonb
|
|
from app.mcp.errors import CalculationCapacityExceeded
|
|
|
|
|
|
# Explicit metadata projection prevents status polling from decoding large JSON results.
|
|
JOB_COLUMNS = "job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds,status,error,lease_generation,lease_owner,lease_expires_at,created_at,updated_at,started_at,finished_at,queue_expires_at,result_expires_at"
|
|
|
|
|
|
PUBLIC_JOB_COLUMNS = "job_id,status,tool,error,created_at,updated_at,started_at,finished_at,queue_expires_at,result_expires_at"
|
|
|
|
|
|
class CalculationJobRepository:
|
|
def __init__(self, connection, project_id):
|
|
self.connection = connection
|
|
self.project_id = str(project_id)
|
|
|
|
async def maintain(self):
|
|
await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET status='timed_out',
|
|
error='{"code":"QUEUE_TIMEOUT"}'::jsonb,finished_at=clock_timestamp(),updated_at=clock_timestamp()
|
|
WHERE project_id=%s AND status='queued' AND queue_expires_at<=clock_timestamp()""",
|
|
(self.project_id,),
|
|
)
|
|
await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET result=NULL
|
|
WHERE project_id=%s AND result IS NOT NULL AND result_expires_at<=clock_timestamp()""",
|
|
(self.project_id,),
|
|
)
|
|
|
|
async def enqueue(self, *, owner_user_id, request_key, tool, parameters, binding, timeout_seconds,
|
|
queue_wait_seconds=3600, max_active_jobs=100):
|
|
async with self.connection.transaction():
|
|
return await self._enqueue(owner_user_id=owner_user_id, request_key=request_key,
|
|
tool=tool, parameters=parameters, binding=binding, timeout_seconds=timeout_seconds,
|
|
queue_wait_seconds=queue_wait_seconds, max_active_jobs=max_active_jobs)
|
|
|
|
async def _enqueue(self, *, owner_user_id, request_key, tool, parameters, binding,
|
|
timeout_seconds, queue_wait_seconds, max_active_jobs):
|
|
# Serialize admission within the caller's transaction, across every API process.
|
|
await self.connection.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 1748531781))", (self.project_id,))
|
|
await self.maintain()
|
|
cursor = await self.connection.execute(
|
|
f"SELECT {JOB_COLUMNS} FROM workbench.calculation_jobs WHERE project_id=%s AND owner_user_id=%s AND request_key=%s",
|
|
(self.project_id, owner_user_id, request_key),
|
|
)
|
|
existing = await cursor.fetchone()
|
|
if existing:
|
|
row = dict(existing)
|
|
if (row['tool'] != tool
|
|
or {key: value for key, value in row['parameters'].items() if key != 'timeout_seconds'}
|
|
!= {key: value for key, value in parameters.items() if key != 'timeout_seconds'}
|
|
or row['binding'] != binding):
|
|
raise HTTPException(409, 'Calculation request_key was used with different inputs')
|
|
return row
|
|
cursor = await self.connection.execute(
|
|
"""SELECT count(*) AS active FROM workbench.calculation_jobs WHERE project_id=%s
|
|
AND status IN ('queued','running','cancel_requested')""", (self.project_id,))
|
|
if (await cursor.fetchone())['active'] >= max_active_jobs:
|
|
raise CalculationCapacityExceeded()
|
|
cursor = await self.connection.execute(
|
|
f"""INSERT INTO workbench.calculation_jobs
|
|
(job_id,project_id,owner_user_id,request_key,tool,parameters,binding,timeout_seconds,queue_expires_at)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,clock_timestamp()+%s*interval '1 second')
|
|
RETURNING {JOB_COLUMNS}""",
|
|
(uuid4(), self.project_id, owner_user_id, request_key, tool,
|
|
Jsonb(parameters), Jsonb(binding), timeout_seconds, queue_wait_seconds),
|
|
)
|
|
return dict(await cursor.fetchone())
|
|
|
|
async def get(self, job_id, owner_user_id=None, *, include_result=False):
|
|
await self.maintain()
|
|
columns = JOB_COLUMNS + (",result" if include_result else "")
|
|
cursor = await self.connection.execute(
|
|
f"""SELECT {columns} FROM workbench.calculation_jobs WHERE project_id=%s AND job_id=%s
|
|
AND (%s::uuid IS NULL OR owner_user_id=%s)""",
|
|
(self.project_id, job_id, owner_user_id, owner_user_id),
|
|
)
|
|
row = await cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
async def list(self, *, owner_user_id, binding, limit, offset, status=None):
|
|
await self.maintain()
|
|
cursor = await self.connection.execute(
|
|
f"""SELECT {PUBLIC_JOB_COLUMNS} FROM workbench.calculation_jobs WHERE project_id=%s AND owner_user_id=%s
|
|
AND binding @> %s AND (%s::text IS NULL OR status=%s) ORDER BY created_at DESC,job_id LIMIT %s OFFSET %s""",
|
|
(self.project_id, owner_user_id, Jsonb(binding), status, status, limit, offset),
|
|
)
|
|
return [dict(row) for row in await cursor.fetchall()]
|
|
|
|
async def cancel(self, job_id, owner_user_id):
|
|
await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET status=CASE WHEN status='queued' THEN 'cancelled'
|
|
ELSE 'cancel_requested' END,finished_at=CASE WHEN status='queued' THEN clock_timestamp() ELSE finished_at END,updated_at=clock_timestamp()
|
|
WHERE project_id=%s AND job_id=%s AND owner_user_id=%s AND status IN ('queued','running')""",
|
|
(self.project_id, job_id, owner_user_id),
|
|
)
|
|
return await self.get(job_id, owner_user_id)
|
|
|
|
async def recover(self):
|
|
await self.maintain()
|
|
# A crashed write is never automatically replayed.
|
|
await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET status='interrupted',
|
|
error='{"code":"WORKER_INTERRUPTED"}'::jsonb,finished_at=clock_timestamp(),updated_at=clock_timestamp()
|
|
WHERE project_id=%s AND status IN ('running','cancel_requested')
|
|
AND lease_expires_at < clock_timestamp()""", (self.project_id,),
|
|
)
|
|
|
|
async def claim(self, owner, lease_seconds):
|
|
await self.maintain()
|
|
cursor = await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs j SET status='running',started_at=clock_timestamp(),lease_owner=%s,
|
|
lease_generation=j.lease_generation+1,
|
|
lease_expires_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp()
|
|
WHERE j.job_id=(SELECT job_id FROM workbench.calculation_jobs
|
|
WHERE project_id=%s AND status='queued' AND queue_expires_at>clock_timestamp() ORDER BY created_at,job_id
|
|
FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING j.*""",
|
|
(owner, lease_seconds, self.project_id),
|
|
)
|
|
row = await cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
async def heartbeat(self, row, owner, lease_seconds):
|
|
cursor = await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET lease_expires_at=clock_timestamp()+%s*interval '1 second',
|
|
updated_at=clock_timestamp() WHERE project_id=%s AND job_id=%s AND lease_owner=%s
|
|
AND lease_generation=%s AND lease_expires_at>clock_timestamp()
|
|
AND status IN ('running','cancel_requested') RETURNING status""",
|
|
(lease_seconds, self.project_id, row['job_id'], owner, row['lease_generation']),
|
|
)
|
|
result = await cursor.fetchone()
|
|
return result['status'] if result else None
|
|
|
|
async def finish(self, row, owner, status, *, result=None, error=None, result_retention_seconds=604800):
|
|
cursor = await self.connection.execute(
|
|
"""UPDATE workbench.calculation_jobs SET status=%s,result=%s,error=%s,finished_at=clock_timestamp(),
|
|
result_expires_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp()
|
|
WHERE project_id=%s AND job_id=%s AND lease_owner=%s AND lease_generation=%s
|
|
AND lease_expires_at>clock_timestamp() AND status IN ('running','cancel_requested')
|
|
AND (status!='cancel_requested' OR %s IN ('cancelled','interrupted')
|
|
OR (%s='failed' AND %s='CLEANUP_FAILED')) RETURNING job_id""",
|
|
(status, Jsonb(result) if result is not None else None, Jsonb(error) if error else None, result_retention_seconds,
|
|
self.project_id, row['job_id'], owner, row['lease_generation'], status, status, (error or {}).get('code')),
|
|
)
|
|
return await cursor.fetchone() is not None
|