feat(projects): automate project infrastructure provisioning
This commit is contained in:
@@ -241,6 +241,67 @@ class MetadataRepository:
|
||||
await self.session.refresh(project)
|
||||
return project
|
||||
|
||||
async def create_provisioned_project(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
code: str,
|
||||
description: str | None,
|
||||
gs_workspace: str,
|
||||
map_extent: dict,
|
||||
creator_user_id: UUID,
|
||||
business_dsn: str,
|
||||
timescale_dsn: str,
|
||||
pool_min_size: int = 1,
|
||||
pool_max_size: int = 4,
|
||||
) -> models.Project:
|
||||
"""Atomically expose a fully provisioned project and both DB routes."""
|
||||
business_secret = _encrypt_database_secret(business_dsn)
|
||||
timescale_secret = _encrypt_database_secret(timescale_dsn)
|
||||
project = models.Project(
|
||||
id=uuid4(),
|
||||
name=name,
|
||||
code=code,
|
||||
description=description,
|
||||
gs_workspace=gs_workspace,
|
||||
map_extent=map_extent,
|
||||
status="active",
|
||||
created_at=_utcnow(),
|
||||
updated_at=_utcnow(),
|
||||
)
|
||||
records = (
|
||||
project,
|
||||
models.ProjectDatabase(
|
||||
id=uuid4(),
|
||||
project_id=project.id,
|
||||
db_role="biz_data",
|
||||
db_type="postgresql",
|
||||
dsn_encrypted=business_secret,
|
||||
pool_min_size=pool_min_size,
|
||||
pool_max_size=pool_max_size,
|
||||
),
|
||||
models.ProjectDatabase(
|
||||
id=uuid4(),
|
||||
project_id=project.id,
|
||||
db_role="iot_data",
|
||||
db_type="timescaledb",
|
||||
dsn_encrypted=timescale_secret,
|
||||
pool_min_size=pool_min_size,
|
||||
pool_max_size=pool_max_size,
|
||||
),
|
||||
models.UserProjectMembership(
|
||||
id=uuid4(),
|
||||
user_id=creator_user_id,
|
||||
project_id=project.id,
|
||||
project_role="member",
|
||||
),
|
||||
)
|
||||
for record in records:
|
||||
self.session.add(record)
|
||||
await self.session.commit()
|
||||
await self.session.refresh(project)
|
||||
return project
|
||||
|
||||
async def update_project(
|
||||
self,
|
||||
project_id: UUID,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.native.wndb.core.connection import project_connection
|
||||
|
||||
|
||||
def get_project_map_bbox(project: str) -> tuple[float, float, float, float]:
|
||||
"""Return the combined published node/link bounds in EPSG:3857."""
|
||||
with project_connection(project) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
with project_geometries as (
|
||||
select geom from gis.junctions
|
||||
union all
|
||||
select geom from gis.pipes
|
||||
), bounds as (
|
||||
select ST_Extent(geom) as extent from project_geometries
|
||||
)
|
||||
select ST_XMin(extent) as minx,
|
||||
ST_YMin(extent) as miny,
|
||||
ST_XMax(extent) as maxx,
|
||||
ST_YMax(extent) as maxy
|
||||
from bounds
|
||||
"""
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None or any(
|
||||
row[key] is None for key in ("minx", "miny", "maxx", "maxy")
|
||||
):
|
||||
raise RuntimeError("Imported model has no publishable GIS geometry")
|
||||
return tuple(float(row[key]) for key in ("minx", "miny", "maxx", "maxy"))
|
||||
@@ -62,12 +62,18 @@ def get_project_database_name(name: str) -> str:
|
||||
|
||||
|
||||
def get_project_template_database_name(name: str | None = None) -> str:
|
||||
"""Return the configured immutable template for the WNDB schema version.
|
||||
"""Return the network-data template paired with one physical BizDB.
|
||||
|
||||
The template belongs to the database schema version, not to an individual
|
||||
logical project or its temporary physical database name.
|
||||
Project routing is resolved before the suffix is applied, so a logical
|
||||
project such as ``tjwater_v2`` uses ``tjwater_v2_template``.
|
||||
"""
|
||||
return settings.WNDB_TEMPLATE_DB_NAME
|
||||
database_name = get_project_database_name(name or settings.DB_NAME)
|
||||
return f"{database_name}_template"
|
||||
|
||||
|
||||
def get_schema_template_database_name() -> str:
|
||||
"""Return the immutable, data-free template used to create BizDB schemas."""
|
||||
return settings.WNDB_SCHEMA_TEMPLATE_DB_NAME
|
||||
|
||||
|
||||
def get_project_pgconn_string(db_name: str | None = None) -> str:
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
import re
|
||||
from threading import RLock
|
||||
from typing import Iterator
|
||||
|
||||
from psycopg import Connection, sql
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from app.core.config import get_timescaledb_pgconn_string, settings
|
||||
|
||||
from .sync_pool import close_timescale_pool
|
||||
|
||||
|
||||
_DATABASE_NAME = re.compile(r"^[a-z][a-z0-9_]{0,49}$")
|
||||
_SERVER_DATABASES = frozenset({"template0", "template1", "postgres"})
|
||||
_admin_pool: ConnectionPool | None = None
|
||||
_admin_conninfo: str | None = None
|
||||
_lock = RLock()
|
||||
|
||||
|
||||
def validate_timescale_database_name(name: str, *, allow_template: bool = False) -> str:
|
||||
if not _DATABASE_NAME.fullmatch(name):
|
||||
raise ValueError(
|
||||
"TimescaleDB database name must start with a lowercase letter and "
|
||||
"contain only lowercase letters, digits, and underscores"
|
||||
)
|
||||
protected = {*_SERVER_DATABASES, settings.TIMESCALEDB_SCHEMA_TEMPLATE_DB_NAME}
|
||||
if name in protected and not (
|
||||
allow_template and name == settings.TIMESCALEDB_SCHEMA_TEMPLATE_DB_NAME
|
||||
):
|
||||
raise ValueError(f"TimescaleDB database {name!r} is protected")
|
||||
return name
|
||||
|
||||
|
||||
def _get_admin_pool() -> ConnectionPool:
|
||||
global _admin_pool, _admin_conninfo
|
||||
conninfo = get_timescaledb_pgconn_string(db_name="postgres")
|
||||
with _lock:
|
||||
if (
|
||||
_admin_pool is not None
|
||||
and not _admin_pool.closed
|
||||
and _admin_conninfo == conninfo
|
||||
):
|
||||
return _admin_pool
|
||||
if _admin_pool is not None and not _admin_pool.closed:
|
||||
_admin_pool.close()
|
||||
_admin_pool = ConnectionPool(
|
||||
conninfo=conninfo,
|
||||
min_size=0,
|
||||
max_size=2,
|
||||
kwargs={"autocommit": True, "row_factory": dict_row},
|
||||
check=ConnectionPool.check_connection,
|
||||
open=True,
|
||||
)
|
||||
_admin_conninfo = conninfo
|
||||
return _admin_pool
|
||||
|
||||
|
||||
@contextmanager
|
||||
def timescale_admin_connection() -> Iterator[Connection]:
|
||||
with _get_admin_pool().connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def timescale_database_exists(name: str) -> bool:
|
||||
with timescale_admin_connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("select 1 from pg_database where datname = %s", (name,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def require_timescale_schema_template() -> str:
|
||||
template = settings.TIMESCALEDB_SCHEMA_TEMPLATE_DB_NAME
|
||||
validate_timescale_database_name(template, allow_template=True)
|
||||
if not timescale_database_exists(template):
|
||||
raise RuntimeError(
|
||||
f"TimescaleDB schema template {template!r} does not exist"
|
||||
)
|
||||
return template
|
||||
|
||||
|
||||
def create_timescale_database(name: str) -> None:
|
||||
validate_timescale_database_name(name)
|
||||
template = require_timescale_schema_template()
|
||||
close_timescale_pool(name)
|
||||
with timescale_admin_connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"select pg_advisory_lock(hashtextextended(%s, 0))",
|
||||
(f"tjwater:timescaledb:{name}",),
|
||||
)
|
||||
try:
|
||||
cur.execute("select 1 from pg_database where datname = %s", (name,))
|
||||
if cur.fetchone() is not None:
|
||||
raise ValueError(f"TimescaleDB database {name!r} already exists")
|
||||
cur.execute(
|
||||
"select datallowconn from pg_database where datname = %s",
|
||||
(template,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError(
|
||||
f"TimescaleDB schema template {template!r} does not exist"
|
||||
)
|
||||
template_allowed = bool(row["datallowconn"])
|
||||
if template_allowed:
|
||||
cur.execute(
|
||||
"update pg_database set datallowconn = false where datname = %s",
|
||||
(template,),
|
||||
)
|
||||
try:
|
||||
cur.execute(
|
||||
"select pg_terminate_backend(pid) from pg_stat_activity "
|
||||
"where datname = %s and pid <> pg_backend_pid()",
|
||||
(template,),
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("create database {} with template = {}").format(
|
||||
sql.Identifier(name),
|
||||
sql.Identifier(template),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if template_allowed:
|
||||
cur.execute(
|
||||
"update pg_database set datallowconn = true where datname = %s",
|
||||
(template,),
|
||||
)
|
||||
finally:
|
||||
cur.execute(
|
||||
"select pg_advisory_unlock(hashtextextended(%s, 0))",
|
||||
(f"tjwater:timescaledb:{name}",),
|
||||
)
|
||||
|
||||
|
||||
def delete_timescale_database(name: str) -> None:
|
||||
validate_timescale_database_name(name)
|
||||
close_timescale_pool(name)
|
||||
with timescale_admin_connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"select pg_advisory_lock(hashtextextended(%s, 0))",
|
||||
(f"tjwater:timescaledb:{name}",),
|
||||
)
|
||||
try:
|
||||
cur.execute("select 1 from pg_database where datname = %s", (name,))
|
||||
if cur.fetchone() is None:
|
||||
return
|
||||
cur.execute(
|
||||
"update pg_database set datallowconn = false where datname = %s",
|
||||
(name,),
|
||||
)
|
||||
cur.execute(
|
||||
"select pg_terminate_backend(pid) from pg_stat_activity "
|
||||
"where datname = %s and pid <> pg_backend_pid()",
|
||||
(name,),
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("drop database {}").format(sql.Identifier(name))
|
||||
)
|
||||
finally:
|
||||
cur.execute(
|
||||
"select pg_advisory_unlock(hashtextextended(%s, 0))",
|
||||
(f"tjwater:timescaledb:{name}",),
|
||||
)
|
||||
Reference in New Issue
Block a user