feat(projects): automate project infrastructure provisioning

app/native/wndb/core/model_replace.py

@@ -89,6 +89,56 @@ def _table_columns(
         return [row["column_name"] for row in cur.fetchall()]
 
 
+def _geometry_srids(
+    conn: Connection, schema_name: str, table_name: str
+) -> dict[str, int]:
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            select f_geometry_column as column_name, srid
+            from geometry_columns
+            where f_table_schema = %s and f_table_name = %s
+            order by f_geometry_column
+            """,
+            (schema_name, table_name),
+        )
+        return {row["column_name"]: int(row["srid"]) for row in cur.fetchall()}
+
+
+def _copy_out_statement(
+    schema_name: str,
+    table_name: str,
+    columns: list[str],
+    *,
+    source_geometry_srids: dict[str, int],
+    target_geometry_srids: dict[str, int],
+) -> sql.Composed:
+    relation = sql.Identifier(schema_name, table_name)
+    column_list = sql.SQL(", ").join(map(sql.Identifier, columns))
+    srid_changes = {
+        column_name: target_geometry_srids[column_name]
+        for column_name, source_srid in source_geometry_srids.items()
+        if target_geometry_srids[column_name] != source_srid
+    }
+    if not srid_changes:
+        return sql.SQL("copy {} ({}) to stdout").format(relation, column_list)
+
+    select_list = sql.SQL(", ").join(
+        sql.SQL("st_setsrid({}, {}) as {}").format(
+            sql.Identifier(column_name),
+            sql.Literal(srid_changes[column_name]),
+            sql.Identifier(column_name),
+        )
+        if column_name in srid_changes
+        else sql.Identifier(column_name)
+        for column_name in columns
+    )
+    return sql.SQL("copy (select {} from {}) to stdout").format(
+        select_list,
+        relation,
+    )
+
+
 def _external_references(conn: Connection) -> set[tuple[str, str, str, str]]:
     with conn.cursor() as cur:
         cur.execute(
@@ -128,7 +178,20 @@ def _copy_table(
 ) -> None:
     relation = sql.Identifier(schema_name, table_name)
     column_list = sql.SQL(", ").join(map(sql.Identifier, columns))
-    copy_out = sql.SQL("copy {} ({}) to stdout").format(relation, column_list)
+    source_geometry_srids = _geometry_srids(source_conn, schema_name, table_name)
+    target_geometry_srids = _geometry_srids(target_conn, schema_name, table_name)
+    if source_geometry_srids.keys() != target_geometry_srids.keys():
+        raise RuntimeError(
+            "Source/target geometry columns differ for "
+            f"{schema_name}.{table_name}"
+        )
+    copy_out = _copy_out_statement(
+        schema_name,
+        table_name,
+        columns,
+        source_geometry_srids=source_geometry_srids,
+        target_geometry_srids=target_geometry_srids,
+    )
     copy_in = sql.SQL("copy {} ({}) from stdin").format(relation, column_list)
     with source_conn.cursor().copy(copy_out) as source_copy:
         with target_conn.cursor().copy(copy_in) as target_copy:
@@ -136,6 +199,47 @@ def _copy_table(
                 target_copy.write(chunk)
 
 
+def copy_network_tables(source_project: str, target_project: str) -> None:
+    """Copy only the immutable simulation input tables into a project template."""
+    with project_connection(source_project) as source_conn, source_conn.transaction():
+        with source_conn.cursor() as cur:
+            cur.execute("set transaction isolation level repeatable read, read only")
+        source_tables = [
+            table for table in _model_tables(source_conn) if table[0] == "network"
+        ]
+        source_columns = {
+            table: _table_columns(source_conn, *table) for table in source_tables
+        }
+        copy_order = _copy_order(source_conn, source_tables)
+
+        with project_transaction(target_project) as target_conn:
+            target_tables = [
+                table for table in _model_tables(target_conn) if table[0] == "network"
+            ]
+            if set(target_tables) != set(source_tables):
+                raise RuntimeError("Source/target network schemas differ")
+            for table, columns in source_columns.items():
+                if _table_columns(target_conn, *table) != columns:
+                    raise RuntimeError(
+                        f"Source/target columns differ for {table[0]}.{table[1]}"
+                    )
+            with target_conn.cursor() as cur:
+                for schema_name, table_name in reversed(copy_order):
+                    cur.execute(
+                        sql.SQL("delete from {}").format(
+                            sql.Identifier(schema_name, table_name)
+                        )
+                    )
+            for schema_name, table_name in copy_order:
+                _copy_table(
+                    source_conn,
+                    target_conn,
+                    schema_name,
+                    table_name,
+                    source_columns[(schema_name, table_name)],
+                )
+
+
 def replace_project_model(
     target_project: str,
     source_project: str,

app/native/wndb/core/project_templates.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import time
+
+from psycopg import sql
+from psycopg.conninfo import make_conninfo
+
+from app.core.config import settings
+from app.infra.db.project_routing import get_project_template_database_name
+
+from .connection import admin_connection, close_project_pool, project_connection
+from .model_replace import copy_network_tables
+from .projects import (
+    _ensure_project_model_template_ready,
+    copy_project,
+    delete_project,
+    have_project,
+)
+
+
+PUBLICATION_NAME = "wndb_network_pub"
+SUBSCRIPTION_NAME = "wndb_network_sub"
+
+
+def _slot_name(project: str) -> str:
+    return f"{project}_network_slot"
+
+
+def _publisher_conninfo(project: str) -> str:
+    return make_conninfo(
+        dbname=project,
+        host=settings.DB_HOST,
+        port=settings.DB_PORT,
+        user=settings.DB_USER,
+        password=settings.DB_PASSWORD,
+    )
+
+
+def ensure_replication_worker_capacity() -> None:
+    """Keep one worker slot spare while admitting one new project subscription."""
+    with admin_connection() as conn, conn.cursor() as cur:
+        cur.execute(
+            """
+            select current_setting('max_worker_processes')::integer as maximum,
+                   count(*) filter (
+                       where backend_type in (
+                           'logical replication launcher',
+                           'logical replication worker'
+                       )
+                   ) as used
+            from pg_stat_activity
+            """
+        )
+        row = cur.fetchone()
+    maximum = int(row["maximum"])
+    used = int(row["used"])
+    if used + 1 >= maximum:
+        raise RuntimeError(
+            "PostgreSQL has no reserved logical-replication worker capacity "
+            f"(used={used}, max_worker_processes={maximum}); increase "
+            "max_worker_processes before provisioning another project"
+        )
+
+
+def _create_publication_and_slot(project: str) -> None:
+    slot_name = _slot_name(project)
+    with project_connection(project) as conn, conn.cursor() as cur:
+        cur.execute(
+            "select n.nspname as schema_name, c.relname as table_name "
+            "from pg_class c join pg_namespace n on n.oid = c.relnamespace "
+            "where n.nspname = 'network' and c.relkind in ('r', 'p') "
+            "order by c.relname"
+        )
+        tables = [(row["schema_name"], row["table_name"]) for row in cur.fetchall()]
+        if not tables:
+            raise RuntimeError("Project database has no network tables to publish")
+        cur.execute(
+            sql.SQL("create publication {} for table {}").format(
+                sql.Identifier(PUBLICATION_NAME),
+                sql.SQL(", ").join(
+                    sql.Identifier(schema_name, table_name)
+                    for schema_name, table_name in tables
+                ),
+            )
+        )
+        cur.execute(
+            "select slot_name from pg_create_logical_replication_slot(%s, 'pgoutput')",
+            (slot_name,),
+        )
+
+
+def _create_subscription(project: str, template: str) -> None:
+    with project_connection(template) as conn, conn.cursor() as cur:
+        cur.execute(
+            sql.SQL(
+                "create subscription {} connection {} publication {} "
+                "with (create_slot = false, copy_data = false, slot_name = {})"
+            ).format(
+                sql.Identifier(SUBSCRIPTION_NAME),
+                sql.Literal(_publisher_conninfo(project)),
+                sql.Identifier(PUBLICATION_NAME),
+                sql.Literal(_slot_name(project)),
+            )
+        )
+
+
+def _wait_until_ready(template: str) -> None:
+    last_error: RuntimeError | None = None
+    for _ in range(50):
+        try:
+            _ensure_project_model_template_ready(template)
+            return
+        except RuntimeError as exc:
+            last_error = exc
+            time.sleep(0.1)
+    raise last_error or RuntimeError(
+        f"Project model template {template!r} did not become ready"
+    )
+
+
+def create_project_model_template(project: str) -> str:
+    template = get_project_template_database_name(project)
+    if have_project(template):
+        raise ValueError(f"Project model template {template!r} already exists")
+    template_created = False
+    try:
+        _create_publication_and_slot(project)
+        copy_project(
+            settings.WNDB_SCHEMA_TEMPLATE_DB_NAME,
+            template,
+            allow_template_source=True,
+            allow_template_target=True,
+        )
+        template_created = True
+        copy_network_tables(project, template)
+        _create_subscription(project, template)
+        _wait_until_ready(template)
+        return template
+    except Exception:
+        if template_created:
+            delete_project_model_template(project)
+        else:
+            _drop_publisher_objects(project)
+        raise
+
+
+def _drop_subscription(template: str) -> None:
+    if not have_project(template):
+        return
+    try:
+        with project_connection(template) as conn, conn.cursor() as cur:
+            cur.execute(
+                sql.SQL("drop subscription if exists {}").format(
+                    sql.Identifier(SUBSCRIPTION_NAME)
+                )
+            )
+    finally:
+        close_project_pool(template)
+
+
+def _drop_publisher_objects(
+    project: str,
+    *,
+    drop_publication: bool = True,
+    drop_slot: bool = True,
+) -> None:
+    if not have_project(project):
+        return
+    with project_connection(project) as conn, conn.cursor() as cur:
+        if drop_slot:
+            cur.execute(
+                "select pg_drop_replication_slot(slot_name) "
+                "from pg_replication_slots where slot_name = %s and active = false",
+                (_slot_name(project),),
+            )
+        if drop_publication:
+            cur.execute(
+                sql.SQL("drop publication if exists {}").format(
+                    sql.Identifier(PUBLICATION_NAME)
+                )
+            )
+
+
+def delete_project_model_template(project: str) -> None:
+    template = get_project_template_database_name(project)
+    try:
+        _drop_subscription(template)
+    finally:
+        _drop_publisher_objects(project)
+        if have_project(template):
+            delete_project(template, allow_template=True)

app/native/wndb/core/projects.py

@@ -9,22 +9,24 @@ from psycopg.rows import dict_row
 from app.core.config import settings
 from app.infra.db.project_routing import (
     get_project_database_name,
+    get_schema_template_database_name,
     get_project_template_database_name,
 )
 
 from .connection import (
     admin_connection,
     close_project_pool,
+    project_connection,
 )
 
-_SERVER_DATABASES = frozenset({"template0", "template1", "postgres", "project"})
+_SERVER_DATABASES = frozenset({"template0", "template1", "postgres"})
 _TEMPORARY_DATABASE_PREFIX = "tjw_tmp_"
 
 
 def _protected_databases() -> frozenset[str]:
     return _SERVER_DATABASES | {
         settings.METADATA_DB_NAME,
-        settings.WNDB_TEMPLATE_DB_NAME,
+        settings.WNDB_SCHEMA_TEMPLATE_DB_NAME,
     }
 
 
@@ -34,10 +36,7 @@ def _validate_project_database(name: str, *, allow_template_source: bool = False
 
     protected = {database.casefold() for database in _protected_databases()}
     is_template = name.casefold().endswith("_template")
-    if (
-        allow_template_source
-        and name.casefold() == settings.WNDB_TEMPLATE_DB_NAME.casefold()
-    ):
+    if allow_template_source and is_template:
         return
     if name.casefold() in protected or is_template:
         raise ValueError(f"Database {name!r} is protected and cannot be managed as a project")
@@ -120,6 +119,72 @@ def _set_database_connections(cur, database_name: str, *, allowed: bool) -> None
     )
 
 
+def _is_project_model_template(database_name: str) -> bool:
+    return (
+        database_name.casefold().endswith("_template")
+        and database_name.casefold()
+        != get_schema_template_database_name().casefold()
+    )
+
+
+def _ensure_project_model_template_ready(database_name: str) -> None:
+    """Reject cloning a missing, disabled, or initially syncing subscription."""
+    if not _is_project_model_template(database_name):
+        return
+
+    try:
+        with project_connection(database_name) as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    """
+                    select count(distinct subscription_row.oid) as subscriptions,
+                           count(distinct subscription_row.oid)
+                               filter (where subscription_row.subenabled)
+                               as enabled_subscriptions,
+                           count(subscription_status.pid)
+                               filter (where subscription_status.pid is not null)
+                               as active_workers
+                    from pg_subscription subscription_row
+                    left join pg_stat_subscription subscription_status
+                      on subscription_status.subid = subscription_row.oid
+                    where subscription_row.subdbid = (
+                        select oid from pg_database
+                        where datname = current_database()
+                    )
+                    """
+                )
+                status = cur.fetchone()
+                cur.execute(
+                    """
+                    select count(*) as relations,
+                           count(*) filter (where srsubstate <> 'r')
+                               as pending_relations
+                    from pg_subscription_rel
+                    """
+                )
+                relations = cur.fetchone()
+    finally:
+        close_project_pool(database_name)
+
+    if (
+        status is None
+        or int(status["subscriptions"]) != 1
+        or int(status["enabled_subscriptions"]) != 1
+        or int(status["active_workers"]) < 1
+    ):
+        raise RuntimeError(
+            f"Project model template {database_name!r} subscription is not active"
+        )
+    if (
+        relations is None
+        or int(relations["relations"]) < 1
+        or int(relations["pending_relations"]) > 0
+    ):
+        raise RuntimeError(
+            f"Project model template {database_name!r} is still synchronizing"
+        )
+
+
 def temporary_project_name(project: str, purpose: str) -> str:
     """Return a collision-resistant physical database name for one run."""
     physical_name = get_project_database_name(project)
@@ -137,18 +202,11 @@ def temporary_project_database(project: str, purpose: str):
     """Clone one project's runnable model into an isolated temporary database."""
     temporary_name = temporary_project_name(project, purpose)
     try:
-        copy_project(get_project_template_database_name(project), temporary_name)
-        # Import lazily to keep physical database lifecycle independent from
-        # model-copy implementation details at module import time.
-        from .database import refresh_materialized_views_after_commit
-        from .model_replace import replace_project_model
-
-        replace_project_model(
+        copy_project(
+            get_project_template_database_name(project),
             temporary_name,
-            project,
-            copy_source_scada=True,
+            allow_template_source=True,
         )
-        refresh_materialized_views_after_commit(temporary_name)
         yield temporary_name
     finally:
         if have_project(temporary_name):
@@ -160,7 +218,11 @@ def temporary_template_database(name_hint: str, purpose: str):
     """Create an empty schema-only temporary database from the fixed template."""
     temporary_name = temporary_project_name(name_hint, purpose)
     try:
-        copy_project(get_project_template_database_name(name_hint), temporary_name)
+        copy_project(
+            get_schema_template_database_name(),
+            temporary_name,
+            allow_template_source=True,
+        )
         yield temporary_name
     finally:
         if have_project(temporary_name):
@@ -175,11 +237,28 @@ def have_project(name: str) -> bool:
             return cur.fetchone() is not None
 
 
-def copy_project(source: str, new: str) -> None:
+def copy_project(
+    source: str,
+    new: str,
+    *,
+    allow_template_source: bool = False,
+    allow_template_target: bool = False,
+) -> None:
     physical_source = get_project_database_name(source)
     physical_new = get_project_database_name(new)
-    _validate_project_database(physical_source, allow_template_source=True)
-    _validate_project_database(physical_new)
+    _validate_project_database(
+        physical_source,
+        allow_template_source=allow_template_source,
+    )
+    if physical_new.casefold() == get_schema_template_database_name().casefold():
+        raise ValueError(
+            f"Database {physical_new!r} is the protected schema template"
+        )
+    _validate_project_database(
+        physical_new,
+        allow_template_source=allow_template_target,
+    )
+    _ensure_project_model_template_ready(physical_source)
     close_project_pool(source)
     close_project_pool(new)
 
@@ -216,12 +295,23 @@ def copy_project(source: str, new: str) -> None:
 
 
 def create_project(name: str) -> None:
-    return copy_project(get_project_template_database_name(name), name)
+    return copy_project(
+        get_schema_template_database_name(),
+        name,
+        allow_template_source=True,
+    )
 
 
-def delete_project(name: str) -> None:
+def delete_project(name: str, *, allow_template: bool = False) -> None:
     database_name = get_project_database_name(name)
-    _validate_project_database(database_name)
+    if database_name.casefold() == get_schema_template_database_name().casefold():
+        raise ValueError(
+            f"Database {database_name!r} is the protected schema template"
+        )
+    _validate_project_database(
+        database_name,
+        allow_template_source=allow_template,
+    )
     close_project_pool(name)
     with admin_connection() as conn:
         with conn.cursor() as cur:

app/native/wndb/inp/importer.py

@@ -12,7 +12,7 @@ from ..core.projects import (
     temporary_project_name,
     temporary_template_database,
 )
-from app.infra.db.project_routing import get_project_template_database_name
+from app.infra.db.project_routing import get_schema_template_database_name
 from ..core.connection import project_transaction
 from ..core.model_replace import replace_project_model
 from ..core.database import (
@@ -404,7 +404,11 @@ def read_inp(project: str, inp: str, version: str = "3") -> bool:
     staging_project = temporary_project_name(project, "model_import")
     replacement_committed = False
     try:
-        copy_project(get_project_template_database_name(project), staging_project)
+        copy_project(
+            get_schema_template_database_name(),
+            staging_project,
+            allow_template_source=True,
+        )
         with project_transaction(staging_project):
             parse_file(staging_project, inp, version)
         replace_project_model(project, staging_project)

app/native/wndb/model/options_v3.py

@@ -5,7 +5,10 @@ from .options import get_option_schema, get_option_v3_schema, generate_v2, gener
 def _parse_v2(v2_lines: list[str]) -> dict[str, str]:
     cs_v2 = g_update_prefix | { 'type' : 'option' }
     for s in v2_lines:
-        tokens = s.split()
+        stripped = s.strip()
+        if not stripped or stripped.startswith(';'):
+            continue
+        tokens = stripped.split()
         if tokens[0].upper() == 'PATTERN': # can not upper id
             value = tokens[1] if len(tokens) > 1 else ''
             cs_v2 |= { 'PATTERN' : value }
@@ -23,6 +26,22 @@ def _parse_v2(v2_lines: list[str]) -> dict[str, str]:
     return cs_v2
 
 
+def _option_changes(option_type: str, *change_sets: ChangeSet) -> ChangeSet:
+    values: dict[str, str] = {}
+    for change_set in change_sets:
+        for operation in change_set.operations:
+            values.update(
+                {
+                    key: str(value)
+                    for key, value in operation.items()
+                    if key not in {'operation', 'type'}
+                }
+            )
+    if not values:
+        return ChangeSet()
+    return ChangeSet(g_update_prefix | {'type': option_type} | values)
+
+
 def _inp_in_option_v3(section: list[str]) -> ChangeSet:
     if len(section) <= 0:
         return ChangeSet()
@@ -30,10 +49,11 @@ def _inp_in_option_v3(section: list[str]) -> ChangeSet:
     cs_v3 = g_update_prefix | { 'type' : 'option_v3' }
     v2_lines = []
     for s in section:
-        if s.startswith(';'):
+        stripped = s.strip()
+        if not stripped or stripped.startswith(';'):
             continue
 
-        tokens = s.strip().split()
+        tokens = stripped.split()
         key = tokens[0]
         if key in get_option_v3_schema('').keys():
             value = ''
@@ -43,14 +63,17 @@ def _inp_in_option_v3(section: list[str]) -> ChangeSet:
                 value = ' '.join(tokens[1:])
             cs_v3 |= { key : value }
         else:
-            v2_lines.append(s.strip())
+            v2_lines.append(stripped)
 
-    # unlikely...
     cs_v2 = _parse_v2(v2_lines)
+    direct_v2 = _option_changes('option', ChangeSet(cs_v2))
+    direct_v3 = _option_changes('option_v3', ChangeSet(cs_v3))
+    generated_v2 = generate_v2(direct_v3) if direct_v3.operations else ChangeSet()
+    generated_v3 = generate_v3(direct_v2) if direct_v2.operations else ChangeSet()
 
-    result = ChangeSet(cs_v3)
-    result.merge(generate_v3(ChangeSet(cs_v2)))
-    result.merge(generate_v2(result))
+    result = ChangeSet()
+    result.merge(_option_changes('option', generated_v2, direct_v2))
+    result.merge(_option_changes('option_v3', generated_v3, direct_v3))
     return result