TJWaterServerBinary/app/native/wndb/inp/importer.py

import datetime
import os

from psycopg import sql

from ..core.projects import (
    close_project,
    create_project,
    delete_project,
    have_project,
    is_project_open,
    open_project,
)
from ..core.connection import project_transaction
from ..core.database import ChangeSet, refresh_materialized_views, sql_literal, write
from .sections import (
    BACKDROP,
    BOUND,
    CONTROLS,
    COORDINATES,
    CURVES,
    DEMANDS,
    EMITTERS,
    ENERGY,
    JUNCTIONS,
    LABELS,
    MIXING,
    OPTIONS,
    PATTERNS,
    PIPES,
    PUMPS,
    QUALITY,
    REACTIONS,
    REGION,
    REGION_NODES,
    REPORT,
    RESERVOIRS,
    RULES,
    SOURCES,
    STATUS,
    TAGS,
    TANKS,
    TIMES,
    TITLE,
    VALVES,
    VERTICES,
    section_name,
)
from ..model.title import inp_in_title
from ..model.junctions import inp_in_junction
from ..model.reservoirs import inp_in_reservoir
from ..model.tanks import inp_in_tank
from ..model.pipes import inp_in_pipe
from ..model.pumps import inp_in_pump
from ..model.valves import inp_in_valve
from ..model.tags import inp_in_tag
from ..model.demands import inp_in_demand
from ..model.status import inp_in_status
from ..model.patterns import pattern_v3_types, inp_in_pattern
from ..model.curves import curve_types, inp_in_curve
from ..model.controls import inp_in_control
from ..model.rules import inp_in_rule
from ..model.energy import inp_in_energy
from ..model.emitters import inp_in_emitter
from ..model.quality import inp_in_quality
from ..model.sources import inp_in_source
from ..model.reactions import inp_in_reaction
from ..model.mixing import inp_in_mixing
from ..model.times import inp_in_time
from ..model.reports import inp_in_report
from ..model.options_legacy import inp_in_option
from ..model.options_v3 import inp_in_option_v3
from ..gis.coordinates import inp_in_coord
from ..gis.vertices import inp_in_vertex
from ..gis.labels import inp_in_label
from ..gis.backdrop import inp_in_backdrop
from ..gis.regions import inp_in_region, inp_in_bound, inp_in_regionnodes
from ..gis.region_geometry import to_postgis_polygon

# DingZQ, 2024-12-28, export inp
from .exporter import export_inp

_S = "S"
_L = "L"


def _inp_in_option(section: list[str], version: str = "3") -> str:
    return inp_in_option_v3(section) if version == "3" else inp_in_option(section)


_handler = {
    TITLE: (_S, inp_in_title),
    JUNCTIONS: (_L, inp_in_junction),  # line, demand_outside
    RESERVOIRS: (_L, inp_in_reservoir),
    TANKS: (_L, inp_in_tank),
    PIPES: (_L, inp_in_pipe),
    PUMPS: (_L, inp_in_pump),
    VALVES: (_L, inp_in_valve),
    TAGS: (_L, inp_in_tag),
    DEMANDS: (_L, inp_in_demand),
    STATUS: (_L, inp_in_status),
    PATTERNS: (_L, inp_in_pattern),  # line, fixed
    CURVES: (_L, inp_in_curve),
    CONTROLS: (_L, inp_in_control),
    RULES: (_L, inp_in_rule),
    ENERGY: (_L, inp_in_energy),
    EMITTERS: (_L, inp_in_emitter),
    QUALITY: (_L, inp_in_quality),
    SOURCES: (_L, inp_in_source),
    REACTIONS: (_L, inp_in_reaction),
    MIXING: (_L, inp_in_mixing),
    TIMES: (_S, inp_in_time),
    REPORT: (_S, inp_in_report),
    OPTIONS: (_S, _inp_in_option),  # line, version
    COORDINATES: (_L, inp_in_coord),
    VERTICES: (_L, inp_in_vertex),
    REGION: (_L, inp_in_region),
    BOUND: (_L, inp_in_bound),
    REGION_NODES: (_L, inp_in_regionnodes),
    LABELS: (_L, inp_in_label),
    BACKDROP: (_S, inp_in_backdrop),
    # END         : 'END',
}

_level_1 = [
    TITLE,
    PATTERNS,
    CURVES,
    CONTROLS,
    RULES,
    TIMES,
    REPORT,
    OPTIONS,
    BACKDROP,
]

_level_2 = [
    JUNCTIONS,
    RESERVOIRS,
    TANKS,
]

_level_3 = [
    PIPES,
    PUMPS,
    VALVES,
    DEMANDS,
    EMITTERS,
    QUALITY,
    SOURCES,
    MIXING,
    COORDINATES,
    LABELS,
]

_level_4 = [
    TAGS,
    STATUS,
    ENERGY,
    REACTIONS,
    VERTICES,
    REGION,
    BOUND,
    REGION_NODES,
]

map_regiontype = {
    # map the region types from desktop to server
    "DISTRIBUTION": "WDA",
    "DMA": "DMA",
    "PMA": "PMA",
    "VD": "VD",
    "SA": "SA",
}


class SQLBatch:
    def __init__(self, project: str, count: int = 100) -> None:
        self.batch: list[str] = []
        self.project = project
        self.count = count

    def add(self, sql: str) -> None:
        self.batch.append(sql)
        if len(self.batch) == self.count:
            self.flush()

    def flush(self) -> None:
        write(self.project, "".join(self.batch))
        self.batch.clear()


def _print_time(desc: str) -> datetime.datetime:
    now = datetime.datetime.now()
    time = now.strftime("%Y-%m-%d %H:%M:%S")
    print(f"{time}: {desc}")
    return now


def _get_file_offset(inp: str) -> tuple[dict[str, list[int]], bool]:
    offset: dict[str, list[int]] = {}

    current = ""
    demand_outside = False

    with open(inp, encoding="utf-8") as f:
        while True:
            line = f.readline()
            if not line:
                break

            line = line.strip()
            if line.startswith("["):
                for s in section_name:
                    if line.startswith(f"[{s}"):
                        if s not in offset:
                            offset[s] = []
                        offset[s].append(f.tell())
                        current = s
                        break
            elif line != "" and line.startswith(";") == False:
                if current == DEMANDS:
                    demand_outside = True

    return (offset, demand_outside)


def parse_file(project: str, inp: str, version: str = "3") -> None:
    start = _print_time(f'Start reading file "{inp}"...')

    _print_time("First scan...")
    offset, demand_outside = _get_file_offset(inp)

    levels = _level_1 + _level_2 + _level_3 + _level_4

    # parse the whole section rather than line
    sections: dict[str, list[str]] = {}
    for [s, t] in _handler.items():
        if t[0] == _S:
            sections[s] = []

    variable_patterns = []
    current_pattern = None
    current_curve = None
    curve_type_desc_line = None
    current_region = None
    current_bound = []
    current_bound.clear()
    region_list = {}

    sql_batch = SQLBatch(project)
    _print_time("Second scan...")
    with open(inp, encoding="utf-8") as f:
        for s in levels:
            if s not in offset:
                continue

            if s == DEMANDS and demand_outside == False:
                continue

            _print_time(f"[{s}]")

            is_s = _handler[s][0] == _S
            handler = _handler[s][1]

            for ptr in offset[s]:
                f.seek(ptr)

                while True:
                    line = f.readline()
                    if not line:
                        break

                    line = line.strip()
                    if line.startswith("["):
                        break
                    elif line == "":
                        continue

                    if is_s:
                        sections[s].append(line)
                    else:
                        if line.startswith(";"):
                            if version != "3":  # v2
                                line = line.removeprefix(";")
                                if s == PATTERNS:  # ;desc
                                    pass
                                elif s == CURVES:  # ;type: desc
                                    curve_type_desc_line = line
                            continue

                        if s == PATTERNS:
                            tokens = line.split()

                            if tokens[1].upper() in pattern_v3_types:  # v3
                                sql_batch.add(
                                    f"insert into network.patterns (id) values ({sql_literal(tokens[0])});"
                                )
                                current_pattern = tokens[0]
                                if tokens[1].upper() == "VARIABLE":
                                    variable_patterns.append(tokens[0])
                                continue

                            if current_pattern != tokens[0]:
                                sql_batch.add(
                                    f"insert into network.patterns (id) values ({sql_literal(tokens[0])});"
                                )
                                current_pattern = tokens[0]

                        elif s == CURVES:
                            tokens = line.split()

                            if tokens[1].upper() in curve_types:  # v3
                                sql_batch.add(
                                    f"insert into network.curves (id, curve_type) values ({sql_literal(tokens[0])}, {sql_literal(tokens[1].upper())});"
                                )
                                current_curve = tokens[0]
                                continue

                            if current_curve != tokens[0]:
                                type = curve_types[0]
                                if curve_type_desc_line != None:
                                    type = curve_type_desc_line.split(":")[0].strip()
                                sql_batch.add(
                                    f"insert into network.curves (id, curve_type) values ({sql_literal(tokens[0])}, {sql_literal(type)});"
                                )
                                current_curve = tokens[0]
                                curve_type_desc_line = None
                        elif s == REGION:
                            tokens = line.split()
                            region_list[tokens[0]] = tokens[1]
                            continue
                        elif s == BOUND:
                            tokens = line.split()
                            if tokens[0] != current_region and len(current_bound) > 0:
                                current_bound.append(current_bound[0])
                                current_geometry = to_postgis_polygon(current_bound)
                                region_type = map_regiontype.get(
                                    region_list[current_region],
                                    region_list[current_region],
                                )
                                sql_batch.add(
                                    "insert into gis.regions(id, region_type, boundary) "
                                    f"values ({sql_literal(current_region)}, {sql_literal(region_type)}, "
                                    f"st_geomfromtext({sql_literal(current_geometry)}, 900914));"
                                )
                                current_bound.clear()
                            vertex_point = (float(tokens[1]), float(tokens[2]))
                            current_bound.append(vertex_point)
                            current_region = tokens[0]
                        if s == JUNCTIONS:
                            sql_batch.add(handler(line, demand_outside))
                        elif s == PATTERNS:
                            sql_batch.add(
                                handler(line, current_pattern not in variable_patterns)
                            )
                        elif s == BOUND:
                            continue
                        else:
                            sql_batch.add(handler(line))

                f.seek(0)

            if is_s:
                if s == OPTIONS:
                    sql_batch.add(handler(sections[s], version))
                else:
                    sql_batch.add(handler(sections[s]))
        # need to insert the last region into database
        if len(current_bound) > 0:
            current_bound.append(current_bound[0])
            current_geometry = to_postgis_polygon(current_bound)
            region_type = map_regiontype.get(
                region_list[current_region],
                region_list[current_region],
            )
            sql_batch.add(
                "insert into gis.regions(id, region_type, boundary) "
                f"values ({sql_literal(current_region)}, {sql_literal(region_type)}, "
                f"st_geomfromtext({sql_literal(current_geometry)}, 900914));"
            )
        sql_batch.flush()

    end = _print_time(f'End reading file "{inp}"')
    print(f"Total (in second): {(end-start).seconds}(s)")


def read_inp(project: str, inp: str, version: str = "3") -> bool:
    if version != "3" and version != "2":
        version = "2"

    if is_project_open(project):
        close_project(project)

    if have_project(project):
        delete_project(project)

    create_project(project)
    open_project(project)

    with project_transaction(project):
        parse_file(project, inp, version)
    refresh_materialized_views(project)

    """try:
        parse_file(project, inp, version)
    except:
        close_project(project)
        delete_project(project)
        return False"""

    close_project(project)
    return True


# DingZQ, 2024-12-28, convert v3 to v2
def convert_inp_v3_to_v2(inp: str) -> ChangeSet:
    project = "v3Tov2"

    if is_project_open(project):
        close_project(project)

    if have_project(project):
        delete_project(project)

    create_project(project)
    open_project(project)

    filename = f"inp/{project}_temp.inp"
    if os.path.exists(filename):
        os.remove(filename)

    with open(filename, "w", encoding="utf-8") as f:
        f.write(inp)

    parse_file(project, filename, "3")

    """try:
        parse_file(project, inp, version)
    except:
        close_project(project)
        delete_project(project)
        return False"""

    return export_inp(project, "2")


def import_inp(project: str, cs: ChangeSet, version: str = "3") -> bool:
    if version != "3" and version != "2":
        version = "2"

    if "inp" not in cs.operations[0]:
        return False

    filename = f"inp/{project}_temp.inp"
    if os.path.exists(filename):
        os.remove(filename)

    _print_time(f'Start writing temp file "{filename}"...')
    with open(filename, "w", encoding="utf-8") as f:
        f.write(str(cs.operations[0]["inp"]))
    _print_time(f'End writing temp file "{filename}"...')

    result = read_inp(project, filename, version)

    # os.remove(filename)

    return result