Use sql batch to optimize

api/inp_in_new.py

@@ -13,7 +13,7 @@ from .s8_tags import inp_in_tag_new
 from .s9_demands import inp_in_demand_new
 from .s10_status import inp_in_status_new
 from .s11_patterns import inp_in_pattern_new
-from .s12_curves import inp_in_curve_new
+from .s12_curves import CURVE_TYPE_PUMP, inp_in_curve_new
 from .s13_controls import inp_in_control_new
 from .s14_rules import inp_in_rule_new
 from .s15_energy import inp_in_energy_new
@@ -103,10 +103,12 @@ _level_4 = [
     VERTICES,
 ]
 
-
-def _get_offset(inp: str) -> dict[str, list[int]]:
+def _get_offset(inp: str) -> tuple[dict[str, list[int]], bool]:
     offset: dict[str, list[int]] = {}
 
+    current = ''
+    demand_outside = False
+
     with open(inp) as f:
         while True:
             line = f.readline()
@@ -120,20 +122,43 @@ def _get_offset(inp: str) -> dict[str, list[int]]:
                         if s not in offset:
                             offset[s] = []
                         offset[s].append(f.tell())
+                        current = s
                         break
+            elif line != '' and line.startswith(';') == False:
+                if current == DEMANDS:
+                    demand_outside = True
 
-    return offset
+    return (offset, demand_outside)
 
 
-def print_time(desc: str) -> None:
-    time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+def print_time(desc: str) -> datetime.datetime:
+    now = datetime.datetime.now()
+    time = now.strftime('%Y-%m-%d %H:%M:%S')
     print(f"{time}: {desc}")
+    return now
+
+
+class SQLBatch:
+    def __init__(self, project: str, count: int = 100) -> None:
+        self.batch: list[str] = []
+        self.project = project
+        self.count = count
+
+    def add(self, sql: str) -> None:
+        self.batch.append(sql)
+        if len(self.batch) == self.count:
+            self.flush()
+
+    def flush(self) -> None:
+        write(self.project, ''.join(self.batch))
+        self.batch.clear()
 
 
 def parse_file(project: str, inp: str) -> None:
-    print_time(f"start reading {inp}...")
+    start = print_time(f'Start reading "{inp}"...')
 
-    offset = _get_offset(inp)
+    print_time("First scan...")
+    offset, demand_outside = _get_offset(inp)
 
     levels = _level_1 + _level_2 + _level_3 + _level_4
 
@@ -143,15 +168,21 @@ def parse_file(project: str, inp: str) -> None:
         if t[0] == _S:
             sections[s] = []
 
-    pattern_desc_line = None
+    current_pattern = None
+    current_curve = None
     curve_type_desc_line = None
-    demand_junction = None
 
+    sql_batch = SQLBatch(project)
+
+    print_time("Second scan...")
     with open(inp) as f:
         for s in levels:
             if s not in offset:
                 continue
 
+            if s == DEMANDS and demand_outside == False:
+                continue
+
             print_time(f"[{s}]")
 
             is_s = _handler[s][0] == _S
@@ -177,39 +208,40 @@ def parse_file(project: str, inp: str) -> None:
                         if line.startswith(';'):
                             line = line.removeprefix(';')
                             if s == PATTERNS: # ;desc
-                                pattern_desc_line = line
+                                pass
                             elif s == CURVES: # ;type: desc
                                 curve_type_desc_line = line
                             continue
 
                         if s == PATTERNS:
-                            if pattern_desc_line != None:
-                                tokens = line.split()
-                                write(project, f"insert into _pattern (id) values ('{tokens[0]}');")
-                                pattern_desc_line = None
-                        elif s == CURVES:
-                            if curve_type_desc_line != None:
-                                type_and_desc = curve_type_desc_line.split(':')
-                                tokens = line.split()
-                                write(project, f"insert into _curve (id, type) values ('{tokens[0]}', '{type_and_desc[0].strip()}');")
-                                curve_type_desc_line = None
-                        elif s == DEMANDS:
                             tokens = line.split()
-                            junction = str(tokens[0])
-                            if demand_junction != junction:
-                                if try_read(project, f"select * from demands where junction = '{junction}'") != None:
-                                    write(project, f"delete from demands where junction = '{junction}';")
-                                demand_junction = junction
-
-                        handler(project, line)
+                            if current_pattern != tokens[0]:
+                                write(project, f"insert into _pattern (id) values ('{tokens[0]}');")
+                                current_pattern = tokens[0]
+                        elif s == CURVES:
+                            tokens = line.split()
+                            if current_curve != tokens[0]:
+                                type = CURVE_TYPE_PUMP
+                                if curve_type_desc_line != None:
+                                    type = curve_type_desc_line.split(':')[0].strip()
+                                write(project, f"insert into _curve (id, type) values ('{tokens[0]}', '{type}');")
+                                current_curve = tokens[0]
+                                curve_type_desc_line = None
 
+                        if s == JUNCTIONS:
+                            sql_batch.add(handler(line, demand_outside))
+                        else:
+                            sql_batch.add(handler(line))
 
                 f.seek(0)
 
             if is_s:
-                handler(project, sections[s])
+                sql_batch.add(handler(sections[s]))
 
-    print_time(f"end reading {inp}")
+        sql_batch.flush()
+
+    end = print_time(f'End reading "{inp}"')
+    print(f"Total (in second): {(end-start).seconds}(s)")
 
 
 def parse_inp(project: str, inp: str) -> None: