From f6e7d887993e442ae3b206d31550feec993c6cb9 Mon Sep 17 00:00:00 2001 From: DingZQ Date: Wed, 29 Mar 2023 23:36:42 +0800 Subject: [PATCH 01/10] Add methods to get scada elemnets and scada devices --- main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.py b/main.py index fe410c0..cf83010 100644 --- a/main.py +++ b/main.py @@ -1379,6 +1379,10 @@ async def fastapi_set_backdrop_properties(network: str, req: Request) -> ChangeS async def fastapi_get_scada_device_schema(network: str) -> dict[str, dict[str, Any]]: return get_scada_device_schema(network) +@app.get('/getscadadevices/') +async def fastapi_get_scada_devices(network: str) -> list[str]: + return get_scada_devices(network) + @app.get('/getscadadevice/') async def fastapi_get_scada_device(network: str, id: str) -> dict[str, Any]: return get_scada_device(network, id) @@ -1445,6 +1449,10 @@ async def fastapi_clean_scada_device_data(network: str) -> ChangeSet: async def fastapi_get_scada_element_schema(network: str) -> dict[str, dict[str, Any]]: return get_scada_element_schema(network) +@app.get('/getscadaelements/') +async def fastapi_get_scada_elements(network: str) -> list[str]: + return get_scada_elements(network) + @app.get('/getscadaelement/') async def fastapi_get_scada_element(network: str, id: str) -> dict[str, Any]: return get_scada_element(network, id) From 68f30cf171ca5328dc083c0c1862d4e882d96736 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 09:57:22 +0800 Subject: [PATCH 02/10] Move script out to create template --- create_template.py | 108 +++++++++++++++++++++++++++++++++++++++++++++ script/template.py | 108 --------------------------------------------- 2 files changed, 108 insertions(+), 108 deletions(-) create mode 100644 create_template.py delete mode 100644 script/template.py diff --git a/create_template.py b/create_template.py new file mode 100644 index 0000000..50ff02b --- /dev/null +++ b/create_template.py @@ -0,0 +1,108 @@ +import psycopg as pg + +sql_create = [ + "script/sql/create/0.base.sql", + "script/sql/create/1.title.sql", + "script/sql/create/2.junctions.sql", + "script/sql/create/3.reservoirs.sql", + "script/sql/create/4.tanks.sql", + "script/sql/create/5.pipes.sql", + "script/sql/create/6.pumps.sql", + "script/sql/create/7.valves.sql", + "script/sql/create/8.tags.sql", + "script/sql/create/9.demands.sql", + "script/sql/create/10.status.sql", + "script/sql/create/11.patterns.sql", + "script/sql/create/12.curves.sql", + "script/sql/create/13.controls.sql", + "script/sql/create/14.rules.sql", + "script/sql/create/15.energy.sql", + "script/sql/create/16.emitters.sql", + "script/sql/create/17.quality.sql", + "script/sql/create/18.sources.sql", + "script/sql/create/19.reactions.sql", + "script/sql/create/20.mixing.sql", + "script/sql/create/21.times.sql", + "script/sql/create/22.report.sql", + "script/sql/create/23.options.sql", + "script/sql/create/24.coordinates.sql", + "script/sql/create/25.vertices.sql", + "script/sql/create/26.labels.sql", + "script/sql/create/27.backdrop.sql", + "script/sql/create/28.end.sql", + "script/sql/create/29.scada_device.sql", + "script/sql/create/30.scada_device_data.sql", + "script/sql/create/31.scada_element.sql", + "script/sql/create/operation.sql" +] + +sql_drop = [ + "script/sql/drop/operation.sql", + "script/sql/drop/31.scada_element.sql", + "script/sql/drop/30.scada_device_data.sql", + "script/sql/drop/29.scada_device.sql", + "script/sql/drop/28.end.sql", + "script/sql/drop/27.backdrop.sql", + "script/sql/drop/26.labels.sql", + "script/sql/drop/25.vertices.sql", + "script/sql/drop/24.coordinates.sql", + "script/sql/drop/23.options.sql", + "script/sql/drop/22.report.sql", + "script/sql/drop/21.times.sql", + "script/sql/drop/20.mixing.sql", + "script/sql/drop/19.reactions.sql", + "script/sql/drop/18.sources.sql", + "script/sql/drop/17.quality.sql", + "script/sql/drop/16.emitters.sql", + "script/sql/drop/15.energy.sql", + "script/sql/drop/14.rules.sql", + "script/sql/drop/13.controls.sql", + "script/sql/drop/12.curves.sql", + "script/sql/drop/11.patterns.sql", + "script/sql/drop/10.status.sql", + "script/sql/drop/9.demands.sql", + "script/sql/drop/8.tags.sql", + "script/sql/drop/7.valves.sql", + "script/sql/drop/6.pumps.sql", + "script/sql/drop/5.pipes.sql", + "script/sql/drop/4.tanks.sql", + "script/sql/drop/3.reservoirs.sql", + "script/sql/drop/2.junctions.sql", + "script/sql/drop/1.title.sql", + "script/sql/drop/0.base.sql" +] + +def create_template(): + with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute("create database project") + with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: + with conn.cursor() as cur: + for sql in sql_create: + with open(sql, "r") as f: + cur.execute(f.read()) + print(f'executed {sql}') + conn.commit() + +def have_template(): + with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute("select * from pg_database where datname = 'project'") + return cur.rowcount > 0 + +def delete_template(): + with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: + with conn.cursor() as cur: + for sql in sql_drop: + with open(sql, "r") as f: + cur.execute(f.read()) + print(f'executed {sql}') + conn.commit() + with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute("drop database project") + +if __name__ == "__main__": + if (have_template()): + delete_template() + create_template() diff --git a/script/template.py b/script/template.py deleted file mode 100644 index a15c882..0000000 --- a/script/template.py +++ /dev/null @@ -1,108 +0,0 @@ -import psycopg as pg - -sql_create = [ - "sql/create/0.base.sql", - "sql/create/1.title.sql", - "sql/create/2.junctions.sql", - "sql/create/3.reservoirs.sql", - "sql/create/4.tanks.sql", - "sql/create/5.pipes.sql", - "sql/create/6.pumps.sql", - "sql/create/7.valves.sql", - "sql/create/8.tags.sql", - "sql/create/9.demands.sql", - "sql/create/10.status.sql", - "sql/create/11.patterns.sql", - "sql/create/12.curves.sql", - "sql/create/13.controls.sql", - "sql/create/14.rules.sql", - "sql/create/15.energy.sql", - "sql/create/16.emitters.sql", - "sql/create/17.quality.sql", - "sql/create/18.sources.sql", - "sql/create/19.reactions.sql", - "sql/create/20.mixing.sql", - "sql/create/21.times.sql", - "sql/create/22.report.sql", - "sql/create/23.options.sql", - "sql/create/24.coordinates.sql", - "sql/create/25.vertices.sql", - "sql/create/26.labels.sql", - "sql/create/27.backdrop.sql", - "sql/create/28.end.sql", - "sql/create/29.scada_device.sql", - "sql/create/30.scada_device_data.sql", - "sql/create/31.scada_element.sql", - "sql/create/operation.sql" -] - -sql_drop = [ - "sql/drop/operation.sql", - "sql/drop/31.scada_element.sql", - "sql/drop/30.scada_device_data.sql", - "sql/drop/29.scada_device.sql", - "sql/drop/28.end.sql", - "sql/drop/27.backdrop.sql", - "sql/drop/26.labels.sql", - "sql/drop/25.vertices.sql", - "sql/drop/24.coordinates.sql", - "sql/drop/23.options.sql", - "sql/drop/22.report.sql", - "sql/drop/21.times.sql", - "sql/drop/20.mixing.sql", - "sql/drop/19.reactions.sql", - "sql/drop/18.sources.sql", - "sql/drop/17.quality.sql", - "sql/drop/16.emitters.sql", - "sql/drop/15.energy.sql", - "sql/drop/14.rules.sql", - "sql/drop/13.controls.sql", - "sql/drop/12.curves.sql", - "sql/drop/11.patterns.sql", - "sql/drop/10.status.sql", - "sql/drop/9.demands.sql", - "sql/drop/8.tags.sql", - "sql/drop/7.valves.sql", - "sql/drop/6.pumps.sql", - "sql/drop/5.pipes.sql", - "sql/drop/4.tanks.sql", - "sql/drop/3.reservoirs.sql", - "sql/drop/2.junctions.sql", - "sql/drop/1.title.sql", - "sql/drop/0.base.sql" -] - -def create_template(): - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("create database project") - with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: - with conn.cursor() as cur: - for sql in sql_create: - with open(sql, "r") as f: - cur.execute(f.read()) - print(f'executed {sql}') - conn.commit() - -def have_template(): - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("select * from pg_database where datname = 'project'") - return cur.rowcount > 0 - -def delete_template(): - with pg.connect(conninfo="dbname=project host=127.0.0.1") as conn: - with conn.cursor() as cur: - for sql in sql_drop: - with open(sql, "r") as f: - cur.execute(f.read()) - print(f'executed {sql}') - conn.commit() - with pg.connect(conninfo="dbname=postgres host=127.0.0.1", autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute("drop database project") - -if __name__ == "__main__": - if (have_template()): - delete_template() - create_template() From 976361a3bd5cfdd1e4eb0b2e13b4f6b626b29cde Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 10:00:06 +0800 Subject: [PATCH 03/10] Rename script --- dev_script.py => devops.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev_script.py => devops.py (100%) diff --git a/dev_script.py b/devops.py similarity index 100% rename from dev_script.py rename to devops.py From 924dbc802cb9934568320ec499e685af9c22e060 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 10:27:21 +0800 Subject: [PATCH 04/10] Add more test cases for restore --- test_tjnetwork.py | 216 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/test_tjnetwork.py b/test_tjnetwork.py index 1628e2d..2ab4ad0 100644 --- a/test_tjnetwork.py +++ b/test_tjnetwork.py @@ -1,4 +1,5 @@ import pytest +import random from tjnetwork import * class TestApi: @@ -242,6 +243,221 @@ class TestApi: self.leave(p) + def test_delete_nodes_then_restore(self): + p = 'test_delete_nodes_then_restore' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nls : list[tuple[str, str]] = [] + + nodes = get_nodes(p) + for node in nodes: + nls.append(('node', node)) + + links = get_links(p) + + for _ in range(100): + random.shuffle(nls) + for nl in nls: + if nl[0] == 'node': + node = nl[1] + if is_junction(p, node): + delete_junction(p, ChangeSet({'id': node})) + if is_reservoir(p, node): + delete_reservoir(p, ChangeSet({'id': node})) + if is_tank(p, node): + delete_tank(p, ChangeSet({'id': node})) + else: + link = nl[1] + if is_pipe(p, link): + delete_pipe(p, ChangeSet({'id': link})) + if is_pump(p, link): + delete_pump(p, ChangeSet({'id': link})) + if is_valve(p, link): + delete_valve(p, ChangeSet({'id': link})) + + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False + + assert get_nodes(p) == [] + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_links_then_restore(self): + p = 'test_delete_links_then_restore' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nls : list[tuple[str, str]] = [] + + nodes = get_nodes(p) + + links = get_links(p) + for link in links: + nls.append(('link', link)) + + for _ in range(100): + random.shuffle(nls) + for nl in nls: + if nl[0] == 'node': + node = nl[1] + if is_junction(p, node): + delete_junction(p, ChangeSet({'id': node})) + if is_reservoir(p, node): + delete_reservoir(p, ChangeSet({'id': node})) + if is_tank(p, node): + delete_tank(p, ChangeSet({'id': node})) + else: + link = nl[1] + if is_pipe(p, link): + delete_pipe(p, ChangeSet({'id': link})) + if is_pump(p, link): + delete_pump(p, ChangeSet({'id': link})) + if is_valve(p, link): + delete_valve(p, ChangeSet({'id': link})) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) == False + + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_nodes_links_then_restore_v2(self): + p = 'test_delete_nodes_links_then_restore_v2' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nls : list[tuple[str, str]] = [] + + nodes = get_nodes(p) + for node in nodes: + nls.append(('node', node)) + + links = get_links(p) + for link in links: + nls.append(('link', link)) + + for _ in range(100): + random.shuffle(nls) + for nl in nls: + if nl[0] == 'node': + node = nl[1] + if is_junction(p, node): + delete_junction(p, ChangeSet({'id': node})) + if is_reservoir(p, node): + delete_reservoir(p, ChangeSet({'id': node})) + if is_tank(p, node): + delete_tank(p, ChangeSet({'id': node})) + else: + link = nl[1] + if is_pipe(p, link): + delete_pipe(p, ChangeSet({'id': link})) + if is_pump(p, link): + delete_pump(p, ChangeSet({'id': link})) + if is_valve(p, link): + delete_valve(p, ChangeSet({'id': link})) + + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False + + assert get_nodes(p) == [] + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_nodes_links_then_restore_v3(self): + p = 'test_delete_nodes_links_then_restore_v3' + read_inp(p, f'./inp/net3.inp', '3') + + open_project(p) + + nls : list[tuple[str, str]] = [] + + nodes = get_nodes(p) + for node in nodes: + nls.append(('node', node)) + + links = get_links(p) + for link in links: + nls.append(('link', link)) + + for _ in range(100): + random.shuffle(nls) + for nl in nls: + if nl[0] == 'node': + node = nl[1] + if is_junction(p, node): + delete_junction(p, ChangeSet({'id': node})) + if is_reservoir(p, node): + delete_reservoir(p, ChangeSet({'id': node})) + if is_tank(p, node): + delete_tank(p, ChangeSet({'id': node})) + else: + link = nl[1] + if is_pipe(p, link): + delete_pipe(p, ChangeSet({'id': link})) + if is_pump(p, link): + delete_pump(p, ChangeSet({'id': link})) + if is_valve(p, link): + delete_valve(p, ChangeSet({'id': link})) + + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False + + assert get_nodes(p) == [] + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + # 1 title From 610e7ee561bc9800200d704a5eac23a013173c36 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 10:44:15 +0800 Subject: [PATCH 05/10] Add api and script to restore --- api/__init__.py | 3 ++- api/database.py | 5 +++++ restore_project.py | 12 ++++++++++++ restore_projects.py | 8 ++++++++ tjnetwork.py | 3 +++ 5 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 restore_project.py create mode 100644 restore_projects.py diff --git a/api/__init__.py b/api/__init__.py index 02f4cc8..e222df7 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -16,7 +16,8 @@ from .database import update_snapshot, update_snapshot_for_current_operation from .database import delete_snapshot, delete_snapshot_by_operation from .database import get_operation_by_snapshot, get_snapshot_by_operation from .database import pick_snapshot -from .database import pick_operation, sync_with_server, get_restore_operation, set_restore_operation, set_restore_operation_to_current +from .database import pick_operation, sync_with_server +from .database import get_restore_operation, set_restore_operation, set_restore_operation_to_current, restore from .batch_cmd import execute_batch_command from .batch_cmds import execute_batch_commands diff --git a/api/database.py b/api/database.py index 6d9089f..a3e4788 100644 --- a/api/database.py +++ b/api/database.py @@ -339,3 +339,8 @@ def set_restore_operation(name: str, operation: int) -> None: def set_restore_operation_to_current(name: str) -> None: return set_restore_operation(name, get_current_operation(name)) + + +def restore(name: str, discard: bool) -> ChangeSet: + op = get_restore_operation(name) + return pick_operation(name, op, discard) diff --git a/restore_project.py b/restore_project.py new file mode 100644 index 0000000..af92e9c --- /dev/null +++ b/restore_project.py @@ -0,0 +1,12 @@ +import sys +from tjnetwork import * + +def main(): + if len(sys.argv) != 2: + print("restore_project name") + return + + restore(sys.argv[1]) + +if __name__ == '__main__': + main() diff --git a/restore_projects.py b/restore_projects.py new file mode 100644 index 0000000..bd1126e --- /dev/null +++ b/restore_projects.py @@ -0,0 +1,8 @@ +from tjnetwork import * + +def main(): + for p in list_project(): + restore(p) + +if __name__ == '__main__': + main() diff --git a/tjnetwork.py b/tjnetwork.py index d291b6e..8890429 100644 --- a/tjnetwork.py +++ b/tjnetwork.py @@ -291,6 +291,9 @@ def set_restore_operation(name: str, operation: int) -> None: def set_restore_operation_to_current(name: str) -> None: return api.set_restore_operation_to_current(name) +def restore(name: str, discard: bool = False) -> ChangeSet: + return api.restore(name, discard) + ############################################################ # type From f13d32bb0a28494c30cc006c4439e224b9dd46a9 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 10:46:51 +0800 Subject: [PATCH 06/10] Open and close when restore --- restore_project.py | 5 ++++- restore_projects.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/restore_project.py b/restore_project.py index af92e9c..d7cbcef 100644 --- a/restore_project.py +++ b/restore_project.py @@ -6,7 +6,10 @@ def main(): print("restore_project name") return - restore(sys.argv[1]) + p = sys.argv[1] + open_project(p) + restore(p) + close_project(p) if __name__ == '__main__': main() diff --git a/restore_projects.py b/restore_projects.py index bd1126e..ae7af0a 100644 --- a/restore_projects.py +++ b/restore_projects.py @@ -2,7 +2,9 @@ from tjnetwork import * def main(): for p in list_project(): + open_project(p) restore(p) + close_project(p) if __name__ == '__main__': main() From 1f2733aa90960565af44d7714dc6f0ea5bb26092 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 10:48:11 +0800 Subject: [PATCH 07/10] Add info when restore --- restore_projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/restore_projects.py b/restore_projects.py index ae7af0a..d984968 100644 --- a/restore_projects.py +++ b/restore_projects.py @@ -2,6 +2,7 @@ from tjnetwork import * def main(): for p in list_project(): + print(f'restore {p}...') open_project(p) restore(p) close_project(p) From 71f7dbc233fe7cb0cc6145daeec71ee14d205f50 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 13:02:59 +0800 Subject: [PATCH 08/10] Refine test case --- test_tjnetwork.py | 102 +++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/test_tjnetwork.py b/test_tjnetwork.py index 2ab4ad0..0aab61e 100644 --- a/test_tjnetwork.py +++ b/test_tjnetwork.py @@ -257,7 +257,7 @@ class TestApi: links = get_links(p) - for _ in range(100): + for _ in range(10): random.shuffle(nls) for nl in nls: if nl[0] == 'node': @@ -277,21 +277,21 @@ class TestApi: if is_valve(p, link): delete_valve(p, ChangeSet({'id': link})) - for node in nodes: - assert is_node(p, node) == False - for link in links: - assert is_link(p, link) == False + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False - assert get_nodes(p) == [] - assert get_links(p) == [] + assert get_nodes(p) == [] + assert get_links(p) == [] - op = get_restore_operation(p) - pick_operation(p, op) + op = get_restore_operation(p) + pick_operation(p, op) - for node in nodes: - assert is_node(p, node) - for link in links: - assert is_link(p, link) + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) self.leave(p) @@ -310,7 +310,7 @@ class TestApi: for link in links: nls.append(('link', link)) - for _ in range(100): + for _ in range(10): random.shuffle(nls) for nl in nls: if nl[0] == 'node': @@ -330,20 +330,20 @@ class TestApi: if is_valve(p, link): delete_valve(p, ChangeSet({'id': link})) - for node in nodes: - assert is_node(p, node) - for link in links: - assert is_link(p, link) == False + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) == False - assert get_links(p) == [] + assert get_links(p) == [] - op = get_restore_operation(p) - pick_operation(p, op) + op = get_restore_operation(p) + pick_operation(p, op) - for node in nodes: - assert is_node(p, node) - for link in links: - assert is_link(p, link) + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) self.leave(p) @@ -364,7 +364,7 @@ class TestApi: for link in links: nls.append(('link', link)) - for _ in range(100): + for _ in range(10): random.shuffle(nls) for nl in nls: if nl[0] == 'node': @@ -384,21 +384,21 @@ class TestApi: if is_valve(p, link): delete_valve(p, ChangeSet({'id': link})) - for node in nodes: - assert is_node(p, node) == False - for link in links: - assert is_link(p, link) == False + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False - assert get_nodes(p) == [] - assert get_links(p) == [] + assert get_nodes(p) == [] + assert get_links(p) == [] - op = get_restore_operation(p) - pick_operation(p, op) + op = get_restore_operation(p) + pick_operation(p, op) - for node in nodes: - assert is_node(p, node) - for link in links: - assert is_link(p, link) + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) self.leave(p) @@ -419,7 +419,7 @@ class TestApi: for link in links: nls.append(('link', link)) - for _ in range(100): + for _ in range(10): random.shuffle(nls) for nl in nls: if nl[0] == 'node': @@ -439,21 +439,21 @@ class TestApi: if is_valve(p, link): delete_valve(p, ChangeSet({'id': link})) - for node in nodes: - assert is_node(p, node) == False - for link in links: - assert is_link(p, link) == False + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False - assert get_nodes(p) == [] - assert get_links(p) == [] + assert get_nodes(p) == [] + assert get_links(p) == [] - op = get_restore_operation(p) - pick_operation(p, op) + op = get_restore_operation(p) + pick_operation(p, op) - for node in nodes: - assert is_node(p, node) - for link in links: - assert is_link(p, link) + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) self.leave(p) From 8de01a3e02177bf30bea2dd6d5c2a280889e9fa4 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 13:19:09 +0800 Subject: [PATCH 09/10] Add test case for batch commands --- test_tjnetwork.py | 143 +++++++++++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 46 deletions(-) diff --git a/test_tjnetwork.py b/test_tjnetwork.py index 0aab61e..6faee5c 100644 --- a/test_tjnetwork.py +++ b/test_tjnetwork.py @@ -249,33 +249,18 @@ class TestApi: open_project(p) - nls : list[tuple[str, str]] = [] - nodes = get_nodes(p) - for node in nodes: - nls.append(('node', node)) - links = get_links(p) for _ in range(10): - random.shuffle(nls) - for nl in nls: - if nl[0] == 'node': - node = nl[1] - if is_junction(p, node): - delete_junction(p, ChangeSet({'id': node})) - if is_reservoir(p, node): - delete_reservoir(p, ChangeSet({'id': node})) - if is_tank(p, node): - delete_tank(p, ChangeSet({'id': node})) - else: - link = nl[1] - if is_pipe(p, link): - delete_pipe(p, ChangeSet({'id': link})) - if is_pump(p, link): - delete_pump(p, ChangeSet({'id': link})) - if is_valve(p, link): - delete_valve(p, ChangeSet({'id': link})) + random.shuffle(nodes) + for node in nodes: + if is_junction(p, node): + delete_junction(p, ChangeSet({'id': node})) + if is_reservoir(p, node): + delete_reservoir(p, ChangeSet({'id': node})) + if is_tank(p, node): + delete_tank(p, ChangeSet({'id': node})) for node in nodes: assert is_node(p, node) == False @@ -302,33 +287,99 @@ class TestApi: open_project(p) - nls : list[tuple[str, str]] = [] - nodes = get_nodes(p) - links = get_links(p) - for link in links: - nls.append(('link', link)) for _ in range(10): - random.shuffle(nls) - for nl in nls: - if nl[0] == 'node': - node = nl[1] - if is_junction(p, node): - delete_junction(p, ChangeSet({'id': node})) - if is_reservoir(p, node): - delete_reservoir(p, ChangeSet({'id': node})) - if is_tank(p, node): - delete_tank(p, ChangeSet({'id': node})) - else: - link = nl[1] - if is_pipe(p, link): - delete_pipe(p, ChangeSet({'id': link})) - if is_pump(p, link): - delete_pump(p, ChangeSet({'id': link})) - if is_valve(p, link): - delete_valve(p, ChangeSet({'id': link})) + random.shuffle(links) + for link in links: + if is_pipe(p, link): + delete_pipe(p, ChangeSet({'id': link})) + if is_pump(p, link): + delete_pump(p, ChangeSet({'id': link})) + if is_valve(p, link): + delete_valve(p, ChangeSet({'id': link})) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) == False + + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_nodes_then_restore_commands(self): + p = 'test_delete_nodes_then_restore_commands' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nodes = get_nodes(p) + links = get_links(p) + + for _ in range(10): + random.shuffle(nodes) + + batch = ChangeSet() + for node in nodes: + if is_junction(p, node): + batch.delete({'type' : 'junction', 'id': node }) + if is_reservoir(p, node): + batch.delete({'type' : 'reservoir', 'id': node }) + if is_tank(p, node): + batch.delete({'type' : 'tank', 'id': node }) + execute_batch_commands(p, batch) + + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False + + assert get_nodes(p) == [] + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_links_then_restore_commands(self): + p = 'test_delete_links_then_restore_commands' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nodes = get_nodes(p) + links = get_links(p) + + for _ in range(10): + random.shuffle(links) + + batch = ChangeSet() + for link in links: + if is_pipe(p, link): + batch.delete({'type' : 'pipe', 'id': link }) + if is_pump(p, link): + batch.delete({'type' : 'pump', 'id': link }) + if is_valve(p, link): + batch.delete({'type' : 'valve', 'id': link }) + execute_batch_commands(p, batch) for node in nodes: assert is_node(p, node) From a579272ea9be41e17caea4ac5216ae5f77102a42 Mon Sep 17 00:00:00 2001 From: "WQY\\qiong" Date: Fri, 31 Mar 2023 14:51:49 +0800 Subject: [PATCH 10/10] Support batch operation table --- api/__init__.py | 3 +- api/batch_cmds.py | 71 +++++++++++++++++++++++-- api/database.py | 9 ++-- script/sql/create/operation.sql | 23 +++++++++ script/sql/drop/operation.sql | 6 +++ test_tjnetwork.py | 91 +++++++++++++++++++++++++++++---- 6 files changed, 186 insertions(+), 17 deletions(-) diff --git a/api/__init__.py b/api/__init__.py index e222df7..e12dd91 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -19,8 +19,7 @@ from .database import pick_snapshot from .database import pick_operation, sync_with_server from .database import get_restore_operation, set_restore_operation, set_restore_operation_to_current, restore -from .batch_cmd import execute_batch_command -from .batch_cmds import execute_batch_commands +from .batch_cmds import execute_batch_commands, execute_batch_command from .s0_base import JUNCTION, RESERVOIR, TANK, PIPE, PUMP, VALVE, PATTERN, CURVE from .s0_base import is_node, is_junction, is_reservoir, is_tank diff --git a/api/batch_cmds.py b/api/batch_cmds.py index 4ee0688..6657524 100644 --- a/api/batch_cmds.py +++ b/api/batch_cmds.py @@ -1,5 +1,6 @@ +from typing import Any from .sections import * -from .database import API_ADD, API_UPDATE, API_DELETE, ChangeSet +from .database import API_ADD, API_UPDATE, API_DELETE, ChangeSet, write, read, read_all, get_current_operation from .s1_title import set_title from .s2_junctions import set_junction, add_junction, delete_junction from .s3_reservoirs import set_reservoir, add_reservoir, delete_reservoir @@ -290,7 +291,71 @@ def execute_batch_commands(name: str, cs: ChangeSet) -> ChangeSet: elif operation == API_DELETE: result.merge(execute_delete_command(name, ChangeSet(op))) except: - print(f'ERROR: Fail to execute {todo}!') - pass + print(f'ERROR: Fail to execute {todo}') + + return result + + +def execute_batch_command(name: str, cs: ChangeSet) -> ChangeSet: + write(name, 'delete from batch_operation where id > 0') + write(name, "update operation_table set option = 'batch_operation' where option = 'operation'") + + # for delete, generate cascade command + new_cs = ChangeSet() + for op in cs.operations: + if op['operation'] == API_DELETE: + new_cs.merge(del_cascade_cmd(name, ChangeSet(op))) + else: + new_cs.merge(ChangeSet(op)) + + result = ChangeSet() + + todo = {} + + try: + for op in new_cs.operations: + todo = op + operation = op['operation'] + if operation == API_ADD: + result.merge(execute_add_command(name, ChangeSet(op))) + elif operation == API_UPDATE: + result.merge(execute_update_command(name, ChangeSet(op))) + elif operation == API_DELETE: + result.merge(execute_delete_command(name, ChangeSet(op))) + except: + print(f'ERROR: Fail to execute {todo}!') + + count = read(name, 'select count(*) as count from batch_operation')['count'] + if count == 1: + write(name, 'delete from batch_operation where id > 0') + write(name, "update operation_table set option = 'operation' where option = 'batch_operation'") + return ChangeSet() + + redo_list: list[str] = [] + redo_cs_list: list[dict[str, Any]] = [] + redo_rows = read_all(name, 'select redo, redo_cs from batch_operation where id > 0 order by id asc') + for row in redo_rows: + redo_list.append(row['redo']) + redo_cs_list += eval(row['redo_cs']) + + undo_list: list[str] = [] + undo_cs_list: list[dict[str, Any]] = [] + undo_rows = read_all(name, 'select undo, undo_cs from batch_operation where id > 0 order by id desc') + for row in undo_rows: + undo_list.append(row['undo']) + undo_cs_list += eval(row['undo_cs']) + + redo = '\n'.join(redo_list).replace("'", "''") + redo_cs = str(redo_cs_list).replace("'", "''") + undo = '\n'.join(undo_list).replace("'", "''") + undo_cs = str(undo_cs_list).replace("'", "''") + + parent = get_current_operation(name) + write(name, f"insert into operation (id, redo, undo, parent, redo_cs, undo_cs) values (default, '{redo}', '{undo}', {parent}, '{redo_cs}', '{undo_cs}')") + current = read(name, 'select max(id) as id from operation')['id'] + write(name, f"update current_operation set id = {current}") + + write(name, 'delete from batch_operation where id > 0') + write(name, "update operation_table set option = 'operation' where option = 'batch_operation'") return result diff --git a/api/database.py b/api/database.py index a3e4788..6fd268b 100644 --- a/api/database.py +++ b/api/database.py @@ -113,6 +113,8 @@ def get_current_operation(name: str) -> int: def execute_command(name: str, command: DbChangeSet) -> ChangeSet: + op_table = read(name, "select * from operation_table")['option'] + write(name, command.redo_sql) parent = get_current_operation(name) @@ -120,10 +122,11 @@ def execute_command(name: str, command: DbChangeSet) -> ChangeSet: undo_sql = command.undo_sql.replace("'", "''") redo_cs_str = str(command.redo_cs).replace("'", "''") undo_cs_str = str(command.undo_cs).replace("'", "''") - write(name, f"insert into operation (id, redo, undo, parent, redo_cs, undo_cs) values (default, '{redo_sql}', '{undo_sql}', {parent}, '{redo_cs_str}', '{undo_cs_str}')") + write(name, f"insert into {op_table} (id, redo, undo, parent, redo_cs, undo_cs) values (default, '{redo_sql}', '{undo_sql}', {parent}, '{redo_cs_str}', '{undo_cs_str}')") - current = read(name, 'select max(id) as id from operation')['id'] - write(name, f"update current_operation set id = {current}") + if op_table == 'operation': + current = read(name, 'select max(id) as id from operation')['id'] + write(name, f"update current_operation set id = {current}") return ChangeSet.from_list(command.redo_cs) diff --git a/script/sql/create/operation.sql b/script/sql/create/operation.sql index dc4f220..3c7f952 100644 --- a/script/sql/create/operation.sql +++ b/script/sql/create/operation.sql @@ -30,3 +30,26 @@ create table restore_operation ); insert into restore_operation (id) values (0); + + +create table batch_operation +( + id bigserial primary key +, redo text not null +, undo text not null +, parent integer references operation(id) on delete cascade +, redo_child integer references operation(id) -- must update before delete +, redo_cs text not null +, undo_cs text not null +); + +insert into batch_operation (id, redo, undo, redo_cs, undo_cs) values (0, '', '', '', ''); + +create type operation_table_option as enum ('operation', 'batch_operation'); + +create table operation_table +( + option operation_table_option primary key +); + +insert into operation_table (option) values ('operation'); diff --git a/script/sql/drop/operation.sql b/script/sql/drop/operation.sql index 06a7765..ca063b2 100644 --- a/script/sql/drop/operation.sql +++ b/script/sql/drop/operation.sql @@ -1,3 +1,9 @@ +drop table if exists operation_table; + +drop type if exists operation_table_option; + +drop table if exists batch_operation; + drop table if exists restore_operation; drop table if exists snapshot_operation; diff --git a/test_tjnetwork.py b/test_tjnetwork.py index 6faee5c..a4df208 100644 --- a/test_tjnetwork.py +++ b/test_tjnetwork.py @@ -157,15 +157,7 @@ class TestApi: cs.add({'type': JUNCTION, 'id': 'j2', 'x': 0.0, 'y': 10.0, 'elevation': 20.0}) # fail cs = execute_batch_command(p, cs) - assert len(cs.operations) == 0 - - assert get_current_operation(p) == 0 - - cs = ChangeSet() - cs.add({'type': JUNCTION, 'id': 'j1', 'x': 0.0, 'y': 10.0, 'elevation': 20.0}) - cs.add({'type': JUNCTION, 'id': 'j2', 'x': 0.0, 'y': 10.0, 'elevation': 20.0}) - - cs = execute_batch_command(p, cs) + assert len(cs.operations) == 2 assert get_current_operation(p) == 1 @@ -399,6 +391,87 @@ class TestApi: self.leave(p) + def test_delete_nodes_then_restore_command(self): + p = 'test_delete_nodes_then_restore_commands' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nodes = get_nodes(p) + links = get_links(p) + + for _ in range(10): + random.shuffle(nodes) + + batch = ChangeSet() + for node in nodes: + if is_junction(p, node): + batch.delete({'type' : 'junction', 'id': node }) + if is_reservoir(p, node): + batch.delete({'type' : 'reservoir', 'id': node }) + if is_tank(p, node): + batch.delete({'type' : 'tank', 'id': node }) + execute_batch_command(p, batch) + + for node in nodes: + assert is_node(p, node) == False + for link in links: + assert is_link(p, link) == False + + assert get_nodes(p) == [] + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + + def test_delete_links_then_restore_command(self): + p = 'test_delete_links_then_restore_commands' + read_inp(p, f'./inp/net3.inp', '2') + + open_project(p) + + nodes = get_nodes(p) + links = get_links(p) + + for _ in range(10): + random.shuffle(links) + + batch = ChangeSet() + for link in links: + if is_pipe(p, link): + batch.delete({'type' : 'pipe', 'id': link }) + if is_pump(p, link): + batch.delete({'type' : 'pump', 'id': link }) + if is_valve(p, link): + batch.delete({'type' : 'valve', 'id': link }) + execute_batch_command(p, batch) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) == False + + assert get_links(p) == [] + + op = get_restore_operation(p) + pick_operation(p, op) + + for node in nodes: + assert is_node(p, node) + for link in links: + assert is_link(p, link) + + self.leave(p) + + def test_delete_nodes_links_then_restore_v2(self): p = 'test_delete_nodes_links_then_restore_v2' read_inp(p, f'./inp/net3.inp', '2')