From 6b09662de6921f7fa2e3d61ceb36e3516d4c47ca Mon Sep 17 00:00:00 2001 From: Jiang Date: Tue, 18 Aug 2026 18:29:09 +0800 Subject: [PATCH] refactor(storage): route project DSNs and remove legacy backends --- BACKEND_NAMING_AUDIT.md | 2 +- README.md | 14 +- app/api/v1/endpoints/cache.py | 57 - app/api/v1/endpoints/model_import.py | 27 +- app/api/v1/endpoints/network/junctions.py | 1 - app/api/v1/endpoints/network/pipes.py | 1 - app/api/v1/endpoints/network/pumps.py | 1 - app/api/v1/endpoints/network/tanks.py | 1 - app/api/v1/endpoints/network/valves.py | 1 - app/api/v1/rest_router.py | 47 +- app/api/v1/router.py | 8 - app/auth/project_dependencies.py | 57 + app/core/config.py | 6 - app/infra/cache/__init__.py | 0 app/infra/cache/redis_client.py | 19 - app/infra/db/influxdb/__init__.py | 0 app/infra/db/influxdb/api.py | 4964 ----------------- app/infra/db/influxdb/info.py | 5 - app/infra/db/influxdb/query.py | 33 - .../repositories/metadata_repository.py | 9 +- app/infra/db/postgresql/database.py | 14 +- app/infra/db/project_routing.py | 65 + app/infra/db/timescaledb/database.py | 18 +- app/infra/db/timescaledb/internal_queries.py | 26 +- app/native/wndb/connection.py | 22 +- app/native/wndb/project.py | 19 +- app/services/globals.py | 1 - app/services/network_import.py | 13 +- app/services/scheme_management.py | 30 +- app/services/simulation.py | 21 +- contracts/manifest.json | 2 +- contracts/server-v1.openapi.json | 393 -- infra/docker/docker-compose.yml | 15 - requirements.txt | 6 +- resources/old_requirements.txt | Bin 4554 -> 4476 bytes scripts/all_auto_task.py | 25 - scripts/auto_cache.py | 115 - scripts/auto_realtime.py | 156 - scripts/auto_store_non_realtime_SCADA_data.py | 139 - scripts/build_pyd.py | 7 +- scripts/get_data.py | 2 - scripts/install.py | 2 - scripts/main.py | 4481 --------------- scripts/main_api_endpoints.md | 395 -- scripts/redis_clear_all_keys.py | 5 - tests/api/test_model_import_endpoints.py | 46 + tests/api/test_openapi_contract.py | 94 +- .../test_metadata_repository_dsn_decrypt.py | 11 +- tests/unit/test_project_routing.py | 85 + tests/unit/test_scheme_list_filter.py | 4 +- tests/unit/test_wndb_connection.py | 28 +- 51 files changed, 542 insertions(+), 10951 deletions(-) delete mode 100644 app/api/v1/endpoints/cache.py delete mode 100644 app/infra/cache/__init__.py delete mode 100644 app/infra/cache/redis_client.py delete mode 100644 app/infra/db/influxdb/__init__.py delete mode 100644 app/infra/db/influxdb/api.py delete mode 100644 app/infra/db/influxdb/info.py delete mode 100644 app/infra/db/influxdb/query.py create mode 100644 app/infra/db/project_routing.py delete mode 100644 scripts/all_auto_task.py delete mode 100644 scripts/auto_cache.py delete mode 100644 scripts/auto_realtime.py delete mode 100644 scripts/auto_store_non_realtime_SCADA_data.py delete mode 100644 scripts/main.py delete mode 100644 scripts/main_api_endpoints.md delete mode 100644 scripts/redis_clear_all_keys.py create mode 100644 tests/unit/test_project_routing.py diff --git a/BACKEND_NAMING_AUDIT.md b/BACKEND_NAMING_AUDIT.md index dc5572b..4e76cc6 100644 --- a/BACKEND_NAMING_AUDIT.md +++ b/BACKEND_NAMING_AUDIT.md @@ -49,7 +49,7 @@ These route groups expose many command-style concatenated paths. They should not - Network object CRUD: `addjunction`, `getjunctionelevation`, `setpipediameter`, `getvalvesetting`, and similar junction/pipe/pump/tank/reservoir/valve routes - Region/DMA/VD commands: `calculatedistrictmeteringareaforregion`, `getdistrictmeteringarea`, `generatevirtualdistrict`, and related routes - SCADA native CRUD: `getscadadevice`, `setscadadevicedata`, `cleanscadaelement`, and related routes -- Snapshot/cache utilities: `takesnapshotforoperation`, `syncwithserver`, `clearrediskey`, `queryredis` +- Snapshot/synchronization utilities: `takesnapshotforoperation`, `syncwithserver` - Advanced simulation endpoints with underscore paths: `pressure_regulation`, `daily_scheduling_analysis`, `network_update`, `pressure_sensor_placement_kmeans` ### Direct Cleanup Candidates diff --git a/README.md b/README.md index 1662b04..c852278 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ - Python 3.12 - FastAPI / Uvicorn - Pydantic / SQLAlchemy / psycopg -- Redis、PostgreSQL、PostGIS、TimescaleDB +- PostgreSQL、PostGIS、TimescaleDB - WNTR、EPANET、Cython、科学计算与空间分析依赖 - pytest @@ -19,7 +19,7 @@ app/api/ HTTP API 路由 app/auth/ 认证和权限上下文 app/core/ 配置、日志和基础设施初始化 app/domain/ 领域模型和 Pydantic schema -app/infra/ 数据库、缓存、EPANET 和外部集成 +app/infra/ 数据库、EPANET 和外部集成 app/services/ 业务服务编排 app/algorithms/ 管网算法、模拟、爆管、漏损、清洗和健康分析 app/native/ 本地管网数据读写与转换 @@ -65,6 +65,16 @@ docker compose -f infra/docker/docker-compose.yml config - 优先复用现有 FastAPI/service/repository 边界。 - 不要把临时数据、数据库 dump、日志或本地运行产物纳入提交。 +## 项目数据库路由 + +项目级 REST 请求通过 `X-Project-Id` 解析元数据中的数据库配置： + +- `biz_data` DSN 用于管网业务数据；`{project_code}_template` 和模拟临时库沿用该 DSN 的主机、端口与凭据，仅替换数据库名。 +- `iot_data` DSN 用于 TimescaleDB，始终使用元数据配置的完整 DSN，不再从项目代码推导数据库名。 +- 元数据、业务库和 TimescaleDB 可以部署在同一主机，也可以分别部署。 + +使用模板复制或临时方案库的模拟功能时，`biz_data` 账号必须具备现有数据库创建、删除和连接终止操作所需的 PostgreSQL 权限。 + ## 测试与发布 提交前根据改动范围运行最小有效测试： diff --git a/app/api/v1/endpoints/cache.py b/app/api/v1/endpoints/cache.py deleted file mode 100644 index fee4ddc..0000000 --- a/app/api/v1/endpoints/cache.py +++ /dev/null @@ -1,57 +0,0 @@ -from fastapi import APIRouter, Query -from app.infra.cache.redis_client import redis_client - -router = APIRouter() - -@router.delete("/redis-keys/detail", summary="清除单个缓存键", description="根据键名清除单个Redis缓存") -async def fastapi_clear_redis_key(key: str = Query(..., description="缓存键名")): - """ - 清除单个缓存键 - - 根据指定的键名删除Redis中对应的缓存 - """ - redis_client.delete(key) - return True - - -@router.delete("/redis-keys", summary="清除匹配的缓存键", description="根据模式清除匹配的Redis缓存键") -async def fastapi_clear_redis_keys(keys: str = Query(..., description="缓存键模式（支持通配符）")): - """ - 清除匹配的缓存键 - - 根据指定的模式删除Redis中所有匹配的缓存键 - """ - # delete keys contains the key - matched_keys = redis_client.keys(f"*{keys}*") - if matched_keys: - redis_client.delete(*matched_keys) - - return True - - -@router.delete("/all-redis", summary="清除所有缓存", description="清空整个Redis数据库的所有缓存") -async def fastapi_clear_all_redis(): - """ - 清除所有缓存 - - 清空Redis数据库中的所有缓存键值对 - """ - redis_client.flushdb() - return True - - -@router.get("/redis", summary="查询缓存键列表", description="获取Redis中所有的缓存键") -async def fastapi_query_redis(): - """ - 查询缓存键列表 - - 获取Redis数据库中所有的缓存键列表 - """ - # Helper to decode bytes to str for JSON response if needed, - # but original just returned keys (which might be bytes in redis-py unless decode_responses=True) - # create_redis_client usually sets decode_responses=False by default. - # We will assume user handles bytes or we should decode. - # Original just returned redis_client.keys("*") - keys = redis_client.keys("*") - # Clean output for API - return [k.decode('utf-8') if isinstance(k, bytes) else k for k in keys] diff --git a/app/api/v1/endpoints/model_import.py b/app/api/v1/endpoints/model_import.py index ffd84e8..9eb28ae 100644 --- a/app/api/v1/endpoints/model_import.py +++ b/app/api/v1/endpoints/model_import.py @@ -17,8 +17,13 @@ from app.auth.metadata_dependencies import ( get_current_metadata_admin, get_metadata_repository, ) +from app.auth.project_dependencies import ( + ProjectContext, + resolve_project_business_routing, +) from app.core.audit import AuditAction, log_audit_event from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository +from app.infra.db.project_routing import activate_project_routing from app.services.network_import import network_update from app.services.tjnetwork import run_inp @@ -118,21 +123,21 @@ async def _run_uploaded_inp(content: bytes) -> str: return run_inp(model_name) -async def _update_from_inp(content: bytes) -> None: +async def _update_from_inp(content: bytes, project_code: str) -> None: temp_path: Path | None = None try: with NamedTemporaryFile(suffix=".inp", delete=False) as temp_file: temp_file.write(content) temp_path = Path(temp_file.name) - network_update(str(temp_path)) + network_update(str(temp_path), project_code) finally: if temp_path is not None: temp_path.unlink(missing_ok=True) -async def _apply_model_update(content: bytes) -> None: +async def _apply_model_update(content: bytes, project_code: str) -> None: try: - await _update_from_inp(content) + await _update_from_inp(content, project_code) except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -177,7 +182,19 @@ async def update_project_model( ) -> dict: project = await _get_active_project(project_id, metadata_repo) content, filename = await _read_upload(file) - await _apply_model_update(content) + routing = await resolve_project_business_routing( + ProjectContext( + project_id=project.id, + project_code=project.code, + user_id=current_user.id, + project_role="owner", + system_role=current_user.role, + is_superuser=current_user.is_superuser, + ), + metadata_repo, + ) + with activate_project_routing(routing): + await _apply_model_update(content, project.code) await _audit_model_change( request=request, current_user=current_user, diff --git a/app/api/v1/endpoints/network/junctions.py b/app/api/v1/endpoints/network/junctions.py index 4959dcd..b317ae5 100644 --- a/app/api/v1/endpoints/network/junctions.py +++ b/app/api/v1/endpoints/network/junctions.py @@ -333,7 +333,6 @@ async def fastapi_get_all_junction_properties( list: 包含所有节点属性的列表 """ # 缓存查询结果提高性能 - # global redis_client # Redis logic removed for clean split, can be re-added if needed or imported results = get_all_junctions(network) return results diff --git a/app/api/v1/endpoints/network/pipes.py b/app/api/v1/endpoints/network/pipes.py index d65a83b..c149771 100644 --- a/app/api/v1/endpoints/network/pipes.py +++ b/app/api/v1/endpoints/network/pipes.py @@ -385,7 +385,6 @@ async def fastapi_get_all_pipe_properties( 包含所有管道属性的字典列表 """ # 缓存查询结果提高性能 - # global redis_client results = get_all_pipes(network) return results diff --git a/app/api/v1/endpoints/network/pumps.py b/app/api/v1/endpoints/network/pumps.py index d947f67..2ef9431 100644 --- a/app/api/v1/endpoints/network/pumps.py +++ b/app/api/v1/endpoints/network/pumps.py @@ -177,7 +177,6 @@ async def fastapi_get_all_pump_properties( 包含所有水泵属性的字典列表 """ # 缓存查询结果提高性能 - # global redis_client results = get_all_pumps(network) return results diff --git a/app/api/v1/endpoints/network/tanks.py b/app/api/v1/endpoints/network/tanks.py index 9d579b0..1d5e84d 100644 --- a/app/api/v1/endpoints/network/tanks.py +++ b/app/api/v1/endpoints/network/tanks.py @@ -540,7 +540,6 @@ async def fastapi_get_all_tank_properties( 包含所有水箱属性的字典列表 """ # 缓存查询结果提高性能 - # global redis_client results = get_all_tanks(network) return results diff --git a/app/api/v1/endpoints/network/valves.py b/app/api/v1/endpoints/network/valves.py index 43acf30..d9244e8 100644 --- a/app/api/v1/endpoints/network/valves.py +++ b/app/api/v1/endpoints/network/valves.py @@ -307,7 +307,6 @@ async def fastapi_get_all_valve_properties( 返回指定水网中所有阀门的完整属性列表。 """ # 缓存查询结果提高性能 - # global redis_client results = get_all_valves(network) return results diff --git a/app/api/v1/rest_router.py b/app/api/v1/rest_router.py index 00ea840..941523a 100644 --- a/app/api/v1/rest_router.py +++ b/app/api/v1/rest_router.py @@ -17,7 +17,16 @@ from app.api.problem_details import ProblemDetails from app.api.pagination import PaginatedList from app.api.v1.router import api_router as handler_api_router from app.auth.metadata_dependencies import get_current_metadata_user -from app.auth.project_dependencies import ProjectContext, get_project_context +from app.auth.project_dependencies import ( + ProjectContext, + get_project_business_routing, + get_project_context, + get_project_simulation_routing, +) +from app.infra.db.project_routing import ( + ActiveProjectRouting, + activate_project_routing, +) T = TypeVar("T") @@ -44,6 +53,13 @@ _PUBLIC_PARAMETER_RENAMES = { } _MODEL_NAME_IS_NETWORK = {"RunSimulationManuallyByDate", "PressureSensorPlacement"} _MODEL_USERNAME_FROM_AUTH = {"PressureSensorPlacement"} +_TIMESCALE_ROUTED_ENDPOINT_MODULES = { + "app.api.v1.endpoints.burst_detection", + "app.api.v1.endpoints.burst_location", + "app.api.v1.endpoints.leakage", + "app.api.v1.endpoints.simulation", +} +_TIMESCALE_ROUTED_ENDPOINT_NAMES = {"open_project_endpoint"} def _clean_name(name: str) -> str: @@ -128,10 +144,14 @@ def _with_header_project_context(endpoint, route_name: str): None, ) injected_context_name = existing_context_parameter or "_rest_project_context" + injected_routing_name = "_rest_project_routing" injected_user_name = "_rest_current_user" @wraps(endpoint) async def wrapper(*args, **kwargs): + project_routing = kwargs.pop(injected_routing_name, None) + if not isinstance(project_routing, ActiveProjectRouting): + raise RuntimeError("REST project database routing was not resolved") project_context = kwargs.get(injected_context_name) if not isinstance(project_context, ProjectContext): raise RuntimeError("REST project context was not resolved") @@ -162,10 +182,11 @@ def _with_header_project_context(endpoint, route_name: str): kwargs[parameter_name] = original_model.model_validate(data) if model_has_username: kwargs.pop(injected_user_name, None) - result = endpoint(*args, **kwargs) - if inspect.isawaitable(result): - return await result - return result + with activate_project_routing(project_routing): + result = endpoint(*args, **kwargs) + if inspect.isawaitable(result): + return await result + return result parameters = [] for name, parameter in signature.parameters.items(): @@ -181,6 +202,14 @@ def _with_header_project_context(endpoint, route_name: str): if name in body_models: parameter = parameter.replace(annotation=body_models[name][1]) parameters.append(parameter) + routing_dependency = ( + get_project_simulation_routing + if ( + endpoint.__module__ in _TIMESCALE_ROUTED_ENDPOINT_MODULES + or endpoint.__name__ in _TIMESCALE_ROUTED_ENDPOINT_NAMES + ) + else get_project_business_routing + ) if not existing_context_parameter: parameters.append( inspect.Parameter( @@ -190,6 +219,14 @@ def _with_header_project_context(endpoint, route_name: str): default=Depends(get_project_context), ) ) + parameters.append( + inspect.Parameter( + injected_routing_name, + kind=inspect.Parameter.KEYWORD_ONLY, + annotation=ActiveProjectRouting, + default=Depends(routing_dependency), + ) + ) if username_parameter or model_has_username: parameters.append( inspect.Parameter( diff --git a/app/api/v1/router.py b/app/api/v1/router.py index f2b73ef..89133fe 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -7,7 +7,6 @@ from app.api.v1.endpoints import ( audit, burst_detection, burst_location, - cache, extension, geocoding, leakage, @@ -53,7 +52,6 @@ from app.api.v1.endpoints.timeseries import ( ) from app.auth.permissions import ( BURST_RUN, - ENVIRONMENT_MANAGE, OPTIMIZATION_RUN, RISK_RUN, SCADA_CLEAN, @@ -89,7 +87,6 @@ simulation_access = Depends( webgis_view_access = Depends(require_permission(WEBGIS_VIEW)) simulation_run_access = Depends(require_permission(SIMULATION_RUN)) -environment_manage_access = Depends(require_permission(ENVIRONMENT_MANAGE)) burst_run_access = Depends(require_permission(BURST_RUN)) risk_run_access = Depends(require_permission(RISK_RUN)) optimization_run_access = Depends(require_permission(OPTIMIZATION_RUN)) @@ -168,11 +165,6 @@ api_router.include_router( tags=["Risk"], dependencies=[risk_run_access], ) -api_router.include_router( - cache.router, - tags=["Cache"], - dependencies=[environment_manage_access], -) api_router.include_router( web_search.router, tags=["Web Search"], diff --git a/app/auth/project_dependencies.py b/app/auth/project_dependencies.py index 6a2d387..85895ff 100644 --- a/app/auth/project_dependencies.py +++ b/app/auth/project_dependencies.py @@ -16,6 +16,7 @@ from app.infra.db.metadb.repositories.metadata_repository import ( MetadataRepository, ProjectDbRouting, ) +from app.infra.db.project_routing import ActiveProjectRouting DB_ROLE_BIZ_DATA = "biz_data" DB_ROLE_IOT_DATA = "iot_data" @@ -99,6 +100,62 @@ async def get_project_context( return await resolve_project_context(x_project_id, current_user, metadata_repo) +async def get_project_business_routing( + ctx: ProjectContext = Depends(get_project_context), + metadata_repo: MetadataRepository = Depends(get_metadata_repository), +) -> ActiveProjectRouting: + return await resolve_project_business_routing(ctx, metadata_repo) + + +async def resolve_project_business_routing( + ctx: ProjectContext, + metadata_repo: MetadataRepository, +) -> ActiveProjectRouting: + business = await _get_project_routing( + metadata_repo, + ctx.project_id, + DB_ROLE_BIZ_DATA, + DB_TYPE_POSTGRES, + "PostgreSQL", + ) + return ActiveProjectRouting( + project_code=ctx.project_code, + business_dsn=business.dsn, + ) + + +async def get_project_simulation_routing( + ctx: ProjectContext = Depends(get_project_context), + metadata_repo: MetadataRepository = Depends(get_metadata_repository), +) -> ActiveProjectRouting: + return await resolve_project_simulation_routing(ctx, metadata_repo) + + +async def resolve_project_simulation_routing( + ctx: ProjectContext, + metadata_repo: MetadataRepository, +) -> ActiveProjectRouting: + business = await _get_project_routing( + metadata_repo, + ctx.project_id, + DB_ROLE_BIZ_DATA, + DB_TYPE_POSTGRES, + "PostgreSQL", + ) + timescale = await _get_project_routing( + metadata_repo, + ctx.project_id, + DB_ROLE_IOT_DATA, + DB_TYPE_TIMESCALE, + "TimescaleDB", + ) + return ActiveProjectRouting( + project_code=ctx.project_code, + business_dsn=business.dsn, + timescale_dsn=timescale.dsn, + ) + + async def _get_project_routing( metadata_repo: MetadataRepository, project_id: UUID, diff --git a/app/core/config.py b/app/core/config.py index 9a521bf..abb8d14 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -26,12 +26,6 @@ class Settings(BaseSettings): TIMESCALEDB_DB_PORT: str = "5433" TIMESCALEDB_DB_USER: str = "postgres" TIMESCALEDB_DB_PASSWORD: str = "password" - # InfluxDB - INFLUXDB_URL: str = "http://localhost:8086" - INFLUXDB_TOKEN: str = "token" - INFLUXDB_ORG: str = "org" - INFLUXDB_BUCKET: str = "bucket" - # Metadata Database Config (PostgreSQL) METADATA_DB_NAME: str = "system_hub" METADATA_DB_HOST: str = "localhost" diff --git a/app/infra/cache/__init__.py b/app/infra/cache/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/infra/cache/redis_client.py b/app/infra/cache/redis_client.py deleted file mode 100644 index c9b58f8..0000000 --- a/app/infra/cache/redis_client.py +++ /dev/null @@ -1,19 +0,0 @@ -import redis -import msgpack -from datetime import datetime -from typing import Any - -# Initialize Redis connection -redis_client = redis.Redis(host="127.0.0.1", port=6379, db=0) - -def encode_datetime(obj: Any) -> Any: - """Serialize datetime objects to dictionary format.""" - if isinstance(obj, datetime): - return {"__datetime__": True, "as_str": obj.strftime("%Y%m%dT%H:%M:%S.%f")} - return obj - -def decode_datetime(obj: Any) -> Any: - """Deserialize dictionary format to datetime objects.""" - if "__datetime__" in obj: - return datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") - return obj diff --git a/app/infra/db/influxdb/__init__.py b/app/infra/db/influxdb/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/infra/db/influxdb/api.py b/app/infra/db/influxdb/api.py deleted file mode 100644 index 9aa20b0..0000000 --- a/app/infra/db/influxdb/api.py +++ /dev/null @@ -1,4964 +0,0 @@ -from influxdb_client import ( - InfluxDBClient, - BucketsApi, - WriteApi, - OrganizationsApi, - Point, - QueryApi, - WriteOptions, - DeleteApi, - WritePrecision, -) -from typing import List, Dict -from datetime import datetime, timedelta, timezone -from influxdb_client.client.write_api import SYNCHRONOUS, ASYNCHRONOUS -from dateutil import parser -# import get_realValue -# import get_data -import psycopg -import time -import app.services.simulation as simulation -from app.services.tjnetwork import close_project, get_time, is_project_open, open_project -import schedule -import threading -import app.services.globals as globals -import csv -import pandas as pd -import openpyxl -import pytz -import app.infra.db.influxdb.info as influxdb_info -import app.services.project_info as project_info -import app.services.time_api as time_api -from app.core.config import get_pgconn_string - -# influxdb数据库连接信息 -url = influxdb_info.url -token = influxdb_info.token -org_name = influxdb_info.org -client = InfluxDBClient( - url=url, token=token, org=org_name, timeout=600 * 1000 -) # 600 seconds - - -def query_pg_scada_info_realtime(name: str) -> None: - """ - 查询pg数据库中，scada_info中，属于realtime的数据 - :param name: 数据库名称 - :return: - """ - # 连接数据库 - conn_string = get_pgconn_string(db_name=name) - try: - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 查询 transmission_mode 为 'realtime' 的记录 - cur.execute( - """ - SELECT type, api_query_id - FROM scada_info - WHERE transmission_mode = 'realtime'; - """ - ) - records = cur.fetchall() - # 清空全局列表 - globals.reservoir_liquid_level_realtime_ids.clear() - globals.tank_liquid_level_realtime_ids.clear() - globals.fixed_pump_realtime_ids.clear() - globals.variable_pump_realtime_ids.clear() - globals.source_outflow_realtime_ids.clear() - globals.pipe_flow_realtime_ids.clear() - globals.pressure_realtime_ids.clear() - globals.demand_realtime_ids.clear() - globals.quality_realtime_ids.clear() - # 根据 type 分类存储 api_query_id - for record in records: - record_type, api_query_id = record - if api_query_id is not None: # 确保 api_query_id 不为空 - if record_type == "reservoir_liquid_level": - globals.reservoir_liquid_level_realtime_ids.append( - api_query_id - ) - elif record_type == "tank_liquid_level": - globals.tank_liquid_level_realtime_ids.append(api_query_id) - elif record_type == "fixed_pump": - globals.fixed_pump_realtime_ids.append(api_query_id) - elif record_type == "variable_pump": - globals.variable_pump_realtime_ids.append(api_query_id) - elif record_type == "source_outflow": - globals.source_outflow_realtime_ids.append(api_query_id) - elif record_type == "pipe_flow": - globals.pipe_flow_realtime_ids.append(api_query_id) - elif record_type == "pressure": - globals.pressure_realtime_ids.append(api_query_id) - elif record_type == "demand": - globals.demand_realtime_ids.append(api_query_id) - elif record_type == "quality": - globals.quality_realtime_ids.append(api_query_id) - # 打印结果，方便调试 - # print("Query completed. Results:") - # print("Reservoir Liquid Level IDs:", globals.reservoir_liquid_level_realtime_ids) - # print("Tank Liquid Level IDs:", globals.tank_liquid_level_realtime_ids) - # print("Fixed Pump IDs:", globals.fixed_pump_realtime_ids) - # print("Variable Pump IDs:", globals.variable_pump_realtime_ids) - # print("Source Outflow IDs:", globals.source_outflow_realtime_ids) - # print("Pipe Flow IDs:", globals.pipe_flow_realtime_ids) - # print("Pressure IDs:", globals.pressure_realtime_ids) - # print("Demand IDs:", globals.demand_realtime_ids) - # print("Quality IDs:", globals.quality_realtime_ids) - except Exception as e: - print(f"查询时发生错误：{e}") - - -def query_pg_scada_info_non_realtime(name: str) -> None: - """ - 查询pg数据库中，scada_info中，属于non_realtime的数据，以及这些数据transmission_frequency的最大值 - :param name: 数据库名称 - :return: - """ - # 重新打开数据库 - if is_project_open(name): - close_project(name) - open_project(name) - dic_time = get_time(name) - globals.hydraulic_timestep = dic_time["HYDRAULIC TIMESTEP"] - # DingZQ, 2025-03-21 - # close_project(name) - # 连接数据库 - conn_string = get_pgconn_string(db_name=name) - try: - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 查询 transmission_mode 为 'non_realtime' 的记录 - cur.execute( - """ - SELECT type, api_query_id, transmission_frequency - FROM scada_info - WHERE transmission_mode = 'non_realtime'; - """ - ) - records = cur.fetchall() - # 清空全局列表 - globals.reservoir_liquid_level_non_realtime_ids.clear() - globals.fixed_pump_non_realtime_ids.clear() - globals.variable_pump_non_realtime_ids.clear() - globals.source_outflow_non_realtime_ids.clear() - globals.pipe_flow_non_realtime_ids.clear() - globals.pressure_non_realtime_ids.clear() - globals.demand_non_realtime_ids.clear() - globals.quality_non_realtime_ids.clear() - # 用于计算 transmission_frequency 最大值 - transmission_frequencies = [] - # 根据 type 分类存储 api_query_id - for record in records: - record_type, api_query_id, freq = record - if api_query_id is not None: # 确保 api_query_id 不为空 - if record_type == "reservoir_liquid_level": - globals.reservoir_liquid_level_non_realtime_ids.append( - api_query_id - ) - elif record_type == "fixed_pump": - globals.fixed_pump_non_realtime_ids.append(api_query_id) - elif record_type == "variable_pump": - globals.variable_pump_non_realtime_ids.append(api_query_id) - elif record_type == "source_outflow": - globals.source_outflow_non_realtime_ids.append(api_query_id) - elif record_type == "pipe_flow": - globals.pipe_flow_non_realtime_ids.append(api_query_id) - elif record_type == "pressure": - globals.pressure_non_realtime_ids.append(api_query_id) - elif record_type == "demand": - globals.demand_non_realtime_ids.append(api_query_id) - elif record_type == "quality": - globals.quality_non_realtime_ids.append(api_query_id) - # 收集 transmission_frequency，用于计算最大值 - if freq is not None: - transmission_frequencies.append(freq) - # 计算 transmission_frequency 最大值 - globals.transmission_frequency = ( - max(transmission_frequencies) if transmission_frequencies else None - ) - # 打印结果，方便调试 - # print("Query completed. Results:") - # print("Reservoir Liquid Level Non-Realtime IDs:", globals.reservoir_liquid_level_non_realtime_ids) - # print("Fixed Pump Non-Realtime IDs:", globals.fixed_pump_non_realtime_ids) - # print("Variable Pump Non-Realtime IDs:", globals.variable_pump_non_realtime_ids) - # print("Source Outflow Non-Realtime IDs:", globals.source_outflow_non_realtime_ids) - # print("Pipe Flow Non-Realtime IDs:", globals.pipe_flow_non_realtime_ids) - # print("Pressure Non-Realtime IDs:", globals.pressure_non_realtime_ids) - # print("Demand Non-Realtime IDs:", globals.demand_non_realtime_ids) - # print("Quality Non-Realtime IDs:", globals.quality_non_realtime_ids) - # print("Maximum Transmission Frequency:", globals.transmission_frequency) - # print("Hydraulic Timestep:", globals.hydraulic_timestep) - except Exception as e: - print(f"查询时发生错误：{e}") - - -def query_pg_scada_info(name: str) -> list[dict]: - """ - 查询pg数据库中,scada_info 的所有记录 - :param name: 数据库名称 - :return: 包含所有记录的列表,每条记录为一个字典 - """ - # 连接数据库 - conn_string = get_pgconn_string(db_name=name) - records_list = [] - - try: - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 查询 scada_info 表的所有记录 - cur.execute( - """ - SELECT id, type, transmission_mode, transmission_frequency, reliability - FROM public.scada_info; - """ - ) - records = cur.fetchall() - - # 将查询结果转换为字典列表 - for record in records: - record_dict = { - "id": record[0], - "type": record[1], - "transmission_mode": record[2], - "transmission_frequency": record[3], - "reliability": record[4], - } - records_list.append(record_dict) - - except Exception as e: - print(f"查询时发生错误:{e}") - return [] - - return records_list - - -# 2025/03/23 -def get_new_client() -> InfluxDBClient: - """每次调用返回一个新的 InfluxDBClient 实例。""" - return InfluxDBClient( - url=url, token=token, org=org_name, enable_gzip=True, timeout=600 * 1000 - ) # 600 seconds - - -# 2025/04/11, DingZQ -def create_write_options() -> WriteOptions: - """ - 创建一个写入选项 - """ - return WriteOptions( - jitter_interval=200, # 添加抖动以避免同时写入 - max_retry_delay=30000, # 最大重试延迟（毫秒） - max_retries=5, # 最大重试次数（0 表示不重试） - batch_size=10_000, # 每批次发送10,000个点 - flush_interval=10_000, # 10秒强制刷新 - retry_interval=5_000, # 失败重试间隔5秒 - ) - - -# 2025/02/01 -def delete_buckets(org_name: str) -> None: - """ - 删除InfluxDB中指定organization下的所有buckets。 - :param org_name: InfluxDB中organization的名称。 - :return: None - """ - client = get_new_client() - # 定义需要删除的 bucket 名称列表 - buckets_to_delete = [ - "SCADA_data", - "realtime_simulation_result", - "scheme_simulation_result", - ] - buckets_api = client.buckets_api() - buckets_obj = buckets_api.find_buckets(org=org_name) - # 确保 buckets_obj 拥有 buckets 属性 - if hasattr(buckets_obj, "buckets"): - for bucket in buckets_obj.buckets: - if bucket.name in buckets_to_delete: # 只删除特定名称的 bucket - try: - buckets_api.delete_bucket(bucket) - print(f"Bucket {bucket.name} has been deleted successfully.") - except Exception as e: - print(f"Failed to delete bucket {bucket.name}: {e}") - else: - print(f"Skipping bucket {bucket.name}. Not in the deletion list.") - else: - print("未找到 buckets 属性，无法迭代 buckets。") - client.close() - - -# 2025/02/01 -def create_and_initialize_buckets(org_name: str) -> None: - """ - 初始化influxdb的三个数据存储库，分别为SCADA_data、realtime_simulation_result、scheme_simulation_result - :param org_name: InfluxDB中organization的名称 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - # 先删除原有的，然后再进行初始化 - # delete_buckets(org_name) - - bucket_api = BucketsApi(client) - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - write_api = client.write_api( - write_options=WriteOptions(batch_size=1000, flush_interval=1000), - success_callback=success_callback, - error_callback=error_callback, - ) - org_api = OrganizationsApi(client) - # 获取 org_id - org = next((o for o in org_api.find_organizations() if o.name == org_name), None) - if not org: - raise ValueError(f"Organization '{org_name}' not found.") - org_id = org.id - print(f"Using Organization ID: {org_id}") - # 定义 Buckets 信息 - buckets = [ - {"name": "SCADA_data", "retention_rules": []}, - {"name": "realtime_simulation_result", "retention_rules": []}, - {"name": "scheme_simulation_result", "retention_rules": []}, - ] - # 创建一个临时存储点数据的列表 - points_to_write = [] - # 创建 Buckets 并初始化数据 - for bucket in buckets: - # 创建 Bucket - created_bucket = bucket_api.create_bucket( - bucket_name=bucket["name"], - retention_rules=bucket["retention_rules"], - org_id=org_id, - ) - print(f"Bucket '{bucket['name']}' created with ID: {created_bucket.id}") - # 根据 Bucket 初始化数据 - if bucket["name"] == "SCADA_data": - point = ( - Point("SCADA") - .tag("date", None) - .tag("description", None) - .tag("device_ID", None) - .field("monitored_value", 0.0) - .field("datacleaning_value", 0.0) - .field("simulation_value", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket="SCADA_data", org=org_name, record=point) - # print("Initialized SCADA_data with default structure.") - elif ( - bucket["name"] == "realtime_simulation_result" - ): # realtime_simulation_result - link_point = ( - Point("link") - .tag("date", None) - .tag("ID", None) - .field("flow", 0.0) - .field("leakage", 0.0) - .field("velocity", 0.0) - .field("headloss", 0.0) - .field("status", None) - .field("setting", 0.0) - .field("quality", 0.0) - .field("reaction", 0.0) - .field("friction", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(link_point) - node_point = ( - Point("node") - .tag("date", None) - .tag("ID", None) - .field("head", 0.0) - .field("pressure", 0.0) - .field("actualdemand", 0.0) - .field("demanddeficit", 0.0) - .field("totalExternalOutflow", 0.0) - .field("quality", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(node_point) - # write_api.write(bucket="realtime_simulation_result", org=org_name, record=link_point) - # write_api.write(bucket="realtime_simulation_result", org=org_name, record=node_point) - # print("Initialized realtime_simulation_result with default structure.") - elif bucket["name"] == "scheme_simulation_result": - link_point = ( - Point("link") - .tag("date", None) - .tag("ID", None) - .tag("scheme_type", None) - .tag("scheme_name", None) - .field("flow", 0.0) - .field("leakage", 0.0) - .field("velocity", 0.0) - .field("headloss", 0.0) - .field("status", None) - .field("setting", 0.0) - .field("quality", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(link_point) - node_point = ( - Point("node") - .tag("date", None) - .tag("ID", None) - .tag("scheme_type", None) - .tag("scheme_name", None) - .field("head", 0.0) - .field("pressure", 0.0) - .field("actualdemand", 0.0) - .field("demanddeficit", 0.0) - .field("totalExternalOutflow", 0.0) - .field("quality", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(node_point) - SCADA_point = ( - Point("SCADA") - .tag("date", None) - .tag("description", None) - .tag("device_ID", None) - .tag("scheme_type", None) - .tag("scheme_name", None) - .field("monitored_value", 0.0) - .field("datacleaning_value", 0.0) - .field("scheme_simulation_value", 0.0) - .time("2024-11-21T00:00:00Z", write_precision="s") - ) - points_to_write.append(SCADA_point) - # write_api.write(bucket="scheme_simulation_result", org=org_name, record=link_point) - # write_api.write(bucket="scheme_simulation_result", org=org_name, record=node_point) - # write_api.write(bucket="scheme_simulation_result", org=org_name, record=SCADA_point) - # print("Initialized scheme_simulation_result with default structure.") - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - print("All buckets created and initialized successfully.") - time.sleep(10) - print("Total points written:", points_written) - client.close() - - -def store_realtime_SCADA_data_to_influxdb( - get_real_value_time: str, bucket: str = "SCADA_data" -) -> None: - """ - 将SCADA数据通过数据接口导入数据库 - :param get_real_value_time: 获取数据的时间，格式如'2024-11-25T09:00:00+08:00' - :param bucket: (str): InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # 使用异步写入模式配置写入选项和回调函数 - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - - # 创建一个临时存储点数据的列表 - points_to_write = [] - - try_count = 0 - reservoir_liquid_level_realtime_data_list = [] - tank_liquid_level_realtime_data_list = [] - fixed_pump_realtime_data_list = [] - variable_pump_realtime_data_list = [] - source_outflow_realtime_data_list = [] - pipe_flow_realtime_data_list = [] - pressure_realtime_data_list = [] - demand_realtime_data_list = [] - quality_realtime_data_list = [] - while try_count <= 5: # 尝试6次 ******* - try: - try_count += 1 - if globals.reservoir_liquid_level_realtime_ids: - # print(globals.reservoir_liquid_level_realtime_ids) - reservoir_liquid_level_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.reservoir_liquid_level_realtime_ids) - ) - # print(reservoir_liquid_level_realtime_data_list) - if globals.tank_liquid_level_realtime_ids: - tank_liquid_level_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.tank_liquid_level_realtime_ids) - ) - if globals.fixed_pump_realtime_ids: - fixed_pump_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.fixed_pump_realtime_ids) - ) - if globals.variable_pump_realtime_ids: - variable_pump_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.variable_pump_realtime_ids) - ) - if globals.source_outflow_realtime_ids: - source_outflow_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.source_outflow_realtime_ids) - ) - if globals.pipe_flow_realtime_ids: - pipe_flow_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.pipe_flow_realtime_ids) - ) - if globals.pressure_realtime_ids: - pressure_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.pressure_realtime_ids) - ) - if globals.demand_realtime_ids: - demand_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.demand_realtime_ids) - ) - if globals.quality_realtime_ids: - quality_realtime_data_list = get_realValue.get_realValue( - ids=",".join(globals.quality_realtime_ids) - ) - except Exception as e: - print(e) - time.sleep(10) - else: - try_count = 100 - # 写入数据 - if reservoir_liquid_level_realtime_data_list: - for data in reservoir_liquid_level_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过3分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("reservoir_liquid_level_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if tank_liquid_level_realtime_data_list: - for data in tank_liquid_level_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("tank_liquid_level_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if fixed_pump_realtime_data_list: - for data in fixed_pump_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("fixed_pump_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if variable_pump_realtime_data_list: - for data in variable_pump_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("variable_pump_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if source_outflow_realtime_data_list: - for data in source_outflow_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("source_outflow_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if pipe_flow_realtime_data_list: - for data in pipe_flow_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("pipe_flow_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if pressure_realtime_data_list: - for data in pressure_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("pressure_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if demand_realtime_data_list: - for data in demand_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("demand_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - if quality_realtime_data_list: - for data in quality_realtime_data_list: - # 将 data['time'] 和 get_realValue_time 转换为 datetime 对象 - data_time = datetime.fromisoformat(data["time"]) - get_real_value_time_dt = datetime.fromisoformat( - get_real_value_time - ).replace(tzinfo=None) - # 将获取的时间转换为 UTC 时间 - get_real_value_time_utc = get_real_value_time_dt.astimezone(timezone.utc) - # 计算时间差（绝对值） - time_difference = abs((data_time - get_real_value_time_dt).total_seconds()) - # 判断时间差是否超过1分钟 - if time_difference > 60: # 超过1分钟 - monitored_value = None - else: # 小于等于3分钟 - monitored_value = float(data["monitored_value"]) - # 创建Point对象 - point = ( - Point("quality_realtime") - .tag( - "date", - datetime.fromisoformat(get_real_value_time).strftime("%Y-%m-%d"), - ) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", monitored_value) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(get_real_value_time_utc, write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # write_api.flush() - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # - - time.sleep(10) - print("Total points written:", points_written) - client.close() - - -def convert_time_format(original_time: str) -> str: - """ - 格式转换，将“2024-04-13T08:00:00+08:00"转为“2024-04-13 08:00:00” - :param original_time: str， “2024-04-13T08:00:00+08:00"格式的时间 - :return: str，“2024-04-13 08:00:00”格式的时间 - """ - new_time = original_time.replace("T", " ") - new_time = new_time.replace("+08:00", "") - return new_time - - -# 2025/01/10 -def store_non_realtime_SCADA_data_to_influxdb( - get_history_data_end_time: str, bucket: str = "SCADA_data" -) -> None: - """ - 获取某段时间内传回的scada数据 - :param get_history_data_end_time: 获取历史数据的终止时间时间，格式如'2024-11-25T09:00:00+08:00' - :param bucket: (str): InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # write_options = WriteOptions( - # jitter_interval=200, # 添加抖动以避免同时写入 - # max_retry_delay=30000 # 最大重试延迟（毫秒） - # ) - # 使用异步写入模式配置写入选项和回调函数 - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - - # 创建一个临时存储点数据的列表 - points_to_write = [] - - # 将end_date字符串转换为datetime对象 - end_date_dt = datetime.strptime( - convert_time_format(get_history_data_end_time), "%Y-%m-%d %H:%M:%S" - ) - end_date = end_date_dt.strftime("%Y-%m-%d %H:%M:%S") - # 将transmission_frequency字符串转换为timedelta对象 - transmission_frequency_dt = datetime.strptime( - globals.transmission_frequency, "%H:%M:%S" - ) - datetime(1900, 1, 1) - get_history_data_start_time = end_date_dt - transmission_frequency_dt - begin_date = get_history_data_start_time.strftime("%Y-%m-%d %H:%M:%S") - # print(begin_date) - # print(end_date) - reservoir_liquid_level_non_realtime_data_list = [] - tank_liquid_level_non_realtime_data_list = [] - fixed_pump_non_realtime_data_list = [] - variable_pump_non_realtime_data_list = [] - source_outflow_non_realtime_data_list = [] - pipe_flow_non_realtime_data_list = [] - pressure_non_realtime_data_list = [] - demand_non_realtime_data_list = [] - quality_non_realtime_data_list = [] - try_count = 0 - while try_count < 5: - try: - try_count += 1 - # reservoir_liquid_level_non_realtime_data_list = get_data.get_history_data( - # ids=','.join(reservoir_liquid_level_non_realtime_ids), begin_date=begin_date, end_date=end_date, downsample='1m') - if globals.reservoir_liquid_level_non_realtime_ids: - reservoir_liquid_level_non_realtime_data_list = ( - get_data.get_history_data( - ids=",".join(globals.reservoir_liquid_level_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - ) - if globals.tank_liquid_level_non_realtime_ids: - tank_liquid_level_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.tank_liquid_level_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.fixed_pump_non_realtime_ids: - fixed_pump_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.fixed_pump_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.variable_pump_non_realtime_ids: - variable_pump_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.variable_pump_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.source_outflow_non_realtime_ids: - source_outflow_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.source_outflow_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.pipe_flow_non_realtime_ids: - pipe_flow_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pipe_flow_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - # print(pipe_flow_non_realtime_data_list) - if globals.pressure_non_realtime_ids: - pressure_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pressure_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - # print(pressure_non_realtime_data_list) - if globals.demand_non_realtime_ids: - demand_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.demand_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.quality_non_realtime_ids: - quality_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.quality_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - except Exception as e: - print(f"Attempt {try_count} failed with error: {e}") - if try_count < 5: - print("Retrying in 10 seconds...") - time.sleep(10) - else: - print("Max retries reached. Exiting.") - else: - print("Data fetched successfully.") - break # 成功后退出循环 - if reservoir_liquid_level_non_realtime_data_list: - for data in reservoir_liquid_level_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("reservoir_liquid_level_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if tank_liquid_level_non_realtime_data_list: - for data in tank_liquid_level_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("tank_liquid_level_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if fixed_pump_non_realtime_data_list: - for data in fixed_pump_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("fixed_pump_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if variable_pump_non_realtime_data_list: - for data in variable_pump_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("variable_pump_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if source_outflow_non_realtime_data_list: - for data in source_outflow_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("source_outflow_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pipe_flow_non_realtime_data_list: - for data in pipe_flow_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("pipe_flow_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pressure_non_realtime_data_list: - for data in pressure_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("pressure_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if demand_non_realtime_data_list: - for data in demand_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("demand_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if quality_non_realtime_data_list: - for data in quality_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("quality_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - - time.sleep(10) - - print("Total points written:", points_written) - - client.close() - - -# 2025/03/01 -def download_history_data_manually( - begin_time: str, end_time: str, bucket: str = "SCADA_data" -) -> None: - """ - 获取某个时间段内所有SCADA设备的历史数据，非实时执行，手动补充数据版 - :param begin_time: 获取历史数据的开始时间，格式如'2024-11-25T09:00:00+08:00' - :param end_time: 获取历史数据的结束时间，格式如'2024-11-25T09:00:00+08:00' - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # write_options = WriteOptions( - # jitter_interval=200, # 添加抖动以避免同时写入 - # max_retry_delay=30000 # 最大重试延迟（毫秒） - # ) - # write_api = client.write_api(write_options=SYNCHRONOUS, success_callback=success_callback, error_callback=error_callback) - # 使用异步写入模式配置写入选项和回调函数 - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - # 创建一个临时存储点数据的列表 - points_to_write = [] - - begin_date = convert_time_format(begin_time) - end_date = convert_time_format(end_time) - - reservoir_liquid_level_realtime_data_list = [] - tank_liquid_level_realtime_data_list = [] - fixed_pump_realtime_data_list = [] - variable_pump_realtime_data_list = [] - source_outflow_realtime_data_list = [] - pipe_flow_realtime_data_list = [] - pressure_realtime_data_list = [] - demand_realtime_data_list = [] - quality_realtime_data_list = [] - - reservoir_liquid_level_non_realtime_data_list = [] - tank_liquid_level_non_realtime_data_list = [] - fixed_pump_non_realtime_data_list = [] - variable_pump_non_realtime_data_list = [] - source_outflow_non_realtime_data_list = [] - pipe_flow_non_realtime_data_list = [] - pressure_non_realtime_data_list = [] - demand_non_realtime_data_list = [] - quality_non_realtime_data_list = [] - - try_count = 0 - while try_count < 5: - try: - try_count += 1 - if globals.reservoir_liquid_level_realtime_ids: - reservoir_liquid_level_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.reservoir_liquid_level_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.tank_liquid_level_realtime_ids: - tank_liquid_level_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.tank_liquid_level_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.fixed_pump_realtime_ids: - fixed_pump_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.fixed_pump_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.variable_pump_realtime_ids: - variable_pump_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.variable_pump_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.source_outflow_realtime_ids: - source_outflow_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.source_outflow_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.pipe_flow_realtime_ids: - pipe_flow_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pipe_flow_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.pressure_realtime_ids: - pressure_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pressure_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.demand_realtime_ids: - demand_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.demand_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.quality_realtime_ids: - quality_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.quality_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - # reservoir_liquid_level_non_realtime_data_list = get_data.get_history_data( - # ids=','.join(reservoir_liquid_level_non_realtime_ids), begin_date=begin_date, end_date=end_date, downsample='1m') - if globals.reservoir_liquid_level_non_realtime_ids: - reservoir_liquid_level_non_realtime_data_list = ( - get_data.get_history_data( - ids=",".join(globals.reservoir_liquid_level_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - ) - if globals.tank_liquid_level_non_realtime_ids: - tank_liquid_level_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.tank_liquid_level_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.fixed_pump_non_realtime_ids: - fixed_pump_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.fixed_pump_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.variable_pump_non_realtime_ids: - variable_pump_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.variable_pump_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.source_outflow_non_realtime_ids: - source_outflow_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.source_outflow_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.pipe_flow_non_realtime_ids: - pipe_flow_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pipe_flow_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - # print(pipe_flow_non_realtime_data_list) - if globals.pressure_non_realtime_ids: - pressure_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.pressure_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - # print(pressure_non_realtime_data_list) - if globals.demand_non_realtime_ids: - demand_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.demand_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - if globals.quality_non_realtime_ids: - quality_non_realtime_data_list = get_data.get_history_data( - ids=",".join(globals.quality_non_realtime_ids), - begin_date=begin_date, - end_date=end_date, - downsample="1m", - ) - except Exception as e: - print(f"Attempt {try_count} failed with error: {e}") - if try_count < 5: - print("Retrying in 10 seconds...") - time.sleep(10) - else: - print("Max retries reached. Exiting.") - else: - print("Data fetched successfully.") - break # 成功后退出循环 - - if reservoir_liquid_level_realtime_data_list: - for data in reservoir_liquid_level_realtime_data_list: - # 创建Point对象 - point = ( - Point("reservoir_liquid_level_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if tank_liquid_level_realtime_data_list: - for data in tank_liquid_level_realtime_data_list: - # 创建Point对象 - point = ( - Point("tank_liquid_level_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if fixed_pump_realtime_data_list: - for data in fixed_pump_realtime_data_list: - # 创建Point对象 - point = ( - Point("fixed_pump_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if variable_pump_realtime_data_list: - for data in variable_pump_realtime_data_list: - # 创建Point对象 - point = ( - Point("variable_pump_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if source_outflow_realtime_data_list: - for data in source_outflow_realtime_data_list: - # 创建Point对象 - point = ( - Point("source_outflow_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pipe_flow_realtime_data_list: - for data in pipe_flow_realtime_data_list: - # 创建Point对象 - point = ( - Point("pipe_flow_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pressure_realtime_data_list: - for data in pressure_realtime_data_list: - # 创建Point对象 - point = ( - Point("pressure_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if demand_realtime_data_list: - for data in demand_realtime_data_list: - # 创建Point对象 - point = ( - Point("demand_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if quality_realtime_data_list: - for data in quality_realtime_data_list: - # 创建Point对象 - point = ( - Point("quality_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if reservoir_liquid_level_non_realtime_data_list: - for data in reservoir_liquid_level_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("reservoir_liquid_level_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if tank_liquid_level_non_realtime_data_list: - for data in tank_liquid_level_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("tank_liquid_level_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if fixed_pump_non_realtime_data_list: - for data in fixed_pump_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("fixed_pump_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if variable_pump_non_realtime_data_list: - for data in variable_pump_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("variable_pump_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if source_outflow_non_realtime_data_list: - for data in source_outflow_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("source_outflow_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pipe_flow_non_realtime_data_list: - for data in pipe_flow_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("pipe_flow_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if pressure_non_realtime_data_list: - for data in pressure_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("pressure_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if demand_non_realtime_data_list: - for data in demand_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("demand_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - if quality_non_realtime_data_list: - for data in quality_non_realtime_data_list: - # 创建Point对象 - point = ( - Point("quality_non_realtime") - .tag("date", data["time"].strftime("%Y-%m-%d")) - .tag("description", data["description"]) - .tag("device_ID", data["device_ID"]) - .field("monitored_value", float(data["monitored_value"])) - .field("datacleaning_value", None) - .field("simulation_value", None) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - - time.sleep(10) - - print("Total points written:", points_written) - - client.close() - - -########################SCADA############################################################################################################ - - -# DingZQ, 2025-03-08 -def query_all_SCADA_records_by_date( - query_date: str, bucket: str = "SCADA_data" -) -> list[dict[str, float]]: - """ - 根据日期查询所有SCADA数据 - - :param query_date: 输入的日期，格式为 '2024-11-24', 日期是北京时间的日期 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :param client: 已初始化的 InfluxDBClient 实例。 - - :return: - """ - client = get_new_client() - - if client.ping(): - print( - "{} -- Successfully connected to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - else: - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - - bg_start_time, bg_end_time = time_api.parse_beijing_date_range(query_date) - # bg_end_time = bg_start_time + timedelta(hours=2) # 服务器性能不行，暂时返回2个小时的数据 - utc_start_time = bg_start_time.astimezone(timezone.utc) - utc_end_time = bg_end_time.astimezone(timezone.utc) - - print(f"utc_start_time: {utc_start_time}, utc_end_time: {utc_end_time}") - - # 构建查询字典 - SCADA_results = [] - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_end_time.isoformat()}) - |> filter(fn: (r) => r["_field"] == "monitored_value") - |> sort(columns: ["_time"], desc: false) - """ - - # 执行查询 - try: - result = query_api.query(flux_query) - - # 从查询结果中提取 monitored_value - if result: - # 假设返回的结果为一行数据 - for table in result: - for record in table.records: - # 获取字段 "_value" 即为 monitored_value - monitored_value = record.get_value() - rec = { - "ID": record["device_ID"], # 是api_query 而不是 普通的Id - "time": record.get_time(), - record["_measurement"]: monitored_value, - } - SCADA_results.append(rec) - - except Exception as e: - print(f"Error querying InfluxDB for date {query_date}: {e}") - - client.close() - - return SCADA_results - - -def query_SCADA_data_by_device_ID_and_time( - query_ids_list: List[str], query_time: str, bucket: str = "SCADA_data" -) -> Dict[str, float]: - """ - 根据SCADA设备的ID和时间查询值 - :param query_ids_list: SCADA设备ID的列表 - :param query_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建查询字典 - SCADA_result_dict = {} - for device_id in query_ids_list: - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "monitored_value") - """ - # 执行查询 - try: - result = query_api.query(flux_query) - # 从查询结果中提取 monitored_value - if result: - # 假设返回的结果为一行数据 - for table in result: - for record in table.records: - # 获取字段 "_value" 即为 monitored_value - monitored_value = record.get_value() - SCADA_result_dict[device_id] = monitored_value - else: - # 如果没有结果，默认设置为 None 或其他值 - SCADA_result_dict[device_id] = None - except Exception as e: - print(f"Error querying InfluxDB for device ID {device_id}: {e}") - SCADA_result_dict[device_id] = None - client.close() - - return SCADA_result_dict - - -def query_scheme_SCADA_data_by_device_ID_and_time( - query_ids_list: List[str], - query_time: str, - scheme_type: str, - scheme_name: str, - bucket: str = "scheme_simulation_result", -) -> Dict[str, float]: - """ - 根据SCADA设备的ID和时间查询方案中的值 - :param query_ids_list: SCADA设备ID的列表 - :param query_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建查询字典 - SCADA_result_dict = {} - for device_id in query_ids_list: - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "monitored_value" and r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}") - """ - # 执行查询 - try: - result = query_api.query(flux_query) - # 从查询结果中提取 monitored_value - if result: - # 假设返回的结果为一行数据 - for table in result: - for record in table.records: - # 获取字段 "_value" 即为 monitored_value - monitored_value = record.get_value() - SCADA_result_dict[device_id] = monitored_value - else: - # 如果没有结果，默认设置为 None 或其他值 - SCADA_result_dict[device_id] = None - except Exception as e: - print(f"Error querying InfluxDB for device ID {device_id}: {e}") - SCADA_result_dict[device_id] = None - - client.close() - - return SCADA_result_dict - - -# 2025/03/14 -# DingZQ -# 返回SCADA数据的原始值，其中可能包含了异常值跟缺失值，我们需要再后续曲线中修复 -# 缺失值 -# 异常值 -def query_SCADA_data_by_device_ID_and_timerange( - query_ids_list: List[str], - start_time: str, - end_time: str, - bucket: str = "SCADA_data", -): - """ - 查询指定时间范围内，多个SCADA设备的数据，用于漏损定位 - :param query_ids_list: SCADA设备ID的列表 - :param start_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param end_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - print("start_time", start_time) - print("end_time", end_time) - # 将北京时间转换为 UTC 时间 - # beijing_start_time = datetime.fromisoformat(start_time) - # utc_start_time = beijing_start_time.astimezone(timezone.utc) - timedelta(seconds=1) - # print(utc_start_time) - # beijing_end_time = datetime.fromisoformat(end_time) - # utc_end_time = beijing_end_time.astimezone(timezone.utc) + timedelta(seconds=1) - # print(utc_end_time) - beijing_start_time = datetime.fromisoformat(start_time) - print("beijing_start_time", beijing_start_time) - utc_start_time = time_api.to_utc_time(beijing_start_time) - print("utc_start_time", utc_start_time) - beijing_end_time = datetime.fromisoformat(end_time) - print("beijing_end_time", beijing_end_time) - utc_stop_time = time_api.to_utc_time(beijing_end_time) - print("utc_stop_time", utc_stop_time) - SCADA_dict = {} - for device_id in query_ids_list: - # flux_query = f''' - # from(bucket: "{bucket}") - # |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - # |> filter(fn: (r) => r["_measurement"] == "SCADA_data" and r["device_ID"] == {device_id} and r["_field"] == "monitored_value") - # |> pivot(rowKey: ["_time"], columnKey: ["device_ID"], valueColumn: "_value") - # |> sort(columns: ["_time"]) - # ''' - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "monitored_value") - |> sort(columns: ["_time"]) - """ - # 执行查询，返回一个 FluxTable 列表 - tables = query_api.query(flux_query) - records_list = [] - for table in tables: - for record in table.records: - # 获取记录的时间和监测值 - records_list.append( - {"time": record["_time"], "value": record["_value"]} - ) - SCADA_dict[device_id] = records_list - - client.close() - - return SCADA_dict - - -# 2025/05/04 DingZQ -# SCADA 原始数据有异常偏离，返回的是一个list，list的内容是清洗后的正常值，表示为 time + value -def query_cleaning_SCADA_data_by_device_ID_and_timerange( - query_ids_list: List[str], - start_time: str, - end_time: str, - bucket: str = "SCADA_data", -): - """ - 查询指定时间范围内，多个SCADA设备的修复的单个的数据 - :param query_ids_list: SCADA设备ID的列表 - :param start_time: 输入的北京时间，格式为 '2023-11-24T17:30:00+08:00'。 - :param end_time: 输入的北京时间，格式为 '2023-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - print("start_time", start_time) - print("end_time", end_time) - # 将北京时间转换为 UTC 时间 - beijing_start_time = datetime.fromisoformat(start_time) - print("beijing_start_time", beijing_start_time) - utc_start_time = time_api.to_utc_time(beijing_start_time) - print("utc_start_time", utc_start_time) - beijing_end_time = datetime.fromisoformat(end_time) - print("beijing_end_time", beijing_end_time) - utc_stop_time = time_api.to_utc_time(beijing_end_time) - print("utc_stop_time", utc_stop_time) - - SCADA_dict = {} - for device_id in query_ids_list: - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "datacleaning_value") - |> sort(columns: ["_time"]) - """ - # 执行查询，返回一个 FluxTable 列表 - tables = query_api.query(flux_query) - print(tables) - records_list = [] - for table in tables: - for record in table.records: - # 获取记录的时间和监测值 - records_list.append( - {"time": record["_time"], "value": record["_value"]} - ) - SCADA_dict[device_id] = records_list - - client.close() - - return SCADA_dict - -# 查找 SCADA 模拟数据，返回的是一个list，list的内容是清洗后的正常值，表示为 time + value -def query_simulation_SCADA_data_by_device_ID_and_timerange( - query_ids_list: List[str], - start_time: str, - end_time: str, - bucket: str = "SCADA_data", -): - """ - 查询指定时间范围内，多个SCADA设备的修复的单个的数据 - :param query_ids_list: SCADA设备ID的列表 - :param start_time: 输入的北京时间，格式为 '2023-11-24T17:30:00+08:00'。 - :param end_time: 输入的北京时间，格式为 '2023-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - print("start_time", start_time) - print("end_time", end_time) - # 将北京时间转换为 UTC 时间 - beijing_start_time = datetime.fromisoformat(start_time) - print("beijing_start_time", beijing_start_time) - utc_start_time = time_api.to_utc_time(beijing_start_time) - print("utc_start_time", utc_start_time) - beijing_end_time = datetime.fromisoformat(end_time) - print("beijing_end_time", beijing_end_time) - utc_stop_time = time_api.to_utc_time(beijing_end_time) - print("utc_stop_time", utc_stop_time) - - SCADA_dict = {} - for device_id in query_ids_list: - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "datacleaning_value") - |> sort(columns: ["_time"]) - """ - # 执行查询，返回一个 FluxTable 列表 - tables = query_api.query(flux_query) - print(tables) - records_list = [] - for table in tables: - for record in table.records: - # 获取记录的时间和监测值 - records_list.append( - {"time": record["_time"], "value": record["_value"]} - ) - SCADA_dict[device_id] = records_list - - client.close() - - return SCADA_dict - - -# 2025/05/04 DingZQ -# SCADA 数据原版缺失，根据历史数据的平均值补上缺失的部分 -def query_filling_SCADA_data_by_device_ID_and_timerange( - query_ids_list: List[str], - start_time: str, - end_time: str, - bucket: str = "SCADA_data", -): - """ - 查询指定时间范围内，多个SCADA设备的填补的单个的数据 - :param query_ids_list: SCADA设备ID的列表 - :param start_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param end_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - print("start_time", start_time) - print("end_time", end_time) - # 将北京时间转换为 UTC 时间 - beijing_start_time = datetime.fromisoformat(start_time) - print("beijing_start_time", beijing_start_time) - utc_start_time = time_api.to_utc_time(beijing_start_time) - print("utc_start_time", utc_start_time) - beijing_end_time = datetime.fromisoformat(end_time) - print("beijing_end_time", beijing_end_time) - utc_stop_time = time_api.to_utc_time(beijing_end_time) - print("utc_stop_time", utc_stop_time) - - SCADA_dict = {} - for device_id in query_ids_list: - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "datafilling_value") - |> sort(columns: ["_time"]) - """ - # 执行查询，返回一个 FluxTable 列表 - tables = query_api.query(flux_query) - print(tables) - records_list = [] - for table in tables: - for record in table.records: - # 获取记录的时间和监测值 - records_list.append( - {"time": record["_time"], "value": record["_value"]} - ) - SCADA_dict[device_id] = records_list - - client.close() - - return SCADA_dict - - -# 2025/05/04 DingZQ -# 是把原始数据跟清洗后的数据合并到一起，暂时不需要用这个API -def query_cleaned_SCADA_data_by_device_ID_and_timerange( - query_ids_list: List[str], - start_time: str, - end_time: str, - bucket: str = "SCADA_data", -): - """ - 查询指定时间范围内，多个SCADA设备的清洗完毕后的完整数据 - :param query_ids_list: SCADA设备ID的列表 - :param start_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param end_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - print("start_time", start_time) - print("end_time", end_time) - # 将北京时间转换为 UTC 时间 - beijing_start_time = datetime.fromisoformat(start_time) - print("beijing_start_time", beijing_start_time) - utc_start_time = time_api.to_utc_time(beijing_start_time) - print("utc_start_time", utc_start_time) - beijing_end_time = datetime.fromisoformat(end_time) - print("beijing_end_time", beijing_end_time) - utc_stop_time = time_api.to_utc_time(beijing_end_time) - print("utc_stop_time", utc_stop_time) - SCADA_dict = {} - for device_id in query_ids_list: - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["device_ID"] == "{device_id}" and r["_field"] == "cleaned_value") - |> sort(columns: ["_time"]) - """ - # 执行查询，返回一个 FluxTable 列表 - tables = query_api.query(flux_query) - print(tables) - records_list = [] - for table in tables: - for record in table.records: - # 获取记录的时间和监测值 - records_list.append( - {"time": record["_time"], "value": record["_value"]} - ) - SCADA_dict[device_id] = records_list - - client.close() - - return SCADA_dict - - -# DingZQ, 2025-02-15 -def query_SCADA_data_by_device_ID_and_date( - query_ids_list: List[str], query_date: str, bucket: str = "SCADA_data" -) -> list[dict[str, float]]: - """ - 根据SCADA设备的ID和日期查询值 - :param query_ids_list: SCADA设备ID的列表, 是api_query 而不是 普通的Id - :param query_date: 输入的日期，格式为 '2024-11-24', 日期是北京时间的日期 - :param bucket: InfluxDB 的 bucket 名称，默认值为 "SCADA_data"。 - :param client: 已初始化的 InfluxDBClient 实例。 - :return: - """ - - start_time, end_time = time_api.parse_beijing_date_range(query_date) - - return query_SCADA_data_by_device_ID_and_timerange( - query_ids_list, str(start_time), str(end_time), bucket - ) - - -# 2025/02/01 -def store_realtime_simulation_result_to_influxdb( - node_result_list: List[Dict[str, any]], - link_result_list: List[Dict[str, any]], - result_start_time: str, - bucket: str = "realtime_simulation_result", -): - """ - 将实时模拟计算结果数据存储到 InfluxDB 的realtime_simulation_result这个bucket中。 - :param node_result_list: (List[Dict[str, any]]): 包含节点和结果数据的字典列表。 - :param link_result_list: (List[Dict[str, any]]): 包含连接和结果数据的字典列表。 - :param result_start_time: (str): 计算结果的模拟开始时间。 - :param bucket: (str): InfluxDB 的 bucket 名称，默认值为 "realtime_simulation_result"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - print( - "store_realtime_simulation_result_to_influxdb : result_start_time ", - result_start_time, - ) - - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # 开始写入数据 - try: - # 使用异步写入模式配置写入选项和回调函数 - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - # 创建一个临时存储点数据的列表 - points_to_write = [] - date_str = result_start_time.split("T")[0] - print("store_realtime_simulation_result_to_influxdb : date_str ", date_str) - - time_beijing = datetime.strptime( - result_start_time, "%Y-%m-%dT%H:%M:%S%z" - ).isoformat() - for result in node_result_list: - # 提取节点信息和结果数据 - node_id = result.get("node") - data_list = result.get("result", []) - for data in data_list: - # 构建 Point 数据，多个 field 存在于一个数据点中 - node_point = ( - Point("node") - .tag("date", date_str) - .tag("ID", node_id) - .field("head", data.get("head", 0.0)) - .field("pressure", data.get("pressure", 0.0)) - .field("actualdemand", data.get("demand", 0.0)) - .field("demanddeficit", None) - .field("totalExternalOutflow", None) - .field("quality", data.get("quality", 0.0)) - .time(time_beijing, write_precision="s") - ) - points_to_write.append(node_point) - # 写入数据到 InfluxDB，多个 field 在同一个 point 中 - # write_api.write(bucket=bucket, org=org_name, record=node_point) - # write_api.flush() - # print(f"成功将 {len(node_result_list)} 条node数据写入 InfluxDB。") - for result in link_result_list: - link_id = result.get("link") - data_list = result.get("result", []) - for data in data_list: - link_point = ( - Point("link") - .tag("date", date_str) - .tag("ID", link_id) - .field("flow", data.get("flow", 0.0)) - .field("velocity", data.get("velocity", 0.0)) - .field("headloss", data.get("headloss", 0.0)) - .field("quality", data.get("quality", 0.0)) - .field("status", data.get("status", "UNKNOWN")) - .field("setting", data.get("setting", 0.0)) - .field("reaction", data.get("reaction", 0.0)) - .field("friction", data.get("friction", 0.0)) - .time(time_beijing, write_precision="s") - ) - points_to_write.append(link_point) - # write_api.write(bucket=bucket, org=org_name, record=link_point) - # write_api.flush() - # print(f"成功将 {len(link_result_list)} 条link数据写入 InfluxDB。") - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - except Exception as e: - client.close() - raise RuntimeError(f"数据写入 InfluxDB 时发生错误: {e}") - - time.sleep(10) - - print("Total points written:", points_written) - - client.close() - - -# 2025/02/01 -def query_latest_record_by_ID( - ID: str, type: str, bucket: str = "realtime_simulation_result" -) -> dict: - """ - 查询指定ID的最新的一条记录 - :param ID: (str): 要查询的 ID。 - :param type: (str): "node"或“link” - :param bucket: (str): 数据存储的 bucket 名称。 - :return: dict: 最新记录的数据，如果没有找到则返回 None。 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - if type == "node": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: -1d, stop: now()) // 查找最近七天的记录 - |> filter(fn: (r) => r["_measurement"] == "node" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - |> group() // 将所有数据聚合到同一个 group - |> sort(columns: ["_time"], desc: true) - |> limit(n: 1) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - return { - "time": record["_time"], - "ID": ID, - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - # "demanddeficit": record["demanddeficit"], - # "totalExternalOutflow": record["totalExternalOutflow"], - "quality": record["quality"], - } - elif type == "link": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: -1d, stop: now()) // 查找最近七天的记录 - |> filter(fn: (r) => r["_measurement"] == "link" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - |> group() // 将所有数据聚合到同一个 group - |> sort(columns: ["_time"], desc: true) - |> limit(n: 1) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - return { - "time": record["_time"], - "ID": ID, - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - client.close() - return None # 如果没有找到记录 - - -# 2025/02/01 -def query_all_records_by_time( - query_time: str, bucket: str = "realtime_simulation_result" -) -> tuple: - """ - 查询指定北京时间的所有记录，包括 'node' 和 'link' measurement，分别以指定格式返回。 - :param query_time: (str): 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: (str): 数据存储的 bucket 名称。 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "node" or r["_measurement"] == "link") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - # 执行查询 - tables = query_api.query(flux_query) - node_records = [] - link_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - node_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - link_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - client.close() - return node_records, link_records - - -# 2025/03/03 -def query_all_record_by_time_property( - query_time: str, - type: str, - property: str, - bucket: str = "realtime_simulation_result", -) -> list: - """ - 查询指定北京时间的所有记录，查询 'node' 或 'link' 的某一属性值，以指定格式返回。 - :param query_time: (str): 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param type: (str): 查询的类型（决定 measurement） - :param property: (str): 查询的字段名称（field） - :param bucket: (str): 数据存储的 bucket 名称。 - :return: list(dict): result_records - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "{measurement}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - result_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - result_records.append({"ID": record["ID"], "value": record["_value"]}) - client.close() - return result_records - - -def query_all_scheme_record_by_time_property( - query_time: str, - type: str, - property: str, - scheme_name: str, - bucket: str = "scheme_simulation_result", -) -> list: - """ - 查询指定北京时间的所有记录,查询 'node' 或 'link' 的某一属性值,以指定格式返回(新版本)。 - - :param query_time: (str): 输入的北京时间,格式为 '2024-11-24T17:30:00+08:00'。 - :param type: (str): 查询的类型(决定 measurement),'node' 或 'link' - :param property: (str): 查询的字段名称(field) - :param scheme_name: (str): 方案名称(如 "FANGAN1761124840355") - :param bucket: (str): 数据存储的 bucket 名称。 - :return: list(dict): result_records - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "{measurement}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - - result_records = [] - - # 解析查询结果 - for table in tables: - for record in table.records: - result_records.append({"ID": record["ID"], "value": record["_value"]}) - - client.close() - return result_records - - -def query_scheme_simulation_result_by_ID_time( - scheme_name: str, - ID: str, - type: str, - query_time: str, - bucket: str = "scheme_simulation_result", -) -> list[dict]: - """ - 查询指定ID在指定时间的记录 - :param ID: (str): 要查询的 ID。 - :param type: (str): "node"或“link” - :param query_time: (str): 查询的时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: (str): 数据存储的 bucket 名称。 - :return: list[dict]: 指定时间的记录数据列表，如果没有找到则返回空列表。 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - results = [] - if type == "node": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "node" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - results.append( - { - "time": record["_time"], - "ID": ID, - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - elif type == "link": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "link" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - results.append( - { - "time": record["_time"], - "ID": ID, - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - client.close() - return results # 返回列表，如果没有记录则为空列表 - - -def query_simulation_result_by_ID_time( - ID: str, type: str, query_time: str, bucket: str = "realtime_simulation_result" -) -> list[dict]: - """ - 查询指定ID在指定时间的记录 - :param ID: (str): 要查询的 ID。 - :param type: (str): "node"或“link” - :param query_time: (str): 查询的时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: (str): 数据存储的 bucket 名称。 - :return: list[dict]: 指定时间的记录数据列表，如果没有找到则返回空列表。 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - results = [] - if type == "node": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "node" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - results.append( - { - "time": record["_time"], - "ID": ID, - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - elif type == "link": - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "link" and r["ID"] == "{ID}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - tables = query_api.query(flux_query) - # 解析查询结果 - for table in tables: - for record in table.records: - results.append( - { - "time": record["_time"], - "ID": ID, - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - client.close() - return results # 返回列表，如果没有记录则为空列表 - - -# 2025/02/21 -def query_all_records_by_date( - query_date: str, bucket: str = "realtime_simulation_result" -) -> tuple: - """ - 查询指定日期的所有记录，包括‘node’和‘link’，分别以指定的格式返回 - :param query_date: 输入的日期，格式为‘2025-02-14’ - :param bucket: 数据存储的bucket名称 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - # 记录开始时间 - time_cost_start = time.perf_counter() - print( - "{} -- query_all_records_by_date started.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - bg_start_time, bg_end_time = time_api.parse_beijing_date_range( - query_date=query_date - ) - utc_start_time = time_api.to_utc_time(bg_start_time) - utc_stop_time = time_api.to_utc_time(bg_end_time) - - print("bg_start_time", bg_start_time) - print("bg_end_time", bg_end_time) - print("utc_start_time", utc_start_time) - print("utc_stop_time", utc_stop_time) - - print("utc_start_time.isoformat", utc_start_time.isoformat()) - print("utc_stop_time.isoformat", utc_stop_time.isoformat()) - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "node" or r["_measurement"] == "link" and r["date"] == "{query_date}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - # 执行查询 - tables = query_api.query(flux_query) - node_records = [] - link_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - node_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - link_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - time_cost_end = time.perf_counter() - print( - "{} -- query_all_records_by_date finished, cost time: {:.2f} s.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S"), - time_cost_end - time_cost_start, - ) - ) - - client.close() - - return node_records, link_records - - -# 2025/04/12 DingZQ -def query_all_records_by_time_range( - starttime: str, endtime: str, bucket: str = "realtime_simulation_result" -) -> tuple: - """ - 查询指定时间范围内的所有记录，包括‘node’和‘link’，分别以指定的格式返回 - :param starttime: 输入的开始时间，格式为‘2025-02-14T16:00:00+08:00’ - :param endtime: 输入的结束时间，格式为‘2025-02-14T16:00:00+08:00’ - :param bucket: 数据存储的bucket名称 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - - # 记录开始时间 - time_cost_start = time.perf_counter() - print( - "{} -- query_all_records_by_date started.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - bg_start_time = time_api.parse_beijing_time(starttime) - bg_end_time = time_api.parse_beijing_time(endtime) - utc_start_time = time_api.to_utc_time(bg_start_time) - utc_stop_time = time_api.to_utc_time(bg_end_time) - - print("bg_start_time", bg_start_time) - print("bg_end_time", bg_end_time) - print("utc_start_time", utc_start_time) - print("utc_stop_time", utc_stop_time) - - print("utc_start_time.isoformat", utc_start_time.isoformat()) - print("utc_stop_time.isoformat", utc_stop_time.isoformat()) - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "node" or r["_measurement"] == "link" and r["date"] == "{query_date}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - - # 执行查询 - tables = query_api.query(flux_query) - - node_records = [] - link_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - node_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - link_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - - time_cost_end = time.perf_counter() - print( - "{} -- query_all_records_by_date finished, cost time: {:.2f} s.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S"), - time_cost_end - time_cost_start, - ) - ) - - client.close() - - return node_records, link_records - - -# 2025/03/15 DingZQ -def query_all_records_by_date_with_type( - query_date: str, query_type: str, bucket: str = "realtime_simulation_result" -) -> list: - """ - 查询指定日期的所有记录，包括‘node’和‘link’，分别以指定的格式返回 - :param query_date: 输入的日期，格式为‘2025-02-14’ - :param query_type: type 可以是 node 或者 link - :param bucket: 数据存储的bucket名称 - :param client: 已初始化的InfluxDBClient 实例。 - :return: dict: tuple: (node_records, link_records) - """ - # 记录开始时间 - client = get_new_client() - - time_cost_start = time.perf_counter() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - bg_start_time, bg_end_time = time_api.parse_beijing_date_range( - query_date=query_date - ) - utc_start_time = time_api.to_utc_time(bg_start_time) - utc_stop_time = time_api.to_utc_time(bg_end_time) - - print("bg_start_time", bg_start_time) - print("bg_end_time", bg_end_time) - print("utc_start_time", utc_start_time) - print("utc_stop_time", utc_stop_time) - - print("utc_start_time.isoformat", utc_start_time.isoformat()) - print("utc_stop_time.isoformat", utc_stop_time.isoformat()) - - print("measurement", query_type) - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["_measurement"] == "{query_type}" and r["date"] == "{query_date}") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - # 执行查询 - tables = query_api.query(flux_query) - result_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - result_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - result_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - time_cost_end = time.perf_counter() - - client.close() - - return result_records - - -# 2025/02/21 -def query_all_record_by_date_property( - query_date: str, - type: str, - property: str, - bucket: str = "realtime_simulation_result", -) -> list: - """ - 查询指定日期的‘node’或‘link’的某一属性值的所有记录，以指定的格式返回 - :param query_date: 输入的日期，格式为‘2025-02-14’ - :param type: (str): 查询的类型（决定 measurement） - :param property: (str): 查询的字段名称（field） - :param bucket: 数据存储的bucket名称 - :return: list(dict): result_records - """ - client = get_new_client() - # 记录开始时间 - time_cost_start = time.perf_counter() - print( - "{} -- Hydraulic simulation started.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S") - ) - ) - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - # 将 start_date 的北京时间转换为 UTC 时间 - start_time = ( - (datetime.strptime(query_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(query_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "{measurement}" and r["date"] == "{query_date}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - result_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - result_records.append( - {"ID": record["ID"], "time": record["_time"], "value": record["_value"]} - ) - time_cost_end = time.perf_counter() - print( - "{} -- Hydraulic simulation finished, cost time: {:.2f} s.".format( - datetime.now(pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S"), - time_cost_end - time_cost_start, - ) - ) - client.close() - return result_records - - -# 2025/02/01 -def query_curve_by_ID_property_daterange( - ID: str, - type: str, - property: str, - start_date: str, - end_date: str, - bucket: str = "realtime_simulation_result", -) -> list: - """ - 根据 type 查询对应的 measurement，根据 ID 和 date 查询对应的 tag，根据 property 查询对应的 field。 - :param ID: (str): 要查询的 ID（tag） - :param type: (str): 查询的类型（决定 measurement） - :param property: (str): 查询的字段名称（field） - :param start_date: (str): 查询的开始日期，格式为 'YYYY-MM-DD' - :param end_date: (str): 查询的结束日期，格式为 'YYYY-MM-DD' - :param bucket: (str): 数据存储的 bucket 名称，默认值为 "realtime_simulation_result" - :return: 查询结果的列表 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - # 解析日期范围（当天的 UTC 开始和结束时间） - # previous_day = datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1) - # start_time = previous_day.isoformat() + "T16:00:00Z" - # stop_time = datetime.strptime(end_date, "%Y-%m-%d").isoformat() + "T15:59:59Z" - # 将 start_date 的北京时间转换为 UTC 时间范围 - start_time = ( - (datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "{measurement}" and r["ID"] == "{ID}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - # 解析查询结果 - results = [] - for table in tables: - for record in table.records: - results.append({"time": record["_time"], "value": record["_value"]}) - client.close() - return results - - -# 2025/02/13 -def store_scheme_simulation_result_to_influxdb( - node_result_list: List[Dict[str, any]], - link_result_list: List[Dict[str, any]], - scheme_start_time: str, - num_periods: int = 1, - scheme_type: str = None, - scheme_name: str = None, - bucket: str = "scheme_simulation_result", -): - """ - 将方案模拟计算结果存入 InfluxuDb 的scheme_simulation_result这个bucket中。 - :param node_result_list: (List[Dict[str, any]]): 包含节点和结果数据的字典列表。 - :param link_result_list: (List[Dict[str, any]]): 包含连接和结果数据的字典列表。 - :param scheme_start_time: (str): 方案模拟开始时间。 - :param num_periods: (int): 方案模拟的周期数 - :param scheme_type: (str): 方案类型 - :param scheme_name: (str): 方案名称 - :param bucket: (str): InfluxDB 的 bucket 名称，默认值为 "scheme_simulation_result"。 - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - try: - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # write_options = WriteOptions( - # jitter_interval=200, # 添加抖动以避免同时写入 - # max_retry_delay=30000 # 最大重试延迟（毫秒） - # ) - # 使用异步写入模式配置写入选项和回调函数 - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - # 创建一个临时存储点数据的列表 - points_to_write = [] - date_str = scheme_start_time.split("T")[0] - time_beijing = datetime.strptime(scheme_start_time, "%Y-%m-%dT%H:%M:%S%z") - timestep_parts = globals.hydraulic_timestep.split(":") - timestep = timedelta( - hours=int(timestep_parts[0]), - minutes=int(timestep_parts[1]), - seconds=int(timestep_parts[2]), - ) - for node_result in node_result_list: - # 提取节点信息和数据结果 - node_id = node_result.get("node") - # 从period 0 到 period num_period - 1 - for period_index in range(num_periods): - scheme_time = (time_beijing + (timestep * period_index)).isoformat() - data_list = [node_result.get("result", [])[period_index]] - for data in data_list: - # 构建 Point 数据，多个 field 存在于一个数据点中 - node_point = ( - Point("node") - .tag("date", date_str) - .tag("ID", node_id) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("head", data.get("head", 0.0)) - .field("pressure", data.get("pressure", 0.0)) - .field("actualdemand", data.get("demand", 0.0)) - .field("demanddeficit", None) - .field("totalExternalOutflow", None) - .field("quality", data.get("quality", 0.0)) - .time(scheme_time, write_precision="s") - ) - points_to_write.append(node_point) - # 写入数据到 InfluxDB，多个 field 在同一个 point 中 - # write_api.write(bucket=bucket, org=org_name, record=node_point) - # write_api.flush() - for link_result in link_result_list: - link_id = link_result.get("link") - for period_index in range(num_periods): - scheme_time = (time_beijing + (timestep * period_index)).isoformat() - data_list = [link_result.get("result", [])[period_index]] - for data in data_list: - link_point = ( - Point("link") - .tag("date", date_str) - .tag("ID", link_id) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("flow", data.get("flow", 0.0)) - .field("velocity", data.get("velocity", 0.0)) - .field("headloss", data.get("headloss", 0.0)) - .field("quality", data.get("quality", 0.0)) - .field("status", data.get("status", "UNKNOWN")) - .field("setting", data.get("setting", 0.0)) - .field("reaction", data.get("reaction", 0.0)) - .field("friction", data.get("friction", 0.0)) - .time(scheme_time, write_precision="s") - ) - points_to_write.append(link_point) - # write_api.write(bucket=bucket, org=org_name, record=link_point) - # write_api.flush() - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - except Exception as e: - client.close() - raise RuntimeError(f"数据写入 InfluxDB 时发生错误: {e}") - - time.sleep(10) - - print("Total points written:", points_written) - - client.close() - - -# 2025/03/12 -def query_corresponding_query_id_and_element_id(name: str) -> None: - """ - 查询scada_info这张表中，api_query_id与associated_element_id的对应关系，用于下一步fill_scheme_simulation_result_to_SCADA - :param name: 数据库名称 - :return: - """ - # 连接数据库 - conn_string = get_pgconn_string(db_name=name) - try: - with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - # 查询 transmission_mode 为 'realtime' 的记录 - cur.execute( - """ - SELECT type, associated_element_id, api_query_id - FROM scada_info - WHERE type IN ('source_outflow', 'pipe_flow', 'demand', 'pressure', 'quality'); - """ - ) - records = cur.fetchall() - # 遍历查询结果，根据 type 分类存入对应的字典 - for record in records: - record_type, associated_element_id, api_query_id = record - if record_type == "source_outflow": - globals.scheme_source_outflow_ids[api_query_id] = ( - associated_element_id - ) - elif record_type == "pipe_flow": - globals.scheme_pipe_flow_ids[api_query_id] = ( - associated_element_id - ) - elif record_type == "pressure": - globals.scheme_pressure_ids[api_query_id] = ( - associated_element_id - ) - elif record_type == "demand": - globals.scheme_demand_ids[api_query_id] = associated_element_id - elif record_type == "quality": - globals.scheme_quality_ids[api_query_id] = associated_element_id - # 如果需要调试，可以打印该字典 - # print("scheme_source_outflow_ids:", globals.scheme_source_outflow_ids) - # print("scheme_pipe_flow_ids:", globals.scheme_pipe_flow_ids) - # print("scheme_pressure_ids:", globals.scheme_pressure_ids) - # print("scheme_demand_ids:", globals.scheme_demand_ids) - # print("scheme_quality_ids:", globals.scheme_quality_ids) - except psycopg.Error as e: - print(f"数据库连接或查询出错: {e}") - - -# 2025/03/22 -# def auto_get_burst_flow(): - - -# 2025/03/11 -def fill_scheme_simulation_result_to_SCADA( - scheme_type: str = None, - scheme_name: str = None, - query_date: str = None, - bucket: str = "scheme_simulation_result", -): - """ - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_date: 查询日期，格式为 'YYYY-MM-DD' - :param bucket: InfluxDB 的 bucket 名称，默认值为 "scheme_simulation_result" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - # 本地变量，用于记录成功写入的数据点数量 - points_written = 0 - lock = threading.Lock() - - # 回调函数中使用 nonlocal 来修改外层的变量 points_written - def success_callback(batch, response): - nonlocal points_written - count = len(batch) if isinstance(batch, list) else 1 - with lock: - points_written += count - - def error_callback(exception): - print("Error writing batch:", exception) - - # write_options = WriteOptions( - # jitter_interval=200, # 添加抖动以避免同时写入 - # max_retry_delay=30000 # 最大重试延迟（毫秒） - # ) - write_api = client.write_api( - write_options=create_write_options(), - success_callback=success_callback, - error_callback=error_callback, - ) - # 创建一个临时存储点数据的列表 - points_to_write = [] - # 查找associated_element_id的对应值 - for key, value in globals.scheme_source_outflow_ids.items(): - scheme_source_outflow_result = query_scheme_curve_by_ID_property( - scheme_type=scheme_type, - scheme_name=scheme_name, - query_date=query_date, - ID=value, - type="link", - property="flow", - ) - # print(f"Key: {key}, Query result: {scheme_source_outflow_result}") # 调试输出 - for data in scheme_source_outflow_result: - point = ( - Point("scheme_source_outflow") - .tag("date", query_date) - .tag("device_ID", key) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("monitored_value", data["value"]) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - - for key, value in globals.scheme_pipe_flow_ids.items(): - scheme_pipe_flow_result = query_scheme_curve_by_ID_property( - scheme_type=scheme_type, - scheme_name=scheme_name, - query_date=query_date, - ID=value, - type="link", - property="flow", - ) - for data in scheme_pipe_flow_result: - point = ( - Point("scheme_pipe_flow") - .tag("date", query_date) - .tag("device_ID", key) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("monitored_value", data["value"]) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - - for key, value in globals.scheme_pressure_ids.items(): - scheme_pressure_result = query_scheme_curve_by_ID_property( - scheme_type=scheme_type, - scheme_name=scheme_name, - query_date=query_date, - ID=value, - type="node", - property="pressure", - ) - for data in scheme_pressure_result: - point = ( - Point("scheme_pressure") - .tag("date", query_date) - .tag("device_ID", key) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("monitored_value", data["value"]) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - - for key, value in globals.scheme_demand_ids.items(): - scheme_demand_result = query_scheme_curve_by_ID_property( - scheme_type=scheme_type, - scheme_name=scheme_name, - query_date=query_date, - ID=value, - type="node", - property="actualdemand", - ) - for data in scheme_demand_result: - point = ( - Point("scheme_demand") - .tag("date", query_date) - .tag("device_ID", key) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("monitored_value", data["value"]) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - - for key, value in globals.scheme_quality_ids.items(): - scheme_quality_result = query_scheme_curve_by_ID_property( - scheme_type=scheme_type, - scheme_name=scheme_name, - query_date=query_date, - ID=value, - type="node", - property="quality", - ) - for data in scheme_quality_result: - point = ( - Point("scheme_quality") - .tag("date", query_date) - .tag("device_ID", key) - .tag("scheme_type", scheme_type) - .tag("scheme_name", scheme_name) - .field("monitored_value", data["value"]) - .time(data["time"], write_precision="s") - ) - points_to_write.append(point) - # write_api.write(bucket=bucket, org=org_name, record=point) - # 批量写入数据 - print("points to write:", len(points_to_write)) - if points_to_write: - write_api.write(bucket=bucket, org=org_name, record=points_to_write) - write_api.flush() # 刷新缓存一次 - - time.sleep(10) - - print("Total points written:", points_written) - - client.close() - - -# 2025/02/15 -def query_SCADA_data_curve( - api_query_id: str, start_date: str, end_date: str, bucket: str = "SCADA_data" -) -> list: - """ - 根据SCADA设备的api_query_id和时间范围，查询得到曲线，查到的数据为0时区时间 - :param api_query_id: SCADA设备的api_query_id - :param start_date: 查询开始的时间，格式为 'YYYY-MM-DD' - :param end_date: 查询结束的时间，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: 查询结果的列表 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将 start_date 的北京时间转换为 UTC 时间范围 - start_time = ( - (datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["device_ID"] == "{api_query_id}" and r["_field"] == "monitored_value") - """ - # 执行查询 - tables = query_api.query(flux_query) - # 解析查询结果 - results = [] - for table in tables: - for record in table.records: - results.append({"time": record["_time"], "value": record["_value"]}) - client.close() - return results - - -# 2025/02/18 -def query_scheme_all_record_by_time( - scheme_type: str, - scheme_name: str, - query_time: str, - bucket: str = "scheme_simulation_result", -) -> tuple: - """ - 查询指定方案某一时刻的所有记录，包括‘node'和‘link’，分别以指定格式返回。 - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param bucket: 数据存储的 bucket 名称。 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "node" or r["_measurement"] == "link") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - # 执行查询 - tables = query_api.query(flux_query) - node_records = [] - link_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - node_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - link_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - client.close() - return node_records, link_records - - -# 2025/03/04 -def query_scheme_all_record_by_time_property( - scheme_type: str, - scheme_name: str, - query_time: str, - type: str, - property: str, - bucket: str = "scheme_simulation_result", -) -> list: - """ - 查询指定方案某一时刻‘node'或‘link’某一属性值，以指定格式返回。 - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_time: 输入的北京时间，格式为 '2024-11-24T17:30:00+08:00'。 - :param type: 查询的类型（决定 measurement） - :param property: 查询的字段名称（field） - :param bucket: 数据存储的 bucket 名称。 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - # 将北京时间转换为 UTC 时间 - beijing_time = datetime.fromisoformat(query_time) - utc_time = beijing_time.astimezone(timezone.utc) - utc_start_time = utc_time - timedelta(seconds=1) - utc_stop_time = utc_time + timedelta(seconds=1) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "{measurement}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - result_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - result_records.append({"ID": record["ID"], "value": record["_value"]}) - client.close() - return result_records - - -# 2025/02/19 -def query_scheme_curve_by_ID_property( - scheme_type: str, - scheme_name: str, - query_date: str, - ID: str, - type: str, - property: str, - bucket: str = "scheme_simulation_result", -) -> list: - """ - 根据scheme_Type和scheme_name,查询该模拟方案中，某一node或link的某一属性值的所有时间的结果 - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_date: 查询日期，格式为 'YYYY-MM-DD' - :param ID: 元素的ID - :param type: 元素的类型，node或link - :param property: 元素的属性值 - :param bucket: 数据存储的 bucket 名称，默认值为 "scheme_simulation_result" - :return: 查询结果的列表 - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - start_time = ( - (datetime.strptime(query_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(query_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "{measurement}" and r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}" and r["ID"] == "{ID}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - # 解析查询结果 - results = [] - for table in tables: - for record in table.records: - results.append({"time": record["_time"], "value": record["_value"]}) - client.close() - return results - - -# 2025/02/21 -def query_scheme_all_record( - scheme_type: str, - scheme_name: str, - query_date: str, - bucket: str = "scheme_simulation_result", -) -> tuple: - """ - 查询指定方案的所有记录，包括‘node'和‘link’，分别以指定格式返回。 - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_date: 查询日期，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称。 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - - bg_start_time, bg_end_time = time_api.parse_beijing_date_range( - query_date=query_date - ) - utc_start_time = time_api.to_utc_time(bg_start_time) - utc_stop_time = time_api.to_utc_time(bg_end_time) - - print(utc_start_time, utc_stop_time) - - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {utc_start_time.isoformat()}, stop: {utc_stop_time.isoformat()}) - |> filter(fn: (r) => r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}" and r["_measurement"] == "node" or r["_measurement"] == "link") - |> pivot( - rowKey:["_time"], - columnKey:["_field"], - valueColumn:"_value" - ) - """ - # 执行查询 - tables = query_api.query(flux_query) - node_records = [] - link_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - # print(record.values) # 打印完整记录内容 - measurement = record["_measurement"] - # 处理 node 数据 - if measurement == "node": - node_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "head": record["head"], - "pressure": record["pressure"], - "actualdemand": record["actualdemand"], - "quality": record["quality"], - } - ) - # 处理 link 数据 - elif measurement == "link": - link_records.append( - { - "time": record["_time"], - "ID": record["ID"], - "flow": record["flow"], - "velocity": record["velocity"], - "headloss": record["headloss"], - "quality": record["quality"], - "status": record["status"], - "setting": record["setting"], - "reaction": record["reaction"], - "friction": record["friction"], - } - ) - client.close() - return node_records, link_records - - -# 2025/03/04 -def query_scheme_all_record_property( - scheme_type: str, - scheme_name: str, - query_date: str, - type: str, - property: str, - bucket: str = "scheme_simulation_result", -) -> list: - """ - 查询指定方案的‘node'或‘link’的某一属性值，以指定格式返回。 - :param scheme_type: 方案类型 - :param scheme_name: 方案名称 - :param query_date: 查询日期，格式为 'YYYY-MM-DD' - :param type: 查询的类型（决定 measurement） - :param property: 查询的字段名称（field） - :param bucket: 数据存储的 bucket 名称。 - :return: dict: tuple: (node_records, link_records) - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 确定 measurement - if type == "node": - measurement = "node" - elif type == "link": - measurement = "link" - else: - raise ValueError(f"不支持的类型: {type}") - start_time = ( - (datetime.strptime(query_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(query_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}" and r["date"] == "{query_date}" and r["_measurement"] == "{measurement}" and r["_field"] == "{property}") - """ - # 执行查询 - tables = query_api.query(flux_query) - result_records = [] - # 解析查询结果 - for table in tables: - for record in table.records: - result_records.append( - {"time": record["_time"], "ID": record["ID"], "value": record["_value"]} - ) - client.close() - return result_records - - -# 2025/02/16 -def export_SCADA_data_to_csv( - start_date: str, end_date: str, bucket: str = "SCADA_data" -) -> None: - """ - 导出influxdb中SCADA_data这个bucket的数据到csv中 - :param start_date: 查询开始的时间，格式为 'YYYY-MM-DD' - :param end_date: 查询结束的时间，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将 start_date 的北京时间转换为 UTC 时间范围 - start_time = ( - (datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句 - flux_query = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - """ - # 执行查询 - tables = query_api.query(flux_query) - # 存储查询结果 - rows = [] - for table in tables: - for record in table.records: - row = { - "time": record.get_time(), - "measurement": record.get_measurement(), - "date": record.values.get("date", None), - "description": record.values.get("description", None), - "device_ID": record.values.get("device_ID", None), - "monitored_value": ( - record.get_value() - if record.get_field() == "monitored_value" - else None - ), - "datacleaning_value": ( - record.get_value() - if record.get_field() == "datacleaning_value" - else None - ), - "simulation_value": ( - record.get_value() - if record.get_field() == "simulation_value" - else None - ), - } - rows.append(row) - # 动态生成 CSV 文件名 - csv_filename = f"SCADA_data_{start_date}至{end_date}.csv" - # 写入到 CSV 文件 - with open(csv_filename, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "description", - "device_ID", - "monitored_value", - "datacleaning_value", - "simulation_value", - ], - ) - writer.writeheader() - writer.writerows(rows) - print(f"Data exported to {csv_filename} successfully.") - client.close() - - -# 2025/02/17 -def export_realtime_simulation_result_to_csv( - start_date: str, end_date: str, bucket: str = "realtime_simulation_result" -) -> None: - """ - 导出influxdb中realtime_simulation_result这个bucket的数据到csv中 - :param start_date: 查询开始的时间，格式为 'YYYY-MM-DD' - :param end_date: 查询结束的时间，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将 start_date 的北京时间转换为 UTC 时间范围 - start_time = ( - (datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_link = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "link") - """ - # 执行查询 - link_tables = query_api.query(flux_query_link) - # 存储link类的数据 - link_rows = [] - link_data = {} - for table in link_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in link_data: - link_data[key] = {} - field = record.get_field() - link_data[key][field] = record.get_value() - link_data[key]["measurement"] = record.get_measurement() - link_data[key]["date"] = record.values.get("date", None) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_node = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "node") - """ - # 执行查询 - node_tables = query_api.query(flux_query_node) - # 存储node类的数据 - node_rows = [] - node_data = {} - for table in node_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in node_data: - node_data[key] = {} - field = record.get_field() - node_data[key][field] = record.get_value() - node_data[key]["measurement"] = record.get_measurement() - node_data[key]["date"] = record.values.get("date", None) - - for key in set(link_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(link_data.get(key, {})) - link_rows.append(row) - for key in set(node_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(node_data.get(key, {})) - node_rows.append(row) - # 动态生成 CSV 文件名 - csv_filename_link = f"realtime_simulation_link_result_{start_date}至{end_date}.csv" - csv_filename_node = f"realtime_simulation_node_result_{start_date}至{end_date}.csv" - # 写入到 CSV 文件 - with open(csv_filename_link, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "ID", - "flow", - "leakage", - "velocity", - "headloss", - "status", - "setting", - "quality", - "friction", - "reaction", - ], - ) - writer.writeheader() - writer.writerows(link_rows) - with open(csv_filename_node, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "ID", - "head", - "pressure", - "actualdemand", - "demanddeficit", - "totalExternalOutflow", - "quality", - ], - ) - writer.writeheader() - writer.writerows(node_rows) - print(f"Data exported to {csv_filename_link} and {csv_filename_node} successfully.") - client.close() - - -# 2025/02/18 -def export_scheme_simulation_result_to_csv_time( - start_date: str, end_date: str, bucket: str = "scheme_simulation_result" -) -> None: - """ - 导出influxdb中scheme_simulation_result这个bucket的数据到csv中 - :param start_date: 查询开始的时间，格式为 'YYYY-MM-DD' - :param end_date: 查询结束的时间，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - # 将 start_date 的北京时间转换为 UTC 时间范围 - start_time = ( - (datetime.strptime(start_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(end_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_link = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "link") - """ - # 执行查询 - link_tables = query_api.query(flux_query_link) - # 存储link类的数据 - link_rows = [] - link_data = {} - for table in link_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in link_data: - link_data[key] = {} - field = record.get_field() - link_data[key][field] = record.get_value() - link_data[key]["measurement"] = record.get_measurement() - link_data[key]["date"] = record.values.get("date", None) - link_data[key]["scheme_type"] = record.values.get("scheme_type", None) - link_data[key]["scheme_name"] = record.values.get("scheme_name", None) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_node = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "node") - """ - # 执行查询 - node_tables = query_api.query(flux_query_node) - # 存储node类的数据 - node_rows = [] - node_data = {} - for table in node_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in node_data: - node_data[key] = {} - field = record.get_field() - node_data[key][field] = record.get_value() - node_data[key]["measurement"] = record.get_measurement() - node_data[key]["date"] = record.values.get("date", None) - node_data[key]["scheme_type"] = record.values.get("scheme_type", None) - node_data[key]["scheme_name"] = record.values.get("scheme_name", None) - for key in set(link_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(link_data.get(key, {})) - link_rows.append(row) - for key in set(node_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(node_data.get(key, {})) - node_rows.append(row) - # 动态生成 CSV 文件名 - csv_filename_link = f"scheme_simulation_link_result_{start_date}至{end_date}.csv" - csv_filename_node = f"scheme_simulation_node_result_{start_date}至{end_date}.csv" - # 写入到 CSV 文件 - with open(csv_filename_link, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "scheme_type", - "scheme_name", - "ID", - "flow", - "leakage", - "velocity", - "headloss", - "status", - "setting", - "quality", - "friction", - "reaction", - ], - ) - writer.writeheader() - writer.writerows(link_rows) - with open(csv_filename_node, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "scheme_type", - "scheme_name", - "ID", - "head", - "pressure", - "actualdemand", - "demanddeficit", - "totalExternalOutflow", - "quality", - ], - ) - writer.writeheader() - writer.writerows(node_rows) - print(f"Data exported to {csv_filename_link} and {csv_filename_node} successfully.") - client.close() - - -# 2025/02/18 -def export_scheme_simulation_result_to_csv_scheme( - scheme_type: str, - scheme_name: str, - query_date: str, - bucket: str = "scheme_simulation_result", -) -> None: - """ - 导出influxdb中scheme_simulation_result这个bucket的数据到csv中 - :param scheme_type: 查询的方案类型 - :param scheme_name: 查询的方案名 - :param query_date: 查询日期，格式为 'YYYY-MM-DD' - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - query_api = client.query_api() - start_time = ( - (datetime.strptime(query_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(query_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_link = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "link" and r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}") - """ - # 执行查询 - link_tables = query_api.query(flux_query_link) - # 存储link类的数据 - link_rows = [] - link_data = {} - for table in link_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in link_data: - link_data[key] = {} - field = record.get_field() - link_data[key][field] = record.get_value() - link_data[key]["measurement"] = record.get_measurement() - link_data[key]["date"] = record.values.get("date", None) - link_data[key]["scheme_type"] = record.values.get("scheme_type", None) - link_data[key]["scheme_name"] = record.values.get("scheme_name", None) - # 构建 Flux 查询语句，查询指定时间范围内的数据 - flux_query_node = f""" - from(bucket: "{bucket}") - |> range(start: {start_time}, stop: {stop_time}) - |> filter(fn: (r) => r["_measurement"] == "node" and r["scheme_type"] == "{scheme_type}" and r["scheme_name"] == "{scheme_name}") - """ - # 执行查询 - node_tables = query_api.query(flux_query_node) - # 存储node类的数据 - node_rows = [] - node_data = {} - for table in node_tables: - for record in table.records: - key = (record.get_time(), record.values.get("ID", None)) - if key not in node_data: - node_data[key] = {} - field = record.get_field() - node_data[key][field] = record.get_value() - node_data[key]["measurement"] = record.get_measurement() - node_data[key]["date"] = record.values.get("date", None) - node_data[key]["scheme_type"] = record.values.get("scheme_type", None) - node_data[key]["scheme_name"] = record.values.get("scheme_name", None) - for key in set(link_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(link_data.get(key, {})) - link_rows.append(row) - for key in set(node_data.keys()): - row = {"time": key[0], "ID": key[1]} - row.update(node_data.get(key, {})) - node_rows.append(row) - # 动态生成 CSV 文件名 - csv_filename_link = ( - f"scheme_simulation_link_result_{scheme_name}_of_{scheme_type}.csv" - ) - csv_filename_node = ( - f"scheme_simulation_node_result_{scheme_name}_of_{scheme_type}.csv" - ) - # 写入到 CSV 文件 - with open(csv_filename_link, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "scheme_type", - "scheme_name", - "ID", - "flow", - "leakage", - "velocity", - "headloss", - "status", - "setting", - "quality", - "friction", - "reaction", - ], - ) - writer.writeheader() - writer.writerows(link_rows) - with open(csv_filename_node, mode="w", newline="") as file: - writer = csv.DictWriter( - file, - fieldnames=[ - "time", - "measurement", - "date", - "scheme_type", - "scheme_name", - "ID", - "head", - "pressure", - "actualdemand", - "demanddeficit", - "totalExternalOutflow", - "quality", - ], - ) - writer.writeheader() - writer.writerows(node_rows) - print(f"Data exported to {csv_filename_link} and {csv_filename_node} successfully.") - client.close() - - -def upload_cleaned_SCADA_data_to_influxdb( - file_path: str, bucket: str = "SCADA_data" -) -> None: - """ - 将清洗后的SCADA数据导入influxdb，有标准化导入格式 - :param file_path: 导入数据的文件 - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - - data_list = [] - with open(file_path, mode="r", encoding="utf-8-sig") as csv_file: - csv_reader = csv.DictReader(csv_file) - for row in csv_reader: - # 解析日期和时间字段 - datetime_value = datetime.strptime(row["time"], "%Y-%m-%d %H:%M:%S%z") - # 处理datacleaning_value为空的情况 - datacleaning_value = ( - float(row["datacleaning_value"]) if row["datacleaning_value"] else None - ) - # 处理monitored_value字段类型错误 - try: - monitored_value = ( - float(row["monitored_value"]) if row["monitored_value"] else None - ) - except ValueError: - monitored_value = None # 如果转换失败，则设为None（或其他适当的默认值） - - data_list.append( - { - "measurement": row["measurement"], - "device_ID": row["device_ID"], - "date": datetime_value.strftime("%Y-%m-%d"), - "description": row["description"], - "monitored_value": monitored_value, - "datacleaning_value": datacleaning_value, - "datetime": datetime_value, - } - ) - - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - try: - write_api = client.write_api(write_options=SYNCHRONOUS) - # 写入数据 - for data in data_list: - print(data) - # 创建Point对象 - point = ( - Point(data["measurement"]) # measurement为mpointName - .tag("device_ID", data["device_ID"]) # tag key为mpointId - .tag("date", data["date"]) # 具体日期tag，方便查询 - .tag("description", data["description"]) - .field( - "monitored_value", data["monitored_value"] - ) # field key为dataValue - .field("datacleaning_value", data["datacleaning_value"]) - .time(data["datetime"]) # 时间以datetime为准 - ) - - write_api.write(bucket=bucket, record=point) - - except InfluxDBError as e: - print(f"InfluxDB错误: {str(e)}") - except Exception as e: - print(f"未知错误: {str(e)}") - finally: - if "write_api" in locals(): - write_api.close() - client.close() - - -# 2025/05/05 DingZQ -# 删除某一天的数据 -def delete_data(delete_date: str, bucket: str) -> None: - """ - 删除某一天的数据 - :param delete_date: 删除的日期，格式为 'YYYY-MM-DD' - :param bucket: 选择要删除数据的bucket - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - start_time = ( - (datetime.strptime(delete_date, "%Y-%m-%d") - timedelta(days=1)) - .replace(hour=16, minute=0, second=0, tzinfo=timezone.utc) - .isoformat() - ) - stop_time = ( - datetime.strptime(delete_date, "%Y-%m-%d") - .replace(hour=15, minute=59, second=59, tzinfo=timezone.utc) - .isoformat() - ) - - # 构造删除谓词（InfluxDB Delete API 要求的 SQL-like 语句） - # 注意：字段名用 _field，measurement 用 _measurement，标签直接写标签名 - predicate = f'date="{delete_date}"' - - delete_api: DeleteApi = client.delete_api() - delete_api.delete( - start=start_time, stop=stop_time, predicate=predicate, bucket=bucket - ) - - # 2025/08/18 从文件导入scada数据，xkl - - -def import_data_from_file(file_path: str, bucket: str = "SCADA_data") -> None: - """ - 从指定的CSV文件导入数据到InfluxDB的指定bucket中。 - :param file_path: CSV文件的路径 - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - ) - - # 清空指定bucket的数据 - # delete_api = DeleteApi(client) - # start = "1970-01-01T00:00:00Z" - # stop = "2100-01-01T00:00:00Z" - # delete_api.delete(start, stop, '', bucket="SCADA_data", org="TJWATERORG") - - df = pd.read_csv(file_path) - write_api = client.write_api(write_options=SYNCHRONOUS) - points_to_write = [] - for _, row in df.iterrows(): - scada_id = row["ScadaId"] - value = row["Value"] - time_str = row["Time"] - date_str = str(time_str)[:10] # 取前10位作为日期 - try: - raw_value = float(value) - except (ValueError, TypeError): - raw_value = 0.0 - point = ( - Point("SCADA") - .tag("date", date_str) - .tag("description", None) - .tag("device_ID", scada_id) - .field("monitored_value", raw_value) - .field("datacleaning_value", 0.0) - .field("simulation_value", 0.0) - .time(time_str, write_precision="s") - ) - points_to_write.append(point) - # 批量写入数据 - batch_size = 500 - for i in range(0, len(points_to_write), batch_size): - batch = points_to_write[i : i + batch_size] - write_api.write(bucket=bucket, record=batch) - print(f"Data imported from {file_path} to bucket {bucket} successfully.") - print(f"Total points written: {len(points_to_write)}") - write_api.close() - client.close() - - -# 2025/08/28 从多列格式文件导入SCADA数据，xkl -def import_multicolumn_data_from_file( - file_path: str, raw: bool = True, bucket: str = "SCADA_data" -) -> None: - """ - 从指定的多列格式CSV文件导入数据到InfluxDB的指定bucket中。 - :param file_path: CSV文件的路径 - :param bucket: 数据存储的 bucket 名称，默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - write_api = client.write_api(write_options=SYNCHRONOUS) - points_to_write = [] - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y/%m/%d %H:%M") - ) - ) - - def convert_to_iso(timestr): - # 假设原格式为 '2025-08-03 00:00:00'，将其解析为北京时间 (+08:00) - dt = datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S") - dt_beijing = pytz.timezone('Asia/Shanghai').localize(dt) - return dt_beijing.isoformat() - - with open(file_path, encoding="utf-8") as f: - reader = csv.reader(f) - header = next(reader) - device_ids = header[1:] # 第一列是time，后面是device_ID - if raw: - for row in reader: - time_str = row[0] - iso_time = convert_to_iso(time_str) - for idx, value in enumerate(row[1:]): - try: - raw_value = float(value) - except (ValueError, TypeError): - raw_value = None - scada_id = device_ids[idx] - # 如果是原始数据，直接使用Value列 - if(raw_value is None or raw_value == ''): - continue - point = ( - Point("SCADA") - .tag("date", iso_time.split("T")[0]) - .tag("description", None) - .tag("device_ID", scada_id) - .field("monitored_value", raw_value) - # .field("datacleaning_value", 0.0) - # .field("simulation_value", 0.0) - .time(iso_time, WritePrecision.S) - ) - points_to_write.append(point) - else: - for row in reader: - time_str = row[0] - iso_time = convert_to_iso(time_str) - # 如果不是原始数据，直接使用datacleaning_value列 - for idx, value in enumerate(row[1:]): - scada_id = device_ids[idx] - try: - datacleaning_value = float(value) - except (ValueError, TypeError): - datacleaning_value = 0.0 - # 如果是清洗数据，直接使用datacleaning_value列 - point = ( - Point("SCADA") - .tag("date", iso_time.split("T")[0]) - .tag("description", "None") - .tag("device_ID", scada_id) - # .field("monitored_value", 0.0) - .field("datacleaning_value", datacleaning_value) - # .field("simulation_value", 0.0) - .time(iso_time, WritePrecision.S) - ) - points_to_write.append(point) - # 批量写入数据 - batch_size = 1000 - for i in range(0, len(points_to_write), batch_size): - batch = points_to_write[i : i + batch_size] - write_api.write(bucket=bucket, record=batch) - print(f"Data imported from {file_path} to bucket {bucket} successfully.") - print(f"Total points written: {len(points_to_write)}") - write_api.close() - client.close() - - -# 从多列格式字典导入SCADA数据 -def import_multicolumn_data_from_dict( - data_dict: dict[str, list], raw: bool = True, bucket: str = "SCADA_data" -) -> None: - """ - 从指定的多列格式字典导入数据到InfluxDB的指定bucket中。 - :param data_dict: 字典格式数据,键为列名(第一个为'time',其他为设备ID),值为对应列的值列表 - :param raw: 是否为原始数据,True则写入monitored_value,False则写入datacleaning_value - :param bucket: 数据存储的 bucket 名称,默认值为 "SCADA_data" - :return: - """ - client = get_new_client() - write_api = client.write_api(write_options=SYNCHRONOUS) - points_to_write = [] - if not client.ping(): - print( - "{} -- Failed to connect to InfluxDB.".format( - datetime.now().strftime("%Y/%m/%d %H:%M:%S") - ) - ) - - # 获取时间列表和设备ID列表 - time_list = data_dict.get("time", []) - device_ids = [key for key in data_dict.keys() if key != "time"] - - # 遍历每个时间点 - for i, time_str in enumerate(time_list): - # 确保 time_str 是字符串格式 - if not isinstance(time_str, str): - time_str = str(time_str) - - for device_id in device_ids: - value = data_dict[device_id][i] - try: - float_value = float(value) - except (ValueError, TypeError): - float_value = 0.0 - - if raw: - # 如果是原始数据,写入monitored_value - point = ( - Point("SCADA") - .tag("date", datetime.fromisoformat(time_str).date().isoformat()) - .tag("description", None) - .tag("device_ID", device_id) - .field("monitored_value", float_value) - # .field("datacleaning_value", 0.0) - # .field("simulation_value", 0.0) - .time(time_str, WritePrecision.S) - ) - else: - # 如果是清洗数据,写入datacleaning_value - point = ( - Point("SCADA") - .tag("date", datetime.fromisoformat(time_str).date().isoformat()) - .tag("description", "None") - .tag("device_ID", device_id) - # .field("monitored_value", 0.0) - .field("datacleaning_value", float_value) - # .field("simulation_value", 0.0) - .time(time_str, WritePrecision.S) - ) - points_to_write.append(point) - - # 批量写入数据 - batch_size = 1000 - for i in range(0, len(points_to_write), batch_size): - batch = points_to_write[i : i + batch_size] - write_api.write(bucket=bucket, record=batch) - print(f"Data imported from dict to bucket {bucket} successfully.") - print(f"Total points written: {len(points_to_write)}") - write_api.close() - client.close() - - -# 示例调用 -if __name__ == "__main__": - url = influxdb_info.url - token = influxdb_info.token - org_name = influxdb_info.org - - # client = InfluxDBClient(url=url, token=token) - # # step1: 检查连接状态，初始化influxdb的buckets - # try: - # delete_buckets(org_name) - # create_and_initialize_buckets(org_name) - # except Exception as e: - # print(f"连接失败: {e}") - - # step2: 先查询pg数据库中scada_info的信息，然后存储SCADA数据到SCADA_data这个bucket里 - # query_pg_scada_info_realtime('bb') - # query_pg_scada_info_non_realtime('bb') - # query_corresponding_query_id_and_element_id('bb') - - # 手动执行存储测试 - # 示例1：store_realtime_SCADA_data_to_influxdb - # store_realtime_SCADA_data_to_influxdb(get_real_value_time='2025-03-16T11:13:00+08:00') - - # 示例2：store_non_realtime_SCADA_data_to_influxdb - # store_non_realtime_SCADA_data_to_influxdb(get_history_data_end_time='2025-03-08T12:00:00+08:00') - - # 示例3：download_history_data_manually - # download_history_data_manually(begin_time='2025-04-16T00:00:00+08:00', end_time='2025-04-16T23:59:00+08:00') - # download_history_data_manually(begin_time='2025-05-04T00:00:00+08:00', end_time='2025-05-04T23:59:00+08:00') - - # step3: 查询测试示例 - - # 示例1：query_latest_record_by_ID - # bucket_name = "realtime_simulation_result" # 数据存储的 bucket 名称 - # node_id = "ZBBDTZDP000022" # 查询的节点 ID - # link_id = "ZBBGXSZW000002" - # - # latest_record = query_latest_record_by_ID(ID=node_id, type="node", bucket=bucket_name)uodao - # # # latest_record = query_latest_record_by_ID(ID=link_id, type="link", bucket=bucket_name) - # # - # if latest_record: - # print("最新记录：", latest_record) - # else: - # print("未找到符合条件的记录。") - - # 示例2：query_all_record_by_time - # node_records, link_records = query_all_record_by_time(query_time="2025-04-04T00:00:00+08:00") - # print("Node 数据：", node_records) - # print("Link 数据：", link_records) - - # 示例3：query_curve_by_ID_property_daterange - # curve_result = query_curve_by_ID_property_daterange(ID=node_id, type="node", property="head", - # start_date="2024-11-25", end_date="2024-11-25") - # print(curve_result) - - # 示例4：query_SCADA_data_by_device_ID_and_time - # SCADA_result_dict = query_SCADA_data_by_device_ID_and_time(globals.fixed_pump_realtime_ids, query_time='2025-03-09T23:45:00+08:00') - # print(SCADA_result_dict) - - # 示例5：query_SCADA_data_curve - # SCADA_result = query_SCADA_data_curve(api_query_id='9485', start_date='2024-03-25', end_date='2024-03-25') - # print(SCADA_result) - - # 示例6：export_SCADA_data_to_csv - # export_SCADA_data_to_csv(start_date='2025-03-30', end_date='2025-03-30') - - # 示例7：export_realtime_simulation_result_to_csv - # export_realtime_simulation_result_to_csv(start_date='2025-02-13', end_date='2025-02-15') - - # 示例8：export_scheme_simulation_result_to_csv_time - # export_scheme_simulation_result_to_csv_time(start_date='2025-02-13', end_date='2025-02-15') - - # 示例9：export_scheme_simulation_result_to_csv_scheme - # export_scheme_simulation_result_to_csv_scheme(scheme_type='burst_Analysis', scheme_name='scheme1', query_date='2025-03-10') - - # 示例10：query_scheme_all_record_by_time - # node_records, link_records = query_scheme_all_record_by_time(scheme_type='burst_Analysis', scheme_name='scheme1', query_time="2025-02-14T10:30:00+08:00") - # print("Node 数据：", node_records) - # print("Link 数据：", link_records) - - # 示例11：query_scheme_curve_by_ID_property - # curve_result = query_scheme_curve_by_ID_property(scheme_type='burst_Analysis', scheme_name='scheme1', ID='ZBBDTZDP000022', - # type='node', property='head') - # print(curve_result) - - # 示例12：query_all_record_by_date - # node_records, link_records = query_all_record_by_date(query_date='2025-02-27') - # print("Node 数据：", node_records) - # print("Link 数据：", link_records) - - # 示例13：query_scheme_all_record - # node_records, link_records = query_scheme_all_record(scheme_type='burst_Analysis', scheme_name='scheme1', query_date='2025-03-10') - # print("Node 数据：", node_records) - # print("Link 数据：", link_records) - - # 示例14：query_all_record_by_time_property - # result_records = query_all_record_by_time_property(query_time='2025-03-30T12:00:00+08:00', type='node', property='pressure') - # print(result_records) - - # 示例15：query_all_record_by_date_property - # result_records = query_all_record_by_date_property(query_date='2025-02-14', type='node', property='head') - # print(result_records) - - # 示例16：query_scheme_all_record_by_time_property - # result_records = query_scheme_all_record_by_time_property(scheme_type='burst_Analysis', scheme_name='scheme1', - # query_time='2025-02-14T10:30:00+08:00', type='node', property='head') - # print(result_records) - - # 示例17：query_scheme_all_record_property - # result_records = query_scheme_all_record_property(scheme_type='burst_Analysis', scheme_name='scheme1', query_date='2025-03-10', type='node', property='head') - # print(result_records) - - # 示例18：fill_scheme_simulation_result_to_SCADA - # fill_scheme_simulation_result_to_SCADA(scheme_type='burst_Analysis', scheme_name='burst0330', query_date='2025-03-30') - - # 示例19：query_SCADA_data_by_device_ID_and_timerange - # result = query_SCADA_data_by_device_ID_and_timerange(query_ids_list=globals.pressure_non_realtime_ids, start_time='2025-04-16T00:00:00+08:00', - # end_time='2025-04-16T23:59:00+08:00') - # print(result) - - # 示例：manually_get_burst_flow - # leakage = manually_get_burst_flow(scheme_type='burst_Analysis', scheme_name='burst_scheme', scheme_start_time='2025-03-10T12:00:00+08:00') - # print(leakage) - - # 示例：upload_cleaned_SCADA_data_to_influxdb - import_multicolumn_data_from_file(file_path='data/szh_pressure_scada.csv', raw=True, bucket='SCADA_data') - import_multicolumn_data_from_file(file_path='data/szh_flow_scada_converted.csv', raw=True, bucket='SCADA_data') - - # 示例：delete_data - # delete_data(delete_date='2025-05-04', bucket='SCADA_data') - - # 示例：query_cleaned_SCADA_data_by_device_ID_and_timerange - # result = query_cleaned_SCADA_data_by_device_ID_and_timerange(query_ids_list=['9485'], start_time='2024-03-24T00:00:00+08:00', - # end_time='2024-03-26T23:59:00+08:00') - # print(result) - - # 示例：import_data_from_file - # import_data_from_file(file_path='data/Flow_Timedata.csv', bucket='SCADA_data') - - # # 示例：query_all_records_by_type_date - # result = query_all__records_by_type__date(type='node', query_date='2025-08-04') - - # 示例：query_all_records_by_date_hour - # result = query_all_records_by_date_hour(query_date='2025-08-04', query_hour=1) - - # 示例：import_multicolumn_data_from_file - # import_multicolumn_data_from_file(file_path='data/selected_Flow_Timedata2025_new_format_cleaned.csv', raw=False, bucket='SCADA_data') - - # client = InfluxDBClient(url="http://127.0.0.1:8086", token=token, org=org_name) - # delete_api = client.delete_api() - - # start = "2025-08-02T00:00:00Z" # 要删除的起始时间 - # stop = "2025-08-11T00:00:00Z" # 结束时间（可设为未来） - # predicate = '_measurement="SCADA"' # 指定 measurement - - # delete_api.delete(start, stop, predicate, bucket="SCADA_data", org=org_name) - # client.close() diff --git a/app/infra/db/influxdb/info.py b/app/infra/db/influxdb/info.py deleted file mode 100644 index b330bc3..0000000 --- a/app/infra/db/influxdb/info.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.core.config import settings - -url = settings.INFLUXDB_URL -token = settings.INFLUXDB_TOKEN -org = settings.INFLUXDB_ORG diff --git a/app/infra/db/influxdb/query.py b/app/infra/db/influxdb/query.py deleted file mode 100644 index 300b471..0000000 --- a/app/infra/db/influxdb/query.py +++ /dev/null @@ -1,33 +0,0 @@ -from influxdb_client import InfluxDBClient, Point, WriteOptions -from influxdb_client.client.query_api import QueryApi -import influxdb_info - -# 配置 InfluxDB 连接 -url = influxdb_info.url -token = influxdb_info.token -org = influxdb_info.org -bucket = "SCADA_data" - -# 创建 InfluxDB 客户端 -client = InfluxDBClient(url=url, token=token, org=org) - -# 创建查询 API 对象 -query_api = client.query_api() - -# 构建查询语句 -query = f''' -from(bucket: "{bucket}") - |> range(start: -1h) -''' - -# 执行查询 -result = query_api.query(query) -print(result) - -# 处理查询结果 -for table in result: - for record in table.records: - print(f"Time: {record.get_time()}, Value: {record.get_value()}, Measurement: {record.get_measurement()}, Field: {record.get_field()}") - -# 关闭客户端连接 -client.close() diff --git a/app/infra/db/metadb/repositories/metadata_repository.py b/app/infra/db/metadb/repositories/metadata_repository.py index b620ba5..7345c82 100644 --- a/app/infra/db/metadb/repositories/metadata_repository.py +++ b/app/infra/db/metadb/repositories/metadata_repository.py @@ -20,14 +20,17 @@ def _normalize_postgres_dsn(dsn: str) -> str: scheme, rest = dsn.split("://", 1) if scheme not in ("postgresql", "postgres", "postgresql+psycopg"): return dsn + if scheme == "postgresql+psycopg": + scheme = "postgresql" + normalized_dsn = f"{scheme}://{rest}" if "@" not in rest: - return dsn + return normalized_dsn userinfo, hostinfo = rest.rsplit("@", 1) if ":" not in userinfo: - return dsn + return normalized_dsn username, password = userinfo.split(":", 1) if "@" not in password: - return dsn + return normalized_dsn password = password.replace("@", "%40") return f"{scheme}://{username}:{password}@{hostinfo}" diff --git a/app/infra/db/postgresql/database.py b/app/infra/db/postgresql/database.py index 938db68..ad19f15 100644 --- a/app/infra/db/postgresql/database.py +++ b/app/infra/db/postgresql/database.py @@ -3,7 +3,7 @@ from contextlib import asynccontextmanager from typing import AsyncGenerator, Dict, Optional import psycopg_pool from psycopg.rows import dict_row -import app.core.config as postgresql_info +from app.infra.db.project_routing import get_project_pgconn_string # Configure logging logger = logging.getLogger(__name__) @@ -13,6 +13,7 @@ class Database: def __init__(self, db_name=None): self.pool = None self.db_name = db_name + self.conninfo = None def init_pool(self, db_name=None): """Initialize the connection pool.""" @@ -21,9 +22,10 @@ class Database: # Get connection string, handling default case where target_db_name might be None if target_db_name: - conn_string = postgresql_info.get_pgconn_string(db_name=target_db_name) + conn_string = get_project_pgconn_string(db_name=target_db_name) else: - conn_string = postgresql_info.get_pgconn_string() + conn_string = get_project_pgconn_string() + self.conninfo = conn_string try: self.pool = psycopg_pool.AsyncConnectionPool( @@ -75,6 +77,12 @@ async def get_database_instance(db_name: Optional[str] = None) -> Database: if not db_name: return db # 返回默认数据库实例 + expected_conninfo = get_project_pgconn_string(db_name=db_name) + existing = _database_instances.get(db_name) + if existing is not None and existing.conninfo != expected_conninfo: + await existing.close() + del _database_instances[db_name] + if db_name not in _database_instances: # 创建新的数据库实例 instance = create_database_instance(db_name) diff --git a/app/infra/db/project_routing.py b/app/infra/db/project_routing.py new file mode 100644 index 0000000..47e417e --- /dev/null +++ b/app/infra/db/project_routing.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar, Token +from dataclasses import dataclass +from typing import Iterator + +from psycopg.conninfo import make_conninfo + +from app.core.config import get_pgconn_string, get_timescaledb_pgconn_string + + +@dataclass(frozen=True) +class ActiveProjectRouting: + project_code: str + business_dsn: str + timescale_dsn: str | None = None + + +_active_project_routing: ContextVar[ActiveProjectRouting | None] = ContextVar( + "active_project_routing", + default=None, +) + + +def get_active_project_routing() -> ActiveProjectRouting | None: + return _active_project_routing.get() + + +@contextmanager +def activate_project_routing( + routing: ActiveProjectRouting, +) -> Iterator[ActiveProjectRouting]: + token: Token[ActiveProjectRouting | None] = _active_project_routing.set(routing) + try: + yield routing + finally: + _active_project_routing.reset(token) + + +def _dsn_for_database(dsn: str, database_name: str) -> str: + return make_conninfo(dsn, dbname=database_name) + + +def get_project_pgconn_string(db_name: str | None = None) -> str: + routing = get_active_project_routing() + if routing is None: + return get_pgconn_string(db_name=db_name) + if db_name is None or db_name == routing.project_code: + return routing.business_dsn + return _dsn_for_database(routing.business_dsn, db_name) + + +def get_project_timescale_pgconn_string(db_name: str | None = None) -> str: + routing = get_active_project_routing() + if routing is None: + return get_timescaledb_pgconn_string(db_name=db_name) + if routing.timescale_dsn is None: + raise RuntimeError( + f"TimescaleDB routing is not configured for project {routing.project_code}" + ) + # Legacy simulation code used to derive the Timescale database name from + # the project code. Project-scoped requests must instead use the complete + # iot_data DSN selected by metadata routing. + return routing.timescale_dsn diff --git a/app/infra/db/timescaledb/database.py b/app/infra/db/timescaledb/database.py index 1d2c9de..fd2bcd4 100644 --- a/app/infra/db/timescaledb/database.py +++ b/app/infra/db/timescaledb/database.py @@ -3,7 +3,7 @@ from contextlib import asynccontextmanager from typing import AsyncGenerator, Dict, Optional import psycopg_pool from psycopg.rows import dict_row -from app.core.config import get_timescaledb_pgconn_string +from app.infra.db.project_routing import get_project_timescale_pgconn_string # Configure logging logger = logging.getLogger(__name__) @@ -13,6 +13,7 @@ class Database: def __init__(self, db_name=None): self.pool = None self.db_name = db_name + self.conninfo = None def init_pool(self, db_name=None): """Initialize the connection pool.""" @@ -21,9 +22,10 @@ class Database: # Get connection string, handling default case where target_db_name might be None if target_db_name: - conn_string = get_timescaledb_pgconn_string(db_name=target_db_name) + conn_string = get_project_timescale_pgconn_string(db_name=target_db_name) else: - conn_string = get_timescaledb_pgconn_string() + conn_string = get_project_timescale_pgconn_string() + self.conninfo = conn_string try: self.pool = psycopg_pool.AsyncConnectionPool( @@ -54,8 +56,8 @@ class Database: """Get the TimescaleDB connection string.""" target_db_name = db_name or self.db_name if target_db_name: - return get_timescaledb_pgconn_string(db_name=target_db_name) - return get_timescaledb_pgconn_string() + return get_project_timescale_pgconn_string(db_name=target_db_name) + return get_project_timescale_pgconn_string() @asynccontextmanager async def get_connection(self) -> AsyncGenerator: @@ -84,6 +86,12 @@ async def get_database_instance(db_name: Optional[str] = None) -> Database: if not db_name: return db # 返回默认数据库实例 + expected_conninfo = get_project_timescale_pgconn_string(db_name=db_name) + existing = _database_instances.get(db_name) + if existing is not None and existing.conninfo != expected_conninfo: + await existing.close() + del _database_instances[db_name] + if db_name not in _database_instances: # 创建新的数据库实例 instance = create_database_instance(db_name) diff --git a/app/infra/db/timescaledb/internal_queries.py b/app/infra/db/timescaledb/internal_queries.py index 5126cf8..b6d4cbb 100644 --- a/app/infra/db/timescaledb/internal_queries.py +++ b/app/infra/db/timescaledb/internal_queries.py @@ -6,7 +6,7 @@ import psycopg from psycopg import sql from psycopg.rows import dict_row import time -from app.core.config import get_timescaledb_pgconn_string +from app.infra.db.project_routing import get_project_timescale_pgconn_string from app.infra.db.timescaledb.repositories.scheme import SchemeRepository from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository from app.infra.db.timescaledb.repositories.scada import ScadaRepository @@ -26,9 +26,9 @@ class InternalStorage: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: RealtimeRepository.store_realtime_simulation_result_sync( @@ -58,9 +58,9 @@ class InternalStorage: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: SchemeRepository.store_scheme_simulation_result_sync( @@ -99,9 +99,9 @@ class InternalQueries: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: rows = ScadaRepository.get_scada_by_ids_time_range_sync( @@ -140,9 +140,9 @@ class InternalQueries: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: rows = ScadaRepository.get_scada_by_ids_time_range_sync( @@ -185,9 +185,9 @@ class InternalQueries: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: return ScadaRepository.get_latest_scada_time_sync( @@ -286,9 +286,9 @@ class InternalQueries: for attempt in range(max_retries): try: conn_string = ( - get_timescaledb_pgconn_string(db_name=db_name) + get_project_timescale_pgconn_string(db_name=db_name) if db_name - else get_timescaledb_pgconn_string() + else get_project_timescale_pgconn_string() ) with psycopg.Connection.connect(conn_string) as conn: with conn.cursor(row_factory=dict_row) as cur: diff --git a/app/native/wndb/connection.py b/app/native/wndb/connection.py index c8d1a67..db4cce1 100644 --- a/app/native/wndb/connection.py +++ b/app/native/wndb/connection.py @@ -4,9 +4,10 @@ from threading import RLock import psycopg as pg -from app.core.config import get_pgconn_string +from app.infra.db.project_routing import get_project_pgconn_string g_conn_dict: dict[str, pg.Connection] = {} +g_conninfo_dict: dict[str, str] = {} _registry_lock = RLock() _project_locks: dict[str, RLock] = {} @@ -42,14 +43,18 @@ def _get_project_lock(name: str) -> RLock: def open_connection(name: str) -> pg.Connection: with _get_project_lock(name): + conninfo = get_project_pgconn_string(db_name=name) connection = g_conn_dict.get(name) - if connection is None or not _is_healthy(connection): + if ( + connection is None + or g_conninfo_dict.get(name) != conninfo + or not _is_healthy(connection) + ): if connection is not None: _close_connection(connection) - connection = pg.connect( - conninfo=get_pgconn_string(db_name=name), autocommit=True - ) + connection = pg.connect(conninfo=conninfo, autocommit=True) g_conn_dict[name] = connection + g_conninfo_dict[name] = conninfo return connection @@ -60,6 +65,12 @@ def is_connection_open(name: str) -> bool: return False if not _is_healthy(connection): del g_conn_dict[name] + g_conninfo_dict.pop(name, None) + _close_connection(connection) + return False + if g_conninfo_dict.get(name) != get_project_pgconn_string(db_name=name): + del g_conn_dict[name] + g_conninfo_dict.pop(name, None) _close_connection(connection) return False return True @@ -68,6 +79,7 @@ def is_connection_open(name: str) -> bool: def close_connection(name: str) -> None: with _get_project_lock(name): connection = g_conn_dict.pop(name, None) + g_conninfo_dict.pop(name, None) if connection is not None: _close_connection(connection) diff --git a/app/native/wndb/project.py b/app/native/wndb/project.py index 5403d0f..2a7bca6 100644 --- a/app/native/wndb/project.py +++ b/app/native/wndb/project.py @@ -7,7 +7,8 @@ from .connection import ( is_connection_open, open_connection, ) -from app.core.config import get_pgconn_string, get_pg_config, get_pg_password +from app.core.config import get_pg_config, get_pg_password +from app.infra.db.project_routing import get_project_pgconn_string # no undo/redo @@ -16,7 +17,7 @@ _server_databases = ["template0", "template1", "postgres", "project"] def list_project() -> list[str]: ps = [] - with pg.connect(conninfo=get_pgconn_string(), autocommit=True) as conn: + with pg.connect(conninfo=get_project_pgconn_string(), autocommit=True) as conn: with conn.cursor(row_factory=dict_row) as cur: for p in cur.execute( f"select datname from pg_database where datname <> 'postgres' and datname <> 'template0' and datname <> 'template1' and datname <> 'project'" @@ -27,7 +28,7 @@ def list_project() -> list[str]: def have_project(name: str) -> bool: with pg.connect( - conninfo=get_pgconn_string(db_name="postgres"), autocommit=True + conninfo=get_project_pgconn_string(db_name="postgres"), autocommit=True ) as conn: with conn.cursor() as cur: cur.execute("select 1 from pg_database where datname = %s", (name,)) @@ -38,7 +39,7 @@ def copy_project(source: str, new: str) -> None: close_connection(source) with pg.connect( - conninfo=get_pgconn_string(db_name="postgres"), autocommit=True + conninfo=get_project_pgconn_string(db_name="postgres"), autocommit=True ) as admin_conn: with admin_conn.cursor() as cur: cur.execute( @@ -131,7 +132,9 @@ class CopyProjectEx: connection.commit() def __call__(self, source: str, new_db: str, excluded_tables: [str] = None) -> None: - source_connection = pg.connect(conninfo=get_pgconn_string(), autocommit=True) + source_connection = pg.connect( + conninfo=get_project_pgconn_string(), autocommit=True + ) self.create_database(source_connection, new_db) @@ -140,7 +143,7 @@ class CopyProjectEx: source_connection.close() new_db_connection = pg.connect( - conninfo=get_pgconn_string(db_name=new_db), autocommit=True + conninfo=get_project_pgconn_string(db_name=new_db), autocommit=True ) self.init_operation_table(new_db_connection, excluded_tables) new_db_connection.close() @@ -151,7 +154,7 @@ def create_project(name: str) -> None: def delete_project(name: str) -> None: - with pg.connect(conninfo=get_pgconn_string(), autocommit=True) as conn: + with pg.connect(conninfo=get_project_pgconn_string(), autocommit=True) as conn: with conn.cursor() as cur: cur.execute( f"select pg_terminate_backend(pid) from pg_stat_activity where datname = '{name}'" @@ -161,7 +164,7 @@ def delete_project(name: str) -> None: def clean_project(excluded: list[str] = []) -> None: projects = list_project() - with pg.connect(conninfo=get_pgconn_string(), autocommit=True) as conn: + with pg.connect(conninfo=get_project_pgconn_string(), autocommit=True) as conn: with conn.cursor(row_factory=dict_row) as cur: row = cur.execute(f"select current_database()").fetchone() if row != None: diff --git a/app/services/globals.py b/app/services/globals.py index d71c63d..681ed05 100644 --- a/app/services/globals.py +++ b/app/services/globals.py @@ -23,7 +23,6 @@ non_realtime_region_patterns = {} # 基于source_outflow_region进行区分 realtime_region_pipe_flow_and_demand_id = {} # 基于source_outflow_region搜索该分区中的实时pipe_flow和demand的api_query_id，后续用region的流量 - 实时流量计的流量 realtime_region_pipe_flow_and_demand_patterns = {} # 基于source_outflow_region搜索该分区中的实时pipe_flow和demand的associated_pattern，后续用region的流量 - 实时流量计的流量 # --------------------------------------------------------- -# influxdb_api.py中的全局变量 # 全局变量，用于存储不同类型的realtime api_query_id reservoir_liquid_level_realtime_ids = [] tank_liquid_level_realtime_ids = [] diff --git a/app/services/network_import.py b/app/services/network_import.py index 3c8e5df..378ffa2 100644 --- a/app/services/network_import.py +++ b/app/services/network_import.py @@ -5,8 +5,7 @@ import chardet import psycopg from psycopg import sql -import app.services.project_info as project_info -from app.core.config import get_pgconn_string +from app.infra.db.project_routing import get_project_pgconn_string from app.services.tjnetwork import read_inp @@ -15,13 +14,14 @@ from app.services.tjnetwork import read_inp ############################################################ -def network_update(file_path: str) -> None: +def network_update(file_path: str, project_code: str) -> None: """ 更新pg数据库中的inp文件 :param file_path: inp文件 + :param project_code: 元数据项目代码 :return: """ - read_inp("szh", file_path) + read_inp(project_code, file_path) csv_path = "./history_pattern_flow.csv" @@ -51,8 +51,7 @@ def network_update(file_path: str) -> None: if os.path.exists(csv_path): print(f"history_patterns_flows文件存在，开始处理...") - # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） - with psycopg.connect(f"dbname={project_info.name} host=127.0.0.1") as conn: + with psycopg.connect(get_project_pgconn_string(project_code)) as conn: with conn.cursor() as cur: with open(csv_path, newline="", encoding="utf-8-sig") as csvfile: reader = csv.DictReader(csvfile) @@ -92,7 +91,7 @@ def submit_scada_info(name: str, coord_id: str) -> None: print(f"检测到的文件编码：{file_encoding}") try: # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） with psycopg.connect(conn_string) as conn: diff --git a/app/services/scheme_management.py b/app/services/scheme_management.py index 1cab298..b0bfe06 100644 --- a/app/services/scheme_management.py +++ b/app/services/scheme_management.py @@ -7,7 +7,7 @@ import pandas as pd import psycopg from sqlalchemy import create_engine -from app.core.config import get_pgconn_string +from app.infra.db.project_routing import get_project_pgconn_string from app.services.time_api import parse_utc_time @@ -20,7 +20,7 @@ def scheme_name_exists(name: str, scheme_name: str) -> bool: :return: 如果存在返回 True，否则返回 False """ try: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: cur.execute( @@ -57,7 +57,7 @@ def store_scheme_info( :return: """ try: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: sql = """ @@ -93,7 +93,7 @@ def delete_scheme_info(name: str, scheme_name: str) -> None: :param scheme_name: 要删除的方案名称 """ try: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: # 使用参数化查询删除方案记录 @@ -121,7 +121,7 @@ def query_scheme_list( """ try: # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) # 连接到 PostgreSQL 数据库（这里是数据库 "bb"） with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -203,7 +203,7 @@ def query_scheme_detail( scheme_type, ) - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: if scheme_type: @@ -255,7 +255,7 @@ def store_leakage_identify_result( run_status: str = "completed", error_message: str | None = None, ) -> None: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: cur.execute( @@ -299,7 +299,7 @@ def query_leakage_identify_schemes( scheme_type: str = "dma_leak_identification", query_date: date | None = None, ) -> list[dict]: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: if query_date is None: @@ -343,7 +343,7 @@ def query_leakage_identify_schemes( def query_leakage_identify_scheme_detail(name: str, scheme_name: str) -> dict: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: cur.execute( @@ -400,7 +400,7 @@ def query_burst_location_schemes( scheme_type: str = "burst_location", query_date: date | None = None, ) -> list[dict]: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: if query_date is None: @@ -444,7 +444,7 @@ def query_burst_location_schemes( def query_burst_location_scheme_detail(name: str, scheme_name: str) -> dict: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: cur.execute( @@ -479,7 +479,7 @@ def query_burst_detection_schemes( scheme_type: str = "burst_detection", query_date: date | None = None, ) -> list[dict]: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: if query_date is None: @@ -523,7 +523,7 @@ def query_burst_detection_schemes( def query_burst_detection_scheme_detail(name: str, scheme_name: str) -> dict: - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: cur.execute( @@ -564,7 +564,7 @@ def upload_shp_to_pg(name: str, table_name: str, role: str, shp_file_path: str): """ try: # 动态连接到指定的数据库 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) with psycopg.connect(conn_string) as conn: # 读取 Shapefile 文件 gdf = gpd.read_file(shp_file_path) @@ -604,7 +604,7 @@ def submit_risk_probability_result(name: str, result_file_path: str) -> None: try: # 动态替换数据库名称 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) # 连接到 PostgreSQL 数据库 with psycopg.connect(conn_string) as conn: diff --git a/app/services/simulation.py b/app/services/simulation.py index d56776c..0ee8b4b 100644 --- a/app/services/simulation.py +++ b/app/services/simulation.py @@ -28,14 +28,13 @@ import pytz import requests import time from typing import Optional, Tuple -import app.infra.db.influxdb.api as influxdb_api import typing import psycopg import logging import app.services.globals as globals import app.services.project_info as project_info from app.services.time_api import parse_beijing_time, parse_clock_duration_seconds -from app.core.config import get_pgconn_string +from app.infra.db.project_routing import get_project_pgconn_string from app.infra.db.timescaledb.internal_queries import ( InternalQueries as TimescaleInternalQueries, ) @@ -55,7 +54,7 @@ def query_corresponding_element_id_and_query_id(name: str) -> None: :return: """ # 连接数据库 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -100,7 +99,7 @@ def query_corresponding_pattern_id_and_query_id(name: str) -> None: :return: """ # 连接数据库 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -140,7 +139,7 @@ def query_non_realtime_region(name: str) -> dict: """ source_outflow_regions = [] # 用于存储所有 region（包含重复的） # 构建连接字符串 - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: # 连接到数据库 with psycopg.connect(conn_string) as conn: @@ -217,7 +216,7 @@ def query_non_realtime_region_patterns( region_tuple_to_key = { frozenset(ids): region for region, ids in globals.source_outflow_region.items() } - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -305,7 +304,7 @@ def query_realtime_region_pipe_flow_and_demand_id( region_tuple_to_key = { frozenset(ids): region for region, ids in globals.source_outflow_region.items() } - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -377,7 +376,7 @@ def query_pipe_flow_region_patterns( :param column_prefix: 需要提取的列的前缀 :return: pipe_flow_region_patterns 字典 """ - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -440,7 +439,7 @@ def query_SCADA_ID_corresponding_info(name: str, SCADA_ID: str) -> dict: :param SCADA_ID: SCADA设备的ID :return: 包含associated_element_id和api_query_id的字典 """ - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: # 使用 psycopg.connect 创建连接 with psycopg.connect(conn_string) as conn: @@ -496,7 +495,7 @@ def get_source_outflow_region_id( "No associated_source_outflow_id found in source_outflow_region." ) return globals.source_outflow_region_id - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: @@ -553,7 +552,7 @@ def get_realtime_region_patterns( globals.realtime_region_pipe_flow_and_demand_patterns = { region: [] for region in globals.realtime_region_pipe_flow_and_demand_id.keys() } - conn_string = get_pgconn_string(db_name=name) + conn_string = get_project_pgconn_string(db_name=name) try: with psycopg.connect(conn_string) as conn: with conn.cursor() as cur: diff --git a/contracts/manifest.json b/contracts/manifest.json index 782a78b..70f691e 100644 --- a/contracts/manifest.json +++ b/contracts/manifest.json @@ -3,7 +3,7 @@ "contracts": { "server": { "file": "server-v1.openapi.json", - "sha256": "df7ae927dcf5ae32c3c1ad9be3245b1b78b984ce1902dd91e6313770860e0d48" + "sha256": "ac9b6fac185dfd999f1791cba51eb482df17a427b361963250aafa5fb1a276b4" } } } diff --git a/contracts/server-v1.openapi.json b/contracts/server-v1.openapi.json index 1c05cfd..5c176bc 100644 --- a/contracts/server-v1.openapi.json +++ b/contracts/server-v1.openapi.json @@ -5227,97 +5227,6 @@ ] } }, - "/api/v1/all-redis": { - "delete": { - "description": "清空整个Redis数据库的所有缓存", - "operationId": "delete_all_redis", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "清除所有缓存", - "tags": [ - "Cache" - ] - } - }, "/api/v1/all-scada-properties": { "get": { "description": "获取指定水网中所有SCADA点的属性信息", @@ -29452,308 +29361,6 @@ ] } }, - "/api/v1/redis": { - "get": { - "description": "获取Redis中所有的缓存键", - "operationId": "get_redis", - "parameters": [ - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JsonValue" - } - } - }, - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "查询缓存键列表", - "tags": [ - "Cache" - ] - } - }, - "/api/v1/redis-keys": { - "delete": { - "description": "根据模式清除匹配的Redis缓存键", - "operationId": "delete_redis_keys", - "parameters": [ - { - "description": "缓存键模式（支持通配符）", - "in": "query", - "name": "keys", - "required": true, - "schema": { - "description": "缓存键模式（支持通配符）", - "title": "Keys", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "清除匹配的缓存键", - "tags": [ - "Cache" - ] - } - }, - "/api/v1/redis-keys/detail": { - "delete": { - "description": "根据键名清除单个Redis缓存", - "operationId": "delete_redis_keys_detail", - "parameters": [ - { - "description": "缓存键名", - "in": "query", - "name": "key", - "required": true, - "schema": { - "description": "缓存键名", - "title": "Key", - "type": "string" - } - }, - { - "in": "header", - "name": "X-Project-Id", - "required": true, - "schema": { - "title": "X-Project-Id", - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Authentication required" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Insufficient permission" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource not found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Resource conflict" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Validation error" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - }, - "description": "Dependency unavailable" - } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ], - "summary": "清除单个缓存键", - "tags": [ - "Cache" - ] - } - }, "/api/v1/redos": { "post": { "description": "重做网络上被撤销的操作", diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml index 138b374..215ec73 100644 --- a/infra/docker/docker-compose.yml +++ b/infra/docker/docker-compose.yml @@ -16,12 +16,8 @@ services: - ../../resources:/app/resources environment: - PYTHONPATH=/app - - REDIS_HOST=redis - - REDIS_PORT=${REDIS_PORT} - - REDIS_PASSWORD=${REDIS_PASSWORD} # Add other DB connections here as needed by your app depends_on: - - redis - timescaledb - postgis @@ -29,17 +25,6 @@ services: # Infrastructure Services # ========================================== - # --- Redis --- - redis: - image: redis:latest - container_name: redis - restart: always - command: redis-server --requirepass ${REDIS_PASSWORD} - ports: - - "${REDIS_PORT}:6379" - volumes: - - ./redis/data:/data - # --- Keycloak --- keycloakDB: image: postgis/postgis:14-3.5 diff --git a/requirements.txt b/requirements.txt index 04460ac..95bf401 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,6 @@ email-validator==2.3.0 esda==2.7.0 et_xmlfile==2.0.0 exceptiongroup==1.3.1 -fakeredis==2.33.0 fastapi==0.128.0 fastmcp==2.9.2 fonttools==4.58.0 @@ -43,7 +42,6 @@ httpx==0.28.1 httpx-sse==0.4.3 idna==3.10 importlib_metadata==8.7.1 -influxdb-client==1.48.0 iniconfig==2.0.0 jaraco.classes==3.4.0 jaraco.context==6.1.0 @@ -107,7 +105,6 @@ pydantic==2.10.6 pydantic-settings==2.12.0 pydantic_core==2.27.2 pydevd-pycharm==243.16718.36 -pydocket==0.16.6 Pygments==2.18.0 PyJWT==2.10.1 pykalman==0.10.2 @@ -127,7 +124,6 @@ pytz==2025.2 PyYAML==6.0.3 pyzmq==26.2.1 reactivex==4.0.4 -redis==5.2.1 referencing==0.36.2 requests==2.32.3 rich==14.2.0 @@ -168,4 +164,4 @@ zmq==0.0.0 pymoo==0.6.1.6 scikit-learn==1.6.1 scipy==1.15.2 -pyclipper==1.4.0 \ No newline at end of file +pyclipper==1.4.0 diff --git a/resources/old_requirements.txt b/resources/old_requirements.txt index 5b04d859b7842510b325e2b3b630f0ad3bd90c2a..f9a0c3ec3e93088d7fd849869464fb8a01b53c0b 100644 GIT binary patch delta 17 ZcmX@5{6}d+7xU&l%wAlZmH1XL0{}%#2Au!^ delta 77 zcmeyPbV_+c7qfaALk>eJLj^+$LlT27Lo$$;$&kvB$56sx3xtLYdJHBE77Tiu>zPBj Xgj0Z$#b60jpo9?+8*V<$yO9|H!q^a~ diff --git a/scripts/all_auto_task.py b/scripts/all_auto_task.py deleted file mode 100644 index c7de0b8..0000000 --- a/scripts/all_auto_task.py +++ /dev/null @@ -1,25 +0,0 @@ -import auto_realtime -import auto_store_non_realtime_SCADA_data -import asyncio -import influxdb_api -import influxdb_info -import project_info - -# 为了让多个任务并发运行，我们可以用 asyncio.to_thread 分别启动它们 -async def main(): - task1 = asyncio.to_thread(auto_realtime.realtime_task) - task2 = asyncio.to_thread(auto_store_non_realtime_SCADA_data.store_non_realtime_SCADA_data_task) - await asyncio.gather(task1, task2) - - -if __name__ == "__main__": - url = influxdb_info.url - token = influxdb_info.token - org_name = influxdb_info.org - - influxdb_api.query_pg_scada_info_realtime(project_info.name) - influxdb_api.query_pg_scada_info_non_realtime(project_info.name) - - # 用 asyncio 并发启动两个任务 - asyncio.run(main()) - \ No newline at end of file diff --git a/scripts/auto_cache.py b/scripts/auto_cache.py deleted file mode 100644 index 451c584..0000000 --- a/scripts/auto_cache.py +++ /dev/null @@ -1,115 +0,0 @@ -import schedule -import time -import datetime -import shutil -import redis -import urllib.request -import influxdb_api -import msgpack -import datetime - -# 将 Query的信息 序列号到 redis/json， 默认不支持datetime，需要自定义 -# 自定义序列化函数 -# 序列化处理器 -def encode_datetime(obj): - """将datetime转换为可序列化的字典结构""" - if isinstance(obj, datetime.datetime): - return { - '__datetime__': True, - 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f") - } - return obj - -# 反序列化处理器 -def decode_datetime(obj): - """将字典还原为datetime对象""" - if '__datetime__' in obj: - return datetime.datetime.strptime( - obj['as_str'], "%Y%m%dT%H:%M:%S.%f" - ) - return obj - -########################## -# 需要用Python 3.12 来运行才能提高performance -########################## - -def queryallrecordsbydate(querydate: str, redis_client: redis.Redis): - cache_key = f"queryallrecordsbydate_{querydate}" - exists = redis_client.exists(cache_key) - - if not exists: - nodes_links: tuple = influxdb_api.query_all_records_by_date(query_date=querydate) - redis_client.set(cache_key, msgpack.packb(nodes_links, default=encode_datetime)) - -def queryallrecordsbydate_by_url(querydate: str): - print(f'queryallrecordsbydate: {querydate}') - - try: - response = urllib.request.urlopen( - f"http://127.0.0.1/queryallrecordsbydate/?querydate={querydate}" - ) - html = response.read().decode("utf-8") - - except urllib.error.URLError as e: - print("Error") - -def queryallscadarecordsbydate(querydate: str, redis_client: redis.Redis): - cache_key = f"queryallscadarecordsbydate_{querydate}" - exists = redis_client.exists(cache_key) - - if not exists: - result_dict = influxdb_api.query_all_SCADA_records_by_date(query_date=querydate) - redis_client.set(cache_key, msgpack.packb(result_dict, default=encode_datetime)) - -def queryallscadarecordsbydate_by_url(querydate: str): - print(f'queryallscadarecordsbydate: {querydate}') - - try: - response = urllib.request.urlopen( - f"http://127.0.0.1/queryallscadarecordsbydate/?querydate={querydate}" - ) - html = response.read().decode("utf-8") - - except urllib.error.URLError as e: - print("Error") - - -def auto_cache_data(): - # 初始化 Redis 连接 - # 用redis 限制并发访u - redis_client = redis.Redis(host="127.0.0.1", port=6379, db=0) - - # auto cache data for the last 3 days - today = datetime.date.today() - for i in range(1, 4): - prev_day = today - datetime.timedelta(days=i) - str_prev_day = prev_day.strftime('%Y-%m-%d') - print(str_prev_day) - - queryallrecordsbydate(str_prev_day, redis_client) - queryallscadarecordsbydate(str_prev_day, redis_client) - - redis_client.close() - -def auto_cache_data_by_url(): - # auto cache data for the last 3 days - today = datetime.date.today() - for i in range(1, 4): - prev_day = today - datetime.timedelta(days=i) - str_prev_day = prev_day.strftime('%Y-%m-%d') - print(str_prev_day) - - queryallrecordsbydate_by_url(str_prev_day) - queryallscadarecordsbydate_by_url(str_prev_day) - - -if __name__ == "__main__": - - auto_cache_data_by_url() - - # auto run in the midnight - schedule.every().day.at("03:00").do(auto_cache_data_by_url) - - while True: - schedule.run_pending() - time.sleep(1) diff --git a/scripts/auto_realtime.py b/scripts/auto_realtime.py deleted file mode 100644 index 39c34fb..0000000 --- a/scripts/auto_realtime.py +++ /dev/null @@ -1,156 +0,0 @@ -from logging.handlers import TimedRotatingFileHandler -import influxdb_api -import os -import logging -import globals -from datetime import datetime, timedelta, timezone -import schedule -import time -import shutil -from influxdb_client import InfluxDBClient, BucketsApi, WriteApi, OrganizationsApi, Point, QueryApi -import simulation -import influxdb_info -import project_info - -def setup_logger(): - # 创建日志目录 - log_dir = "logs" - os.makedirs(log_dir, exist_ok=True) - - # 配置基础日志格式 - log_format = "%(asctime)s - %(levelname)s - %(message)s" - formatter = logging.Formatter(log_format) - - # 创建主 Logger - logger = logging.getLogger() - logger.setLevel(logging.INFO) # 全局日志级别 - - # --- 1. 按日期分割的日志文件 Handler --- - log_file = os.path.join(log_dir, "simulation.log") - file_handler = TimedRotatingFileHandler( - filename=log_file, - when="midnight", # 每天午夜轮转 - interval=1, - backupCount=7, - encoding="utf-8" - ) - file_handler.suffix = "simulation-%Y-%m-%d.log" # 文件名格式 - file_handler.setFormatter(formatter) - file_handler.setLevel(logging.INFO) # 文件记录所有级别日志 - - # --- 2. 控制台实时输出 Handler --- - console_handler = logging.StreamHandler() # 默认输出到 sys.stderr (控制台) - console_handler.setFormatter(formatter) - console_handler.setLevel(logging.INFO) # 控制台仅显示 INFO 及以上级别 - - # 将 Handler 添加到 Logger - logger.addHandler(file_handler) - #logger.addHandler(console_handler) - - return logger - -logger = setup_logger() - -# 2025/02/01 -def get_next_time() -> str: - """ - 获取下一个1分钟时间点，返回格式为字符串'YYYY-MM-DDTHH:MM:00+08:00' - :return: 返回字符串格式的时间，表示下一个1分钟的时间点 - """ - # 获取当前时间，并设定为北京时间 - now = datetime.now() # now 类型为 datetime，表示当前本地时间 - # 获取当前的分钟，并且将秒和微秒置为零 - current_time = now.replace(second=0, microsecond=0) # current_time 类型为 datetime，时间的秒和微秒部分被清除 - return current_time.strftime('%Y-%m-%dT%H:%M:%S+08:00') - - -# 2025/02/06 -def store_realtime_SCADA_data_job() -> None: - """ - 定义的任务1，每分钟执行1次，每次执行时，更新get_real_value_time并调用store_realtime_SCADA_data_to_influxdb函数 - :return: None - """ - # 获取当前时间并更新get_real_value_time，转换为字符串格式 - get_real_value_time: str = get_next_time() # get_real_value_time 类型为 str，格式为'2025-02-01T18:45:00+08:00' - # 调用函数执行任务 - influxdb_api.store_realtime_SCADA_data_to_influxdb(get_real_value_time) - logger.info('{} -- Successfully store realtime SCADA data.'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) - - -# 2025/02/06 -def get_next_15minute_time() -> str: - """ - 获取下一个15分钟的时间点，返回格式为字符串'YYYY-MM-DDTHH:MM:00+08:00' - :return: 返回字符串格式的时间，表示下一个15分钟执行时间点 - """ - now = datetime.now() - # 向上舍入到下一个15分钟 - next_15minute = (now.minute // 15 + 1) * 15 - 15 - if next_15minute == 60: - next_15minute = 0 - now = now + timedelta(hours=1) - next_time = now.replace(minute=next_15minute, second=0, microsecond=0) - return next_time.strftime('%Y-%m-%dT%H:%M:%S+08:00') - - -# 2025/02/07 -def run_simulation_job() -> None: - """ - 定义的任务3，每15分钟执行一次在store_realtime_SCADA_data_to_influxdb之后执行run_simulation。 - :return: None - """ - # 获取当前时间，并检查是否是整点15分钟 - current_time = datetime.now() - if current_time.minute % 15 == 0: - print(f"{current_time.strftime('%Y-%m-%d %H:%M:%S')} -- Start simulation task.") - # 计算前，获取scada_info中的信息，按照设定的方法修改pg数据库 - simulation.query_corresponding_element_id_and_query_id(project_info.name) - simulation.query_corresponding_pattern_id_and_query_id(project_info.name) - region_result = simulation.query_non_realtime_region(project_info.name) - globals.source_outflow_region_id = simulation.get_source_outflow_region_id(project_info.name, region_result) - globals.realtime_region_pipe_flow_and_demand_id = simulation.query_realtime_region_pipe_flow_and_demand_id(project_info.name, region_result) - globals.pipe_flow_region_patterns = simulation.query_pipe_flow_region_patterns(project_info.name) - globals.non_realtime_region_patterns = simulation.query_non_realtime_region_patterns(project_info.name, region_result) - globals.source_outflow_region_patterns, realtime_region_pipe_flow_and_demand_patterns = simulation.get_realtime_region_patterns(project_info.name, - globals.source_outflow_region_id, - globals.realtime_region_pipe_flow_and_demand_id) - modify_pattern_start_time: str = get_next_15minute_time() # 获取下一个15分钟时间点 - # print(modify_pattern_start_time) - simulation.run_simulation(name=project_info.name, simulation_type="realtime", modify_pattern_start_time=modify_pattern_start_time) - - logger.info('{} -- Successfully run simulation and store realtime simulation result.'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) - else: - logger.info(f"{current_time.strftime('%Y-%m-%d %H:%M:%S')} -- Skipping the simulation task.") - - -# 2025/02/06 -def realtime_task() -> None: - """ - 定时执行任务1和，使用schedule库每1分钟执行一次store_realtime_SCADA_data_job函数。 - 该任务会一直运行，定期调用store_realtime_SCADA_data_job获取SCADA数据。 - :return: - """ - # 等待到整分对齐 - now = datetime.now() - wait_seconds = 60 - now.second - time.sleep(wait_seconds) - # 使用 .at(":00") 指定在每分钟的第0秒执行 - schedule.every(1).minute.at(":00").do(store_realtime_SCADA_data_job) - # 每15分钟执行一次run_simulation_job - schedule.every(1).minute.at(":00").do(run_simulation_job) - # 持续执行任务，检查是否有待执行的任务 - while True: - schedule.run_pending() # 执行所有待处理的定时任务 - time.sleep(1) # 暂停1秒，避免过于频繁的任务检查 - - -if __name__ == "__main__": - url = influxdb_info.url - token = influxdb_info.token - org_name = influxdb_info.org - - client = InfluxDBClient(url=url, token=token) - # step2: 先查询pg数据库中scada_info的信息，然后存储SCADA数据到SCADA_data这个bucket里 - influxdb_api.query_pg_scada_info_realtime(project_info.name) - # 自动执行 - realtime_task() diff --git a/scripts/auto_store_non_realtime_SCADA_data.py b/scripts/auto_store_non_realtime_SCADA_data.py deleted file mode 100644 index 8995222..0000000 --- a/scripts/auto_store_non_realtime_SCADA_data.py +++ /dev/null @@ -1,139 +0,0 @@ -import influxdb_api -import globals -from datetime import datetime, timedelta, timezone -import schedule -import os -import logging -from logging.handlers import TimedRotatingFileHandler -import time -from influxdb_client import InfluxDBClient, BucketsApi, WriteApi, OrganizationsApi, Point, QueryApi -import influxdb_info -import project_info - -def setup_logger(): - # 创建日志目录 - log_dir = "logs" - os.makedirs(log_dir, exist_ok=True) - - # 配置基础日志格式 - log_format = "%(asctime)s - %(levelname)s - %(message)s" - formatter = logging.Formatter(log_format) - - # 创建主 Logger - logger = logging.getLogger() - logger.setLevel(logging.INFO) # 全局日志级别 - - # --- 1. 按日期分割的日志文件 Handler --- - log_file = os.path.join(log_dir, "scada.log") - file_handler = TimedRotatingFileHandler( - filename=log_file, - when="midnight", # 每天午夜轮转 - interval=1, - backupCount=7, - encoding="utf-8" - ) - file_handler.suffix = "scada-%Y-%m-%d.log" # 文件名格式 - file_handler.setFormatter(formatter) - file_handler.setLevel(logging.INFO) # 文件记录 INFO 及以上级别 - - # --- 2. 控制台实时输出 Handler --- - console_handler = logging.StreamHandler() # 默认输出到 sys.stderr (控制台) - console_handler.setFormatter(formatter) - console_handler.setLevel(logging.INFO) # 控制台仅显示 INFO 及以上级别 - - # 将 Handler 添加到 Logger - logger.addHandler(file_handler) - # logger.addHandler(console_handler) - - return logger - - -logger = setup_logger() - -# 2025/02/01 -def get_next_time() -> str: - """ - 获取下一个1分钟时间点，返回格式为字符串'YYYY-MM-DDTHH:MM:00+08:00' - :return: 返回字符串格式的时间，表示下一个1分钟的时间点 - """ - # 获取当前时间，并设定为北京时间 - now = datetime.now() # now 类型为 datetime，表示当前本地时间 - # 获取当前的分钟，并且将秒和微秒置为零 - current_time = now.replace(second=0, microsecond=0) # current_time 类型为 datetime，时间的秒和微秒部分被清除 - return current_time.strftime('%Y-%m-%dT%H:%M:%S+08:00') - - -# 2025/02/06 -def get_next_period_time() -> str: - """ - 获取下一个6小时时间点，返回格式为字符串'YYYY-MM-DDTHH:00:00+08:00' - :return: 返回字符串格式的时间，表示下一个6小时执行时间点 - """ - # 获取当前时间，并设定为北京时间 - now = datetime.now() # now 类型为 datetime，表示当前本地时间 - # 获取当前的小时数并计算下一个6小时时间点 - next_period_hour = (now.hour // 6 + 1) * 6 - 6 # next_period_hour 类型为 int，表示下一个6小时时间点的小时部分 - # 如果计算的小时大于23，表示进入第二天，调整为00:00 - if next_period_hour >= 24: - next_period_hour = 0 - now = now + timedelta(days=1) # 如果超过24小时，日期增加1天 - # 将秒和微秒部分清除，构建出下一个6小时点的datetime对象 - next_period_time = now.replace(hour=next_period_hour, minute=0, second=0, microsecond=0) - return next_period_time.strftime('%Y-%m-%dT%H:%M:%S+08:00') # 格式化为指定的字符串格式并返回 - - -# 2025/02/06 -def store_non_realtime_SCADA_data_job() -> None: - """ - 定义的任务2，每6小时执行一次，在0点、6点、12点、18点执行，执行时，更新get_history_data_end_time并调用store_non_realtime_SCADA_data_to_influxdb函数 - :return: None - """ - # 获取当前时间 - current_time = datetime.now() - # 只在0点、6点、12点、18点执行任务 - # if current_time.hour % 6 == 0 and current_time.minute == 0: - if current_time.minute % 10 == 0: - logger.info(f"{current_time.strftime('%Y-%m-%d %H:%M:%S')} -- Start store non realtime SCADA data task.") - # 获取下一个6小时的时间点，并更新get_history_data_end_time - get_history_data_end_time: str = get_next_time() # get_history_data_end_time 类型为 str，格式为'2025-02-06T12:00:00+08:00' - # print(get_next_time) - # 调用函数执行任务 - influxdb_api.store_non_realtime_SCADA_data_to_influxdb(get_history_data_end_time) - logger.info('{} -- Successfully store non realtime SCADA data.'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) - else: - logger.info(f"{current_time.strftime('%Y-%m-%d %H:%M:%S')} -- Skipping store non realtime SCADA data task.") - - -# 2025/02/06 -def store_non_realtime_SCADA_data_task() -> None: - """ - 定时执行6小时的任务，使用schedule库每分钟执行一次store_non_realtime_SCADA_data_job函数。 - 该任务会一直运行，定期调用store_non_realtime_SCADA_data_job获取SCADA数据。 - :return: - """ - # 等待到整分对齐 - now = datetime.now() - wait_seconds = 60 - now.second - time.sleep(wait_seconds) - try: - # 每分钟检查一次，执行store_non_realtime_SCADA_data_job - schedule.every(1).minute.at(":00").do(store_non_realtime_SCADA_data_job) - # 持续执行任务，检查是否有待执行的任务 - while True: - schedule.run_pending() # 执行所有待处理的定时任务 - time.sleep(1) # 暂停1秒，避免过于频繁的任务检查 - pass - except Exception as e: - logger.error(f"Error occurred in store_non_realtime_SCADA_data_task: {e}") - - -if __name__ == "__main__": - url = influxdb_info.url - token = influxdb_info.token - org_name = influxdb_info.org - - client = InfluxDBClient(url=url, token=token) - # step2: 先查询pg数据库中scada_info的信息，然后存储SCADA数据到SCADA_data这个bucket里 - influxdb_api.query_pg_scada_info_non_realtime(project_info.name) - # 自动执行 - store_non_realtime_SCADA_data_task() diff --git a/scripts/build_pyd.py b/scripts/build_pyd.py index fa91129..a9ec4ec 100644 --- a/scripts/build_pyd.py +++ b/scripts/build_pyd.py @@ -2,9 +2,6 @@ from distutils.core import setup from Cython.Build import cythonize setup(ext_modules=cythonize([ - "main.py", - "auto_realtime.py", - "auto_store_non_realtime_SCADA_data.py", "tjnetwork.py", "online_Analysis.py", "sensitivity.py", @@ -15,10 +12,8 @@ setup(ext_modules=cythonize([ "get_data.py", "get_current_total_Q.py", "get_current_status.py", - "influxdb_api.py", - "influxdb_query_SCADA_data.py", "simulation.py", "time_api.py", "api/*.py", "epanet/*.py" - ])) \ No newline at end of file + ])) diff --git a/scripts/get_data.py b/scripts/get_data.py index 6cfd87d..fc09ceb 100644 --- a/scripts/get_data.py +++ b/scripts/get_data.py @@ -121,7 +121,6 @@ def get_history_data( # print(data) # # 定义 CSV 文件的路径 -# csv_file_path = './influxdb_data_4984.csv' # # 将数据写入 CSV 文件 # # with open(csv_file_path, mode='w', newline='') as file: # # writer = csv.writer(file) @@ -143,7 +142,6 @@ def get_history_data( # # # # print(f"数据已保存到 {csv_file_path}") # -# filtered_csv_file_path = './filtered_influxdb_data_4984.csv' # # # # # # 读取并筛选数据 # data_list1 = [] diff --git a/scripts/install.py b/scripts/install.py index cd63730..c11699c 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -18,14 +18,12 @@ def install(): packages = [ '"psycopg[binary]"', 'pytest', - 'influxdb_client', 'numpy', 'fastapi', "msgpack", 'schedule', 'pandas', 'openpyxl', - 'redis', 'pydantic', 'python-dateutil', 'starlette', diff --git a/scripts/main.py b/scripts/main.py deleted file mode 100644 index 902f9ef..0000000 --- a/scripts/main.py +++ /dev/null @@ -1,4481 +0,0 @@ -import os -import json -import time -import datetime -import logging -import threading -import shutil -import random - -from typing import * -from typing import List, Annotated, Optional, Union - -from urllib.request import Request - -from fastapi import ( - FastAPI, - File, - UploadFile, - Response, - status, - Request, - HTTPException, - Query, - Depends, - Header, -) -from fastapi.responses import PlainTextResponse -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.middleware.cors import CORSMiddleware - -from starlette.responses import FileResponse, JSONResponse -from contextlib import asynccontextmanager - -from pydantic import BaseModel, field_validator - -from multiprocessing import Value - -import redis -import msgpack -from datetime import datetime, timedelta, timezone - -# 第三方/自定义模块 -import app.infra.db.influxdb.api as influxdb_api -import app.infra.db.timescaledb as timescaledb -import app.infra.db.postgresql as postgresql -import py_linq -import app.services.time_api as time_api -import app.services.simulation as simulation -import app.services.globals as globals -import app.services.project_info as project_info -from app.infra.db.timescaledb.database import db as tsdb -from app.infra.db.postgresql.database import db as pgdb -from app.algorithms.online_Analysis import * -from app.services.tjnetwork import ( - Any, - ChangeSet, - PIPE_STATUS_OPEN, - VALVES_TYPE_PRV, - add_curve, - add_district_metering_area, - add_junction, - add_label, - add_mixing, - add_pattern, - add_pipe, - add_pump, - add_region, - add_reservoir, - add_scada_device, - add_scada_device_data, - add_scada_element, - add_service_area, - add_source, - add_tank, - add_valve, - add_vertex, - add_virtual_district, - api, - calculate_demand_to_network, - calculate_demand_to_nodes, - calculate_demand_to_region, - calculate_district_metering_area_for_network, - calculate_district_metering_area_for_nodes, - calculate_district_metering_area_for_region, - calculate_service_area, - calculate_virtual_district, - clean_scada_device, - clean_scada_device_data, - clean_scada_element, - close_project, - convert_inp_v3_to_v2, - copy_project, - create_project, - delete_curve, - delete_district_metering_area, - delete_junction, - delete_label, - delete_mixing, - delete_pattern, - delete_pipe, - delete_project, - delete_pump, - delete_region, - delete_reservoir, - delete_scada_device, - delete_scada_device_data, - delete_scada_element, - delete_service_area, - delete_source, - delete_tank, - delete_valve, - delete_virtual_district, - dump_inp, - dump_output, - execute_batch_command, - execute_batch_commands, - execute_redo, - execute_undo, - export_inp, - generate_district_metering_area, - generate_service_area, - generate_sub_district_metering_area, - generate_virtual_district, - get_all_burst_locate_results, - get_all_district_metering_area_ids, - get_all_district_metering_areas, - get_all_extension_data, - get_all_extension_data_keys, - get_all_junctions, - get_all_pipes, - get_all_pumps, - get_all_reservoirs, - get_all_scada_device_ids, - get_all_scada_devices, - get_all_scada_elements, - get_all_scada_info, - get_all_schemes, - get_all_sensor_placements, - get_all_service_areas, - get_all_tanks, - get_all_valves, - get_all_vertex_links, - get_all_vertices, - get_all_virtual_districts, - get_backdrop, - get_backdrop_schema, - get_control, - get_control_schema, - get_current_operation, - get_curve, - get_curve_schema, - get_curves, - get_demand, - get_demand_schema, - get_district_metering_area, - get_district_metering_area_schema, - get_element_properties, - get_element_properties_with_type, - get_element_type, - get_element_type_value, - get_emitter, - get_emitter_schema, - get_energy, - get_energy_schema, - get_extension_data, - get_junction, - get_junction_schema, - get_label, - get_label_schema, - get_link_properties, - get_link_type, - get_links, - get_major_node_coords, - get_major_pipe_nodes, - get_mixing, - get_mixing_schema, - get_network_link_nodes, - get_network_node_coords, - get_network_pipe_risk_probability_now, - get_node_coord, - get_node_links, - get_node_properties, - get_node_type, - get_nodes, - get_option_v3, - get_option_v3_schema, - get_pattern, - get_pattern_schema, - get_patterns, - get_pipe, - get_pipe_reaction, - get_pipe_reaction_schema, - get_pipe_risk_probability, - get_pipe_risk_probability_geometries, - get_pipe_risk_probability_now, - get_pipe_schema, - get_pipes_risk_probability, - get_pump, - get_pump_energy, - get_pump_energy_schema, - get_pump_schema, - get_quality, - get_quality_schema, - get_reaction, - get_reaction_schema, - get_region, - get_region_schema, - get_reservoir, - get_reservoir_schema, - get_restore_operation, - get_rule, - get_rule_schema, - get_scada_device, - get_scada_device_data, - get_scada_device_data_schema, - get_scada_device_schema, - get_scada_element, - get_scada_element_schema, - get_scada_info, - get_scada_info_schema, - get_scheme, - get_scheme_schema, - get_service_area, - get_service_area_schema, - get_source, - get_source_schema, - get_status, - get_status_schema, - get_tag, - get_tag_schema, - get_tags, - get_tank, - get_tank_reaction, - get_tank_reaction_schema, - get_tank_schema, - get_time, - get_time_schema, - get_title, - get_title_schema, - get_valve, - get_valve_schema, - get_vertex, - get_vertex_schema, - get_virtual_district, - get_virtual_district_schema, - have_project, - have_snapshot, - have_snapshot_for_current_operation, - have_snapshot_for_operation, - import_inp, - is_curve, - is_junction, - is_link, - is_node, - is_pattern, - is_pipe, - is_project_open, - is_pump, - is_reservoir, - is_tank, - is_valve, - list_project, - list_snapshot, - open_project, - pick_operation, - pick_snapshot, - read_inp, - run_inp, - run_project, - run_project_return_dict, - set_backdrop, - set_control, - set_curve, - set_demand, - set_district_metering_area, - set_emitter, - set_energy, - set_extension_data, - set_junction, - set_label, - set_option_v3, - set_pattern, - set_pipe, - set_pipe_reaction, - set_pump, - set_pump_energy, - set_quality, - set_reaction, - set_region, - set_reservoir, - set_restore_operation, - set_rule, - set_scada_device, - set_scada_device_data, - set_scada_element, - set_service_area, - set_source, - set_status, - set_tag, - set_tank, - set_tank_reaction, - set_time, - set_title, - set_valve, - set_vertex, - set_virtual_district, - sync_with_server, - take_snapshot, - take_snapshot_for_current_operation, - take_snapshot_for_operation, -) - - -JUNCTION = 0 -RESERVOIR = 1 -TANK = 2 -PIPE = 1 -NODE_COUNT = 0 -LINK_COUNT = 2 - -prjs = [] -# inpDir = "C:/inpfiles/" -# tmpDir = "C:/tmpfiles/" -# proj_name = project_info.name -# lockedPrjs = {} - -# if not os.path.exists(inpDir): -# os.mkdir(inpDir) - -# if not os.path.exists(tmpDir): -# os.mkdir(tmpDir) - - -# 全局依赖项 -async def global_auth(request: Request): - # 白名单跳过 - # if request.url.path in WHITE_LIST: - # return - # 验证 - token = request.headers.get("Authorization") - if token != "Bearer 567e33c876a2" and token != "Bearer 38b3be72b8af": - raise HTTPException(status_code=401, detail="Invalid token") - - -# 简易令牌验证（实际项目中应替换为 JWT/OAuth2 等） -AUTH_TOKEN = "567e33c876a2" # 预设的有效令牌 - - -async def verify_token(authorization: Annotated[str, Header()] = None): - # 检查请求头是否存在 - if not authorization: - raise HTTPException(status_code=401, detail="Authorization header missing") - - # 提取 Bearer 后的令牌 (格式: Bearer ) - try: - token_type, token = authorization.split(" ", 1) - if token_type.lower() != "bearer": - raise ValueError - except ValueError: - raise HTTPException( - status_code=401, detail="Invalid authorization format. Use: Bearer " - ) - - # 验证令牌 - if token != AUTH_TOKEN: - raise HTTPException(status_code=403, detail="Invalid authentication token") - - return True - - -# 全局依赖项 -# app = FastAPI(dependencies=[Depends(global_auth)]) -# app = FastAPI() - - -# 生命周期管理器 -@asynccontextmanager -async def lifespan(app: FastAPI): - # 初始化数据库连接池 - tsdb.init_pool() - pgdb.init_pool() - - await tsdb.open() - await pgdb.open() - - open_project(project_info.name) - - yield - # 清理资源 - tsdb.close() - pgdb.close() - - -app = FastAPI(lifespan=lifespan) - -app.include_router(timescaledb.router) -app.include_router(postgresql.router) - -access_tokens = [] - - -def generate_access_token(username: str, password: str) -> str: - """ - 根据用户名和密码生成JWT access token - - 参数: - username: 用户名 - password: 密码 - - 返回: - JWT access token字符串 - """ - - if username != "tjwater" or password != "tjwater@123": - raise ValueError("用户名或密码错误") - - token = "567e33c876a2" - return token - - -# 将 Query的信息 序列号到 redis/json， 默认不支持datetime，需要自定义 -# 自定义序列化函数 -# 序列化处理器 -def encode_datetime(obj): - """将datetime转换为可序列化的字典结构""" - if isinstance(obj, datetime): - return {"__datetime__": True, "as_str": obj.strftime("%Y%m%dT%H:%M:%S.%f")} - return obj - - -# 反序列化处理器 -def decode_datetime(obj): - """将字典还原为datetime对象""" - if "__datetime__" in obj: - return datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") - return obj - - -# 初始化 Redis 连接 -# 用redis 限制并发访u -redis_client = redis.Redis(host="127.0.0.1", port=6379, db=0) - -# influxdb数据库连接信息 -# influx_url = influxdb_info.url -# influx_token = influxdb_info.token -# influx_org_name = influxdb_info.org -# influx_client = InfluxDBClient(url=influx_url, token=influx_token, org=influx_org_name, timeout=100*1000) # 100 seconds - - -# 配置 CORS 中间件 -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], # 允许所有来源 - allow_credentials=True, # 允许传递凭证（Cookie、HTTP 头等） - allow_methods=["*"], # 允许所有 HTTP 方法 - allow_headers=["*"], # 允许所有 HTTP 头 -) - -# 定义一个共享变量 -lock_simulation = Value("i", 0) - -app.add_middleware(GZipMiddleware, minimum_size=1000) - -logger = logging.getLogger() -logger.setLevel(logging.INFO) - - -@app.on_event("startup") -async def startup_db(): - logger.info("**********************************************************") - logger.info(str(datetime.now())) - logger.info("TJWater CloudService is starting...") - logger.info("**********************************************************") - - # open proj_name by default - print(project_info.name) - open_project(project_info.name) - - -############################################################ -# auth -############################################################ -@app.post("/login/") -async def fastapi_login(username: str, password: str) -> str: - return generate_access_token(username, password) - - -############################################################ -# extension_data -############################################################ -@app.get("/getallextensiondatakeys/") -async def fastapi_get_all_extension_data_keys(network: str) -> list[str]: - return get_all_extension_data_keys(network) - - -@app.get("/getallextensiondata/") -async def fastapi_get_all_extension_data(network: str) -> dict[str, Any]: - return get_all_extension_data(network) - - -@app.get("/getextensiondata/") -async def fastapi_get_extension_data(network: str, key: str) -> str | None: - return get_extension_data(network, key) - - -@app.post("/setextensiondata", response_model=None) -async def fastapi_set_extension_data(network: str, req: Request) -> ChangeSet: - props = await req.json() - print(props) - cs = set_extension_data(network, ChangeSet(props)) - print(cs.operations[0]) - return cs - - -############################################################ -# project -############################################################ - - -@app.get("/listprojects/") -async def fastapi_list_projects() -> list[str]: - return list_project() - - -@app.get("/haveproject/") -async def fastapi_have_project(network: str): - return have_project(network) - - -@app.post("/createproject/") -async def fastapi_create_project(network: str): - create_project(network) - return network - - -@app.post("/deleteproject/") -async def fastapi_delete_project(network: str): - delete_project(network) - return True - - -@app.get("/isprojectopen/") -async def fastapi_is_project_open(network: str): - return is_project_open(network) - - -@app.post("/openproject/") -async def fastapi_open_project(network: str): - open_project(network) - return network - - -@app.post("/closeproject/") -async def fastapi_close_project(network: str): - close_project(network) - return True - - -@app.post("/copyproject/") -async def fastapi_copy_project(source: str, target: str): - copy_project(source, target) - return True - - -@app.post("/importinp/") -async def fastapi_import_inp(network: str, req: Request): - jo_root = await req.json() - inp_text = jo_root["inp"] - ps = {"inp": inp_text} - ret = import_inp(network, ChangeSet(ps)) - print(ret) - return ret - - -@app.get("/exportinp/", response_model=None) -async def fastapi_export_inp(network: str, version: str) -> ChangeSet: - cs = export_inp(network, version) - op = cs.operations[0] - open_project(network) - op["vertex"] = json.dumps(get_all_vertices(network)) - op["scada"] = json.dumps(get_all_scada_elements(network)) - op["dma"] = json.dumps(get_all_district_metering_areas(network)) - op["sa"] = json.dumps(get_all_service_areas(network)) - op["vd"] = json.dumps(get_all_virtual_districts(network)) - op["legend"] = get_extension_data(network, "legend") - - db = get_extension_data(network, "scada_db") - print(db) - scada_db = "" - if db: - scada_db = db - print(scada_db) - op["scada_db"] = scada_db - - close_project(network) - - return cs - - -@app.post("/readinp/") -async def fastapi_read_inp(network: str, inp: str) -> bool: - read_inp(network, inp) - return True - - -@app.get("/dumpinp/") -async def fastapi_dump_inp(network: str, inp: str) -> bool: - dump_inp(network, inp) - return True - - -# 必须用这个PlainTextResponse，不然每个key都有引号 -@app.get("/runproject/", response_class=PlainTextResponse) -async def fastapi_run_project(network: str) -> str: - lock_key = "exclusive_api_lock" - timeout = 120 # 锁自动过期时间（秒） - - # 尝试获取锁（NX=True: 不存在时设置，EX=timeout: 过期时间） - acquired = redis_client.set(lock_key, "locked", nx=True, ex=timeout) - - if not acquired: - raise HTTPException(status_code=409, detail="is in simulation") - else: - try: - return run_project(network) - finally: - # 手动释放锁（可选，依赖过期时间自动释放更安全） - redis_client.delete(lock_key) - - -# DingZQ, 2025-02-04, 返回dict[str, Any] -# output 和 report -# output 是 json -# report 是 text -@app.get("/runprojectreturndict/") -async def fastapi_run_project_return_dict(network: str) -> dict[str, Any]: - lock_key = "exclusive_api_lock" - timeout = 120 # 锁自动过期时间（秒） - - # 尝试获取锁（NX=True: 不存在时设置，EX=timeout: 过期时间） - acquired = redis_client.set(lock_key, "locked", nx=True, ex=timeout) - - if not acquired: - raise HTTPException(status_code=409, detail="is in simulation") - else: - try: - return run_project_return_dict(network) - finally: - # 手动释放锁（可选，依赖过期时间自动释放更安全） - redis_client.delete(lock_key) - - -# put in inp folder, name without extension -@app.get("/runinp/") -async def fastapi_run_inp(network: str) -> str: - return run_inp(network) - - -# path is absolute path -@app.get("/dumpoutput/") -async def fastapi_dump_output(output: str) -> str: - return dump_output(output) - - -@app.get("/isprojectlocked/") -async def fastapi_is_locked(network: str, req: Request): - return str in lockedPrjs.keys() - - -@app.get("/isprojectlockedbyme/") -async def fastapi_is_locked_by_me(network: str, req: Request): - client_host = req.client.host - return lockedPrjs.get(network) == client_host - - -# 0 successfully locked -# 1 already locked by you -# 2 locked by others -@app.post("/lockproject/") -async def fastapi_lock_project(network: str, req: Request): - client_host = req.client.host - if not network in lockedPrjs.keys(): - lockedPrjs[network] = client_host - return 0 - else: - if lockedPrjs.get(network) == client_host: - return 1 - else: - return 2 - - -@app.post("/unlockproject/") -def fastapi_unlock_project(network: str, req: Request): - client_host = req.client.host - if lockedPrjs[network] == client_host: - print("delete key") - del lockedPrjs[network] - return True - - return False - - -### operations - - -@app.get("/getcurrentoperationid/") -async def fastapi_get_current_operaiton_id(network: str) -> int: - return get_current_operation(network) - - -@app.post("/undo/") -async def fastapi_undo(network: str): - return execute_undo(network) - - -@app.post("/redo/") -async def fastapi_redo(network: str): - return execute_redo(network) - - -@app.get("/getsnapshots/") -def fastapi_list_snapshot(network: str) -> list[tuple[int, str]]: - return list_snapshot(network) - - -@app.get("/havesnapshot/") -async def fastapi_have_snapshot(network: str, tag: str) -> bool: - return have_snapshot(network, tag) - - -@app.get("/havesnapshotforoperation/") -async def fastapi_have_snapshot_for_operation(network: str, operation: int) -> bool: - return have_snapshot_for_operation(network, operation) - - -@app.get("/havesnapshotforcurrentoperation/") -async def fastapi_have_snapshot_for_current_operation(network: str) -> bool: - return have_snapshot_for_current_operation(network) - - -@app.post("/takesnapshotforoperation/") -async def fastapi_take_snapshot_for_operation( - network: str, operation: int, tag: str -) -> None: - return take_snapshot_for_operation(network, operation, tag) - - -@app.post("takenapshotforcurrentoperation") -async def fastapi_take_snapshot_for_current_operation(network: str, tag: str) -> None: - return take_snapshot_for_current_operation(network, tag) - - -@app.post("/takesnapshot/") -def fastapi_take_snapshot(network: str, tag: str) -> None: - return take_snapshot(network, tag) - - -@app.post("/picksnapshot/", response_model=None) -def fastapi_pick_snapshot(network: str, tag: str, discard: bool = False) -> ChangeSet: - return pick_snapshot(network, tag, discard) - - -@app.post("/pickoperation/", response_model=None) -async def fastapi_pick_operation( - network: str, operation: int, discard: bool = False -) -> ChangeSet: - return pick_operation(network, operation, discard) - - -@app.get("/syncwithserver/", response_model=None) -async def fastapi_sync_with_server(network: str, operation: int) -> ChangeSet: - return sync_with_server(network, operation) - - -@app.post("/batch/", response_model=None) -async def fastapi_execute_batch_commands(network: str, req: Request) -> ChangeSet: - jo_root = await req.json() - cs: ChangeSet = ChangeSet() - cs.operations = jo_root["operations"] - rcs = execute_batch_commands(network, cs) - return rcs - - -@app.post("/compressedbatch/", response_model=None) -async def fastapi_execute_compressed_batch_commands( - network: str, req: Request -) -> ChangeSet: - jo_root = await req.json() - cs: ChangeSet = ChangeSet() - cs.operations = jo_root["operations"] - return execute_batch_command(network, cs) - - -@app.get("/getrestoreoperation/") -async def fastapi_get_restore_operation(network: str) -> int: - return get_restore_operation(network) - - -@app.post("/setrestoreoperation/") -async def fastapi_set_restore_operation(network: str, operation: int) -> None: - return set_restore_operation(network, operation) - - -############################################################ -# type -############################################################ - - -@app.get("/isnode/") -async def fastapi_is_node(network: str, node: str) -> bool: - return is_node(network, node) - - -@app.get("/isjunction/") -async def fastapi_is_junction(network: str, node: str) -> bool: - return is_junction(network, node) - - -@app.get("/isreservoir/") -async def fastapi_is_reservoir(network: str, node: str) -> bool: - return is_reservoir(network, node) - - -@app.get("/istank/") -async def fastapi_is_tank(network: str, node: str) -> bool: - return is_tank(network, node) - - -@app.get("/islink/") -async def fastapi_is_link(network: str, link: str) -> bool: - return is_link(network, link) - - -@app.get("/ispipe/") -async def fastapi_is_pipe(network: str, link: str) -> bool: - return is_pipe(network, link) - - -@app.get("/ispump/") -async def fastapi_is_pump(network: str, link: str) -> bool: - return is_pump(network, link) - - -@app.get("/isvalve/") -async def fastapi_is_valve(network: str, link: str) -> bool: - return is_valve(network, link) - - -# DingZQ, 2025-02-05 -@app.get("/getnodetype/") -async def fastapi_get_node_type(network: str, node: str) -> str: - return get_node_type(network, node) - - -@app.get("/getlinktype/") -async def fastapi_get_link_type(network: str, link: str) -> str: - return get_link_type(network, link) - - -@app.get("/getelementtype/") -async def fastapi_get_element_type(network: str, element: str) -> str: - return get_element_type(network, element) - - -@app.get("/getelementtypevalue/") -async def fastapi_get_element_type_value(network: str, element: str) -> int: - return get_element_type_value(network, element) - - -@app.get("/iscurve/") -async def fastapi_is_curve(network: str, curve: str) -> bool: - return is_curve(network, curve) - - -@app.get("/ispattern/") -async def fastapi_is_pattern(network: str, pattern: str) -> bool: - return is_pattern(network, pattern) - - -@app.get("/getnodes/") -async def fastapi_get_nodes(network: str) -> list[str]: - return get_nodes(network) - - -@app.get("/getlinks/") -async def fastapi_get_links(network: str) -> list[str]: - return get_links(network) - - -@app.get("/getcurves/") -async def fastapi_get_curves(network: str) -> list[str]: - return get_curves(network) - - -@app.get("/getpatterns/") -async def fastapi_get_patterns(network: str) -> list[str]: - return get_patterns(network) - - -@app.get("/getnodelinks/") -def get_node_links(network: str, node: str) -> list[str]: - return get_node_links(network, node) - - -############################################################ -# DingZQ, 2025-02-05 -# 用统一的接口来获取 Node & Link properties, Node和Link的Id可以一样，不能进一步统一成获取Element 的 properties -# Node & Link properties -############################################################ -@app.get("/getnodeproperties/") -async def fast_get_node_properties(network: str, node: str) -> dict[str, Any]: - return get_node_properties(network, node) - - -@app.get("/getlinkproperties/") -async def fast_get_link_properties(network: str, link: str) -> dict[str, Any]: - return get_link_properties(network, link) - - -@app.get("/getscadaproperties/") -async def fast_get_scada_properties(network: str, scada: str) -> dict[str, Any]: - return get_scada_info(network, scada) - - -@app.get("/getallscadaproperties/") -async def fast_get_all_scada_properties(network: str) -> list[dict[str, Any]]: - return get_all_scada_info(network) - - -# elementtype can be 'node' or 'link' or 'scada' -@app.get("/getelementpropertieswithtype/") -async def fast_get_element_properties_with_type( - network: str, elementtype: str, element: str -) -> dict[str, Any]: - return get_element_properties_with_type(network, elementtype, element) - - -# type can be 'node' or 'link' or 'scada' -@app.get("/getelementproperties/") -async def fast_get_element_properties(network: str, element: str) -> dict[str, Any]: - return get_element_properties(network, element) - - -############################################################ -# title 1.[TITLE] -############################################################ -@app.get("/gettitleschema/") -async def fast_get_title_schema(network: str) -> dict[str, dict[str, Any]]: - return get_title_schema(network) - - -@app.get("/gettitle/") -async def fast_get_title(network: str) -> dict[str, Any]: - return get_title(network) - - -@app.get("/settitle/", response_model=None) -async def fastapi_set_title(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_title(network, ChangeSet(props)) - - -############################################################ -# junction 2.[JUNCTIONS] -############################################################ -@app.get("/getjunctionschema") -async def fast_get_junction_schema(network: str) -> dict[str, dict[str, Any]]: - return get_junction_schema(network) - - -@app.post("/addjunction/", response_model=None) -async def fastapi_add_junction( - network: str, junction: str, x: float, y: float, z: float -) -> ChangeSet: - ps = {"id": junction, "x": x, "y": y, "elevation": z} - return add_junction(network, ChangeSet(ps)) - - -@app.post("/deletejunction/", response_model=None) -async def fastapi_delete_junction(network: str, junction: str) -> ChangeSet: - ps = {"id": junction} - return delete_junction(network, ChangeSet(ps)) - - -@app.get("/getjunctionelevation/") -async def fastapi_get_junction_elevation(network: str, junction: str) -> float: - ps = get_junction(network, junction) - return ps["elevation"] - - -@app.get("/getjunctionx/") -async def fastapi_get_junction_x(network: str, junction: str) -> float: - ps = get_junction(network, junction) - return ps["x"] - - -@app.get("/getjunctiony/") -async def fastapi_get_junction_x(network: str, junction: str) -> float: - ps = get_junction(network, junction) - return ps["y"] - - -@app.get("/getjunctioncoord/") -async def fastapi_get_junction_coord(network: str, junction: str) -> dict[str, float]: - ps = get_junction(network, junction) - coord = {"x": ps["x"], "y": ps["y"]} - return coord - - -@app.get("/getjunctiondemand/") -async def fastapi_get_junction_demand(network: str, junction: str) -> float: - ps = get_junction(network, junction) - return ps["demand"] - - -@app.get("/getjunctionpattern/") -async def fastapi_get_junction_pattern(network: str, junction: str) -> str: - ps = get_junction(network, junction) - return ps["pattern"] - - -@app.post("/setjunctionelevation/", response_model=None) -async def fastapi_set_junction_elevation( - network: str, junction: str, elevation: float -) -> ChangeSet: - ps = {"id": junction, "elevation": elevation} - return set_junction(network, ChangeSet(ps)) - - -@app.post("/setjunctionx/", response_model=None) -async def fastapi_set_junction_x(network: str, junction: str, x: float) -> ChangeSet: - ps = {"id": junction, "x": x} - return set_junction(network, ChangeSet(ps)) - - -@app.post("/setjunctiony/", response_model=None) -async def fastapi_set_junction_y(network: str, junction: str, y: float) -> ChangeSet: - ps = {"id": junction, "y": y} - return set_junction(network, ChangeSet(ps)) - - -@app.post("/setjunctioncoord/", response_model=None) -async def fastapi_set_junction_coord( - network: str, junction: str, x: float, y: float -) -> ChangeSet: - ps = {"id": junction, "x": x, "y": y} - return set_junction(network, ChangeSet(ps)) - - -@app.post("/setjunctiondemand/", response_model=None) -async def fastapi_set_junction_demand( - network: str, junction: str, demand: float -) -> ChangeSet: - ps = {"id": junction, "demand": demand} - return set_junction(network, ChangeSet(ps)) - - -@app.post("/setjunctionpattern/", response_model=None) -async def fastapi_set_junction_pattern( - network: str, junction: str, pattern: str -) -> ChangeSet: - ps = {"id": junction, "pattern": pattern} - return set_junction(network, ChangeSet(ps)) - - -@app.get("/getjunctionproperties/") -async def fastapi_get_junction_properties( - network: str, junction: str -) -> dict[str, Any]: - return get_junction(network, junction) - - -# DingZQ, 2025-03-29 -@app.get("/getalljunctionproperties/") -async def fastapi_get_all_junction_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getalljunctionproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_junctions(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/setjunctionproperties/", response_model=None) -async def fastapi_set_junction_properties( - network: str, junction: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": junction} | props - return set_junction(network, ChangeSet(ps)) - - -############################################################ -# reservoir 3.[RESERVOIRS] -############################################################ -@app.get("/getreservoirschema") -async def fast_get_reservoir_schema(network: str) -> dict[str, dict[str, Any]]: - return get_reservoir_schema(network) - - -@app.post("/addreservoir/", response_model=None) -async def fastapi_add_reservoir( - network: str, reservoir: str, x: float, y: float, head: float -) -> ChangeSet: - ps = {"id": reservoir, "x": x, "y": y, "head": head} - return add_reservoir(network, ChangeSet(ps)) - - -@app.post("/deletereservoir/", response_model=None) -async def fastapi_delete_reservoir(network: str, reservoir: str) -> ChangeSet: - ps = {"id": reservoir} - return delete_reservoir(network, ChangeSet(ps)) - - -@app.get("/getreservoirhead/") -async def fastapi_get_reservoir_head(network: str, reservoir: str) -> float | None: - ps = get_reservoir(network, reservoir) - return ps["head"] - - -@app.get("/getreservoirpattern/") -async def fastapi_get_reservoir_pattern(network: str, reservoir: str) -> str | None: - ps = get_reservoir(network, reservoir) - return ps["pattern"] - - -@app.get("/getreservoirx/") -async def fastapi_get_reservoir_x( - network: str, reservoir: str -) -> dict[str, float] | None: - ps = get_reservoir(network, reservoir) - return ps["x"] - - -@app.get("/getreservoiry/") -async def fastapi_get_reservoir_y( - network: str, reservoir: str -) -> dict[str, float] | None: - ps = get_reservoir(network, reservoir) - return ps["y"] - - -@app.get("/getreservoircoord/") -async def fastapi_get_reservoir_y( - network: str, reservoir: str -) -> dict[str, float] | None: - ps = get_reservoir(network, reservoir) - coord = {"id": reservoir, "x": ps["x"], "y": ps["y"]} - return coord - - -@app.post("/setreservoirhead/", response_model=None) -async def fastapi_set_reservoir_head( - network: str, reservoir: str, head: float -) -> ChangeSet: - ps = {"id": reservoir, "head": head} - return set_reservoir(network, ChangeSet(ps)) - - -@app.post("/setreservoirpattern/", response_model=None) -async def fastapi_set_reservoir_pattern( - network: str, reservoir: str, pattern: str -) -> ChangeSet: - ps = {"id": reservoir, "pattern": pattern} - return set_reservoir(network, ChangeSet(ps)) - - -@app.post("/setreservoirx/", response_model=None) -async def fastapi_set_reservoir_x(network: str, reservoir: str, x: float) -> ChangeSet: - ps = {"id": reservoir, "x": x} - return set_reservoir(network, ChangeSet(ps)) - - -@app.post("/setreservoirx/", response_model=None) -async def fastapi_set_reservoir_y(network: str, reservoir: str, y: float) -> ChangeSet: - ps = {"id": reservoir, "y": y} - return set_reservoir(network, ChangeSet(ps)) - - -@app.post("/setreservoircoord/", response_model=None) -async def fastapi_set_reservoir_y( - network: str, reservoir: str, x: float, y: float -) -> ChangeSet: - ps = {"id": reservoir, "x": x, "y": y} - return set_reservoir(network, ChangeSet(ps)) - - -@app.get("/getreservoirproperties/") -async def fastapi_get_reservoir_properties( - network: str, reservoir: str -) -> dict[str, Any]: - return get_reservoir(network, reservoir) - - -# DingZQ, 2025-03-29 -@app.get("/getallreservoirproperties/") -async def fastapi_get_all_reservoir_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getallreservoirproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_reservoirs(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/setreservoirproperties/", response_model=None) -async def fastapi_set_reservoir_properties( - network: str, reservoir: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": reservoir} | props - return set_reservoir(network, ChangeSet(ps)) - - -############################################################ -# tank 4.[TANKS] -############################################################ -@app.get("/gettankschema") -async def fast_get_tank_schema(network: str) -> dict[str, dict[str, Any]]: - return get_tank_schema(network) - - -@app.post("/addtank/", response_model=None) -async def fastapi_add_tank( - network: str, - tank: str, - x: float, - y: float, - elevation: float, - init_level: float = 0, - min_level: float = 0, - max_level: float = 0, - diameter: float = 0, - min_vol: float = 0, -) -> ChangeSet: - ps = { - "id": tank, - "x": x, - "y": y, - "elevation": elevation, - "init_level": init_level, - "min_level": min_level, - "max_level": max_level, - "diameter": diameter, - "min_vol": min_vol, - } - return add_tank(network, ChangeSet(ps)) - - -@app.post("/deletetank/", response_model=None) -async def fastapi_delete_tank(network: str, tank: str) -> ChangeSet: - ps = {"id": tank} - return delete_tank(network, ChangeSet(ps)) - - -@app.get("/gettankelevation/") -async def fastapi_get_tank_elevation(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["elevation"] - - -@app.get("/gettankinitlevel/") -async def fastapi_get_tank_init_level(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["init_level"] - - -@app.get("/gettankminlevel/") -async def fastapi_get_tank_min_level(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["min_level"] - - -@app.get("/gettankmaxlevel/") -async def fastapi_get_tank_max_level(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["max_level"] - - -@app.get("/gettankdiameter/") -async def fastapi_get_tank_diameter(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["diameter"] - - -@app.get("/gettankminvol/") -async def fastapi_get_tank_min_vol(network: str, tank: str) -> float | None: - ps = get_tank(network, tank) - return ps["min_vol"] - - -@app.get("/gettankvolcurve/") -async def fastapi_get_tank_vol_curve(network: str, tank: str) -> str | None: - ps = get_tank(network, tank) - return ps["vol_curve"] - - -@app.get("/gettankoverflow/") -async def fastapi_get_tank_overflow(network: str, tank: str) -> str | None: - ps = get_tank(network, tank) - return ps["overflow"] - - -@app.get("/gettankx/") -async def fastapi_get_tank_x(network: str, tank: str) -> float: - ps = get_tank(network, tank) - return ps["x"] - - -@app.get("/gettanky/") -async def fastapi_get_tank_x(network: str, tank: str) -> float: - ps = get_tank(network, tank) - return ps["y"] - - -@app.get("/gettankcoord/") -async def fastapi_get_tank_coord(network: str, tank: str) -> dict[str, float]: - ps = get_tank(network, tank) - coord = {"x": ps["x"], "y": ps["y"]} - return coord - - -@app.post("/settankelevation/", response_model=None) -async def fastapi_set_tank_elevation( - network: str, tank: str, elevation: float -) -> ChangeSet: - ps = {"id": tank, "elevation": elevation} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankinitlevel/", response_model=None) -async def fastapi_set_tank_init_level( - network: str, tank: str, init_level: float -) -> ChangeSet: - ps = {"id": tank, "init_level": init_level} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankminlevel/", response_model=None) -async def fastapi_set_tank_min_level( - network: str, tank: str, min_level: float -) -> ChangeSet: - ps = {"id": tank, "min_level": min_level} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankmaxlevel/", response_model=None) -async def fastapi_set_tank_max_level( - network: str, tank: str, max_level: float -) -> ChangeSet: - ps = {"id": tank, "max_level": max_level} - return set_tank(network, ChangeSet(ps)) - - -@app.post("settankdiameter//", response_model=None) -async def fastapi_set_tank_diameter( - network: str, tank: str, diameter: float -) -> ChangeSet: - ps = {"id": tank, "diameter": diameter} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankminvol/", response_model=None) -async def fastapi_set_tank_min_vol( - network: str, tank: str, min_vol: float -) -> ChangeSet: - ps = {"id": tank, "min_vol": min_vol} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankvolcurve/", response_model=None) -async def fastapi_set_tank_vol_curve( - network: str, tank: str, vol_curve: str -) -> ChangeSet: - ps = {"id": tank, "vol_curve": vol_curve} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankoverflow/", response_model=None) -async def fastapi_set_tank_overflow( - network: str, tank: str, overflow: str -) -> ChangeSet: - ps = {"id": tank, "overflow": overflow} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankx/", response_model=None) -async def fastapi_set_tank_x(network: str, tank: str, x: float) -> ChangeSet: - ps = {"id": tank, "x": x} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settanky/", response_model=None) -async def fastapi_set_tank_y(network: str, tank: str, y: float) -> ChangeSet: - ps = {"id": tank, "y": y} - return set_tank(network, ChangeSet(ps)) - - -@app.post("/settankcoord/", response_model=None) -async def fastapi_set_tank_coord( - network: str, tank: str, x: float, y: float -) -> ChangeSet: - ps = {"id": tank, "x": x, "y": y} - return set_tank(network, ChangeSet(ps)) - - -@app.get("/gettankproperties/") -async def fastapi_get_tank_properties(network: str, tank: str) -> dict[str, Any]: - return get_tank(network, tank) - - -# DingZQ, 2025-03-29 -@app.get("/getalltankproperties/") -async def fastapi_get_all_tank_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getalltankproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_tanks(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/settankproperties/", response_model=None) -async def fastapi_set_tank_properties( - network: str, tank: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": tank} | props - return set_tank(network, ChangeSet(ps)) - - -############################################################ -# pipe 4.[PIPES] -############################################################ -@app.get("/getpipeschema") -async def fastapi_get_pipe_schema(network: str) -> dict[str, dict[str, Any]]: - return get_pipe_schema(network) - - -@app.post("/addpipe/", response_model=None) -async def fastapi_add_pipe( - network: str, - pipe: str, - node1: str, - node2: str, - length: float = 0, - diameter: float = 0, - roughness: float = 0, - minor_loss: float = 0, - status: str = PIPE_STATUS_OPEN, -) -> ChangeSet: - ps = { - "id": pipe, - "node1": node1, - "node2": node2, - "length": length, - "diameter": diameter, - "roughness": roughness, - "minor_loss": minor_loss, - "status": status, - } - return add_pipe(network, ChangeSet(ps)) - - -@app.post("/deletepipe/", response_model=None) -async def fastapi_delete_pipe(network: str, pipe: str) -> ChangeSet: - ps = {"id": pipe} - return delete_pipe(network, ChangeSet(ps)) - - -@app.get("/getpipenode1/") -async def fastapi_get_pipe_node1(network: str, pipe: str) -> str | None: - ps = get_pipe(network, pipe) - return ps["node1"] - - -@app.get("/getpipenode2/") -async def fastapi_get_pipe_node2(network: str, pipe: str) -> str | None: - ps = get_pipe(network, pipe) - return ps["node2"] - - -@app.get("/getpipelength/") -async def fastapi_get_pipe_length(network: str, pipe: str) -> float | None: - ps = get_pipe(network, pipe) - return ps["length"] - - -@app.get("/getpipediameter/") -async def fastapi_get_pipe_diameter(network: str, pipe: str) -> float | None: - ps = get_pipe(network, pipe) - return ps["diameter"] - - -@app.get("/getpiperoughness/") -async def fastapi_get_pipe_roughness(network: str, pipe: str) -> float | None: - ps = get_pipe(network, pipe) - return ps["roughness"] - - -@app.get("/getpipeminorloss/") -async def fastapi_get_pipe_minor_loss(network: str, pipe: str) -> float | None: - ps = get_pipe(network, pipe) - return ps["minor_loss"] - - -@app.get("/getpipestatus/") -async def fastapi_get_pipe_status(network: str, pipe: str) -> str | None: - ps = get_pipe(network, pipe) - return ps["status"] - - -@app.post("/setpipenode1/", response_model=None) -async def fastapi_set_pipe_node1(network: str, pipe: str, node1: str) -> ChangeSet: - ps = {"id": pipe, "node1": node1} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpipenode2/", response_model=None) -async def fastapi_set_pipe_node2(network: str, pipe: str, node2: str) -> ChangeSet: - ps = {"id": pipe, "node2": node2} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpipelength/", response_model=None) -async def fastapi_set_pipe_length(network: str, pipe: str, length: float) -> ChangeSet: - ps = {"id": pipe, "length": length} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpipediameter/", response_model=None) -async def fastapi_set_pipe_diameter( - network: str, pipe: str, diameter: float -) -> ChangeSet: - ps = {"id": pipe, "diameter": diameter} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpiperoughness/", response_model=None) -async def fastapi_set_pipe_roughness( - network: str, pipe: str, roughness: float -) -> ChangeSet: - ps = {"id": pipe, "roughness": roughness} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpipeminorloss/", response_model=None) -async def fastapi_set_pipe_minor_loss( - network: str, pipe: str, minor_loss: float -) -> ChangeSet: - ps = {"id": pipe, "minor_loss": minor_loss} - return set_pipe(network, ChangeSet(ps)) - - -@app.post("/setpipestatus/", response_model=None) -async def fastapi_set_pipe_status(network: str, pipe: str, status: str) -> ChangeSet: - ps = {"id": pipe, "status": status} - - print(status) - print(ps) - - ret = set_pipe(network, ChangeSet(ps)) - print(ret) - return ret - - -@app.get("/getpipeproperties/") -async def fastapi_get_pipe_properties(network: str, pipe: str) -> dict[str, Any]: - return get_pipe(network, pipe) - - -# DingZQ, 2025-03-29 -@app.get("/getallpipeproperties/") -async def fastapi_get_all_pipe_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getallpipeproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_pipes(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/setpipeproperties/", response_model=None) -async def fastapi_set_pipe_properties( - network: str, pipe: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": pipe} | props - return set_pipe(network, ChangeSet(ps)) - - -############################################################ -# pump 4.[PUMPS] -############################################################ -@app.get("/getpumpschema") -async def fastapi_get_pump_schema(network: str) -> dict[str, dict[str, Any]]: - return get_pump_schema(network) - - -@app.post("/addpump/", response_model=None) -async def fastapi_add_pump( - network: str, pump: str, node1: str, node2: str, power: float = 0.0 -) -> ChangeSet: - ps = {"id": pump, "node1": node1, "node2": node2, "power": power} - return add_pump(network, ChangeSet(ps)) - - -@app.post("/deletepump/", response_model=None) -async def fastapi_delete_pump(network: str, pump: str) -> ChangeSet: - ps = {"id": pump} - return delete_pump(network, ChangeSet(ps)) - - -@app.get("/getpumpnode1/") -async def fastapi_get_pump_node1(network: str, pump: str) -> str | None: - ps = get_pump(network, pump) - return ps["node1"] - - -@app.get("/getpumpnode2/") -async def fastapi_get_pump_node2(network: str, pump: str) -> str | None: - ps = get_pump(network, pump) - return ps["node2"] - - -@app.post("/setpumpnode1/", response_model=None) -async def fastapi_set_pump_node1(network: str, pump: str, node1: str) -> ChangeSet: - ps = {"id": pump, "node1": node1} - return set_pump(network, ChangeSet(ps)) - - -@app.post("/setpumpnode2/", response_model=None) -async def fastapi_set_pump_node2(network: str, pump: str, node2: str) -> ChangeSet: - ps = {"id": pump, "node2": node2} - return set_pump(network, ChangeSet(ps)) - - -@app.get("/getpumpproperties/") -async def fastapi_get_pump_properties(network: str, pump: str) -> dict[str, Any]: - return get_pump(network, pump) - - -# DingZQ, 2025-03-29 -@app.get("/getallpumpproperties/") -async def fastapi_get_all_pump_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getallpumpproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_pumps(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/setpumpproperties/", response_model=None) -async def fastapi_set_pump_properties( - network: str, pump: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": pump} | props - return set_pump(network, ChangeSet(ps)) - - -############################################################ -# valve 4.[VALVES] -############################################################ -@app.get("/getvalveschema") -async def fastapi_get_valve_schema(network: str) -> dict[str, dict[str, Any]]: - return get_valve_schema(network) - - -@app.post("/addvalve/", response_model=None) -async def fastapi_add_valve( - network: str, - valve: str, - node1: str, - node2: str, - diameter: float = 0, - v_type: str = VALVES_TYPE_PRV, - setting: float = 0, - minor_loss: float = 0, -) -> ChangeSet: - ps = { - "id": valve, - "node1": node1, - "node2": node2, - "diameter": diameter, - "v_type": v_type, - "setting": setting, - "minor_loss": minor_loss, - } - - return add_valve(network, ChangeSet(ps)) - - -@app.post("/deletevalve/", response_model=None) -async def fastapi_delete_valve(network: str, valve: str) -> ChangeSet: - ps = {"id": valve} - return delete_valve(network, ChangeSet(ps)) - - -@app.get("/getvalvenode1/") -async def fastapi_get_valve_node1(network: str, valve: str) -> str | None: - ps = get_valve(network, valve) - return ps["node1"] - - -@app.get("/getvalvenode2/") -async def fastapi_get_valve_node2(network: str, valve: str) -> str | None: - ps = get_valve(network, valve) - return ps["node2"] - - -@app.get("/getvalvediameter/") -async def fastapi_get_valve_diameter(network: str, valve: str) -> float | None: - ps = get_valve(network, valve) - return ps["diameter"] - - -@app.get("/getvalvetype/") -async def fastapi_get_valve_type(network: str, valve: str) -> str | None: - ps = get_valve(network, valve) - return ps["type"] - - -@app.get("/getvalvesetting/") -async def fastapi_get_valve_setting(network: str, valve: str) -> float | None: - ps = get_valve(network, valve) - return ps["setting"] - - -@app.get("/getvalveminorloss/") -async def fastapi_get_valve_minor_loss(network: str, valve: str) -> float | None: - ps = get_valve(network, valve) - return ps["minor_loss"] - - -@app.post("/setvalvenode1/", response_model=None) -async def fastapi_set_valve_node1(network: str, valve: str, node1: str) -> ChangeSet: - ps = {"id": valve, "node1": node1} - return set_valve(network, ChangeSet(ps)) - - -@app.post("/setvalvenode2/", response_model=None) -async def fastapi_set_valve_node2(network: str, valve: str, node2: str) -> ChangeSet: - ps = {"id": valve, "node2": node2} - return set_valve(network, ChangeSet(ps)) - - -@app.post("/setvalvenodediameter/", response_model=None) -async def fastapi_set_valve_diameter( - network: str, valve: str, diameter: float -) -> ChangeSet: - ps = {"id": valve, "diameter": diameter} - return set_valve(network, ChangeSet(ps)) - - -@app.post("/setvalvetype/", response_model=None) -async def fastapi_set_valve_type(network: str, valve: str, type: str) -> ChangeSet: - ps = {"id": valve, "type": type} - return set_valve(network, ChangeSet(ps)) - - -@app.post("/setvalvesetting/", response_model=None) -async def fastapi_set_valve_setting( - network: str, valve: str, setting: float -) -> ChangeSet: - ps = {"id": valve, "setting": setting} - return set_valve(network, ChangeSet(ps)) - - -@app.get("/getvalveproperties/") -async def fastapi_get_valve_properties(network: str, valve: str) -> dict[str, Any]: - return get_valve(network, valve) - - -# DingZQ, 2025-03-29 -@app.get("/getallvalveproperties/") -async def fastapi_get_all_valve_properties(network: str) -> list[dict[str, Any]]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"getallvalveproperties_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = get_all_valves(network) - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -@app.post("/setvalveproperties/", response_model=None) -async def fastapi_set_valve_properties( - network: str, valve: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": valve} | props - return set_valve(network, ChangeSet(ps)) - - -# node & link -@app.post("/deletenode/", response_model=None) -async def fastapi_delete_node(network: str, node: str) -> ChangeSet: - ps = {"id": node} - if is_junction(network, node): - return delete_junction(network, ChangeSet(ps)) - elif is_reservoir(network, node): - return delete_reservoir(network, ChangeSet(ps)) - elif is_tank(network, node): - return delete_tank(network, ChangeSet(ps)) - - -@app.post("/deletelink/", response_model=None) -async def fastapi_delete_link(network: str, link: str) -> ChangeSet: - ps = {"id": link} - if is_pipe(network, link): - return delete_pipe(network, ChangeSet(ps)) - elif is_pump(network, link): - return delete_pump(network, ChangeSet(ps)) - elif is_valve(network, link): - return delete_valve(network, ChangeSet(ps)) - - -############################################################ -# tag 8.[TAGS] -############################################################ -# -# TAG_TYPE_NODE = api.TAG_TYPE_NODE -# TAG_TYPE_LINK = api.TAG_TYPE_LINK -# - - -@app.get("/gettagschema/") -async def fastapi_get_tag_schema(network: str) -> dict[str, dict[str, Any]]: - return get_tag_schema(network) - - -@app.get("/gettag/") -async def fastapi_get_tag(network: str, t_type: str, id: str) -> dict[str, Any]: - return get_tag(network, t_type, id) - - -@app.get("/gettags/") -async def fastapi_get_tags(network: str) -> list[dict[str, Any]]: - tags = get_tags(network) - print(tags) - return tags - - -# example: -# set_tag(p, ChangeSet({'t_type': TAG_TYPE_NODE, 'id': 'j1', 'tag': 'j1t' })) -# set_tag(p, ChangeSet({'t_type': TAG_TYPE_LINK, 'id': 'p0', 'tag': 'p0t' })) -@app.post("/settag/", response_model=None) -async def fastapi_set_tag(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_tag(network, ChangeSet(props)) - - -############################################################ -# demand 9.[DEMANDS] -############################################################ -@app.get("/getdemandschema") -async def fastapi_get_demand_schema(network: str) -> dict[str, dict[str, Any]]: - return get_demand_schema(network) - - -@app.get("/getdemandproperties/") -async def fastapi_get_demand_properties(network: str, junction: str) -> dict[str, Any]: - return get_demand(network, junction) - - -# example: set_demand(p, ChangeSet({'junction': 'j1', 'demands': [{'demand': 10.0, 'pattern': None, 'category': 'x'}, {'demand': 20.0, 'pattern': None, 'category': None}]})) -@app.post("/setdemandproperties/", response_model=None) -async def fastapi_set_demand_properties( - network: str, junction: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"junction": junction} | props - return set_demand(network, ChangeSet(ps)) - - -############################################################ -# status 10.[STATUS] init_status -############################################################ -@app.get("/getstatusschema") -async def fastapi_get_status_schema(network: str) -> dict[str, dict[str, Any]]: - return get_status_schema(network) - - -@app.get("/getstatus/") -async def fastapi_get_status(network: str, link: str) -> dict[str, Any]: - return get_status(network, link) - - -# example: set_status(p, ChangeSet({'link': 'p0', 'status': LINK_STATUS_OPEN, 'setting': 10.0})) -@app.post("/setstatus/", response_model=None) -async def fastapi_set_status_properties( - network: str, link: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"link": link} | props - return set_status(network, ChangeSet(ps)) - - -############################################################ -# pattern 11.[PATTERNS] -############################################################ -@app.get("/getpatternschema") -async def fastapi_get_pattern_schema(network: str) -> dict[str, dict[str, Any]]: - return get_pattern_schema(network) - - -@app.post("/addpattern/", response_model=None) -async def fastapi_add_pattern(network: str, pattern: str, req: Request) -> ChangeSet: - props = await req.json() - ps = { - "id": pattern, - } | props - return add_pattern(network, ChangeSet(ps)) - - -@app.post("/deletepattern/", response_model=None) -async def fastapi_delete_pattern(network: str, pattern: str) -> ChangeSet: - ps = {"id": pattern} - return delete_pattern(network, ChangeSet(ps)) - - -@app.get("/getpatternproperties/") -async def fastapi_get_pattern_properties(network: str, pattern: str) -> dict[str, Any]: - return get_pattern(network, pattern) - - -# example: set_pattern(p, ChangeSet({'id' : 'p0', 'factors': [1.0, 2.0, 3.0]})) -@app.post("/setpatternproperties/", response_model=None) -async def fastapi_set_pattern_properties( - network: str, pattern: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": pattern} | props - return set_pattern(network, ChangeSet(ps)) - - -############################################################ -# curve 12.[CURVES] -############################################################ -@app.get("/getcurveschema") -async def fastapi_get_curve_schema(network: str) -> dict[str, dict[str, Any]]: - return get_curve_schema(network) - - -@app.post("/addcurve/", response_model=None) -async def fastapi_add_curve(network: str, curve: str, req: Request) -> ChangeSet: - props = await req.json() - - print(props) - - ps = { - "id": curve, - } | props - - print(ps) - - return add_curve(network, ChangeSet(ps)) - - -@app.post("/deletecurve/", response_model=None) -async def fastapi_delete_curve(network: str, curve: str) -> ChangeSet: - ps = {"id": curve} - return delete_curve(network, ChangeSet(ps)) - - -@app.get("/getcurveproperties/") -async def fastapi_get_curve_properties(network: str, curve: str) -> dict[str, Any]: - return get_curve(network, curve) - - -# example: set_curve(p, ChangeSet({'id' : 'c0', 'c_type' : CURVE_TYPE_PUMP, 'coords': [{'x': 1.0, 'y': 2.0}, {'x': 2.0, 'y': 1.0}]})) -@app.post("/setcurveproperties/", response_model=None) -async def fastapi_set_curve_properties( - network: str, curve: str, req: Request -) -> ChangeSet: - props = await req.json() - # c_type放到request中 - ps = {"id": curve} | props - return set_curve(network, ChangeSet(ps)) - - -############################################################ -# control 13.[CONTROLS] -############################################################ -@app.get("/getcontrolschema/") -async def fastapi_get_control_schema(network: str) -> dict[str, dict[str, Any]]: - return get_control_schema(network) - - -@app.get("/getcontrolproperties/") -async def fastapi_get_control_properties(network: str) -> dict[str, Any]: - return get_control(network) - - -# example: set_control(p, ChangeSet({'control': 'x'})) -@app.post("/setcontrolproperties/", response_model=None) -async def fastapi_set_control_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_control(network, ChangeSet(props)) - - -############################################################ -# rule 14.[RULES] -############################################################ -@app.get("/getruleschema/") -async def fastapi_get_rule_schema(network: str) -> dict[str, dict[str, Any]]: - return get_rule_schema(network) - - -@app.get("/getruleproperties/") -async def fastapi_get_rule_properties(network: str) -> dict[str, Any]: - return get_rule(network) - - -# example: set_rule(p, ChangeSet({'rule': 'x'})) -@app.post("/setruleproperties/", response_model=None) -async def fastapi_set_rule_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_rule(network, ChangeSet(props)) - - -############################################################ -# energy 15.[ENERGY] -############################################################ -@app.get("/getenergyschema/") -async def fastapi_get_energy_schema(network: str) -> dict[str, dict[str, Any]]: - return get_energy_schema(network) - - -@app.get("/getenergyproperties/") -async def fastapi_get_energy_properties(network: str) -> dict[str, Any]: - return get_energy(network) - - -@app.post("/setenergyproperties/", response_model=None) -async def fastapi_set_energy_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_energy(network, ChangeSet(props)) - - -@app.get("/getpumpenergyschema/") -async def fastapi_get_pump_energy_schema(network: str) -> dict[str, dict[str, Any]]: - return get_pump_energy_schema(network) - - -@app.get("/getpumpenergyproperties//") -async def fastapi_get_pump_energy_proeprties(network: str, pump: str) -> dict[str, Any]: - return get_pump_energy(network, pump) - - -@app.get("/setpumpenergyproperties//", response_model=None) -async def fastapi_set_pump_energy_properties( - network: str, pump: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"id": pump} | props - return set_pump_energy(network, ChangeSet(ps)) - - -############################################################ -# emitter 16.[EMITTERS] -############################################################ -@app.get("/getemitterschema") -async def fastapi_get_emitter_schema(network: str) -> dict[str, dict[str, Any]]: - return get_emitter_schema(network) - - -@app.get("/getemitterproperties/") -async def fastapi_get_emitter_properties(network: str, junction: str) -> dict[str, Any]: - return get_emitter(network, junction) - - -# example: set_emitter(p, ChangeSet({'junction': 'j1', 'coefficient': 10.0})) -@app.post("/setemitterproperties/", response_model=None) -async def fastapi_set_emitter_properties( - network: str, junction: str, req: Request -) -> ChangeSet: - props = await req.json() - ps = {"junction": junction} | props - return set_emitter(network, ChangeSet(ps)) - - -############################################################ -# quality 17.[QUALITY] -############################################################ -@app.get("/getqualityschema/") -async def fastapi_get_quality_schema(network: str) -> dict[str, dict[str, Any]]: - return get_quality_schema(network) - - -@app.get("/getqualityproperties/") -async def fastapi_get_quality_properties(network: str, node: str) -> dict[str, Any]: - return get_quality(network, node) - - -# example: set_quality(p, ChangeSet({'node': 'j1', 'quality': 10.0})) -@app.post("/setqualityproperties/", response_model=None) -async def fastapi_set_quality_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_quality(network, ChangeSet(props)) - - -############################################################ -# source 18.[SOURCES] -############################################################ -@app.get("/getsourcechema/") -async def fastapi_get_source_schema(network: str) -> dict[str, dict[str, Any]]: - return get_source_schema(network) - - -@app.get("/getsource/") -async def fastapi_get_source(network: str, node: str) -> dict[str, Any]: - return get_source(network, node) - - -@app.post("/setsource/", response_model=None) -async def fastapi_set_source(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_source(network, ChangeSet(props)) - - -# example: add_source(p, ChangeSet({'node': 'j0', 's_type': SOURCE_TYPE_CONCEN, 'strength': 10.0, 'pattern': 'p0'})) -@app.post("/addsource/", response_model=None) -async def fastapi_add_source(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_source(network, ChangeSet(props)) - - -@app.post("/deletesource/", response_model=None) -async def fastapi_delete_source(network: str, node: str) -> ChangeSet: - props = {"node": node} - return delete_source(network, ChangeSet(props)) - - -############################################################ -# reaction 19.[REACTIONS] -############################################################ -@app.get("/getreactionschema/") -async def fastapi_get_reaction_schema(network: str) -> dict[str, dict[str, Any]]: - return get_reaction_schema(network) - - -@app.get("/getreaction/") -async def fastapi_get_reaction(network: str) -> dict[str, Any]: - return get_reaction(network) - - -@app.post("/setreaction/", response_model=None) -# set_reaction(p, ChangeSet({ 'ORDER BULK' : '10' })) -async def fastapi_set_reaction(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_reaction(network, ChangeSet(props)) - - -@app.get("/getpipereactionschema/") -async def fastapi_get_pipe_reaction_schema(network: str) -> dict[str, dict[str, Any]]: - return get_pipe_reaction_schema(network) - - -@app.get("/getpipereaction/") -async def fastapi_get_pipe_reaction(network: str, pipe: str) -> dict[str, Any]: - return get_pipe_reaction(network, pipe) - - -@app.post("/setpipereaction/", response_model=None) -async def fastapi_set_pipe_reaction(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_pipe_reaction(network, ChangeSet(props)) - - -@app.get("/gettankreactionschema/") -async def fastapi_get_tank_reaction_schema(network: str) -> dict[str, dict[str, Any]]: - return get_tank_reaction_schema(network) - - -@app.get("/gettankreaction/") -async def fastapi_get_tank_reaction(network: str, tank: str) -> dict[str, Any]: - return get_tank_reaction(network, tank) - - -@app.post("/settankreaction/", response_model=None) -async def fastapi_set_tank_reaction(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_tank_reaction(network, ChangeSet(props)) - - -############################################################ -# mixing 20.[MIXING] -############################################################ -@app.get("/getmixingschema/") -async def fastapi_get_mixing_schema(network: str) -> dict[str, dict[str, Any]]: - return get_mixing_schema(network) - - -@app.get("/getmixing/") -async def fastapi_get_mixing(network: str, tank: str) -> dict[str, Any]: - return get_mixing(network, tank) - - -@app.post("/setmixing/", response_model=None) -async def fastapi_set_mixing(network: str, req: Request) -> ChangeSet: - props = await req.json() - return api.set_mixing(network, ChangeSet(props)) - - -# example: add_mixing(p, ChangeSet({'tank': 't0', 'model': MIXING_MODEL_MIXED, 'value': 10.0})) -@app.post("/addmixing/", response_model=None) -async def fastapi_add_mixing(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_mixing(network, ChangeSet(props)) - - -@app.post("/deletemixing/", response_model=None) -async def fastapi_delete_mixing(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_mixing(network, ChangeSet(props)) - - -############################################################ -# time 21.[TIME] -############################################################ -@app.get("/gettimeschema") -async def fastapi_get_time_schema(network: str) -> dict[str, dict[str, Any]]: - return get_time_schema(network) - - -@app.get("/gettimeproperties/") -async def fastapi_get_time_properties(network: str) -> dict[str, Any]: - return get_time(network) - - -@app.post("/settimeproperties/", response_model=None) -async def fastapi_set_time_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_time(network, ChangeSet(props)) - - -############################################################ -# option 23.[OPTIONS] -############################################################ -@app.get("/getoptionschema/") -async def fastapi_get_option_schema(network: str) -> dict[str, dict[str, Any]]: - return get_option_v3_schema(network) - - -@app.get("/getoptionproperties/") -async def fastapi_get_option_properties(network: str) -> dict[str, Any]: - return get_option_v3(network) - - -@app.post("/setoptionproperties/", response_model=None) -async def fastapi_set_option_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_option_v3(network, ChangeSet(props)) - - -############################################################ -# coord 24.[COORDINATES] -############################################################ -@app.get("/getnodecoord/") -async def fastapi_get_node_coord(network: str, node: str) -> dict[str, float] | None: - return get_node_coord(network, node) - - -# DingZQ, 2025-01-27, get all node coord/links -# nodes: id:type:x:y -# links: id:type:node1:node2 -# node type: junction, reservoir, tank -# link type: pipe, pump, valve -@app.get("/getnetworkgeometries/", dependencies=[Depends(verify_token)]) -async def fastapi_get_network_geometries(network: str) -> dict[str, Any] | None: - - # 获取所有节点坐标# 缓存查询结果提高性能 - global redis_client - cache_key = f"getnetworkgeometries_{network}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - coords = get_network_node_coords(network) - nodes = [] - for node_id, coord in coords.items(): - nodes.append(f"{node_id}:{coord['type']}:{coord['x']}:{coord['y']}") - links = get_network_link_nodes(network) - - # return list of scadas. scada : id, x, y - # scadas = get_all_scada_elements(network) - - # data from WMH's scada info - scadas = get_all_scada_info(network) - - results = {"nodes": nodes, "links": links, "scadas": scadas} - - # 缓存查询结果提高性能 - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -# DingZQ, 2024-12-31, get major node coord -# id:type:x:y -# type: junction, reservoir, tank -@app.get("/getmajornodecoords/") -async def fastapi_get_major_node_coords( - network: str, diameter: int -) -> list[str] | None: - start_time = time.time() - coords = get_major_node_coords(network, diameter) - end_time = time.time() - logger.info("get_major_node_coords: %s, time: %s", coords, end_time - start_time) - - result = [] - for node_id, coord in coords.items(): - result.append(f"{node_id}:{coord['type']}:{coord['x']}:{coord['y']}") - return result - - -# DingZQ, 2025-01-03, get network in extent -@app.get("/getnetworkinextent/") -async def fastapi_get_network_in_extent( - network: str, x1: float, y1: float, x2: float, y2: float -) -> dict[str, Any] | None: - nodes = api.get_nodes_in_extent(network, x1, y1, x2, y2) - links = api.get_links_in_extent(network, x1, y1, x2, y2) - return {"nodes": nodes, "links": links} - - -# DingZQ, 2024-12-08, get all links' start and end node -# link_id:link_type:node_id1:node_id2 -@app.get("/getnetworklinknodes/") -async def fastapi_get_network_link_nodes(network: str) -> list[str] | None: - return get_network_link_nodes(network) - - -# DingZQ 2024-12-31 -# 获取直径大于800的管道 -@app.get("/getmajorpipenodes/") -async def fastapi_get_major_pipe_nodes(network: str, diameter: int) -> list[str] | None: - start_time = time.time() - result = get_major_pipe_nodes(network, diameter) - end_time = time.time() - logger.info("get_major_pipe_nodes: %s, time: %s", result, end_time - start_time) - return result - - -############################################################ -# vertex 25.[VERTICES] -############################################################ -@app.get("/getvertexschema/") -async def fastapi_get_vertex_schema(network: str) -> dict[str, dict[str, Any]]: - return get_vertex_schema(network) - - -@app.get("/getvertexproperties/") -async def fastapi_get_vertex_properties(network: str, link: str) -> dict[str, Any]: - return get_vertex(network, link) - - -# set_vertex(p, ChangeSet({'link' : 'p0', 'coords': [{'x': 1.0, 'y': 2.0}, {'x': 2.0, 'y': 1.0}]})) -@app.post("/setvertexproperties/", response_model=None) -async def fastapi_set_vertex_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_vertex(network, ChangeSet(props)) - - -@app.post("/addvertex/", response_model=None) -async def fastapi_add_vertex(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_vertex(network, ChangeSet(props)) - - -@app.post("/deletevertex/", response_model=None) -async def fastapi_delete_vertex(network: str, req: Request) -> ChangeSet: - props = await req.json() - return api.delete_vertex(network, ChangeSet(props)) - - -@app.get("/getallvertexlinks/", response_class=PlainTextResponse) -async def fastapi_get_all_vertex_links(network: str) -> list[str]: - return json.dumps(get_all_vertex_links(network)) - - -@app.get("/getallvertices/", response_class=PlainTextResponse) -async def fastapi_get_all_vertices(network: str) -> list[dict[str, Any]]: - return json.dumps(get_all_vertices(network)) - - -############################################################ -# label 26.[LABELS] -############################################################ -@app.get("/getlabelschema/") -async def fastapi_get_label_schema(network: str) -> dict[str, dict[str, Any]]: - return get_label_schema(network) - - -@app.get("/getlabelproperties/") -async def fastapi_get_label_properties( - network: str, x: float, y: float -) -> dict[str, Any]: - return get_label(network, x, y) - - -@app.post("/setlabelproperties/", response_model=None) -async def fastapi_set_label_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_label(network, ChangeSet(props)) - - -@app.post("/addlabel/", response_model=None) -async def fastapi_add_label(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_label(network, ChangeSet(props)) - - -@app.post("/deletelabel/", response_model=None) -async def fastapi_delete_label(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_label(network, ChangeSet(props)) - - -############################################################ -# backdrop 27.[BACKDROP] -############################################################ -@app.get("/getbackdropschema/") -async def fastapi_get_backdrop_schema(network: str) -> dict[str, dict[str, Any]]: - return get_backdrop_schema(network) - - -@app.get("/getbackdropproperties/") -async def fastapi_get_backdrop_properties(network: str) -> dict[str, Any]: - return get_backdrop(network) - - -@app.post("/setbackdropproperties/", response_model=None) -async def fastapi_set_backdrop_properties(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_backdrop(network, ChangeSet(props)) - - -############################################################ -# scada_device 29 -############################################################ -@app.get("/getscadadeviceschema/") -async def fastapi_get_scada_device_schema(network: str) -> dict[str, dict[str, Any]]: - return get_scada_device_schema(network) - - -@app.get("/getscadadevice/") -async def fastapi_get_scada_device(network: str, id: str) -> dict[str, Any]: - return get_scada_device(network, id) - - -@app.post("/setscadadevice/", response_model=None) -async def fastapi_set_scada_device(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_scada_device(network, ChangeSet(props)) - - -@app.post("/addscadadevice/", response_model=None) -async def fastapi_add_scada_device(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_scada_device(network, ChangeSet(props)) - - -@app.post("/deletescadadevice/", response_model=None) -async def fastapi_delete_scada_device(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_scada_device(network, ChangeSet(props)) - - -@app.post("/cleanscadadevice/", response_model=None) -async def fastapi_clean_scada_device(network: str) -> ChangeSet: - return clean_scada_device(network) - - -@app.get("/getallscadadeviceids/") -async def fastapi_get_all_scada_device_ids(network: str) -> list[str]: - return get_all_scada_device_ids(network) - - -@app.get("/getallscadadevices/", response_class=PlainTextResponse) -async def fastapi_get_all_scada_devices(network: str) -> list[dict[str, Any]]: - return json.dumps(get_all_scada_devices(network)) - - -############################################################ -# scada_device_data 30 -############################################################ -@app.get("/getscadadevicedataschema/") -async def fastapi_get_scada_device_data_schema( - network: str, -) -> dict[str, dict[str, Any]]: - return get_scada_device_data_schema(network) - - -@app.get("/getscadadevicedata/") -async def fastapi_get_scada_device_data(network: str, id: str) -> dict[str, Any]: - return get_scada_device_data(network, id) - - -# example: set_scada_device_data(p, ChangeSet({'device_id': 'sm_device', 'data': [{ 'time': '2023-02-10 00:02:22', 'value': 100.0 }, { 'time': '2023-02-10 00:03:22', 'value': 200.0 }]})) -# time format must be 'YYYY-MM-DD HH:MM:SS' -@app.post("/setscadadevicedata/", response_model=None) -async def fastapi_set_scada_device_data(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_scada_device_data(network, ChangeSet(props)) - - -# example: add_scada_device_data(p, ChangeSet({'device_id': 'sm_device', 'time': '2023-02-10 00:02:22', 'value': 100.0})) -@app.post("/addscadadevicedata/", response_model=None) -async def fastapi_add_scada_device_data(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_scada_device_data(network, ChangeSet(props)) - - -# example: delete_scada_device_data(p, ChangeSet({'device_id': 'sm_device', 'time': '2023-02-12 00:02:22'})) -@app.post("/deletescadadevicedata/", response_model=None) -async def fastapi_delete_scada_device_data(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_scada_device_data(network, ChangeSet(props)) - - -@app.post("/cleanscadadevicedata/", response_model=None) -async def fastapi_clean_scada_device_data(network: str) -> ChangeSet: - return clean_scada_device_data(network) - - -############################################################ -# scada_element 31 -############################################################ -@app.get("/getscadaelementschema/") -async def fastapi_get_scada_element_schema(network: str) -> dict[str, dict[str, Any]]: - return get_scada_element_schema(network) - - -@app.get("/getscadaelements/") -async def fastapi_get_scada_elements(network: str) -> list[str]: - return get_all_scada_elements(network) - - -@app.get("/getscadaelement/") -async def fastapi_get_scada_element(network: str, id: str) -> dict[str, Any]: - return get_scada_element(network, id) - - -@app.post("/setscadaelement/", response_model=None) -async def fastapi_set_scada_element(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_scada_element(network, ChangeSet(props)) - - -@app.post("/addscadaelement/", response_model=None) -async def fastapi_add_scada_element(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_scada_element(network, ChangeSet(props)) - - -@app.post("/deletescadaelement/", response_model=None) -async def fastapi_delete_scada_element(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_scada_element(network, ChangeSet(props)) - - -@app.post("/cleanscadaelement/", response_model=None) -async def fastapi_clean_scada_element(network: str) -> ChangeSet: - return clean_scada_element(network) - - -############################################################ -# general_region 32 -############################################################ -@app.get("/getregionschema/") -async def fastapi_get_region_schema(network: str) -> dict[str, dict[str, Any]]: - return get_region_schema(network) - - -@app.get("/getregion/") -async def fastapi_get_region(network: str, id: str) -> dict[str, Any]: - return get_region(network, id) - - -@app.post("/setregion/", response_model=None) -async def fastapi_set_region(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_region(network, ChangeSet(props)) - - -# example: add_region(p, ChangeSet({'id': 'r', 'boundary': [(0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 0.0)]})) -@app.post("/addregion/", response_model=None) -async def fastapi_add_region(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_region(network, ChangeSet(props)) - - -@app.post("/deleteregion/", response_model=None) -async def fastapi_delete_region(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_region(network, ChangeSet(props)) - - -############################################################ -# district_metering_area 33 -############################################################ -@app.get("/calculatedistrictmeteringareafornodes/") -async def fastapi_calculate_district_metering_area_for_nodes( - network: str, req: Request -) -> list[list[str]]: - props = await req.json() - nodes = props["nodes"] - part_count = props["part_count"] - part_type = props["part_type"] - return calculate_district_metering_area_for_nodes( - network, nodes, part_count, part_type - ) - - -@app.get("/calculatedistrictmeteringareaforregion/") -async def fastapi_calculate_district_metering_area_for_region( - network: str, req: Request -) -> list[list[str]]: - props = await req.json() - region = props["region"] - part_count = props["part_count"] - part_type = props["part_type"] - return calculate_district_metering_area_for_region( - network, region, part_count, part_type - ) - - -@app.get("/calculatedistrictmeteringareafornetwork/") -async def fastapi_calculate_district_metering_area_for_network( - network: str, req: Request -) -> list[list[str]]: - props = await req.json() - part_count = props["part_count"] - part_type = props["part_type"] - return calculate_district_metering_area_for_network(network, part_count, part_type) - - -@app.get("/getdistrictmeteringareaschema/") -async def fastapi_get_district_metering_area_schema( - network: str, -) -> dict[str, dict[str, Any]]: - return get_district_metering_area_schema(network) - - -@app.get("/getdistrictmeteringarea/") -async def fastapi_get_district_metering_area(network: str, id: str) -> dict[str, Any]: - return get_district_metering_area(network, id) - - -@app.post("/setdistrictmeteringarea/", response_model=None) -async def fastapi_set_district_metering_area(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_district_metering_area(network, ChangeSet(props)) - - -@app.post("/adddistrictmeteringarea/", response_model=None) -async def fastapi_add_district_metering_area(network: str, req: Request) -> ChangeSet: - props = await req.json() - - # boundary should be [(x,y), (x,y)] - boundary = props["boundary"] - newBoundary = [] - for pt in boundary: - newBoundary.append((pt[0], pt[1])) - - props["boundary"] = newBoundary - - return add_district_metering_area(network, ChangeSet(props)) - - -@app.post("/deletedistrictmeteringarea/", response_model=None) -async def fastapi_delete_district_metering_area( - network: str, req: Request -) -> ChangeSet: - props = await req.json() - return delete_district_metering_area(network, ChangeSet(props)) - - -@app.get("/getalldistrictmeteringareaids/") -async def fastapi_get_all_district_metering_area_ids(network: str) -> list[str]: - return get_all_district_metering_area_ids(network) - - -@app.get("/getalldistrictmeteringareas/") -async def getalldistrictmeteringareas(network: str) -> list[dict[str, Any]]: - return get_all_district_metering_areas(network) - - -@app.post("/generatedistrictmeteringarea/", response_model=None) -async def fastapi_generate_district_metering_area( - network: str, part_count: int, part_type: int, inflate_delta: float -) -> ChangeSet: - return generate_district_metering_area( - network, part_count, part_type, inflate_delta - ) - - -@app.post("/generatesubdistrictmeteringarea/", response_model=None) -async def fastapi_generate_sub_district_metering_area( - network: str, dma: str, part_count: int, part_type: int, inflate_delta: float -) -> ChangeSet: - print(network) - print(dma) - print(part_count) - print(part_type) - print(inflate_delta) - return generate_sub_district_metering_area( - network, dma, part_count, part_type, inflate_delta - ) - - -############################################################ -# service_area 34 -############################################################ -@app.get("/calculateservicearea/") -async def fastapi_calculate_service_area( - network: str, time_index: int -) -> dict[str, Any]: - return calculate_service_area(network, time_index) - - -@app.get("/getserviceareaschema/") -async def fastapi_get_service_area_schema(network: str) -> dict[str, dict[str, Any]]: - return get_service_area_schema(network) - - -@app.get("/getservicearea/") -async def fastapi_get_service_area(network: str, id: str) -> dict[str, Any]: - return get_service_area(network, id) - - -@app.post("/setservicearea/", response_model=None) -async def fastapi_set_service_area(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_service_area(network, ChangeSet(props)) - - -@app.post("/addservicearea/", response_model=None) -async def fastapi_add_service_area(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_service_area(network, ChangeSet(props)) - - -@app.post("/deleteservicearea/", response_model=None) -async def fastapi_delete_service_area(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_service_area(network, ChangeSet(props)) - - -@app.get("/getallserviceareas/") -async def fastapi_get_all_service_areas(network: str) -> list[dict[str, Any]]: - return get_all_service_areas(network) - - -@app.post("/generateservicearea/", response_model=None) -async def fastapi_generate_service_area( - network: str, inflate_delta: float -) -> ChangeSet: - return generate_service_area(network, inflate_delta) - - -############################################################ -# virtual_district 35 -############################################################ -@app.get("/calculatevirtualdistrict/") -async def fastapi_calculate_virtual_district( - network: str, centers: list[str] -) -> dict[str, list[Any]]: - return calculate_virtual_district(network, centers) - - -@app.get("/getvirtualdistrictschema/") -async def fastapi_get_virtual_district_schema( - network: str, -) -> dict[str, dict[str, Any]]: - return get_virtual_district_schema(network) - - -@app.get("/getvirtualdistrict/") -async def fastapi_get_virtual_district(network: str, id: str) -> dict[str, Any]: - return get_virtual_district(network, id) - - -@app.post("/setvirtualdistrict/", response_model=None) -async def fastapi_set_virtual_district(network: str, req: Request) -> ChangeSet: - props = await req.json() - return set_virtual_district(network, ChangeSet(props)) - - -@app.post("/addvirtualdistrict/", response_model=None) -async def fastapi_add_virtual_district(network: str, req: Request) -> ChangeSet: - props = await req.json() - return add_virtual_district(network, ChangeSet(props)) - - -@app.post("/deletevirtualdistrict/", response_model=None) -async def fastapi_delete_virtual_district(network: str, req: Request) -> ChangeSet: - props = await req.json() - return delete_virtual_district(network, ChangeSet(props)) - - -@app.get("/getallvirtualdistrict/") -async def fastapi_get_all_virtual_district(network: str) -> list[dict[str, Any]]: - return get_all_virtual_districts(network) - - -@app.post("/generatevirtualdistrict/", response_model=None) -async def fastapi_generate_virtual_district( - network: str, inflate_delta: float, req: Request -) -> ChangeSet: - props = await req.json() - return generate_virtual_district(network, props["centers"], inflate_delta) - - -############################################################ -# water_distribution_area 36 -############################################################ -@app.get("/calculatedemandtonodes/") -async def fastapi_calculate_demand_to_nodes( - network: str, req: Request -) -> dict[str, float]: - props = await req.json() - demand = props["demand"] - nodes = props["nodes"] - return calculate_demand_to_nodes(network, demand, nodes) - - -@app.get("/calculatedemandtoregion/") -async def fastapi_calculate_demand_to_region( - network: str, req: Request -) -> dict[str, float]: - props = await req.json() - demand = props["demand"] - region = props["region"] - return calculate_demand_to_region(network, demand, region) - - -@app.get("/calculatedemandtonetwork/") -async def fastapi_calculate_demand_to_network( - network: str, demand: float -) -> dict[str, float]: - return calculate_demand_to_network(network, demand) - - -########################################################### -# scada_info 38 || written by WMH -############################################################ -@app.get("/getscadainfoschema/") -async def fastapi_get_scada_info_schema(network: str) -> dict[str, dict[str, Any]]: - return get_scada_info_schema(network) - - -@app.get("/getscadainfo/") -async def fastapi_get_scada_info(network: str, id: str) -> dict[str, float]: - return get_scada_info(network, id) - - -@app.get("/getallscadainfo/") -async def fastapi_get_all_scada_info(network: str) -> list[dict[str, float]]: - return get_all_scada_info(network) - - -############################################################ -# scheme 40 -############################################################ -@app.get("/getschemeschema/") -async def fastapi_get_scheme_schema(network: str) -> dict[str, dict[Any, Any]]: - return get_scheme_schema(network) - - -@app.get("/getscheme/") -async def fastapi_get_scheme(network: str, schema_name: str) -> dict[Any, Any]: - return get_scheme(network, schema_name) - - -@app.get("/getallschemes/") -async def fastapi_get_all_schemes(network: str) -> list[dict[Any, Any]]: - return get_all_schemes(network) - - -############################################################ -# pipe_risk_probability 41 -############################################################ -@app.get("/getpiperiskprobabilitynow/") -async def fastapi_get_pipe_risk_probability_now( - network: str, pipe_id: str -) -> dict[str, Any]: - return get_pipe_risk_probability_now(network, pipe_id) - - -@app.get("/getpiperiskprobability/") -async def fastapi_get_pipe_risk_probability( - network: str, pipe_id: str -) -> dict[str, Any]: - return get_pipe_risk_probability(network, pipe_id) - - -@app.get("/getpipesriskprobability/") -async def fastapi_get_pipes_risk_probability( - network: str, pipe_ids: str -) -> list[dict[str, Any]]: - pipeids = pipe_ids.split(",") - return get_pipes_risk_probability(network, pipeids) - - -@app.get("/getnetworkpiperiskprobabilitynow/") -async def fastapi_get_network_pipe_risk_probability_now( - network: str, -) -> list[dict[str, Any]]: - return get_network_pipe_risk_probability_now(network) - - -# 返回一个字典，key 是管道的 id，value 是管道的几何信息 -# 几何信息是一个字典，包含 start 和 end 两个 key，value 是管道的起点和终点的坐标 -# 例如： -# "GSD240730154246A51D2C324D1A": { -# "start": [ -# 106.424759007, -# 29.815104642 -# ], -# "end": [ -# 106.424824186, -# 29.814950582 -# ] -# }, -@app.get("/getpiperiskprobabilitygeometries/") -async def fastapi_get_pipe_risk_probability_geometries(network: str) -> dict[str, Any]: - return get_pipe_risk_probability_geometries(network) - - -############################################################ -# sensor_placement 42 -############################################################ -@app.get("/getallsensorplacements/") -async def fastapi_get_all_sensor_placements(network: str) -> list[dict[Any, Any]]: - return get_all_sensor_placements(network) - - -############################################################ -# burst_locate_result 43 -############################################################ -@app.get("/getallburstlocateresults/") -async def fastapi_get_all_burst_locate_results(network: str) -> list[dict[Any, Any]]: - return get_all_burst_locate_results(network) - - -# inp file -@app.post("/uploadinp/", status_code=status.HTTP_200_OK) -async def fastapi_upload_inp(afile: bytes, name: str): - filePath = inpDir + str(name) - f = open(filePath, "wb") - f.write(afile) - f.close() - - return True - - -@app.get("/downloadinp/", status_code=status.HTTP_200_OK) -async def fastapi_download_inp(name: str, response: Response): - filePath = inpDir + name - if os.path.exists(filePath): - return FileResponse( - filePath, media_type="application/octet-stream", filename="inp.inp" - ) - else: - response.status_code = status.HTTP_400_BAD_REQUEST - return True - - -# DingZQ, 2024-12-28, convert v3 to v2 -@app.get("/convertv3tov2/", response_model=None) -async def fastapi_convert_v3_to_v2(req: Request) -> ChangeSet: - network = "v3Tov2" - jo_root = await req.json() - inp = jo_root["inp"] - cs = convert_inp_v3_to_v2(inp) - op = cs.operations[0] - open_project(network) - op["vertex"] = json.dumps(get_all_vertices(network)) - op["scada"] = json.dumps(get_all_scada_elements(network)) - op["dma"] = json.dumps(get_all_district_metering_areas(network)) - op["sa"] = json.dumps(get_all_service_areas(network)) - op["vd"] = json.dumps(get_all_virtual_districts(network)) - op["legend"] = get_extension_data(network, "legend") - - db = get_extension_data(network, "scada_db") - print(db) - scada_db = "" - if db: - scada_db = db - print(scada_db) - op["scada_db"] = scada_db - - close_project(network) - - return cs - - -@app.get("/getjson/") -async def fastapi_get_json(): - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={ - "code": 400, - "message": "this is message", - "data": 123, - }, - ) - - -############################################################ -# DingZQ, 2024-12-09, Add sample API to return real time data/simulation result -# influx db operation -############################################################ -@app.get("/getrealtimedata/") -async def fastapi_get_realtimedata(): - data = [random.randint(0, 100) for _ in range(100)] - return data - - -@app.get("/getsimulationresult/") -async def fastapi_get_simulationresult(): - data = [random.randint(0, 100) for _ in range(100)] - return data - - -# 下面几个query 函数，都是从 influxdb 中查询的，不与 network 绑定，用固定的network 名字 - - -# DingZQ 2025-01-31 -# def query_latest_record_by_ID(ID: str, type: str, bucket: str="realtime_data", client: InfluxDBClient=client) -> dict: -@app.get("/querynodelatestrecordbyid/") -async def fastapi_query_node_latest_record_by_id(id: str): - return influxdb_api.query_latest_record_by_ID(id, type="node") - - -@app.get("/querylinklatestrecordbyid/") -async def fastapi_query_link_latest_record_by_id(id: str): - return influxdb_api.query_latest_record_by_ID(id, type="link") - - -# query scada -@app.get("/queryscadalatestrecordbyid/") -async def fastapi_query_scada_latest_record_by_id(id: str): - return influxdb_api.query_latest_record_by_ID(id, type="scada") - - -# def query_all_record_by_time(query_time: str, bucket: str="realtime_data", client: InfluxDBClient=client) -> tuple: -@app.get("/queryallrecordsbytime/") -async def fastapi_query_all_records_by_time(querytime: str) -> dict[str, list]: - results: tuple = influxdb_api.query_all_records_by_time(query_time=querytime) - return {"nodes": results[0], "links": results[1]} - - -# def query_all_record_by_time_property(querytime: str, type: str, property: str, bucket: str = "realtime_simulation_result") -> tuple: -@app.get("/queryallrecordsbytimeproperty/") -async def fastapi_query_all_record_by_time_property( - querytime: str, type: str, property: str, bucket: str = "realtime_simulation_result" -) -> dict[str, list]: - results: tuple = influxdb_api.query_all_record_by_time_property( - query_time=querytime, type=type, property=property, bucket=bucket - ) - return {"results": results} - - -@app.get("/queryallschemerecordsbytimeproperty/") -async def fastapi_query_all_scheme_record_by_time_property( - querytime: str, - type: str, - property: str, - schemename: str, - bucket: str = "scheme_simulation_result", -) -> dict[str, list]: - """ - 查询指定方案某一时刻的所有记录,查询 'node' 或 'link' 的某一属性值 - - :param querytime: 查询时间,格式为 '2024-11-24T17:30:00+08:00' - :param type: 查询类型 'node' 或 'link' - :param property: 查询的属性字段名 - :param schemename: 方案名称,如 "FANGAN1761124840355" - :param bucket: 数据存储的bucket名称 - :return: 包含查询结果的字典 - """ - results: list = influxdb_api.query_all_scheme_record_by_time_property( - query_time=querytime, - type=type, - property=property, - scheme_name=schemename, - bucket=bucket, - ) - return {"results": results} - - -@app.get("/querysimulationrecordsbyidtime/") -async def fastapi_query_simulation_record_by_ids_time( - id: str, querytime: str, type: str, bucket: str = "realtime_simulation_result" -) -> dict[str, list]: - results: tuple = influxdb_api.query_simulation_result_by_ID_time( - ID=id, type=type, query_time=querytime, bucket=bucket - ) - return {"results": results} - - -@app.get("/queryschemesimulationrecordsbyidtime/") -async def fastapi_query_scheme_simulation_record_by_ids_time( - scheme_name: str, - id: str, - querytime: str, - type: str, - bucket: str = "scheme_simulation_result", -) -> dict[str, list]: - results: tuple = influxdb_api.query_scheme_simulation_result_by_ID_time( - scheme_name=scheme_name, ID=id, type=type, query_time=querytime, bucket=bucket - ) - return {"results": results} - - -@app.get("/queryallrecordsbydate/") -async def fastapi_query_all_records_by_date(querydate: str) -> dict: - # 缓存查询结果提高性能 - global redis_client - - is_today_or_future = time_api.is_today_or_future(querydate) - logger.info(f"isToday or future: {is_today_or_future}") - - # 今天的不要去缓存 - if not is_today_or_future: - cache_key = f"queryallrecordsbydate_{querydate}" - logger.info(f"cache_key: {cache_key}") - - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - results = msgpack.unpackb(data, object_hook=decode_datetime) - logger.info(f"return from cache redis") - return results - - logger.info(f"query from influxdb") - - nodes_links: tuple = influxdb_api.query_all_records_by_date(query_date=querydate) - results = {"nodes": nodes_links[0], "links": nodes_links[1]} - - # 今天的不要去缓存 - if not is_today_or_future: - logger.info(f"save to cache redis") - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - logger.info(f"return results") - - return results - - -@app.get("/queryallrecordsbytimerange/") -async def fastapi_query_all_records_by_time_range( - starttime: str, endtime: str -) -> dict[str, list]: - # 缓存查询结果提高性能 - global redis_client - - # 今天的不要去缓存 - if not time_api.is_today_or_future(starttime): - cache_key = f"queryallrecordsbytimerange_{starttime}_{endtime}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - nodes_links: tuple = influxdb_api.query_all_records_by_time_range( - starttime=starttime, endtime=endtime - ) - results = {"nodes": nodes_links[0], "links": nodes_links[1]} - - # 今天的不要去缓存 - if not time_api.is_today_or_future(starttime): - redis_client.set(cache_key, msgpack.packb(results, default=encode_datetime)) - - return results - - -# 2025-03-15, DingZQ -@app.get("/queryallrecordsbydatewithtype/") -async def fastapi_query_all_records_by_date_with_type( - querydate: str, querytype: str -) -> list: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"queryallrecordsbydatewithtype_{querydate}_{querytype}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = influxdb_api.query_all_records_by_date_with_type( - query_date=querydate, query_type=querytype - ) - - packed = msgpack.packb(results, default=encode_datetime) - redis_client.set(cache_key, packed) - - return results - - -@app.get("/queryallrecordsbyidsdatetype/") -async def fastapi_query_all_records_by_ids_date_type( - ids: str, querydate: str, querytype: str -) -> list: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"queryallrecordsbydatewithtype_{querydate}_{querytype}" - data = redis_client.get(cache_key) - results = [] - if data: - # 使用自定义的反序列化函数 - results = msgpack.unpackb(data, object_hook=decode_datetime) - else: - results = influxdb_api.query_all_records_by_date_with_type( - query_date=querydate, query_type=querytype - ) - packed = msgpack.packb(results, default=encode_datetime) - redis_client.set(cache_key, packed) - - query_ids = ids.split(",") - e_results = py_linq.Enumerable(results) - lst_results = e_results.where(lambda x: x["ID"] in query_ids).to_list() - - return lst_results - - -# 查询指定日期、类型、属性的所有记录 -# 返回 [{'time': '2024-01-01T00:00:00Z', 'ID': '1', 'value': 1.0}, {'time': '2024-01-01T00:00:00Z', 'ID': '2', 'value': 2.0}] -@app.get("/queryallrecordsbydateproperty/") -async def fastapi_query_all_records_by_date_property( - querydate: str, querytype: str, property: str -) -> list[dict]: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"queryallrecordsbydateproperty_{querydate}_{querytype}_{property}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - result_dict = influxdb_api.query_all_record_by_date_property( - query_date=querydate, type=querytype, property=property - ) - packed = msgpack.packb(result_dict, default=encode_datetime) - redis_client.set(cache_key, packed) - - return result_dict - - -# def query_curve_by_ID_property_daterange(ID: str, type: str, property: str, start_date: str, end_date: str, bucket: str="realtime_data", client: InfluxDBClient=client) -> list: -@app.get("/querynodecurvebyidpropertydaterange/") -async def fastapi_query_node_curve_by_id_property_daterange( - id: str, prop: str, startdate: str, enddate: str -): - return influxdb_api.query_curve_by_ID_property_daterange( - id, type="node", property=prop, start_date=startdate, end_date=enddate - ) - - -@app.get("/querylinkcurvebyidpropertydaterange/") -async def fastapi_query_link_curve_by_id_property_daterange( - id: str, prop: str, startdate: str, enddate: str -): - return influxdb_api.query_curve_by_ID_property_daterange( - id, type="link", property=prop, start_date=startdate, end_date=enddate - ) - - -# ids 用,隔开 -# 返回 { 'id': value1, 'id2': value2 } -# def query_SCADA_data_by_device_ID_and_time(query_ids_list: List[str], query_time: str, bucket: str="SCADA_data", client: InfluxDBClient=client) -> Dict[str, float]: -@app.get("/queryscadadatabydeviceidandtime/") -async def fastapi_query_scada_data_by_device_id_and_time(ids: str, querytime: str): - query_ids = ids.split(",") - logger.info(querytime) - return influxdb_api.query_SCADA_data_by_device_ID_and_time( - query_ids_list=query_ids, query_time=querytime - ) - - -# 2025/05/04 DingZQ -# 对于SCAD的曲线数据，我们需要有4 套数据值 -# 1. 原始数据 -# 2. 补充的数据 （补充前面第一步缺失的数据） -# 3. 清洗后的数据点 （用五角星表示） -# 4. 模拟曲线 - - -# 查询到的SCADA原始数据 -# 数据1 -@app.get("/queryscadadatabydeviceidandtimerange/") -async def fastapi_query_scada_data_by_device_id_and_time_range( - ids: str, starttime: str, endtime: str -): - - print(f"query_ids: {ids}, starttime: {starttime}, endtime: {endtime}") - - query_ids = ids.split(",") - return influxdb_api.query_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids, start_time=starttime, end_time=endtime - ) - - -# 查询到的SCADA补充的数据 -# 数据2 -# 注意： 这里的id是 scada_info中的 api_query_id -@app.get("/queryfillingscadadatabydeviceidandtimerange/") -async def fastapi_query_filling_scada_data_by_device_id_and_time_range( - ids: str, starttime: str, endtime: str -): - - print(f"query_ids: {ids}, starttime: {starttime}, endtime: {endtime}") - - query_ids = ids.split(",") - return influxdb_api.query_filling_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids, start_time=starttime, end_time=endtime - ) - - -# 查询到的SCADA清洗后的数据点 -# 数据3 -# 注意： 这里的id是 scada_info中的 api_query_id -@app.get("/querycleaningscadadatabydeviceidandtimerange/") -async def fastapi_query_cleaning_scada_data_by_device_id_and_time_range( - ids: str, starttime: str, endtime: str -): - - print(f"query_ids: {ids}, starttime: {starttime}, endtime: {endtime}") - - query_ids = ids.split(",") - return influxdb_api.query_cleaning_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids, start_time=starttime, end_time=endtime - ) - - -# 查询到的SCADA模拟数据（从 realtime_simulation bucket 中查找） -@app.get("/querysimulationscadadatabydeviceidandtimerange/") -async def fastapi_query_simulation_scada_data_by_device_id_and_time_range( - ids: str, starttime: str, endtime: str -): - - print(f"query_ids: {ids}, starttime: {starttime}, endtime: {endtime}") - - query_ids = ids.split(",") - return influxdb_api.query_simulation_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids, start_time=starttime, end_time=endtime - ) - - -# 查询指定时间范围内，多个SCADA设备的清洗后的数据 -# DingZQ, 2025-04-19 -# 2025/05/04 DingZQ 这个是将原始数据跟清洗后的数据合并到一起，暂时不需要用这个API -@app.get("/querycleanedscadadatabydeviceidandtimerange/") -async def fastapi_query_cleaned_scada_data_by_device_id_and_time_range( - ids: str, starttime: str, endtime: str -): - - print(f"query_ids: {ids}, starttime: {starttime}, endtime: {endtime}") - - query_ids = ids.split(",") - return influxdb_api.query_cleaned_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids, start_time=starttime, end_time=endtime - ) - - -@app.get("/queryscadadatabydeviceidanddate/") -async def fastapi_query_scada_data_by_device_id_and_date(ids: str, querydate: str): - query_ids = ids.split(",") - return influxdb_api.query_SCADA_data_by_device_ID_and_date( - query_ids_list=query_ids, query_date=querydate - ) - - -# DingZQ, 2025-03-08 -# 返回所有SCADA设备在指定日期的所有记录 -@app.get("/queryallscadarecordsbydate/") -async def fastapi_query_all_scada_records_by_date(querydate: str): - global redis_client - - is_today_or_future = time_api.is_today_or_future(querydate) - logger.info(f"isToday or future: {is_today_or_future}") - - # 今天的不要去缓存 - if not is_today_or_future: - cache_key = f"queryallscadarecordsbydate_{querydate}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - logger.info(f"return from cache redis") - return loaded_dict - - logger.info(f"query from influxdb") - result_dict = influxdb_api.query_all_SCADA_records_by_date(query_date=querydate) - - # 今天的不要去缓存 - if not is_today_or_future: - logger.info(f"save to cache redis") - packed = msgpack.packb(result_dict, default=encode_datetime) - redis_client.set(cache_key, packed) - - logger.info(f"return results") - - return result_dict - - -# DingZQ, 2025-03-15 -# Scheme -@app.get("/queryallschemeallrecords/") -async def fastapi_query_all_scheme_all_records( - schemetype: str, schemename: str, querydate: str -) -> tuple: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"queryallschemeallrecords_{schemetype}_{schemename}_{querydate}" - data = redis_client.get(cache_key) - if data: - # 使用自定义的反序列化函数 - loaded_dict = msgpack.unpackb(data, object_hook=decode_datetime) - return loaded_dict - - results = influxdb_api.query_scheme_all_record( - scheme_type=schemetype, scheme_name=schemename, query_date=querydate - ) - packed = msgpack.packb(results, default=encode_datetime) - redis_client.set(cache_key, packed) - - return results - - -# DingZQ, 2025-03-21 -# 缓存是用的queryallschemeallrecords的缓存 -@app.get("/queryschemeallrecordsproperty/") -async def fastapi_query_all_scheme_all_records_property( - schemetype: str, schemename: str, querydate: str, querytype: str, queryproperty: str -) -> list: - # 缓存查询结果提高性能 - global redis_client - cache_key = f"queryallschemeallrecords_{schemetype}_{schemename}_{querydate}" - data = redis_client.get(cache_key) - all_results = None - if data: - # 使用自定义的反序列化函数 - all_results = msgpack.unpackb(data, object_hook=decode_datetime) - else: - all_results = influxdb_api.query_scheme_all_record( - scheme_type=schemetype, scheme_name=schemename, query_date=querydate - ) - packed = msgpack.packb(all_results, default=encode_datetime) - redis_client.set(cache_key, packed) - - results = None - if querytype == "node": - results = all_results[0] - elif querytype == "link": - results = all_results[1] - - return results - - -@app.post("/clearrediskey/") -async def fastapi_clear_redis_key(key: str): - redis_client.delete(key) - return True - - -@app.post("/clearrediskeys/") -async def fastapi_clear_redis_keys(keys: str): - # delete keys contains the key - matched_keys = redis_client.keys(f"*{keys}*") - redis_client.delete(*matched_keys) - - return True - - -@app.post("/clearallredis/") -async def fastapi_clear_all_redis(): - redis_client.flushdb() - return True - - -@app.get("/queryredis/") -async def fastapi_query_redis(): - return redis_client.keys("*") - - -@app.get("/queryinfluxdbbuckets/") -async def fastapi_query_influxdb_buckets(): - return influxdb_api.query_buckets() - - -@app.get("/queryinfluxdbbucketmeasurements/") -async def fastapi_query_influxdb_bucket_measurements(bucket: str): - return influxdb_api.query_measurements(bucket=bucket) - - -# DingZQ, 2024-12-31, generate openapi.json -def generate_openapi_json(): - openapi_json_path = "openapi.json" - with open(openapi_json_path, "w") as file: - json.dump(app.openapi(), file, indent=4) - - -############################################################ -# real_time api 37 -# example: http://127.0.0.1:8000/runsimulation?network=beibeizone&start_time=2024-04-01T08:00:00Z -############################################################ -# 必须用这个PlainTextResponse，不然每个key都有引号 -# @app.get("/runsimulation/", response_class = PlainTextResponse) -# async def fastapi_run_project(network: str,start_time:str,end_time=None) -> str: -# filename = 'c:/lock.simulation' -# filename2 = 'c:/lock.simulation2' -# if os.path.exists(filename2): -# print('file exists') -# raise HTTPException(status_code=409, detail="is in simulation") -# else: -# print('file doesnt exists') -# #os.rename(filename, filename2) -# result = run_simulation(network,start_time,end_time) -# #os.rename(filename2, filename) -# return result - - -############################################################ -# real_time api 37 -# example: http://127.0.0.1:8000/runsimulation?network=beibeizone&start_time=2024-04-01T08:00:00Z -############################################################ - - -# 必须用这个PlainTextResponse，不然每个key都有引号 -# @app.get("/runsimulation/", response_class = PlainTextResponse) -# async def fastapi_run_project(network: str,start_time:str,end_time=None) -> str: -# filename = 'c:/lock.simulation' -# filename2 = 'c:/lock.simulation2' -# if os.path.exists(filename2): -# print('file exists') -# raise HTTPException(status_code=409, detail="is in simulation") -# else: -# print('file doesnt exists') -# #os.rename(filename, filename2) -# result = run_simulation_ex(name=network, simulation_type='realtime', start_datetime=start_time, end_datetime=end_time) -# #os.rename(filename2, filename) -# return result - - -# DingZQ, 2025-05-17 -class Download_History_Data_Manually(BaseModel): - """ - download_date：样式如 datetime(2025, 5, 4) - """ - - download_date: datetime - - -# DingZQ, 2025-05-17 -@app.post("/download_history_data_manually/") -async def fastapi_download_history_data_manually( - data: Download_History_Data_Manually, -) -> None: - item = data.dict() - # 创建东八区时区对象 - tz = timezone(timedelta(hours=8)) - begin_dt = datetime.combine(item["download_date"].date(), time.min).replace( - tzinfo=tz - ) - end_dt = datetime.combine(item["download_date"].date(), time(23, 59, 59)).replace( - tzinfo=tz - ) - - # 2. 转为字符串 - begin_time = begin_dt.isoformat() - end_time = end_dt.isoformat() - - influxdb_api.download_history_data_manually( - begin_time=begin_time, end_time=end_time - ) - - -# DingZQ, 2025-05-17 -# 新增开始时间和持续时间参数 -class Run_Simulation_Manually_by_Date(BaseModel): - """ - name：数据库名称 - start_time：开始时间，样式如 2025-05-04T08:00:00+08:00 - duration：持续时间，单位为分钟 - """ - - name: str - start_time: str - duration: int - - @field_validator("start_time") - @classmethod - def validate_start_time_timezone(cls, value: str) -> str: - time_api.parse_aware_time(value, field_name="start_time") - return value - - -def run_simulation_manually_by_date( - network_name: str, start_time: datetime, duration: int -) -> None: - # 计算结束时间 - end_datetime = start_time + timedelta(minutes=duration) - - # 生成时间点，每15分钟一个 - current_time = start_time - while current_time < end_datetime: - ## 执行函数调用 - simulation.run_simulation( - name=network_name, - simulation_type="realtime", - modify_pattern_start_time=current_time.isoformat(timespec="seconds"), - ) - - # 增加15分钟 - current_time += timedelta(minutes=15) - - -@app.post("/runsimulationmanuallybydate/") -async def fastapi_run_simulation_manually_by_date( - data: Run_Simulation_Manually_by_Date, -) -> dict[str, str]: - item = data.model_dump() - print(f"item: {item}") - - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - try: - simulation.query_corresponding_element_id_and_query_id(item["name"]) - simulation.query_corresponding_pattern_id_and_query_id(item["name"]) - region_result = simulation.query_non_realtime_region(item["name"]) - - globals.source_outflow_region_id = simulation.get_source_outflow_region_id( - item["name"], region_result - ) - globals.realtime_region_pipe_flow_and_demand_id = ( - simulation.query_realtime_region_pipe_flow_and_demand_id( - item["name"], region_result - ) - ) - globals.pipe_flow_region_patterns = ( - simulation.query_pipe_flow_region_patterns(item["name"]) - ) - - globals.non_realtime_region_patterns = ( - simulation.query_non_realtime_region_patterns( - item["name"], region_result - ) - ) - - ( - globals.source_outflow_region_patterns, - globals.realtime_region_pipe_flow_and_demand_patterns, - ) = simulation.get_realtime_region_patterns( - item["name"], - globals.source_outflow_region_id, - globals.realtime_region_pipe_flow_and_demand_id, - ) - - start_time = time_api.parse_utc_time( - item["start_time"], field_name="start_time" - ) - - thread = threading.Thread( - target=lambda: run_simulation_manually_by_date( - item["name"], start_time, item["duration"] - ) - ) - - thread.start() - thread.join() # 等待线程完成 - - return {"status": "success"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) from e - - # thread.join() - # DingZQ 08152025 - # matched_keys = redis_client.keys(...) - # redis_client.delete(*matched_keys) - - -############################################################ -# real_Time api 37.5 -# example: -# response = requests.post("http://127.0.0.1:8000/runsimulation", -# data=json.dumps({'network': 'bb_server', 'simulation_type': 'extended', -# 'start_time': '2024-05-17T09:30:00Z', 'duration': 900, -# 'pump_control': {'1#': [0, 0], '2#': [1, 1], '3#': [1, 1], '4#': [1, 0], -# '5#': [45, 43], '6#': [0, 0], '7#': [0, 0]}}), -# headers={'accept': 'application/json', 'Content-Type': 'application/json'}) -############################################################ - - -# class RunSimuItem(BaseModel): -# network: str -# simulation_type: str -# start_time: str -# end_time: Optional[str] = None -# duration: Optional[int] = 900 -# pump_control: Optional[dict] = None -# -# -# @app.post("/runsimulation/") -# async def fastapi_run_project(item: RunSimuItem) -> str: -# item = item.dict() -# filename = 'c:/lock.simulation' -# filename2 = 'c:/lock.simulation2' -# if os.path.exists(filename2): -# print('file exists') -# raise HTTPException(status_code=409, detail="is in simulation") -# else: -# print('file doesnt exists') -# #os.rename(filename, filename2) -# result = run_simulation_ex(item['network'], item['simulation_type'], -# item['start_time'], item['end_time'], -# item['duration'], item['pump_control']) -# #os.rename(filename2, filename) -# return result - - -############################################################ -# burst analysis api 38 -# example:http://127.0.0.1:8000/burst_analysis?network=beibeizone&start_time=2024-04-01T08:00:00Z&burst_ID=ZBBGXSZW000001&burst_size=200&duration=1800 -############################################################ - -# @app.get("/burst_analysis/", response_class = PlainTextResponse) -# async def fastapi_burst_analysis(network: str,start_time:str,burst_ID:str,burst_size:float,burst_flow:float=None,duration:int=None) -> str: -# filename = 'c:/lock.simulation' -# filename2 = 'c:/lock.simulation2' -# if os.path.exists(filename2): -# print('file exists') -# raise HTTPException(status_code=409, detail="is in simulation") -# else: -# print('file doesnt exists') -# #os.rename(filename, filename2) -# result = burst_analysis(network,start_time,burst_ID,burst_size,burst_flow,duration) -# #os.rename(filename2, filename) -# return result - - -############################################################ -# burst analysis api 38.5 -# example: -# response = requests.post("http://127.0.0.1:8000/burst_analysis", -# data=json.dumps({'network': 'bb_server', -# 'start_time': '2024-05-17T09:30:00Z', -# 'burst_ID': ['ZBBGXSZW000001'], -# 'burst_size': [200], -# 'duration': 1800, -# 'pump_control': {'1#': [0, 0, 0], '2#': [1, 1, 1], '3#': [1, 1, 1], '4#': [1, 1, 1], -# '5#': [45, 45, 45], '6#': [0, 0, 0], '7#': [0, 0, 0]} -# 'valve_closed': ['GSD2307192058576667FF7B41FF']), -# headers={'accept': 'application/json', 'Content-Type': 'application/json'}) -############################################################ - - -class BurstAnalysis(BaseModel): - name: str - modify_pattern_start_time: str - burst_ID: Union[List[str], str] = None - burst_size: Union[List[float], float, int] = None - modify_total_duration: int = 900 - modify_fixed_pump_pattern: Optional[dict[str, list]] = None - modify_variable_pump_pattern: Optional[dict[str, list]] = None - modify_valve_opening: Optional[dict[str, float]] = None - scheme_name: Optional[str] = None - - -@app.post("/burst_analysis/") -async def fastapi_burst_analysis(data: BurstAnalysis) -> str: - item = data.dict() - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - burst_analysis( - name=item["name"], - modify_pattern_start_time=item["modify_pattern_start_time"], - burst_ID=item["burst_ID"], - burst_size=item["burst_size"], - modify_total_duration=item["modify_total_duration"], - modify_fixed_pump_pattern=item["modify_fixed_pump_pattern"], - modify_variable_pump_pattern=item["modify_variable_pump_pattern"], - modify_valve_opening=item["modify_valve_opening"], - scheme_name=item["scheme_name"], - ) - # os.rename(filename2, filename) - - """ - # 将 时间转换成日期，然后缓存这个计算结果 - # 缓存key: burst_analysis__ - global redis_client - schemename = data.scheme_name - - print(data.modify_pattern_start_time) - - querydate = time_api.get_date_from_time(data.modify_pattern_start_time) - - print(f"schemename: {schemename}, querydate: {querydate}") - - cache_key = f"queryallschemeallrecords_burst_Analysis_{schemename}_{querydate}" - data = redis_client.get(cache_key) - if not data: - results = influxdb_api.query_scheme_all_record("burst_Analysis", scheme_name=schemename, query_date=querydate) - packed = msgpack.packb(results, default=encode_datetime) - redis_client.set(cache_key, packed) - """ - - return "success" - - -############################################################ -# valve close analysis api 39 -# example:http://127.0.0.1:8000/valve_close_analysis?network=beibeizone&start_time=2024-04-01T08:00:00Z&valves=GSD2307192058577780A3287D78&valves=GSD2307192058572E953B707226(S2)&duration=1800 -############################################################ - - -@app.get("/valve_close_analysis/", response_class=PlainTextResponse) -async def fastapi_valve_close_analysis( - network: str, - start_time: str, - valves: Annotated[list[str], Query()], - duration: int = None, -) -> str: - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = valve_close_analysis(network, start_time, valves, duration) - # os.rename(filename2, filename) - return result - - -############################################################ -# pipe flushing analysis api 40 -# example:http://127.0.0.1:8000/flushing_analysis?network=beibeizone&start_time=2024-04-01T08:00:00Z&valves=GSD230719205857733F8F5214FF&valves=GSD230719205857C0AF65B6A170&valves_k=0.5&valves_k=0.5&drainage_node_ID=GSD2307192058570DEDF28E4F73&flush_flow=0&duration=1800 -############################################################ - - -@app.get("/flushing_analysis/", response_class=PlainTextResponse) -async def fastapi_flushing_analysis( - network: str, - start_time: str, - valves: Annotated[list[str], Query()], - valves_k: Annotated[list[float], Query()], - drainage_node_ID: str, - flush_flow: float = 0, - duration: int = None, -) -> str: - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = flushing_analysis( - network, - start_time, - valves, - valves_k, - drainage_node_ID, - flush_flow, - duration, - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# contaminant_simulation api 41 -# example:http://127.0.0.1:8000/contaminant_simulation?network=beibeizone&start_time=2024-04-01T08:00:00Z&source=ZBBDTZDP002677&concentration=100&duration=1800 -############################################################ - - -@app.get("/contaminant_simulation/", response_class=PlainTextResponse) -async def fastapi_contaminant_simulation( - network: str, - start_time: str, - source: str, - concentration: float, - duration: int, - pattern: str = None, - scheme_name: str = None, -) -> str: - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = contaminant_simulation( - network, start_time, source, concentration, duration, pattern - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# age analysis api 42 -# example:http://127.0.0.1:8000/age_analysis/?network=bb&start_time=2024-04-01T00:00:00Z&end_time=2024-04-01T08:00:00Z&duration=28800 -############################################################ - - -@app.get("/age_analysis/", response_class=PlainTextResponse) -async def fastapi_age_analysis( - network: str, start_time: str, end_time: str, duration: int -) -> str: - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = age_analysis(network, start_time, end_time, duration) - # os.rename(filename2, filename) - return result - - -############################################################ -# scheduling analysis api 43 -############################################################ - - -class SchedulingAnalysis(BaseModel): - network: str - start_time: str - pump_control: dict - tank_id: str - water_plant_output_id: str - time_delta: Optional[int] = 300 - - -@app.post("/scheduling_analysis/") -async def fastapi_scheduling_analysis(data: SchedulingAnalysis) -> str: - data = data.dict() - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = scheduling_simulation( - data["network"], - data["start_time"], - data["pump_control"], - data["tank_id"], - data["water_plant_output_id"], - data["time_delta"], - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# pressure_regulating api 44 -# example: -# response = requests.post("http://127.0.0.1:8000/pressure_regulating", -# data=json.dumps({'network': 'bb_server', -# 'start_time': '2024-05-17T09:30:00Z', -# 'pump_control': {'1#': [0, 0], '2#': [1, 1], '3#': [1, 1], '4#': [1, 1], -# '5#': [45, 45], '6#': [0, 0], '7#': [0, 0]} -# 'tank_init_level': {'ZBBDTJSC000002': 2, 'ZBBDTJSC000001': 2}}), -# headers={'accept': 'application/json', 'Content-Type': 'application/json'}) -############################################################ - - -class PressureRegulation(BaseModel): - network: str - start_time: str - pump_control: dict - tank_init_level: Optional[dict] = None - - -@app.post("/pressure_regulation/") -async def fastapi_pressure_regulation(data: PressureRegulation) -> str: - item = data.dict() - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = pressure_regulation( - prj_name=item["network"], - start_datetime=item["start_time"], - pump_control=item["pump_control"], - tank_initial_level_control=item["tank_init_level"], - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# project_management api 45 -# example: -# response = requests.post("http://127.0.0.1:8000/project_management", -# data=json.dumps({'network': 'bb_server', -# 'start_time': '2024-05-17T00:00:00Z', -# 'pump_control': -# {'1#':(list:97), '2#':(list:97), '3#':(list:97), '4#':(list:97), -# '5#':(list:97), '6#':(list:97), '7#':(list:97)} -# 'tank_init_level': {'ZBBDTJSC000002': 2, 'ZBBDTJSC000001': 2} -# 'region_demand': {'hp': 150000, 'lp': 40000}}), -# headers={'accept': 'application/json', 'Content-Type': 'application/json'}) -############################################################ - - -class ProjectManagement(BaseModel): - network: str - start_time: str - pump_control: dict - tank_init_level: Optional[dict] = None - region_demand: Optional[dict] = None - - -@app.post("/project_management/") -async def fastapi_project_management(data: ProjectManagement) -> str: - item = data.dict() - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = project_management( - prj_name=item["network"], - start_datetime=item["start_time"], - pump_control=item["pump_control"], - tank_initial_level_control=item["tank_init_level"], - region_demand_control=item["region_demand"], - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# project_management api 46 -# example: -# with open('./inp/bb_temp.inp', 'rb') as file: -# response = requests.post("http://127.0.0.1:8000/network_project", -# files={'file': file}) -############################################################ - - -@app.post("/network_project/") -async def fastapi_network_project(file: UploadFile = File()) -> str: - temp_file_path = "./inp/" - if not os.path.exists(temp_file_path): - os.mkdir(temp_file_path) - temp_file_name = f'network_project_{datetime.now().strftime("%Y%m%d")}' - temp_file_path = f"{temp_file_path}{temp_file_name}.inp" - - with open(temp_file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) - buffer.close() - - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - result = run_inp(temp_file_name) - # os.rename(filename2, filename) - return result - - -############################################################ -# daily scheduling analysis api 47 -############################################################ - - -class DailySchedulingAnalysis(BaseModel): - network: str - start_time: str - pump_control: dict - reservoir_id: str - tank_id: str - water_plant_output_id: str - time_delta: Optional[int] = 300 - - -@app.post("/daily_scheduling_analysis/") -async def fastapi_daily_scheduling_analysis(data: DailySchedulingAnalysis) -> str: - data = data.dict() - filename = "c:/lock.simulation" - filename2 = "c:/lock.simulation2" - if os.path.exists(filename2): - print("file exists") - raise HTTPException(status_code=409, detail="is in simulation") - else: - print("file doesnt exists") - # os.rename(filename, filename2) - result = daily_scheduling_simulation( - data["network"], - data["start_time"], - data["pump_control"], - data["reservoir_id"], - data["tank_id"], - data["water_plant_output_id"], - ) - # os.rename(filename2, filename) - return result - - -############################################################ -# network_update api 48 -############################################################ - - -@app.post("/network_update/") -async def fastapi_network_update(file: UploadFile = File()) -> str: - # 默认文件夹 - default_folder = "./" - - # 使用当前时间生成临时文件名 - temp_file_name = f'network_update_{datetime.now().strftime("%Y%m%d")}' - temp_file_path = os.path.join(default_folder, temp_file_name) - - # 保存上传的文件到服务器 - try: - with open(temp_file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) - buffer.close() - print(f"文件 {temp_file_name} 已成功保存。") - except Exception as e: - raise HTTPException(status_code=500, detail=f"文件保存失败: {e}") - - # 更新数据库 - try: - network_update(temp_file_path) - return json.dumps({"message": "管网更新成功"}) - except Exception as e: - raise HTTPException(status_code=500, detail=f"数据库操作失败: {e}") - - -############################################################ -# pump failure api 49 -############################################################ - - -class PumpFailureState(BaseModel): - time: str - pump_status: dict - - -@app.post("/pump_failure/") -async def fastapi_pump_failure(data: PumpFailureState) -> str: - item = data.dict() - - with open("./pump_failure_message.txt", "a", encoding="utf-8-sig") as f1: - f1.write( - "[{}] {}\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), item) - ) # save message - - status_info = item.copy() - with open("./pump_failure_status.txt", "r", encoding="utf-8-sig") as f2: - lines = f2.readlines() - first_stage_pump_status_dict = json.loads(json.dumps(eval(lines[0]))) - second_stage_pump_status_dict = json.loads( - json.dumps(eval(lines[-1])) - ) # read local file - pump_status_dict = { - "first": first_stage_pump_status_dict, # first-stage pump - "second": second_stage_pump_status_dict, - } # second-stage pump - for pump_type in status_info["pump_status"].keys(): # 'first' or 'second' - if pump_type in pump_status_dict.keys(): # the type of pumps exists - if all( - pump_id in pump_status_dict[pump_type].keys() - for pump_id in status_info["pump_status"][pump_type].keys() - ): # all pump IDs exist - for pump_id in status_info["pump_status"][pump_type].keys(): - pump_status_dict[pump_type][pump_id] = int( - status_info["pump_status"][pump_type][pump_id] - ) # modify status dict - else: - return json.dumps("ERROR: Wrong Pump ID") - else: - return json.dumps("ERROR: Wrong Pump Type") - - with open("./pump_failure_status.txt", "w", encoding="utf-8-sig") as f2_: - f2_.write( - "{}\n{}".format(pump_status_dict["first"], pump_status_dict["second"]) - ) # save local file - - return json.dumps("SUCCESS") - - -############################################################ -# pressure_sensor_placement_sensitivity api 50 -############################################################ -# 2025/05/17 -class Pressure_Sensor_Placement(BaseModel): - name: str - scheme_name: str - sensor_number: int - min_diameter: int = 0 - username: str - - -@app.post("/pressure_sensor_placement_sensitivity/") -async def fastapi_pressure_sensor_placement_sensitivity( - data: Pressure_Sensor_Placement, -) -> None: - item = data.dict() - pressure_sensor_placement_sensitivity( - name=item["name"], - scheme_name=item["scheme_name"], - sensor_number=item["sensor_number"], - min_diameter=item["min_diameter"], - username=item["username"], - ) - - -@app.post("/pressure_sensor_placement_kmeans/") -async def fastapi_pressure_sensor_placement_kmeans( - data: Pressure_Sensor_Placement, -) -> None: - item = data.dict() - pressure_sensor_placement_kmeans( - name=item["name"], - scheme_name=item["scheme_name"], - sensor_number=item["sensor_number"], - min_diameter=item["min_diameter"], - username=item["username"], - ) - - -# 后续改进：合并两个接口为一个，增加method、sensor_type参数选择方法 -@app.post("/sensorplacementscheme/create") -async def fastapi_pressure_sensor_placement( - network: str = Query(...), - scheme_name: str = Query(...), - sensor_type: str = Query(...), - method: str = Query(...), - sensor_count: int = Query(...), - min_diameter: int = Query(0), - user_name: str = Query(...), -) -> str: - item = { - "network": network, - "scheme_name": scheme_name, - "sensor_type": sensor_type, - "method": method, - "sensor_count": sensor_count, - "min_diameter": min_diameter, - "user_name": user_name, - } - - # 验证方法参数 - if item["method"] not in ["sensitivity", "kmeans"]: - raise HTTPException( - status_code=400, detail="Invalid method. Must be 'sensitivity' or 'kmeans'" - ) - - try: - if item["method"] == "sensitivity": - pressure_sensor_placement_sensitivity( - name=item["network"], - scheme_name=item["scheme_name"], - sensor_number=item["sensor_count"], - min_diameter=item["min_diameter"], - username=item["user_name"], - ) - elif item["method"] == "kmeans": - pressure_sensor_placement_kmeans( - name=item["network"], - scheme_name=item["scheme_name"], - sensor_number=item["sensor_count"], - min_diameter=item["min_diameter"], - username=item["user_name"], - ) - - return "success" - - except Exception as e: - raise HTTPException(status_code=500, detail=f"执行失败: {str(e)}") - - -# 新增 SCADA 设备清洗接口 -@app.post("/scadadevicedatacleaning/") -async def fastapi_scada_device_data_cleaning( - network: str = Query(...), - ids_list: List[str] = Query(...), - start_time: str = Query(...), - end_time: str = Query(...), - user_name: str = Query(...), -) -> str: - import pandas as pd # 假设可以使用 pandas 处理表格数据 - - item = { - "network": network, - "ids": ids_list, - "start_time": start_time, - "end_time": end_time, - "user_name": user_name, - } - query_ids_list = item["ids"][0].split(",") - # 先调用 query_SCADA_data_by_device_ID_and_timerange 获取原始数据 - scada_data = influxdb_api.query_SCADA_data_by_device_ID_and_timerange( - query_ids_list=query_ids_list, - start_time=item["start_time"], - end_time=item["end_time"], - ) - - # 获取对应管网的所有 SCADA 设备信息 - scada_device_info = influxdb_api.query_pg_scada_info(item["network"]) - # 将列表转换为字典，以 device_id 为键 - scada_device_info_dict = {info["id"]: info for info in scada_device_info} - - # 按设备类型分组设备 - type_groups = {} - for device_id in query_ids_list: - device_info = scada_device_info_dict.get(device_id, {}) - device_type = device_info.get("type", "unknown") - if device_type not in type_groups: - type_groups[device_type] = [] - type_groups[device_type].append(device_id) - - # 批量处理每种类型的设备 - for device_type, device_ids in type_groups.items(): - if device_type not in ["pressure", "pipe_flow"]: - continue # 跳过未知类型 - - # 过滤该类型的设备数据 - type_scada_data = { - device_id: scada_data[device_id] - for device_id in device_ids - if device_id in scada_data - } - - if not type_scada_data: - continue - - # 假设所有设备的时间点相同，提取 time 列表 - time_list = [record["time"] for record in next(iter(type_scada_data.values()))] - - # 创建 DataFrame，第一列是 time，然后是每个设备的 value 列 - df = pd.DataFrame({"time": time_list}) - for device_id in device_ids: - if device_id in type_scada_data: - values = [record["value"] for record in type_scada_data[device_id]] - df[device_id] = values - - # 移除 time 列，准备输入给清洗方法（清洗方法期望 value 表格） - value_df = df.drop(columns=["time"]) - - # 调用清洗方法 - if device_type == "pressure": - cleaned_value_df = api_ex.Pdataclean.clean_pressure_data_dict_km(value_df) - elif device_type == "pipe_flow": - cleaned_value_df = api_ex.Fdataclean.clean_flow_data_dict(value_df) - - # 添加 time 列到首列 - cleaned_value_df = pd.DataFrame(cleaned_value_df) - # # 只选择以 '_cleaned' 结尾的清洗数据列 - # cleaned_columns = [ - # col for col in cleaned_value_df.columns if col.endswith("_cleaned") - # ] - # cleaned_value_df = cleaned_value_df[cleaned_columns] - # # 重命名列,移除 '_cleaned' 后缀 - # cleaned_value_df = cleaned_value_df.rename( - # columns={ - # col: col.replace("_cleaned", "") for col in cleaned_value_df.columns - # } - # ) - cleaned_df = pd.concat([df["time"], cleaned_value_df], axis=1) - - # 调试输出,确认列名 - print(f"清洗后的列名: {cleaned_df.columns.tolist()}") - - # 将清洗后的数据写回数据库 - influxdb_api.import_multicolumn_data_from_dict( - data_dict=cleaned_df.to_dict("list"), # 转换为 {column_name: [values]} 格式 - raw=False, - ) - - return "success" - - -class Item(BaseModel): - str_info: str - dict_info: Optional[dict] = None - - -@app.post("/test_dict/") -async def get_dict(item: Item): - print(item.dict()) - return item - - -if __name__ == "__main__": - # uvicorn.run(app, host="0.0.0.0", port=8000) - # url='http://127.0.0.1:8000/valve_close_analysis?network=beibeizone&start_time=2024-04-01T08:00:00Z&valve_IDs=GSD2307192058577780A3287D78&valve_IDs=GSD2307192058572E953B707226(S2)&duration=1800' - url = "http://127.0.0.1:8000/burst_analysis?network=beibeizone&start_time=2024-04-01T08:00:00Z&burst_ID=ZBBGXSZW000001&duration=1800" - # url = "http://192.168.1.36:8000/queryallschemeallrecords/?schemename=Fangan0817114448&querydate=2025-08-13&schemetype=burst_Analysis" - # response = Request.get(url) - - import requests - - response = requests.get(url) diff --git a/scripts/main_api_endpoints.md b/scripts/main_api_endpoints.md deleted file mode 100644 index aed26a8..0000000 --- a/scripts/main_api_endpoints.md +++ /dev/null @@ -1,395 +0,0 @@ -# API Endpoints (scripts/main.py) - -Non-commented FastAPI routes defined in `scripts/main.py`. - -- `POST /login/` -- `GET /getallextensiondatakeys/` -- `GET /getallextensiondata/` -- `GET /getextensiondata/` -- `POST /setextensiondata` -- `GET /listprojects/` -- `GET /haveproject/` -- `POST /createproject/` -- `POST /deleteproject/` -- `GET /isprojectopen/` -- `POST /openproject/` -- `POST /closeproject/` -- `POST /copyproject/` -- `POST /importinp/` -- `GET /exportinp/` -- `POST /readinp/` -- `GET /dumpinp/` -- `GET /runproject/` -- `GET /runprojectreturndict/` -- `GET /runinp/` -- `GET /dumpoutput/` -- `GET /isprojectlocked/` -- `GET /isprojectlockedbyme/` -- `POST /lockproject/` -- `POST /unlockproject/` -- `GET /getcurrentoperationid/` -- `POST /undo/` -- `POST /redo/` -- `GET /getsnapshots/` -- `GET /havesnapshot/` -- `GET /havesnapshotforoperation/` -- `GET /havesnapshotforcurrentoperation/` -- `POST /takesnapshotforoperation/` -- `POST takenapshotforcurrentoperation` -- `POST /takesnapshot/` -- `POST /picksnapshot/` -- `POST /pickoperation/` -- `GET /syncwithserver/` -- `POST /batch/` -- `POST /compressedbatch/` -- `GET /getrestoreoperation/` -- `POST /setrestoreoperation/` -- `GET /isnode/` -- `GET /isjunction/` -- `GET /isreservoir/` -- `GET /istank/` -- `GET /islink/` -- `GET /ispipe/` -- `GET /ispump/` -- `GET /isvalve/` -- `GET /getnodetype/` -- `GET /getlinktype/` -- `GET /getelementtype/` -- `GET /getelementtypevalue/` -- `GET /iscurve/` -- `GET /ispattern/` -- `GET /getnodes/` -- `GET /getlinks/` -- `GET /getcurves/` -- `GET /getpatterns/` -- `GET /getnodelinks/` -- `GET /getnodeproperties/` -- `GET /getlinkproperties/` -- `GET /getscadaproperties/` -- `GET /getallscadaproperties/` -- `GET /getelementpropertieswithtype/` -- `GET /getelementproperties/` -- `GET /gettitleschema/` -- `GET /gettitle/` -- `GET /settitle/` -- `GET /getjunctionschema` -- `POST /addjunction/` -- `POST /deletejunction/` -- `GET /getjunctionelevation/` -- `GET /getjunctionx/` -- `GET /getjunctiony/` -- `GET /getjunctioncoord/` -- `GET /getjunctiondemand/` -- `GET /getjunctionpattern/` -- `POST /setjunctionelevation/` -- `POST /setjunctionx/` -- `POST /setjunctiony/` -- `POST /setjunctioncoord/` -- `POST /setjunctiondemand/` -- `POST /setjunctionpattern/` -- `GET /getjunctionproperties/` -- `GET /getalljunctionproperties/` -- `POST /setjunctionproperties/` -- `GET /getreservoirschema` -- `POST /addreservoir/` -- `POST /deletereservoir/` -- `GET /getreservoirhead/` -- `GET /getreservoirpattern/` -- `GET /getreservoirx/` -- `GET /getreservoiry/` -- `GET /getreservoircoord/` -- `POST /setreservoirhead/` -- `POST /setreservoirpattern/` -- `POST /setreservoirx/` -- `POST /setreservoirx/` -- `POST /setreservoircoord/` -- `GET /getreservoirproperties/` -- `GET /getallreservoirproperties/` -- `POST /setreservoirproperties/` -- `GET /gettankschema` -- `POST /addtank/` -- `POST /deletetank/` -- `GET /gettankelevation/` -- `GET /gettankinitlevel/` -- `GET /gettankminlevel/` -- `GET /gettankmaxlevel/` -- `GET /gettankdiameter/` -- `GET /gettankminvol/` -- `GET /gettankvolcurve/` -- `GET /gettankoverflow/` -- `GET /gettankx/` -- `GET /gettanky/` -- `GET /gettankcoord/` -- `POST /settankelevation/` -- `POST /settankinitlevel/` -- `POST /settankminlevel/` -- `POST /settankmaxlevel/` -- `POST settankdiameter//` -- `POST /settankminvol/` -- `POST /settankvolcurve/` -- `POST /settankoverflow/` -- `POST /settankx/` -- `POST /settanky/` -- `POST /settankcoord/` -- `GET /gettankproperties/` -- `GET /getalltankproperties/` -- `POST /settankproperties/` -- `GET /getpipeschema` -- `POST /addpipe/` -- `POST /deletepipe/` -- `GET /getpipenode1/` -- `GET /getpipenode2/` -- `GET /getpipelength/` -- `GET /getpipediameter/` -- `GET /getpiperoughness/` -- `GET /getpipeminorloss/` -- `GET /getpipestatus/` -- `POST /setpipenode1/` -- `POST /setpipenode2/` -- `POST /setpipelength/` -- `POST /setpipediameter/` -- `POST /setpiperoughness/` -- `POST /setpipeminorloss/` -- `POST /setpipestatus/` -- `GET /getpipeproperties/` -- `GET /getallpipeproperties/` -- `POST /setpipeproperties/` -- `GET /getpumpschema` -- `POST /addpump/` -- `POST /deletepump/` -- `GET /getpumpnode1/` -- `GET /getpumpnode2/` -- `POST /setpumpnode1/` -- `POST /setpumpnode2/` -- `GET /getpumpproperties/` -- `GET /getallpumpproperties/` -- `POST /setpumpproperties/` -- `GET /getvalveschema` -- `POST /addvalve/` -- `POST /deletevalve/` -- `GET /getvalvenode1/` -- `GET /getvalvenode2/` -- `GET /getvalvediameter/` -- `GET /getvalvetype/` -- `GET /getvalvesetting/` -- `GET /getvalveminorloss/` -- `POST /setvalvenode1/` -- `POST /setvalvenode2/` -- `POST /setvalvenodediameter/` -- `POST /setvalvetype/` -- `POST /setvalvesetting/` -- `GET /getvalveproperties/` -- `GET /getallvalveproperties/` -- `POST /setvalveproperties/` -- `POST /deletenode/` -- `POST /deletelink/` -- `GET /gettagschema/` -- `GET /gettag/` -- `GET /gettags/` -- `POST /settag/` -- `GET /getdemandschema` -- `GET /getdemandproperties/` -- `POST /setdemandproperties/` -- `GET /getstatusschema` -- `GET /getstatus/` -- `POST /setstatus/` -- `GET /getpatternschema` -- `POST /addpattern/` -- `POST /deletepattern/` -- `GET /getpatternproperties/` -- `POST /setpatternproperties/` -- `GET /getcurveschema` -- `POST /addcurve/` -- `POST /deletecurve/` -- `GET /getcurveproperties/` -- `POST /setcurveproperties/` -- `GET /getcontrolschema/` -- `GET /getcontrolproperties/` -- `POST /setcontrolproperties/` -- `GET /getruleschema/` -- `GET /getruleproperties/` -- `POST /setruleproperties/` -- `GET /getenergyschema/` -- `GET /getenergyproperties/` -- `POST /setenergyproperties/` -- `GET /getpumpenergyschema/` -- `GET /getpumpenergyproperties//` -- `GET /setpumpenergyproperties//` -- `GET /getemitterschema` -- `GET /getemitterproperties/` -- `POST /setemitterproperties/` -- `GET /getqualityschema/` -- `GET /getqualityproperties/` -- `POST /setqualityproperties/` -- `GET /getsourcechema/` -- `GET /getsource/` -- `POST /setsource/` -- `POST /addsource/` -- `POST /deletesource/` -- `GET /getreactionschema/` -- `GET /getreaction/` -- `POST /setreaction/` -- `GET /getpipereactionschema/` -- `GET /getpipereaction/` -- `POST /setpipereaction/` -- `GET /gettankreactionschema/` -- `GET /gettankreaction/` -- `POST /settankreaction/` -- `GET /getmixingschema/` -- `GET /getmixing/` -- `POST /setmixing/` -- `POST /addmixing/` -- `POST /deletemixing/` -- `GET /gettimeschema` -- `GET /gettimeproperties/` -- `POST /settimeproperties/` -- `GET /getoptionschema/` -- `GET /getoptionproperties/` -- `POST /setoptionproperties/` -- `GET /getnodecoord/` -- `GET /getnetworkgeometries/` -- `GET /getmajornodecoords/` -- `GET /getnetworkinextent/` -- `GET /getnetworklinknodes/` -- `GET /getmajorpipenodes/` -- `GET /getvertexschema/` -- `GET /getvertexproperties/` -- `POST /setvertexproperties/` -- `POST /addvertex/` -- `POST /deletevertex/` -- `GET /getallvertexlinks/` -- `GET /getallvertices/` -- `GET /getlabelschema/` -- `GET /getlabelproperties/` -- `POST /setlabelproperties/` -- `POST /addlabel/` -- `POST /deletelabel/` -- `GET /getbackdropschema/` -- `GET /getbackdropproperties/` -- `POST /setbackdropproperties/` -- `GET /getscadadeviceschema/` -- `GET /getscadadevice/` -- `POST /setscadadevice/` -- `POST /addscadadevice/` -- `POST /deletescadadevice/` -- `POST /cleanscadadevice/` -- `GET /getallscadadeviceids/` -- `GET /getallscadadevices/` -- `GET /getscadadevicedataschema/` -- `GET /getscadadevicedata/` -- `POST /setscadadevicedata/` -- `POST /addscadadevicedata/` -- `POST /deletescadadevicedata/` -- `POST /cleanscadadevicedata/` -- `GET /getscadaelementschema/` -- `GET /getscadaelements/` -- `GET /getscadaelement/` -- `POST /setscadaelement/` -- `POST /addscadaelement/` -- `POST /deletescadaelement/` -- `POST /cleanscadaelement/` -- `GET /getregionschema/` -- `GET /getregion/` -- `POST /setregion/` -- `POST /addregion/` -- `POST /deleteregion/` -- `GET /calculatedistrictmeteringareafornodes/` -- `GET /calculatedistrictmeteringareaforregion/` -- `GET /calculatedistrictmeteringareafornetwork/` -- `GET /getdistrictmeteringareaschema/` -- `GET /getdistrictmeteringarea/` -- `POST /setdistrictmeteringarea/` -- `POST /adddistrictmeteringarea/` -- `POST /deletedistrictmeteringarea/` -- `GET /getalldistrictmeteringareaids/` -- `GET /getalldistrictmeteringareas/` -- `POST /generatedistrictmeteringarea/` -- `POST /generatesubdistrictmeteringarea/` -- `GET /calculateservicearea/` -- `GET /getserviceareaschema/` -- `GET /getservicearea/` -- `POST /setservicearea/` -- `POST /addservicearea/` -- `POST /deleteservicearea/` -- `GET /getallserviceareas/` -- `POST /generateservicearea/` -- `GET /calculatevirtualdistrict/` -- `GET /getvirtualdistrictschema/` -- `GET /getvirtualdistrict/` -- `POST /setvirtualdistrict/` -- `POST /addvirtualdistrict/` -- `POST /deletevirtualdistrict/` -- `GET /getallvirtualdistrict/` -- `POST /generatevirtualdistrict/` -- `GET /calculatedemandtonodes/` -- `GET /calculatedemandtoregion/` -- `GET /calculatedemandtonetwork/` -- `GET /getscadainfoschema/` -- `GET /getscadainfo/` -- `GET /getallscadainfo/` -- `GET /getschemeschema/` -- `GET /getscheme/` -- `GET /getallschemes/` -- `GET /getpiperiskprobabilitynow/` -- `GET /getpiperiskprobability/` -- `GET /getpipesriskprobability/` -- `GET /getnetworkpiperiskprobabilitynow/` -- `GET /getpiperiskprobabilitygeometries/` -- `GET /getallsensorplacements/` -- `GET /getallburstlocateresults/` -- `POST /uploadinp/` -- `GET /downloadinp/` -- `GET /convertv3tov2/` -- `GET /getjson/` -- `GET /getrealtimedata/` -- `GET /getsimulationresult/` -- `GET /querynodelatestrecordbyid/` -- `GET /querylinklatestrecordbyid/` -- `GET /queryscadalatestrecordbyid/` -- `GET /queryallrecordsbytime/` -- `GET /queryallrecordsbytimeproperty/` -- `GET /queryallschemerecordsbytimeproperty/` -- `GET /querysimulationrecordsbyidtime/` -- `GET /queryschemesimulationrecordsbyidtime/` -- `GET /queryallrecordsbydate/` -- `GET /queryallrecordsbytimerange/` -- `GET /queryallrecordsbydatewithtype/` -- `GET /queryallrecordsbyidsdatetype/` -- `GET /queryallrecordsbydateproperty/` -- `GET /querynodecurvebyidpropertydaterange/` -- `GET /querylinkcurvebyidpropertydaterange/` -- `GET /queryscadadatabydeviceidandtime/` -- `GET /queryscadadatabydeviceidandtimerange/` -- `GET /queryfillingscadadatabydeviceidandtimerange/` -- `GET /querycleaningscadadatabydeviceidandtimerange/` -- `GET /querysimulationscadadatabydeviceidandtimerange/` -- `GET /querycleanedscadadatabydeviceidandtimerange/` -- `GET /queryscadadatabydeviceidanddate/` -- `GET /queryallscadarecordsbydate/` -- `GET /queryallschemeallrecords/` -- `GET /queryschemeallrecordsproperty/` -- `POST /clearrediskey/` -- `POST /clearrediskeys/` -- `POST /clearallredis/` -- `GET /queryredis/` -- `GET /queryinfluxdbbuckets/` -- `GET /queryinfluxdbbucketmeasurements/` -- `POST /download_history_data_manually/` -- `POST /runsimulationmanuallybydate/` -- `POST /burst_analysis/` -- `GET /valve_close_analysis/` -- `GET /flushing_analysis/` -- `GET /contaminant_simulation/` -- `GET /age_analysis/` -- `POST /scheduling_analysis/` -- `POST /pressure_regulation/` -- `POST /project_management/` -- `POST /network_project/` -- `POST /daily_scheduling_analysis/` -- `POST /network_update/` -- `POST /pump_failure/` -- `POST /pressure_sensor_placement_sensitivity/` -- `POST /pressure_sensor_placement_kmeans/` -- `POST /sensorplacementscheme/create` -- `POST /scadadevicedatacleaning/` -- `POST /test_dict/` diff --git a/scripts/redis_clear_all_keys.py b/scripts/redis_clear_all_keys.py deleted file mode 100644 index 136e14e..0000000 --- a/scripts/redis_clear_all_keys.py +++ /dev/null @@ -1,5 +0,0 @@ -import redis - -redis_client = redis.Redis(host="127.0.0.1", port=6379, db=0) -matched_keys = redis_client.keys(f"**") -redis_client.delete(*matched_keys) diff --git a/tests/api/test_model_import_endpoints.py b/tests/api/test_model_import_endpoints.py index 4fd52de..65c73a9 100644 --- a/tests/api/test_model_import_endpoints.py +++ b/tests/api/test_model_import_endpoints.py @@ -10,6 +10,8 @@ from app.auth.metadata_dependencies import ( get_current_metadata_admin, get_metadata_repository, ) +from app.infra.db.metadb.repositories.metadata_repository import ProjectDbRouting +from app.infra.db.project_routing import get_project_pgconn_string from tests.conftest import build_test_app @@ -98,3 +100,47 @@ def test_model_import_rejects_non_inp_file(monkeypatch): assert response.status_code == 400 assert response.json()["detail"] == "Only .inp model files are accepted" model_import.log_audit_event.assert_not_awaited() + + +def test_model_update_uses_project_business_routing(monkeypatch): + project_id = uuid4() + project = SimpleNamespace(id=project_id, code="demo", status="active") + repo = SimpleNamespace( + session=object(), + get_project_by_id=AsyncMock(return_value=project), + get_project_db_routing=AsyncMock( + return_value=ProjectDbRouting( + project_id=project_id, + db_role="biz_data", + db_type="postgresql", + dsn="postgresql://user:password@biz.example/routed_business", + pool_min_size=1, + pool_max_size=5, + ) + ), + ) + captured: dict[str, str] = {} + + async def fake_apply_model_update(content: bytes, project_code: str) -> None: + assert content == VALID_INP + captured["project_code"] = project_code + captured["dsn"] = get_project_pgconn_string(project_code) + + monkeypatch.setattr(model_import, "_apply_model_update", fake_apply_model_update) + monkeypatch.setattr(model_import, "log_audit_event", AsyncMock()) + client = _client( + admin=SimpleNamespace(id=uuid4(), role="admin", is_superuser=False), + repo=repo, + ) + + response = client.patch( + f"/api/v1/admin/projects/{project_id}/model-imports", + files={"file": ("desktop-model.inp", VALID_INP)}, + ) + + assert response.status_code == 200 + assert captured == { + "project_code": "demo", + "dsn": "postgresql://user:password@biz.example/routed_business", + } + repo.get_project_db_routing.assert_awaited_once_with(project_id, "biz_data") diff --git a/tests/api/test_openapi_contract.py b/tests/api/test_openapi_contract.py index 33aeb40..bfd18d2 100644 --- a/tests/api/test_openapi_contract.py +++ b/tests/api/test_openapi_contract.py @@ -1,8 +1,8 @@ from __future__ import annotations +import inspect from datetime import datetime, timezone from pathlib import Path -from unittest.mock import Mock from uuid import uuid4 import pytest @@ -12,15 +12,44 @@ from fastapi.testclient import TestClient from app.api.v1.endpoints import schemes as schemes_endpoint from app.api.v1.endpoints import simulation as simulation_endpoint -from app.api.v1.endpoints import cache as cache_endpoint from app.api.pagination import PaginatedList from app.api.v1.rest_router import api_router, build_rest_router from app.api.v1.router import api_router as source_api_router from app.auth.metadata_dependencies import get_current_metadata_user -from app.auth.project_dependencies import ProjectContext, get_project_context +from app.auth.project_dependencies import ( + ProjectContext, + get_project_business_routing, + get_project_context, + get_project_simulation_routing, +) +from app.infra.db.project_routing import ( + ActiveProjectRouting, + get_project_pgconn_string, + get_project_timescale_pgconn_string, +) from scripts.check_openapi import current_contract_bytes, validate +def _override_project_routing( + app: FastAPI, + project_context: ProjectContext, +) -> None: + app.dependency_overrides[get_project_context] = lambda: project_context + business = ActiveProjectRouting( + project_code=project_context.project_code, + business_dsn=f"postgresql://user:password@biz/{project_context.project_code}", + ) + simulation = ActiveProjectRouting( + project_code=project_context.project_code, + business_dsn=business.business_dsn, + timescale_dsn=( + f"postgresql://user:password@timescale/{project_context.project_code}" + ), + ) + app.dependency_overrides[get_project_business_routing] = lambda: business + app.dependency_overrides[get_project_simulation_routing] = lambda: simulation + + def test_rest_router_preserves_every_distinct_source_operation() -> None: skipped_names = {"fastapi_get_json", "fastapi_test_dict"} source_names = { @@ -44,6 +73,16 @@ def test_rest_router_has_unique_method_path_pairs() -> None: assert len(pairs) == len(set(pairs)) +def test_removed_redis_management_routes_are_not_published() -> None: + published_paths = { + route.path for route in api_router.routes if isinstance(route, APIRoute) + } + + assert published_paths.isdisjoint( + {"/redis-keys/detail", "/redis-keys", "/all-redis", "/redis"} + ) + + def test_rest_router_rejects_duplicate_method_path_pairs() -> None: first = APIRoute( "/duplicate", @@ -176,6 +215,21 @@ def test_valve_isolation_route_uses_the_isolation_handler() -> None: assert route.name == "valve_isolation_endpoint" +def test_open_project_route_requires_business_and_timescale_routing() -> None: + route = next( + route + for route in api_router.routes + if isinstance(route, APIRoute) + and route.path == "/projects/current" + and route.methods == {"POST"} + ) + routing_parameter = inspect.signature(route.endpoint).parameters[ + "_rest_project_routing" + ] + + assert routing_parameter.default.dependency is get_project_simulation_routing + + def test_valve_isolation_runtime_accepts_frontend_query(monkeypatch) -> None: captured: dict[str, object] = {} @@ -184,6 +238,8 @@ def test_valve_isolation_runtime_accepts_frontend_query(monkeypatch) -> None: network=network, accident_element=accident_element, disabled_valves=disabled_valves, + business_dsn=get_project_pgconn_string(network), + timescale_dsn=get_project_timescale_pgconn_string(network), ) return {"isolatable": True, "must_close_valves": ["V-1"]} @@ -194,12 +250,13 @@ def test_valve_isolation_runtime_accepts_frontend_query(monkeypatch) -> None: ) app = FastAPI(redirect_slashes=False) app.include_router(api_router, prefix="/api/v1") - app.dependency_overrides[get_project_context] = lambda: ProjectContext( + project_context = ProjectContext( project_id=uuid4(), project_code="fengyang", user_id=uuid4(), project_role="member", ) + _override_project_routing(app, project_context) response = TestClient(app, raise_server_exceptions=False).post( "/api/v1/valve-isolation-analyses", @@ -216,6 +273,8 @@ def test_valve_isolation_runtime_accepts_frontend_query(monkeypatch) -> None: "network": "fengyang", "accident_element": ["P-1", "P-2"], "disabled_valves": ["V-9"], + "business_dsn": "postgresql://user:password@biz/fengyang", + "timescale_dsn": "postgresql://user:password@timescale/fengyang", } @@ -247,6 +306,7 @@ def test_rest_runtime_consumes_injected_project_context(monkeypatch) -> None: network=network, scheme_type=scheme_type, query_date=query_date, + business_dsn=get_project_pgconn_string(network), ) return [{"scheme_name": "burst_case", "scheme_type": scheme_type}] @@ -263,7 +323,7 @@ def test_rest_runtime_consumes_injected_project_context(monkeypatch) -> None: user_id=uuid4(), project_role="viewer", ) - app.dependency_overrides[get_project_context] = lambda: project_context + _override_project_routing(app, project_context) response = TestClient(app, raise_server_exceptions=False).get( "/api/v1/schemes", @@ -275,6 +335,7 @@ def test_rest_runtime_consumes_injected_project_context(monkeypatch) -> None: "network": "fengyang", "scheme_type": "burst_analysis", "query_date": None, + "business_dsn": "postgresql://user:password@biz/fengyang", } assert response.json()["items"] == [ {"scheme_name": "burst_case", "scheme_type": "burst_analysis"} @@ -324,12 +385,13 @@ def test_sensor_placement_body_uses_authenticated_project_and_user( ) app = FastAPI(redirect_slashes=False) app.include_router(api_router, prefix="/api/v1") - app.dependency_overrides[get_project_context] = lambda: ProjectContext( + project_context = ProjectContext( project_id=uuid4(), project_code="project_a", user_id=uuid4(), project_role="member", ) + _override_project_routing(app, project_context) app.dependency_overrides[get_current_metadata_user] = lambda: type( "User", (), {"username": "alice"} )() @@ -353,26 +415,6 @@ def test_sensor_placement_body_uses_authenticated_project_and_user( } -def test_cache_management_requires_environment_permission(monkeypatch) -> None: - flushdb = Mock(return_value=True) - monkeypatch.setattr(cache_endpoint.redis_client, "flushdb", flushdb) - app = FastAPI(redirect_slashes=False) - app.include_router(api_router, prefix="/api/v1") - app.dependency_overrides[get_project_context] = lambda: ProjectContext( - project_id=uuid4(), - project_code="project_a", - user_id=uuid4(), - project_role="member", - ) - - response = TestClient(app, raise_server_exceptions=False).delete( - "/api/v1/all-redis" - ) - - assert response.status_code == 403 - flushdb.assert_not_called() - - def test_rest_runtime_json_encodes_untyped_datetime_response() -> None: source_router = APIRouter() diff --git a/tests/unit/test_metadata_repository_dsn_decrypt.py b/tests/unit/test_metadata_repository_dsn_decrypt.py index 0548d9e..00b1b11 100644 --- a/tests/unit/test_metadata_repository_dsn_decrypt.py +++ b/tests/unit/test_metadata_repository_dsn_decrypt.py @@ -6,7 +6,10 @@ from uuid import uuid4 import pytest from cryptography.fernet import InvalidToken -from app.infra.db.metadb.repositories.metadata_repository import MetadataRepository +from app.infra.db.metadb.repositories.metadata_repository import ( + MetadataRepository, + _normalize_postgres_dsn, +) class _DummyResult: @@ -124,6 +127,12 @@ def test_encrypted_dsn_decrypts_without_migration(monkeypatch): session.commit.assert_not_awaited() +def test_psycopg_sqlalchemy_dsn_is_normalized_for_direct_psycopg_clients(): + assert _normalize_postgres_dsn( + "postgresql+psycopg://user:secret@db.example/project" + ) == "postgresql://user:secret@db.example/project" + + def test_upsert_project_database_config_encrypts_plaintext_dsn(monkeypatch): project_id = uuid4() session = SimpleNamespace( diff --git a/tests/unit/test_project_routing.py b/tests/unit/test_project_routing.py new file mode 100644 index 0000000..4e9636d --- /dev/null +++ b/tests/unit/test_project_routing.py @@ -0,0 +1,85 @@ +import pytest +from psycopg.conninfo import conninfo_to_dict + +from app.infra.db.project_routing import ( + ActiveProjectRouting, + activate_project_routing, + get_active_project_routing, + get_project_pgconn_string, + get_project_timescale_pgconn_string, +) + + +def _routing(project_code: str = "project_a") -> ActiveProjectRouting: + return ActiveProjectRouting( + project_code=project_code, + business_dsn=( + "postgresql://biz_user:biz_password@biz.example:5432/biz_database" + "?sslmode=require" + ), + timescale_dsn=( + "postgresql://ts_user:ts_password@timescale.example:5433/ts_database" + "?sslmode=require" + ), + ) + + +def test_project_database_uses_exact_routing_dsn_for_project_code() -> None: + routing = _routing() + + with activate_project_routing(routing): + assert get_project_pgconn_string("project_a") == routing.business_dsn + assert ( + get_project_timescale_pgconn_string("project_a") + == routing.timescale_dsn + ) + + +def test_business_template_keeps_server_and_timescale_ignores_legacy_db_name() -> None: + with activate_project_routing(_routing()): + business = conninfo_to_dict(get_project_pgconn_string("project_a_template")) + timescale = conninfo_to_dict( + get_project_timescale_pgconn_string("temporary_scheme") + ) + + assert business == { + "user": "biz_user", + "password": "biz_password", + "dbname": "project_a_template", + "host": "biz.example", + "port": "5432", + "sslmode": "require", + } + assert timescale == { + "user": "ts_user", + "password": "ts_password", + "dbname": "ts_database", + "host": "timescale.example", + "port": "5433", + "sslmode": "require", + } + + +def test_project_routing_is_nested_and_request_local() -> None: + first = _routing("project_a") + second = _routing("project_b") + + assert get_active_project_routing() is None + with activate_project_routing(first): + assert get_active_project_routing() is first + with activate_project_routing(second): + assert get_active_project_routing() is second + assert get_active_project_routing() is first + assert get_active_project_routing() is None + + +def test_timescale_access_requires_iot_routing_in_project_request() -> None: + business_only = _routing() + business_only = ActiveProjectRouting( + project_code=business_only.project_code, + business_dsn=business_only.business_dsn, + ) + + with activate_project_routing(business_only): + with pytest.raises(RuntimeError, match="TimescaleDB routing is not configured"): + get_project_timescale_pgconn_string() diff --git a/tests/unit/test_scheme_list_filter.py b/tests/unit/test_scheme_list_filter.py index 567c1fb..8631396 100644 --- a/tests/unit/test_scheme_list_filter.py +++ b/tests/unit/test_scheme_list_filter.py @@ -35,7 +35,9 @@ class _FakeConnection: def test_query_scheme_list_pushes_scheme_type_into_sql(monkeypatch): cursor = _FakeCursor() monkeypatch.setattr( - scheme_management, "get_pgconn_string", lambda db_name=None: "postgres://test" + scheme_management, + "get_project_pgconn_string", + lambda db_name=None: "postgres://test", ) monkeypatch.setattr( scheme_management.psycopg, "connect", lambda _conn_string: _FakeConnection(cursor) diff --git a/tests/unit/test_wndb_connection.py b/tests/unit/test_wndb_connection.py index b8fc6f7..0db0002 100644 --- a/tests/unit/test_wndb_connection.py +++ b/tests/unit/test_wndb_connection.py @@ -45,9 +45,11 @@ class _FakeConnection: @pytest.fixture(autouse=True) def clear_native_connections(): connection.g_conn_dict.clear() + connection.g_conninfo_dict.clear() connection._project_locks.clear() yield connection.g_conn_dict.clear() + connection.g_conninfo_dict.clear() connection._project_locks.clear() @@ -61,6 +63,10 @@ def test_is_project_open_drops_closed_cached_connection(): def test_open_connection_reuses_healthy_cached_connection(monkeypatch): cached = _FakeConnection() connection.g_conn_dict["fengyang"] = cached + connection.g_conninfo_dict["fengyang"] = "dbname=fengyang" + monkeypatch.setattr( + connection, "get_project_pgconn_string", lambda db_name: f"dbname={db_name}" + ) def fail_connect(*, conninfo, autocommit): raise AssertionError("cached connection should be reused") @@ -84,7 +90,7 @@ def test_read_all_reopens_closed_cached_connection(monkeypatch): monkeypatch.setattr(connection.pg, "connect", fake_connect) monkeypatch.setattr( - connection, "get_pgconn_string", lambda db_name: f"dbname={db_name}" + connection, "get_project_pgconn_string", lambda db_name: f"dbname={db_name}" ) rows = database.read_all("fengyang", "select * from times") @@ -99,6 +105,7 @@ def test_read_all_reopens_cached_connection_when_health_check_fails(monkeypatch) stale = _FakeConnection(fail_ping=True) fresh = _FakeConnection(rows=[{"scheme_name": "base"}]) connection.g_conn_dict["fengyang"] = stale + connection.g_conninfo_dict["fengyang"] = "dbname=fengyang" opened = [] @@ -108,7 +115,7 @@ def test_read_all_reopens_cached_connection_when_health_check_fails(monkeypatch) monkeypatch.setattr(connection.pg, "connect", fake_connect) monkeypatch.setattr( - connection, "get_pgconn_string", lambda db_name: f"dbname={db_name}" + connection, "get_project_pgconn_string", lambda db_name: f"dbname={db_name}" ) rows = database.read_all("fengyang", "select * from scheme_list") @@ -119,3 +126,20 @@ def test_read_all_reopens_cached_connection_when_health_check_fails(monkeypatch) assert opened == [("dbname=fengyang", True)] assert connection.g_conn_dict["fengyang"] is fresh assert fresh.executed == ["select * from scheme_list"] + + +def test_open_connection_replaces_cache_when_project_dsn_changes(monkeypatch): + cached = _FakeConnection() + fresh = _FakeConnection() + connection.g_conn_dict["fengyang"] = cached + connection.g_conninfo_dict["fengyang"] = "host=old dbname=fengyang" + monkeypatch.setattr( + connection, + "get_project_pgconn_string", + lambda db_name: f"host=new dbname={db_name}", + ) + monkeypatch.setattr(connection.pg, "connect", lambda **_kwargs: fresh) + + assert connection.open_connection("fengyang") is fresh + assert cached.close_calls == 1 + assert connection.g_conninfo_dict["fengyang"] == "host=new dbname=fengyang"