122 lines
5.2 KiB
Python
122 lines
5.2 KiB
Python
"""Complete project SCADA reads with explicit frozen-scope and missing-data accounting."""
|
|
|
|
import json
|
|
from collections import Counter
|
|
|
|
from fastapi import HTTPException
|
|
from fastapi.encoders import jsonable_encoder
|
|
|
|
from app.mcp.models import BoundContext, ScadaInput
|
|
from app.mcp.scope import QueryScope
|
|
from app.infra.db.postgresql import read_queries as reads
|
|
from app.infra.db.postgresql.scada_replays import ScadaReplayRepository
|
|
from app.services.scada_replay_queries import check_replay_window, check_snapshot_start
|
|
|
|
|
|
MAX_COMPLETE_SCADA_BYTES = 8 * 1024 * 1024
|
|
|
|
|
|
async def read_complete_scada(
|
|
scope: QueryScope, binding: BoundContext, args: ScadaInput,
|
|
) -> tuple[dict, dict, list[str], str]:
|
|
"""Exhaust source pages; never treat a first page or selected device subset as a full scan."""
|
|
devices = []
|
|
offset = 0
|
|
size = 0
|
|
|
|
def account(items):
|
|
nonlocal size
|
|
size += len(json.dumps(jsonable_encoder(items), ensure_ascii=False).encode())
|
|
if size > MAX_COMPLETE_SCADA_BYTES:
|
|
raise HTTPException(413, "Complete SCADA result exceeds 8 MiB; split the time window and retain every device")
|
|
|
|
while True:
|
|
page = await reads.scada_devices(scope.connection, limit=501, offset=offset)
|
|
batch = page[:500]
|
|
account(batch)
|
|
devices.extend(batch)
|
|
if len(page) <= 500:
|
|
break
|
|
offset += len(batch)
|
|
|
|
project_device_count = len(devices)
|
|
context = getattr(binding, "replay_context", None)
|
|
catalog = {str(device["device_id"]): device for device in devices}
|
|
outside_scope = []
|
|
if context:
|
|
check_replay_window(context, args.start, args.end)
|
|
repository = ScadaReplayRepository(scope.connection, scope.project.project_id)
|
|
run = await repository.get(context["replay_id"])
|
|
if not run:
|
|
raise HTTPException(404, "SCADA replay was not found in the current project")
|
|
check_snapshot_start(run, args.start)
|
|
frozen = {str(device["device_id"]): device for device in run.get("devices", [])}
|
|
selected = list(dict.fromkeys(run["config"]["device_ids"]))
|
|
outside_scope = sorted(set(catalog) - set(selected))
|
|
# Retain devices removed from the current catalogue but present in the frozen source.
|
|
for device in selected:
|
|
if device not in catalog:
|
|
catalog[device] = frozen.get(device, {"device_id": device})
|
|
devices = list(catalog.values())
|
|
else:
|
|
selected = list(catalog)
|
|
|
|
rows = []
|
|
offset = 0
|
|
# The public limit controls source page size, not completeness of this operation.
|
|
page_size = min(max(args.limit, 1000), 10000)
|
|
|
|
async def read_pages(connection=None):
|
|
nonlocal offset
|
|
while selected:
|
|
if context:
|
|
page = await repository.detected_outputs(
|
|
context["replay_id"], selected, args.start, args.end, context["batch_sequence"],
|
|
limit=page_size + 1, offset=offset,
|
|
)
|
|
else:
|
|
page = await reads.series_rows(
|
|
connection, source="scada", element_type="junction", ids=selected,
|
|
start=args.start, end=args.end, field=None,
|
|
limit=page_size + 1, offset=offset,
|
|
)
|
|
batch = page[:page_size]
|
|
if context:
|
|
# Match the existing frozen-reading contract; internal source hashes are not observations.
|
|
fields = ("time", "device_id", "monitored_value", "cleaned_value", "quality", "reason")
|
|
batch = [{key: row[key] for key in fields if key in row} for row in batch]
|
|
account(batch)
|
|
rows.extend(batch)
|
|
if len(page) <= page_size:
|
|
break
|
|
offset += len(batch)
|
|
|
|
if context:
|
|
await read_pages()
|
|
elif selected:
|
|
async with scope.timescale() as connection:
|
|
await read_pages(connection)
|
|
|
|
counts = Counter(str(row["device_id"]) for row in rows)
|
|
missing = sorted(set(selected) - counts.keys())
|
|
coverage = {
|
|
"mode": "all_scada", "query_complete": True,
|
|
"project_device_count": project_device_count, "queried_device_count": len(selected),
|
|
"devices_with_readings": len(counts), "reading_count": len(rows),
|
|
"queried_device_ids": selected, "missing_device_ids": missing,
|
|
"outside_frozen_scope_device_ids": outside_scope,
|
|
"readings_per_device": {device: counts[device] for device in selected},
|
|
"replay_id": context["replay_id"] if context else None,
|
|
"batch_sequence": context["batch_sequence"] if context else None,
|
|
}
|
|
data = {"devices": devices, "readings": rows, "coverage": coverage}
|
|
account([coverage])
|
|
gaps = []
|
|
if missing:
|
|
gaps.append(f"{len(missing)} queried SCADA devices have no readings in this window; see coverage.missing_device_ids")
|
|
if outside_scope:
|
|
gaps.append(f"{len(outside_scope)} project devices are outside this frozen replay; see coverage.outside_frozen_scope_device_ids")
|
|
units = {device: catalog[device]["measurement_unit"] for device in selected
|
|
if catalog[device].get("measurement_unit")}
|
|
return data, units, gaps, "synthetic" if context else "system_data"
|