feat(investigation): preserve cumulative reference graphs

Accumulate investigation paths and preserve failed attempts across rounds. Keep graph annotations separate from execution state, synchronize the API contract, and consolidate documentation.
This commit is contained in:
2026-11-11 19:10:41 +08:00
parent e47c61c121
commit b853b61a0b
38 changed files with 2445 additions and 147 deletions
+58 -10
View File
@@ -5,7 +5,7 @@ import re
from typing import Annotated, Literal
from uuid import UUID
from pydantic import Field, JsonValue, model_serializer, model_validator
from pydantic import Field, JsonValue, field_validator, model_serializer, model_validator
from app.domain.schemas.answer_ui import (
CHART_SAMPLE_TIME_PATTERN, ReportUiBlock, validate_chart_sample_time, validate_report_ui_blocks,
)
@@ -222,25 +222,61 @@ class InvestigationMapTarget(StrictModel):
feature_id: TaskFeatureId
class InvestigationDiagramNodeRef(StrictModel):
InvestigationNodeAnnotation = Literal["reference", "recorded", "needs_input", "failed", "not_taken"]
LEGACY_INVESTIGATION_NODE_ANNOTATIONS = {
"pending": "reference", "running": "reference", "completed": "recorded",
"waiting": "needs_input", "waiting_human": "needs_input",
"skipped": "not_taken", "cancelled": "not_taken",
}
class _InvestigationDiagramNode(StrictModel):
node_id: Annotated[str, Field(pattern=r"^[A-Za-z_][A-Za-z0-9_-]{0,79}$")]
label: NonEmptyStr
detail: Annotated[str, Field(min_length=1, max_length=2000)] | None = None
evidence_refs: list[NonEmptyStr] = Field(default_factory=list)
action_refs: list[UUID] = Field(default_factory=list)
status: Literal[
"pending", "running", "completed", "waiting", "waiting_human", "failed", "skipped", "cancelled"
] | None = Field(default=None, description=(
"Evidence-confirmed node state; waiting means prerequisites/external/system result, "
"waiting_human requires manual authorization/approval/feedback. "
"Unknown stays omitted/null; host controls colors."
))
map_targets: list[InvestigationMapTarget] = Field(default_factory=list, max_length=100)
class InvestigationDiagramNodeRef(_InvestigationDiagramNode):
status: InvestigationNodeAnnotation | None = Field(default=None, description=(
"A saved process-reference annotation, not live execution state. "
"recorded means a finding exists, not that a task or field action completed."
))
@field_validator("status", mode="before")
@classmethod
def read_legacy_annotation(cls, value: object) -> object:
return LEGACY_INVESTIGATION_NODE_ANNOTATIONS.get(value, value) if isinstance(value, str) else value
class InvestigationGraphNodeUpdate(_InvestigationDiagramNode):
status: InvestigationNodeAnnotation | None = Field(default=None, description=(
"Optional reference annotation: reference, recorded finding, needs_input, failed attempt, or not_taken. "
"No live node/task binding or per-step synchronization is required."
))
class InvestigationGraphEdge(StrictModel):
source: Annotated[str, Field(pattern=r"^[A-Za-z_][A-Za-z0-9_-]{0,79}$")]
target: Annotated[str, Field(pattern=r"^[A-Za-z_][A-Za-z0-9_-]{0,79}$")]
label: Annotated[str, Field(min_length=1, max_length=200)] | None = None
class InvestigationGraphUpdate(StrictModel):
title: Annotated[str, Field(min_length=1, max_length=200)] | None = None
nodes: list[InvestigationGraphNodeUpdate] = Field(default_factory=list, max_length=100)
edges: list[InvestigationGraphEdge] = Field(default_factory=list, max_length=200)
class InvestigationDiagram(StrictModel):
title: NonEmptyStr
mermaid: Annotated[str, Field(min_length=1, max_length=20000)]
node_refs: list[InvestigationDiagramNodeRef] = Field(default_factory=list, max_length=100)
# Present only on a graph maintained by the host. Legacy Mermaid stays readable.
edges: list[InvestigationGraphEdge] | None = Field(default=None, max_length=200)
source_mermaid: Annotated[str, Field(min_length=1, max_length=20000)] | None = None
@model_validator(mode="after")
def validate_description(self):
@@ -323,6 +359,16 @@ class InvestigationProgress(StrictModel):
diagram: InvestigationDiagram | None = None
class InvestigationProgressInput(InvestigationProgress):
graph_update: InvestigationGraphUpdate | None = None
@model_validator(mode="after")
def validate_graph_input(self):
if self.graph_update is not None and self.diagram is not None:
raise ValueError("use graph_update without a generated Mermaid diagram")
return self
class InvestigationProgressSnapshot(StrictModel):
revision: PositiveInt
run_id: UUID
@@ -367,7 +413,7 @@ class AgentArtifactsCommand(StrictModel):
evidence: list[EvidenceInput] = Field(default_factory=list, max_length=100)
plan: ExecutionPlanInput | None = None
report: InvestigationReportInput | None = None
progress: InvestigationProgress | None = None
progress: InvestigationProgressInput | None = None
@model_validator(mode="after")
def validate_artifacts(self):
@@ -377,6 +423,8 @@ class AgentArtifactsCommand(StrictModel):
self.report.diagram if self.report is not None else None,
self.progress.diagram if self.progress is not None else None,
):
if diagram is not None and (diagram.edges is not None or diagram.source_mermaid is not None):
raise ValueError("structured graph metadata is host-owned; use progress.graph_update")
missing = _undeclared_diagram_node_refs(diagram)
if missing:
raise ValueError(
@@ -628,6 +628,37 @@ class OperationalTaskRepository:
row = await cursor.fetchone()
return row["document"] if row else None
async def get_latest_graph_progress(self, task_id: str) -> dict | None:
availability = await self.connection.execute(
"SELECT to_regclass('workbench.operational_task_progress') AS relation"
)
if not (await availability.fetchone())["relation"]:
return None
cursor = await self.connection.execute(
"""SELECT document FROM workbench.operational_task_progress
WHERE project_id=%s AND task_id=%s
AND jsonb_typeof(document#>'{progress,diagram,edges}')='array'
ORDER BY (document->>'execution_generation')::bigint DESC, revision DESC LIMIT 1""",
(self.project_id, task_id),
)
row = await cursor.fetchone()
return row["document"] if row else None
async def get_latest_diagram(self, task_id: str) -> dict | None:
cursor = await self.connection.execute(
"""SELECT diagram FROM (
SELECT document#>'{progress,diagram}' AS diagram, published_at AS occurred_at
FROM workbench.operational_task_progress WHERE project_id=%s AND task_id=%s
UNION ALL
SELECT document#>'{report,diagram}', created_at
FROM workbench.operational_task_reports WHERE project_id=%s AND task_id=%s
) AS diagrams WHERE jsonb_typeof(diagram)='object'
ORDER BY occurred_at DESC LIMIT 1""",
(self.project_id, task_id, self.project_id, task_id),
)
row = await cursor.fetchone()
return row["diagram"] if row else None
async def save_progress(self, task_id: str, snapshot: dict) -> None:
cursor = await self.connection.execute(
"""INSERT INTO workbench.operational_task_progress
+32 -2
View File
@@ -7,7 +7,9 @@ from uuid import UUID
from psycopg import AsyncConnection
from psycopg import sql
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from fastapi import HTTPException
from app.domain.schemas.operational_task import LEGACY_INVESTIGATION_NODE_ANNOTATIONS
from app.infra.db.postgresql.scada import _SCADA_VIEW_SELECT, _device
from app.infra.db.timescaledb.repositories.realtime import RealtimeRepository
from app.infra.db.timescaledb.repositories.analysis import AnalysisResultsRepository
@@ -329,6 +331,29 @@ async def task_context_summary(conn: AsyncConnection, *, project_id: UUID, task_
return result
def _reference_diagram_projection(stored_sql: str, stored_params: tuple, section: str) -> tuple[str, tuple]:
"""Normalize only saved diagram annotations before any data_path or page projection."""
if section not in {"progress", "report"}:
raise ValueError("reference diagrams only belong to progress or report sections")
path = "{data," + section + ",diagram,node_refs}"
statement = f"""
WITH reference_documents AS (
{stored_sql}
), reference_aliases AS (
SELECT %s::jsonb AS annotations
)
SELECT CASE WHEN jsonb_typeof(snapshot#>'{path}')='array' THEN
jsonb_set(snapshot,'{path}',COALESCE((
SELECT jsonb_agg(CASE WHEN annotations ? (node->>'status')
THEN jsonb_set(node,'{{status}}',annotations->(node->>'status'))
ELSE node END ORDER BY position)
FROM jsonb_array_elements(snapshot#>'{path}') WITH ORDINALITY AS nodes(node,position)
),'[]'::jsonb)) ELSE snapshot END AS snapshot
FROM reference_documents CROSS JOIN reference_aliases
"""
return statement, (*stored_params, Jsonb(LEGACY_INVESTIGATION_NODE_ANNOTATIONS))
async def task_context_page(
conn: AsyncConnection, *, project_id: UUID, task_id: UUID, section: str,
report_revision: int | None, data_path: list[str], limit: int, offset: int,
@@ -386,11 +411,16 @@ async def task_context_page(
stored = """SELECT jsonb_build_object('section','progress','data',p.document) AS snapshot
FROM workbench.operational_task_progress p
JOIN workbench.operational_task_runs r ON r.project_id=p.project_id AND r.task_id=p.task_id AND r.run_id=p.run_id
JOIN workbench.operational_tasks t ON t.project_id=p.project_id AND t.task_id=p.task_id AND t.execution_generation=r.generation
WHERE p.project_id=%s AND p.task_id=%s"""
JOIN workbench.operational_tasks t ON t.project_id=p.project_id AND t.task_id=p.task_id
WHERE p.project_id=%s AND p.task_id=%s AND r.generation<=t.execution_generation
AND (t.execution_generation=r.generation
OR jsonb_typeof(p.document#>'{progress,diagram,edges}')='array')
ORDER BY (t.execution_generation=r.generation) DESC,r.generation DESC,p.revision DESC LIMIT 1"""
params = (project_id, task_id)
else:
raise HTTPException(422, "Unknown task context section")
if section in {"progress", "report"}:
stored, params = _reference_diagram_projection(stored, params, section)
return await _snapshot_page(conn, stored_sql=stored, stored_params=params,
data_path=data_path, limit=limit, offset=projection_offset, mcp_sources=[], max_bytes=8000)
+1 -1
View File
@@ -217,7 +217,7 @@ _specs = [
),
(
"get_task_context",
"读取绑定任务的有界摘要及handling_skill元数据;section=skill读取Server按任务workflow分发的业务处置Skill正文与版本,属于流程指引而非事件证据;其他section选择task/report/evidence/actions/progress/access_requests/plan/run,data_path按返回路径读取嵌套字段,limit/offset分页;report_revision保留选定报告版本",
"读取绑定任务的有界摘要及handling_skill元数据;section=skill读取Server按任务workflow分发的业务处置Skill正文与版本,属于流程指引而非事件证据;其他section选择task/report/evidence/actions/progress/access_requests/plan/run,data_path按返回路径读取嵌套字段,limit/offset分页;progress/report图节点仅为流程参考标记,旧执行态兼容转换,不代表实时进度;实际执行状态读run、动作状态读actions;report_revision保留选定报告版本",
m.TaskContextInput,
"operational_task.view",
"task",
+94
View File
@@ -0,0 +1,94 @@
"""Accumulate explanatory investigation paths and generate Mermaid deterministically."""
import re
from app.domain.domain_errors import require
from app.domain.schemas.operational_task import (
InvestigationDiagram, InvestigationDiagramNodeRef, InvestigationGraphUpdate,
)
def _label(value: str) -> str:
# Encode Mermaid delimiters and HTML before placing untrusted text in quotes.
replacements = {"#": "#35;", "&": "#38;", '"': "#quot;", "<": "#lt;",
">": "#gt;", "`": "#96;", "%": "#37;", "|": "#124;", "\\": "#92;"}
return "".join(replacements.get(c, " " if c in "\r\n" else c) for c in value)
def adopt_legacy_investigation_graph(diagram: InvestigationDiagram) -> InvestigationDiagram:
"""Make a saved legacy graph cumulative without parsing or changing its source."""
if diagram.edges is not None:
return diagram
return diagram.model_copy(deep=True, update={"edges": [], "source_mermaid": diagram.mermaid})
def update_investigation_graph(
previous: InvestigationDiagram | None, update: InvestigationGraphUpdate,
) -> InvestigationDiagram:
"""Upsert nodes and append explicit edges; never delete a path or erase a failed attempt.
A retry uses a new node id. Relationships describe activity for readers and
impose no ordering or completion requirements on the Agent.
"""
if previous is not None:
previous = adopt_legacy_investigation_graph(previous)
source = previous.source_mermaid if previous else None
def in_source(node_id: str) -> bool:
return bool(source and re.search(
rf"(?<![A-Za-z0-9_-]){re.escape(node_id)}(?=\s*(?:[\[({{@;]|:::|--|==|-\.|$))",
source, re.MULTILINE,
))
nodes = {node.node_id: node.model_copy(deep=True) for node in previous.node_refs} if previous else {}
require(len({node.node_id for node in update.nodes}) == len(update.nodes),
"GRAPH_NODE_DUPLICATE", "Graph update contains duplicate node ids", status=422)
for node in update.nodes:
require(node.node_id.lower() not in {"end", "graph", "flowchart", "subgraph"},
"GRAPH_NODE_ID_INVALID", "Use a descriptive graph node id", status=422)
existing = nodes.get(node.node_id)
if existing is None:
require(not in_source(node.node_id), "GRAPH_LEGACY_NODE_CONFLICT",
"This id belongs to the original Mermaid; use a new node id", status=409)
nodes[node.node_id] = InvestigationDiagramNodeRef.model_validate(node.model_dump(exclude_unset=True))
continue
require(not in_source(node.node_id) or existing.label == node.label,
"GRAPH_LEGACY_NODE_CONFLICT", "Keep the original node label; use a new node for a new step", status=409)
require(existing.status != "failed" or (
("status" not in node.model_fields_set or node.status == "failed")
and existing.label == node.label
), "GRAPH_FAILED_ATTEMPT_IMMUTABLE", "Keep the failed attempt; use a new node for a retry", status=409)
changes = node.model_dump(exclude_unset=True)
if existing.status == "failed" and existing.detail and not changes.get("detail"):
changes.pop("detail", None)
for field in ("evidence_refs", "action_refs", "map_targets"):
combined = list(getattr(existing, field))
for value in getattr(node, field):
if value not in combined:
combined.append(value)
changes[field] = combined
nodes[node.node_id] = InvestigationDiagramNodeRef.model_validate({
**existing.model_dump(), **changes,
})
require(bool(nodes), "GRAPH_NODE_REQUIRED", "Add a node before publishing a graph", status=422)
edges = list(previous.edges or []) if previous else []
for edge in update.edges:
require(edge.source in nodes and edge.target in nodes, "GRAPH_EDGE_NODE_NOT_FOUND",
"Every graph edge must reference saved or newly added nodes", status=422)
if edge not in edges:
edges.append(edge.model_copy(deep=True))
require(len(nodes) <= 100 and len(edges) <= 200, "GRAPH_CAPACITY_EXCEEDED",
"Investigation graph exceeds 100 nodes or 200 edges; existing paths were retained", status=422)
lines = [source or "flowchart TD"]
for node in nodes.values():
if in_source(node.node_id):
continue # Preserve original branches, shapes and labels byte for byte.
label = node.label + ("(失败)" if node.status == "failed" else "")
lines.append(f' {node.node_id}["{_label(label)}"]')
for edge in edges:
connector = f' -->|"{_label(edge.label)}"| ' if edge.label else " --> "
lines.append(f" {edge.source}{connector}{edge.target}")
mermaid = "\n".join(lines)
require(len(mermaid) <= 20000, "GRAPH_CAPACITY_EXCEEDED",
"Investigation Mermaid exceeds 20000 characters; existing paths were retained", status=422)
return InvestigationDiagram(
title=update.title or (previous.title if previous else "调查脉络"),
mermaid=mermaid, node_refs=list(nodes.values()), edges=edges, source_mermaid=source,
)
+50 -16
View File
@@ -49,6 +49,8 @@ from app.domain.schemas.operational_task import (
InvestigationReportRevision,
InvestigationReportReference,
InvestigationProgressSnapshot,
InvestigationProgress,
InvestigationDiagram,
IssueCommandPayload,
NotificationAcknowledgeCommand,
OperationalTaskCreate,
@@ -71,6 +73,7 @@ from app.domain.schemas.operational_task import (
)
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
from app.services.content_digest import digest
from app.services.investigation_graph import adopt_legacy_investigation_graph, update_investigation_graph
from app.services.operational_presentation import resolve_overlay
from app.services.operational_recovery import accept_completion, track_work_order
from app.services.operational_scada_coverage import require_complete_scada_evidence
@@ -264,6 +267,8 @@ class OperationalTaskService:
active_run = await self._active_run(task)
latest_run = await self.repo.get_latest_run(str(task_id))
progress = await self.repo.get_latest_progress(str(task_id), task["execution_generation"])
if progress is None:
progress = await self.repo.get_latest_graph_progress(str(task_id))
work_order_snapshot = None
if task["task_type"] == "external_work_order" and task["source"].get("external_id"):
feedback = await self.repo.get_feedback(
@@ -850,11 +855,27 @@ class OperationalTaskService:
for item in await self.repo.list_actions(str(task_id))
}
progress_revision = None
saved_diagram = None
if command.progress is not None or command.report is not None:
stored = await self.repo.get_latest_graph_progress(str(task_id))
if stored is not None:
saved_diagram = InvestigationProgressSnapshot.model_validate(stored).progress.diagram
elif command.progress is not None:
legacy = await self.repo.get_latest_diagram(str(task_id))
saved_diagram = InvestigationDiagram.model_validate(legacy) if legacy else None
if command.progress is not None:
require(command.run_id is not None, "RUN_REQUIRED",
"Progress must identify its current execution run", status=422)
if command.progress.diagram:
for node in command.progress.diagram.node_refs:
diagram = command.progress.diagram
if command.progress.graph_update is not None:
diagram = update_investigation_graph(saved_diagram, command.progress.graph_update)
elif saved_diagram is not None:
require(diagram is None or saved_diagram.edges is None, "GRAPH_UPDATE_REQUIRED",
"Use graph_update to extend the saved investigation graph", status=422)
if diagram is None:
diagram = adopt_legacy_investigation_graph(saved_diagram)
if diagram:
for node in diagram.node_refs:
self._require_evidence_refs(node.evidence_refs, known_evidence)
require(all(str(ref) in actions for ref in node.action_refs),
"ACTION_NOT_FOUND", "Progress references an unknown action", status=422)
@@ -863,9 +884,11 @@ class OperationalTaskService:
snapshot = InvestigationProgressSnapshot(
revision=progress_revision, run_id=active_run["run_id"],
execution_generation=task["execution_generation"],
source_session_id=command.source_session_id, published_at=now, progress=command.progress,
source_session_id=command.source_session_id, published_at=now,
progress=InvestigationProgress(summary=command.progress.summary, diagram=diagram),
)
await self.repo.save_progress(str(task_id), _json(snapshot))
saved_diagram = diagram
await self._append_event(
topic="operational.progress.changed", aggregate_type="operational_task_progress",
aggregate_id=str(task_id), aggregate_revision=progress_revision, occurred_at=now,
@@ -917,21 +940,31 @@ class OperationalTaskService:
await self.repo.insert_plan(str(task_id), _json(snapshot))
current_plan = snapshot
if command.report:
self._require_evidence_refs(command.report.evidence_refs, known_evidence)
if command.report.stage == "final":
report_input = command.report
if saved_diagram is not None:
diagram = adopt_legacy_investigation_graph(saved_diagram)
report_input = report_input.model_copy(update={
"diagram": diagram,
"evidence_refs": list(dict.fromkeys([
*report_input.evidence_refs,
*(ref for node in diagram.node_refs for ref in node.evidence_refs),
])),
})
self._require_evidence_refs(report_input.evidence_refs, known_evidence)
if report_input.stage == "final":
require_complete_scada_evidence(
task, await self.repo.list_evidence(str(task_id)), command.report.evidence_refs,
task, await self.repo.list_evidence(str(task_id)), report_input.evidence_refs,
)
for gap in command.report.data_gaps:
require(set(gap.evidence_refs) <= set(command.report.evidence_refs),
for gap in report_input.data_gaps:
require(set(gap.evidence_refs) <= set(report_input.evidence_refs),
"REPORT_GAP_EVIDENCE_INVALID",
"Gap evidence must be included in report evidence", status=422)
if command.report.diagram:
for node in command.report.diagram.node_refs:
self._require_evidence_refs(node.evidence_refs, set(command.report.evidence_refs))
if report_input.diagram:
for node in report_input.diagram.node_refs:
self._require_evidence_refs(node.evidence_refs, set(report_input.evidence_refs))
require(all(str(ref) in actions for ref in node.action_refs),
"ACTION_NOT_FOUND", "Diagram references an unknown action", status=422)
if command.report.outcome == "insufficient_data":
if report_input.outcome == "insufficient_data":
require(not any(action.approval_status == "pending" or (
action.approval_status == "approved" and action.execution_status in {"not_submitted", "submitted", "unknown"}
) for action in actions.values()), "REPORT_ACTION_PENDING",
@@ -941,7 +974,7 @@ class OperationalTaskService:
revision=report_revision,
execution_generation=task["execution_generation"],
source_session_id=command.source_session_id,
published_at=now, published_by=self._actor(), report=command.report,
published_at=now, published_by=self._actor(), report=report_input,
)
report = await self._report_with_progress_diagram(task_id, report)
await self.repo.insert_report(str(task_id), _json(report))
@@ -954,11 +987,12 @@ class OperationalTaskService:
metadata={"task_id": str(task_id), "run_id": str(active_run["run_id"])},
)
previous_result = previous_report["report"] if previous_report else None
if command.report.stage == "final" and (
effective_report = _json(report.report)
if report.report.stage == "final" and (
previous_report is None
or previous_report.get("execution_generation") != task["execution_generation"]
or previous_result.get("stage") != "final"
or any(previous_result.get(key, [] if key == "ui_blocks" else None) != _json(command.report).get(key)
or any(previous_result.get(key, [] if key == "ui_blocks" else None) != effective_report.get(key)
for key in ("current_conclusion", "conclusion_boundary", "outcome", "stop_reason",
"evidence_refs", "recommendations", "pending_items", "data_gaps", "ui_blocks"))
):
@@ -967,7 +1001,7 @@ class OperationalTaskService:
now=now, reason="report_superseded",
)
await self._notify(
task_id, "report_updated", task["title"], command.report.current_conclusion, now,
task_id, "report_updated", task["title"], report.report.current_conclusion, now,
report_reference=InvestigationReportReference(
execution_generation=task["execution_generation"], revision=report_revision,
),