OrgTJWater/next-tjwater-server
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): integrate MCP queries and investigation messaging

Browse Source 
Add project-bound OAuth MCP queries and controlled analysis with scoped evidence,
limits, timeouts and a nearby-model spatial query using EPSG:4326 inputs.
Default nearby searches to 200 meters and five results.

Integrate investigation messaging/presentation, health/readiness and typed history;
remove duplicate web search and fix online simulation and compressed baseline writes.
Include project migrations 023-025; keep local credentials and demo seeding excluded.

Validation: 319 focused pytest checks; real PostGIS temporary-table regression;
read-only Jinqian bridge radius comparison; cross-language MCP discovery/calls.
This commit is contained in:
jiang
2026-10-19 18:46:58 +08:00
parent 2b6fc5aa52
commit be6317eb48
117 changed files with 12703 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/operational_tasks.py
+210 -9
View File
@@ -1,6 +1,7 @@
"""State machines for durable operational tasks and approved external actions."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from typing import Iterable
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
@@ -46,6 +47,7 @@ from app.domain.schemas.operational_task import (
HumanEvidenceCommand,
InvestigationReportRevision,
InvestigationReportReference,
InvestigationProgressSnapshot,
IssueCommandPayload,
NotificationAcknowledgeCommand,
OperationalTaskCreate,
@@ -60,9 +62,13 @@ from app.domain.schemas.operational_task import (
OperationalTaskRun,
RecoverExpiredLeasesResult,
StoredEvidenceRecord,
TaskPresentationEventCommand,
TaskPresentationEventResult,
TaskPresentationCursor,
)
from app.domain.schemas.operational_data import TaskDataSnapshot, TaskDataSource
from app.services.content_digest import digest
from app.services.operational_presentation import resolve_overlay
LEASE_DURATION = timedelta(seconds=60)
@@ -73,7 +79,7 @@ def _json(model) -> dict:
def _task_json(model: OperationalTaskSummary) -> dict:
return model.model_dump(mode="json", by_alias=True, exclude={"active_run"})
return model.model_dump(mode="json", by_alias=True, exclude={"active_run", "latest_run"})
class OperationalTaskService:
@@ -223,6 +229,8 @@ class OperationalTaskService:
require(task is not None, "TASK_NOT_FOUND", "Operational task not found", status=404)
plan = await self.repo.get_latest_plan(str(task_id))
report = await self.repo.get_latest_report(str(task_id))
if report is not None:
report = await self._report_with_progress_diagram(task_id, report)
evidence = [
self._public_evidence(document)
for document in await self.repo.list_evidence(str(task_id))
@@ -230,6 +238,9 @@ class OperationalTaskService:
actions = await self.repo.list_actions(str(task_id))
access_requests = await self._list_access_requests(task_id)
active_run = await self._active_run(task)
latest_run = (await self.repo.get_latest_run(str(task_id))
if hasattr(self.repo, "get_latest_run") else active_run)
progress = await self.repo.get_latest_progress(str(task_id), task["execution_generation"])
work_order_snapshot = None
if task["task_type"] == "external_work_order" and task["source"].get("external_id"):
feedback = await self.repo.get_feedback(
@@ -238,10 +249,12 @@ class OperationalTaskService:
if feedback:
work_order_snapshot = feedback["document"].get("work_order_snapshot")
return OperationalTaskDetail(
**{key: value for key, value in task.items() if key != "active_run"},
**{key: value for key, value in task.items() if key not in {"active_run", "latest_run"}},
active_run=active_run,
latest_run=latest_run,
latest_plan=plan,
latest_report=report,
latest_progress=progress,
evidence=evidence,
actions=actions,
access_requests=access_requests,
@@ -249,18 +262,45 @@ class OperationalTaskService:
work_order_snapshot=work_order_snapshot,
)
async def _report_with_progress_diagram(
self, task_id: UUID, value: dict | InvestigationReportRevision
) -> InvestigationReportRevision:
"""Archive an eligible recorded graph, without inventing completion states.
Also projects older reports that predate graph archival. Read paths never
rewrite their stored document or borrow a newer round's progress.
"""
snapshot = InvestigationReportRevision.model_validate(value)
if (snapshot.report.diagram is not None or snapshot.report.stage != "final"
or snapshot.execution_generation is None):
return snapshot
stored = await self.repo.get_latest_progress(str(task_id), snapshot.execution_generation)
if stored is None:
return snapshot
progress = InvestigationProgressSnapshot.model_validate(stored)
diagram = progress.progress.diagram
if (diagram is None or progress.execution_generation != snapshot.execution_generation
or progress.source_session_id != snapshot.source_session_id
or progress.published_at > snapshot.published_at
or any(not set(node.evidence_refs) <= set(snapshot.report.evidence_refs)
for node in diagram.node_refs)):
return snapshot
return snapshot.model_copy(update={
"report": snapshot.report.model_copy(update={"diagram": diagram}),
})
async def report(self, task_id: UUID, revision: int) -> InvestigationReportRevision:
await self._task(task_id)
value = await self.repo.get_report(str(task_id), revision)
require(value is not None, "REPORT_NOT_FOUND", "Report revision not found", status=404)
return InvestigationReportRevision.model_validate(value)
return await self._report_with_progress_diagram(task_id, value)
async def list_reports(
self, task_id: UUID, limit: int, offset: int
) -> tuple[list[InvestigationReportRevision], int | None]:
await self._task(task_id)
rows = await self.repo.list_reports(str(task_id), limit + 1, offset)
return [InvestigationReportRevision.model_validate(row) for row in rows[:limit]], (
return [await self._report_with_progress_diagram(task_id, row) for row in rows[:limit]], (
offset + limit if len(rows) > limit else None
)
@@ -398,12 +438,15 @@ class OperationalTaskService:
async def list_runs(
self, *, status: str | None, limit: int,
cursor_created_at=None, cursor_run_id=None,
task_type: str | None = None, task_created_from: datetime | None = None,
) -> tuple[list[OperationalTaskRun], str | None]:
rows = await self.repo.list_runs(
status=status,
limit=limit + 1,
cursor_created_at=cursor_created_at,
cursor_run_id=cursor_run_id,
task_type=task_type,
task_created_from=task_created_from,
)
items = [OperationalTaskRun.model_validate(row) for row in rows[:limit]]
next_cursor = None
@@ -418,6 +461,7 @@ class OperationalTaskService:
body = _json(command)
scope = f"run:{run_id}:claim"
async with self.repo.transaction():
await self.repo.require_execution_storage()
replay = await self._request_replay(scope, command.request_id, body)
if replay:
return OperationalTaskRun.model_validate(replay["run"])
@@ -507,6 +551,18 @@ class OperationalTaskService:
) -> OperationalTaskDetail:
return await self._finish_run(task_id, run_id, command, outcome="succeeded")
async def task_runs(self, task_id: UUID, limit: int, offset: int) -> list[OperationalTaskRun]:
await self._task(task_id)
return [OperationalTaskRun.model_validate(row)
for row in await self.repo.list_task_runs(str(task_id), limit, offset)]
async def task_run(self, task_id: UUID, run_id: UUID) -> OperationalTaskRun:
return OperationalTaskRun.model_validate(await self._run(task_id, run_id))
async def _fail_run_messages(self, run_id, reason: str, now: datetime) -> None:
if hasattr(self.repo, "fail_run_messages"):
await self.repo.fail_run_messages(str(run_id), reason, now)
async def fail_run(
self, task_id: UUID, run_id: UUID, command: AgentRunFailureCommand
) -> OperationalTaskDetail:
@@ -539,6 +595,7 @@ class OperationalTaskService:
)
run_model = OperationalTaskRun.model_validate(run)
await self.repo.update_run(_json(run_model), expected)
await self._fail_run_messages(run_id, "Run cancelled", now)
task = await self._task(task_id)
task.update(
lifecycle_status="cancelled", status="cancelled",
@@ -630,6 +687,7 @@ class OperationalTaskService:
"Agent 执行投递连续失败 10 次，请检查服务连接", now,
)
if not command.retryable:
await self._fail_run_messages(run_id, "Run dispatch failed", now)
task = await self._task(task_id)
task.update(
lifecycle_status="failed", status="failed",
@@ -642,11 +700,20 @@ class OperationalTaskService:
await self._save_request(scope, command.request_id, body, {"run": _json(run_model)})
return run_model
async def recover_expired_leases(self, limit: int) -> RecoverExpiredLeasesResult:
async def recover_expired_leases(
self, limit: int, *, task_type: str | None = None, task_created_from: datetime | None = None,
) -> RecoverExpiredLeasesResult:
now = self._now()
recovered: list[UUID] = []
async with self.repo.transaction():
for run in await self.repo.claim_expired_runs(limit):
for run in await self.repo.claim_expired_runs(
limit, task_type=task_type, task_created_from=task_created_from,
):
# Lease recovery already holds a row lock. Try rather than wait
# for the shared run lock to avoid reversing the lock order of
# message/finish transactions; a busy run is checked next tick.
if hasattr(self.repo, "try_lock") and not await self.repo.try_lock(f"run:{run['run_id']}"):
continue
task_id = UUID(str(run["task_id"]))
expected = run["revision"]
run.update(
@@ -656,6 +723,7 @@ class OperationalTaskService:
)
timed_out = OperationalTaskRun.model_validate(run)
await self.repo.update_run(_json(timed_out), expected)
await self._fail_run_messages(run["run_id"], "Run lease expired", now)
await self._run_event(timed_out, now)
task = await self._task(task_id)
timeout_count = await self.repo.consecutive_timed_out_runs(str(task_id))
@@ -738,6 +806,29 @@ class OperationalTaskService:
item["action_id"]: HumanApprovalAction.model_validate(item)
for item in await self.repo.list_actions(str(task_id))
}
progress_revision = None
if command.progress is not None:
require(command.run_id is not None, "RUN_REQUIRED",
"Progress must identify its current execution run", status=422)
if command.progress.diagram:
for node in command.progress.diagram.node_refs:
self._require_evidence_refs(node.evidence_refs, known_evidence)
require(all(str(ref) in actions for ref in node.action_refs),
"ACTION_NOT_FOUND", "Progress references an unknown action", status=422)
previous = await self.repo.get_latest_progress(str(task_id), task["execution_generation"])
progress_revision = (previous["revision"] if previous else 0) + 1
snapshot = InvestigationProgressSnapshot(
revision=progress_revision, run_id=active_run["run_id"],
execution_generation=task["execution_generation"],
source_session_id=command.source_session_id, published_at=now, progress=command.progress,
)
await self.repo.save_progress(str(task_id), _json(snapshot))
await self._append_event(
topic="operational.progress.changed", aggregate_type="operational_task_progress",
aggregate_id=str(task_id), aggregate_revision=progress_revision, occurred_at=now,
metadata={"task_id": str(task_id), "run_id": str(active_run["run_id"]),
"execution_generation": task["execution_generation"]},
)
if command.plan:
self._require_evidence_refs(
(ref for node in command.plan.nodes for ref in node.evidence_refs),
@@ -805,6 +896,7 @@ class OperationalTaskService:
source_session_id=command.source_session_id,
published_at=now, published_by=self._actor(), report=command.report,
)
report = await self._report_with_progress_diagram(task_id, report)
await self.repo.insert_report(str(task_id), _json(report))
await self._append_event(
topic="operational.report.published",
@@ -873,9 +965,76 @@ class OperationalTaskService:
)
await self._task_event(task, now)
response = {"plan_revision": plan_revision, "report_revision": report_revision}
if progress_revision is not None:
response["progress_revision"] = progress_revision
await self._save_request(scope, command.request_id, body, response)
return AgentArtifactsResult(task=await self.detail(task_id), **response)
async def presentation_cursor(self, task_id: UUID) -> TaskPresentationCursor:
# The shared event-order lock gives the run and cursor one committed boundary.
async with self.repo.transaction():
task = await self._task(task_id)
run = await self._active_run(task)
cursor = await self.repo.current_event_cursor()
return TaskPresentationCursor(
task_id=task_id, execution_generation=task["execution_generation"],
active_run=({key: run[key] for key in ("run_id", "generation", "status")} if run else None),
event_cursor=cursor,
)
async def publish_presentation_event(
self, task_id: UUID, command: TaskPresentationEventCommand
) -> TaskPresentationEventResult:
body = _json(command)
scope = f"task:{task_id}:presentation"
async with self.repo.transaction():
replay = await self._request_replay(scope, command.request_id, body)
if replay:
return TaskPresentationEventResult.model_validate(replay)
await self.repo.lock(f"task:{task_id}")
await self.repo.lock(f"run:{command.run_id}")
task = await self._task(task_id)
self._require_actionable_task(task)
run = await self._active_run(task)
require(run is not None and str(run["run_id"]) == str(command.run_id)
and run["generation"] == task["execution_generation"],
"EXECUTION_GENERATION_MISMATCH", "Presentation targets a stale execution run", status=409)
require(run["status"] in {"claimed", "running"}, "INVALID_TRANSITION",
"Only a running run can publish presentation", status=409)
require(run.get("agent_session_id") == command.source_session_id,
"SESSION_MISMATCH", "Presentation must come from the claimed Agent session", status=409)
now = self._now()
self._require_active_run_lease(run, now)
payload = _json(command.command)
if command.command.action == "open_chart":
known = {item["evidence_id"] for item in await self.repo.list_evidence(str(task_id))}
self._require_evidence_refs(command.command.evidence_refs, known)
elif command.command.action == "set_overlay":
stored = await self.repo.get_evidence(str(task_id), command.command.evidence_id)
require(stored is not None, "EVIDENCE_NOT_FOUND", "Overlay evidence does not belong to this task", status=422)
evidence = StoredEvidenceRecord.model_validate(stored)
require(evidence.data_snapshot is not None, "SNAPSHOT_NOT_FOUND",
"Overlay requires a Server-ingested immutable data snapshot", status=422)
payload["data"] = resolve_overlay(evidence.data_snapshot, command.command.mode)
metadata = {"task_id": str(task_id), "run_id": str(run["run_id"]),
"execution_generation": run["generation"], "command": payload}
try:
size = len(json.dumps(metadata, ensure_ascii=False, allow_nan=False).encode("utf-8"))
except (ValueError, TypeError) as exc:
raise DomainError("PRESENTATION_DATA_INVALID", "Presentation data must be finite JSON", status=422) from exc
require(size <= 512 * 1024, "PRESENTATION_TOO_LARGE",
"Presentation event exceeds the 512 KiB limit", status=422)
event_seq = await self._append_event(
topic="operational.presentation.requested", aggregate_type="operational_task_presentation",
aggregate_id=str(task_id), aggregate_revision=run["revision"], occurred_at=now,
metadata=metadata,
)
require(event_seq is not None, "DEPENDENCY_UNAVAILABLE", "Presentation event storage is unavailable", status=503)
result = TaskPresentationEventResult(event_seq=event_seq, run_id=run["run_id"],
execution_generation=run["generation"])
await self._save_request(scope, command.request_id, body, _json(result))
return result
async def propose_action(
self, task_id: UUID, command: ActionProposalCommand
) -> HumanApprovalAction:
@@ -1246,6 +1405,22 @@ class OperationalTaskService:
return ExternalFeedbackResult(task=await self.detail(task_id), **replay)
await self.repo.lock(f"task:{task_id}")
task = await self._task(task_id)
if task["task_type"] == "event_investigation":
self._require_actionable_task(task)
own_source = task["source"].get("external_id")
matching_source = own_source is not None and command.source_event_id in {
own_source, f"peripheral_sample:{own_source}",
}
if task["status"] == "completed" and not matching_source:
actions = await self.repo.list_actions(str(task_id))
require(any(
item["kind"] == "create_work_order" and item.get("receipt")
and item["receipt"].get("adapter") == "peripheral_sample"
and command.source_event_id == f"peripheral_sample:{item['receipt']['receipt_id']}"
for item in actions
), "EXTERNAL_SOURCE_MISMATCH", "Feedback does not match an associated child work order", status=409)
require(command.source_revision > 1, "INITIAL_CHILD_FEEDBACK",
"The child creation receipt is not new field feedback", status=409)
if task["task_type"] == "external_work_order":
require(
task["source"].get("external_id") == command.source_event_id,
@@ -1263,7 +1438,9 @@ class OperationalTaskService:
existing = await self.repo.get_feedback(str(task_id), command.source_event_id)
current_source_revision = (
task.get("source", {}).get("external_revision")
if task["task_type"] == "external_work_order"
if task["task_type"] == "external_work_order" or (
task["task_type"] == "event_investigation" and matching_source
)
else None
)
if current_source_revision is not None:
@@ -1276,7 +1453,8 @@ class OperationalTaskService:
if existing:
prior = existing["document"]
if command.source_revision == existing["source_revision"]:
require(digest(prior) == digest(body), "IDEMPOTENCY_CONFLICT",
require(digest({key: value for key, value in prior.items() if key != "request_id"})
== digest({key: value for key, value in body.items() if key != "request_id"}), "IDEMPOTENCY_CONFLICT",
"External feedback revision was reused with different content", status=409)
response = {
"source_event_id": command.source_event_id,
@@ -1287,6 +1465,10 @@ class OperationalTaskService:
return ExternalFeedbackResult(task=await self.detail(task_id), **response)
require(command.source_revision > existing["source_revision"],
"STALE_EXTERNAL_REVISION", "External feedback revision is stale", status=409)
if (task["task_type"] == "event_investigation" and matching_source
and current_source_revision is not None):
require(command.source_revision > current_source_revision, "STALE_EXTERNAL_REVISION",
"Original event feedback must contain a new source version", status=409)
now = self._now()
await self.repo.upsert_feedback(str(task_id), body, now)
if command.evidence:
@@ -1322,6 +1504,8 @@ class OperationalTaskService:
),
)
if task["task_type"] == "event_investigation":
if matching_source:
task["source"] = {**task["source"], "external_revision": command.source_revision}
await self._requeue_task(task, now)
else:
task.update(revision=task["revision"] + 1, updated_at=now)
@@ -1706,6 +1890,11 @@ class OperationalTaskService:
task = await self._task(task_id)
final_outcome = outcome
if outcome == "succeeded":
if hasattr(self.repo, "pending_run_messages"):
pending_messages = await self.repo.pending_run_messages(str(run_id))
require(not pending_messages, "PENDING_AGENT_MESSAGES",
"Run has messages not yet entered into Agent context",
status=409, recovery="retry_same_request", refs=pending_messages)
plan_document = await self.repo.get_latest_plan(str(task_id))
report_document = await self.repo.get_latest_report(str(task_id))
plan = (
@@ -1770,6 +1959,8 @@ class OperationalTaskService:
)
run_model = OperationalTaskRun.model_validate(run)
await self.repo.update_run(_json(run_model), expected)
if final_outcome != "succeeded":
await self._fail_run_messages(run_id, failure_code or "Run failed", now)
task.update(
lifecycle_status=lifecycle,
status=lifecycle,
@@ -1825,7 +2016,8 @@ class OperationalTaskService:
def _require_actionable_task(task: dict) -> None:
source = task.get("source") or {}
require(
task.get("lifecycle_status") != "cancelled" and source.get("deleted_at") is None,
task.get("lifecycle_status") != "cancelled" and task.get("status") != "cancelled"
and source.get("deleted_at") is None,
"TASK_NOT_ACTIONABLE",
"Cancelled or withdrawn tasks do not accept action changes",
status=409,
@@ -2003,6 +2195,14 @@ class OperationalTaskService:
async def _requeue_task(self, task: dict, now: datetime) -> None:
if self._supports_runs:
active_run = await self.repo.get_active_run(str(task["task_id"]))
while active_run is not None:
await self.repo.lock(f"run:{active_run['run_id']}")
# Completion may have won the run lock after our first read.
refreshed_run = await self.repo.get_active_run(str(task["task_id"]))
if refreshed_run is None or str(refreshed_run["run_id"]) == str(active_run["run_id"]):
active_run = refreshed_run
break
active_run = refreshed_run
if active_run is not None and active_run["status"] == "queued":
# Multiple inputs can arrive before the worker claims the next
# generation. They are all visible when that queued run starts,
@@ -2027,6 +2227,7 @@ class OperationalTaskService:
)
cancelled = OperationalTaskRun.model_validate(active_run)
await self.repo.update_run(_json(cancelled), expected)
await self._fail_run_messages(active_run["run_id"], "Run superseded by requeue", now)
await self._run_event(cancelled, now)
generation = task["execution_generation"] + 1
task_id = UUID(task["task_id"])