feat(agent): add frontend action bridge

src/frontendAction/coordinator.ts

@@ -0,0 +1,172 @@
+import { createHash } from "node:crypto";
+import { z } from "zod";
+
+import { logger } from "../logger.js";
+import { getFrontendActionDefinition } from "./registry.js";
+import type { FrontendActionRequest, FrontendActionResult } from "./types.js";
+
+export const frontendActionResultSchema = z.object({
+  version: z.literal("frontend-action-result@1"),
+  actionId: z.string().trim().min(1).max(128),
+  status: z.enum(["succeeded", "failed", "rejected", "cancelled", "expired"]),
+  output: z.unknown().optional(),
+  error: z.object({
+    code: z.string().trim().min(1).max(128),
+    message: z.string().trim().min(1).max(2_000),
+  }).strict().optional(),
+  completedAt: z.number().finite().positive(),
+}).strict().superRefine((value, context) => {
+  if (value.status !== "succeeded" && !value.error) {
+    context.addIssue({ code: z.ZodIssueCode.custom, message: "non-success results require an error" });
+  }
+});
+
+type PendingAction = {
+  request: FrontendActionRequest;
+  promise: Promise<FrontendActionResult>;
+  resolve: (result: FrontendActionResult) => void;
+  timer: ReturnType<typeof setTimeout>;
+};
+
+export type SubmitResultOutcome = { duplicate: boolean; result: FrontendActionResult };
+export type FrontendActionSink = (request: FrontendActionRequest) => void;
+
+export class FrontendActionError extends Error {
+  constructor(readonly status: number, message: string) {
+    super(message);
+  }
+}
+
+export class FrontendActionCoordinator {
+  private readonly pending = new Map<string, PendingAction>();
+  private readonly completed = new Map<string, { sessionId: string; result: FrontendActionResult }>();
+  private readonly sinks = new Map<string, FrontendActionSink>();
+
+  constructor(private readonly completedLimit = 512) {}
+
+  registerSink(sessionId: string, sink: FrontendActionSink) {
+    this.sinks.set(sessionId, sink);
+    const now = Date.now();
+    for (const pending of this.pending.values()) {
+      if (pending.request.sessionId === sessionId && pending.request.expiresAt > now) sink(pending.request);
+    }
+    return () => {
+      if (this.sinks.get(sessionId) === sink) this.sinks.delete(sessionId);
+    };
+  }
+
+  request(options: {
+    sessionId: string;
+    toolCallId: string;
+    name: string;
+    params: unknown;
+    fallbackText?: string;
+    signal?: AbortSignal;
+  }): Promise<FrontendActionResult> {
+    if (!options.toolCallId.trim()) throw new FrontendActionError(400, "frontend action call ID is required");
+    const definition = getFrontendActionDefinition(options.name);
+    if (!definition) throw new FrontendActionError(400, "unknown frontend action");
+    const parsedParams = definition.inputSchema.safeParse(options.params);
+    if (!parsedParams.success) throw new FrontendActionError(400, "invalid frontend action params");
+    const sink = this.sinks.get(options.sessionId);
+    if (!sink) throw new FrontendActionError(409, "frontend action session has no active browser sink");
+
+    const actionId = createHash("sha256")
+      .update(`${options.sessionId}\0${options.toolCallId}\0${options.name}`)
+      .digest("hex");
+    const completed = this.completed.get(actionId);
+    if (completed) return Promise.resolve(completed.result);
+    const existing = this.pending.get(actionId);
+    if (existing) return existing.promise;
+
+    const issuedAt = Date.now();
+    const request: FrontendActionRequest = {
+      version: "frontend-action@1",
+      actionId,
+      toolCallId: options.toolCallId,
+      sessionId: options.sessionId,
+      name: definition.manifest.id,
+      params: parsedParams.data,
+      risk: definition.manifest.risk,
+      replayPolicy: definition.manifest.risk === "view" ? "never" : "explicit",
+      issuedAt,
+      expiresAt: issuedAt + definition.manifest.timeoutMs,
+      fallbackText: options.fallbackText?.trim() || undefined,
+    };
+    let resolvePending!: (result: FrontendActionResult) => void;
+    const promise = new Promise<FrontendActionResult>((resolve) => { resolvePending = resolve; });
+    const timer = setTimeout(() => this.finish(request, {
+      version: "frontend-action-result@1",
+      actionId,
+      status: "expired",
+      error: { code: "ACTION_TIMEOUT", message: "browser action result was not received before expiry" },
+      completedAt: Date.now(),
+    }), definition.manifest.timeoutMs);
+    this.pending.set(actionId, { request, promise, resolve: resolvePending, timer });
+    options.signal?.addEventListener("abort", () => this.cancelAction(actionId), { once: true });
+    this.audit("proposed", request);
+    this.audit("validated", request);
+    sink(request);
+    this.audit("executing", request);
+    return promise;
+  }
+
+  submitResult(sessionId: string, value: unknown): SubmitResultOutcome {
+    const parsed = frontendActionResultSchema.safeParse(value);
+    if (!parsed.success) throw new FrontendActionError(400, "invalid frontend action result");
+    const result = parsed.data as FrontendActionResult;
+    const completed = this.completed.get(result.actionId);
+    if (completed) {
+      if (sessionId !== completed.sessionId) throw new FrontendActionError(403, "frontend action session mismatch");
+      if (JSON.stringify(completed.result) !== JSON.stringify(result)) throw new FrontendActionError(409, "frontend action result conflicts with terminal result");
+      return { duplicate: true, result: completed.result };
+    }
+    const pending = this.pending.get(result.actionId);
+    if (!pending) throw new FrontendActionError(404, "frontend action not found");
+    if (pending.request.sessionId !== sessionId) throw new FrontendActionError(403, "frontend action session mismatch");
+    const definition = getFrontendActionDefinition(pending.request.name);
+    if (result.status === "succeeded" && definition && !definition.outputSchema.safeParse(result.output).success) {
+      throw new FrontendActionError(400, "invalid frontend action output");
+    }
+    this.finish(pending.request, result);
+    return { duplicate: false, result };
+  }
+
+  cancelSession(sessionId: string) {
+    this.sinks.delete(sessionId);
+    for (const [actionId, pending] of this.pending) {
+      if (pending.request.sessionId === sessionId) this.cancelAction(actionId);
+    }
+  }
+
+  private cancelAction(actionId: string) {
+    const pending = this.pending.get(actionId);
+    if (!pending) return;
+    this.finish(pending.request, {
+      version: "frontend-action-result@1", actionId, status: "cancelled",
+      error: { code: "SESSION_CANCELLED", message: "session ended before the browser action completed" },
+      completedAt: Date.now(),
+    });
+  }
+
+  private finish(request: FrontendActionRequest, result: FrontendActionResult) {
+    const pending = this.pending.get(request.actionId);
+    if (!pending) return;
+    clearTimeout(pending.timer);
+    this.pending.delete(request.actionId);
+    this.completed.set(request.actionId, { sessionId: request.sessionId, result });
+    while (this.completed.size > this.completedLimit) {
+      const oldest = this.completed.keys().next().value as string | undefined;
+      if (!oldest) break;
+      this.completed.delete(oldest);
+    }
+    this.audit(result.status, request, result.error?.code);
+    pending.resolve(result);
+  }
+
+  private audit(stage: string, request: FrontendActionRequest, errorCode?: string) {
+    logger.info({ actionId: request.actionId, action: request.name, stage, sessionId: request.sessionId,
+      toolCallId: request.toolCallId, paramKeys: typeof request.params === "object" && request.params ? Object.keys(request.params) : [], errorCode },
+    "frontend action lifecycle");
+  }
+}

src/frontendAction/registry.ts

@@ -0,0 +1,36 @@
+import { z } from "zod";
+import type { FrontendActionManifest } from "./types.js";
+
+type Definition = { manifest: FrontendActionManifest; inputSchema: z.ZodTypeAny; outputSchema: z.ZodTypeAny };
+const zoomInput = z.object({ x: z.number().finite(), y: z.number().finite(), source_crs: z.enum(["EPSG:3857", "EPSG:4326"]).optional(), zoom: z.number().finite().optional(), duration_ms: z.number().finite().nonnegative().optional() }).strict();
+const id = z.string().trim().min(1).max(256);
+const isoTime = z.string().datetime({ offset: true });
+const timeRange = {
+  start_time: isoTime.optional(),
+  end_time: isoTime.optional(),
+};
+const locateInput = z.object({
+  ids: z.array(id).min(1).max(100),
+  feature_type: z.enum(["junction", "pipe", "valve", "reservoir", "pump", "tank", "conduit", "orifice", "outfall"]),
+}).strict();
+const historyInput = z.object({
+  feature_infos: z.array(z.tuple([id, id])).min(1).max(100),
+  data_type: z.enum(["realtime", "scheme", "none"]),
+  ...timeRange,
+}).strict().refine((value) => !value.start_time || !value.end_time || Date.parse(value.start_time) <= Date.parse(value.end_time), "start_time must not be after end_time");
+const scadaInput = z.object({
+  device_id: id.optional(),
+  device_ids: z.array(id).min(1).max(100).optional(),
+  ...timeRange,
+}).strict().refine((value) => Boolean(value.device_id) !== Boolean(value.device_ids), "provide exactly one of device_id or device_ids")
+  .refine((value) => !value.start_time || !value.end_time || Date.parse(value.start_time) <= Date.parse(value.end_time), "start_time must not be after end_time");
+const definitions: Definition[] = [
+  { manifest: { id: "zoom_to_map", version: "frontend-action@1", risk: "view", timeoutMs: 10_000, supportedSurfaces: ["map_overlay"] }, inputSchema: zoomInput, outputSchema: z.object({}).passthrough() },
+  { manifest: { id: "locate_features", version: "frontend-action@1", risk: "view", timeoutMs: 15_000, supportedSurfaces: ["map_overlay"] }, inputSchema: locateInput, outputSchema: z.object({ locatedIds: z.array(id), missingIds: z.array(id), featureType: id, bounds: z.tuple([z.number(), z.number(), z.number(), z.number()]) }).strict() },
+  { manifest: { id: "view_history", version: "frontend-action@1", risk: "view", timeoutMs: 10_000, supportedSurfaces: ["side_panel", "canvas"] }, inputSchema: historyInput, outputSchema: z.object({ component: z.literal("HistoryPanel"), rendered: z.literal(true), itemCount: z.number().int().nonnegative() }).strict() },
+  { manifest: { id: "view_scada", version: "frontend-action@1", risk: "view", timeoutMs: 10_000, supportedSurfaces: ["side_panel", "canvas"] }, inputSchema: scadaInput, outputSchema: z.object({ component: z.literal("ScadaPanel"), rendered: z.literal(true), itemCount: z.number().int().nonnegative() }).strict() },
+];
+
+export const frontendActionRegistry = definitions.map((item) => item.manifest);
+export const getFrontendActionDefinition = (name: string) => definitions.find((item) => item.manifest.id === name);
+export const frontendActionRegistryResponse = { schema_version: "frontend-action-registry@1" as const, actions: frontendActionRegistry };

src/frontendAction/types.ts

@@ -0,0 +1,37 @@
+export const FRONTEND_ACTION_VERSION = "frontend-action@1" as const;
+export const FRONTEND_ACTION_RESULT_VERSION = "frontend-action-result@1" as const;
+
+export type FrontendActionRisk = "view" | "reversible" | "side_effect";
+export type FrontendActionStatus = "succeeded" | "failed" | "rejected" | "cancelled" | "expired";
+
+export type FrontendActionRequest = {
+  version: typeof FRONTEND_ACTION_VERSION;
+  actionId: string;
+  toolCallId: string;
+  sessionId: string;
+  name: string;
+  params: unknown;
+  risk: FrontendActionRisk;
+  replayPolicy: "never" | "explicit";
+  issuedAt: number;
+  expiresAt: number;
+  fallbackText?: string;
+  replayOf?: string;
+};
+
+export type FrontendActionResult = {
+  version: typeof FRONTEND_ACTION_RESULT_VERSION;
+  actionId: string;
+  status: FrontendActionStatus;
+  output?: unknown;
+  error?: { code: string; message: string };
+  completedAt: number;
+};
+
+export type FrontendActionManifest = {
+  id: string;
+  version: typeof FRONTEND_ACTION_VERSION;
+  risk: FrontendActionRisk;
+  timeoutMs: number;
+  supportedSurfaces: ["map_overlay"] | ["side_panel", "canvas"];
+};

src/routes/chat.ts

@@ -23,9 +23,12 @@ import { type ResultReferenceResolver } from "../results/resolver.js";
 import {
   type OpencodeRuntimeAdapter,
 } from "../runtime/opencode.js";
+import { getRuntimeSessionContext, setRuntimeSessionContext } from "../runtime/sessionContext.js";
 import { type ChatSessionBridge } from "../chat/sessionBridge.js";
 import { type SessionRecord } from "../sessions/metadataStore.js";
 import { uiRegistryResponse } from "../uiEnvelope/registry.js";
+import { FrontendActionCoordinator } from "../frontendAction/coordinator.js";
+import { frontendActionRegistryResponse } from "../frontendAction/registry.js";
 import type { UIEnvelopePayload } from "../uiEnvelope/types.js";
 import { toActorKey, toProjectKey } from "../utils/fileStore.js";
 import {
@@ -80,6 +83,7 @@ const payloadSchema = z.object({
     message: "unsupported model",
   }).optional(),
   approval_mode: z.enum(["request", "always"]).optional().default("request"),
+  capabilities: z.array(z.string().max(64)).max(20).optional().default([]),
 });
 
 const createSessionPayloadSchema = z.object({
@@ -189,6 +193,7 @@ export const buildChatRouter = (
   sessionTranscriptStore: SessionTranscriptStore,
   learningOrchestrator: LearningOrchestrator,
   resultReferenceResolver: ResultReferenceResolver,
+  frontendActionCoordinator: FrontendActionCoordinator,
 ) => {
   const chatRouter = Router();
 
@@ -203,6 +208,26 @@ export const buildChatRouter = (
     res.json(uiRegistryResponse);
   });
 
+  chatRouter.get("/frontend-action-registry", (_req, res) => {
+    res.json(frontendActionRegistryResponse);
+  });
+
+  chatRouter.post("/frontend-actions/:action_id/result", (req, res) => {
+    const sessionId = req.header("x-agent-session-id")?.trim() ?? "";
+    if (!sessionId || req.params.action_id !== req.body?.actionId) {
+      res.status(400).json({ message: "invalid frontend action result" });
+      return;
+    }
+    try {
+      const outcome = frontendActionCoordinator.submitResult(sessionId, req.body);
+      res.json({ accepted: true, duplicate: outcome.duplicate, result: outcome.result });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "frontend action result rejected";
+      const status = typeof error === "object" && error && "status" in error && typeof error.status === "number" ? error.status : 400;
+      res.status(status).json({ message });
+    }
+  });
+
   chatRouter.post("/session", async (req, res) => {
     const parsed = createSessionPayloadSchema.safeParse(req.body ?? {});
     if (!parsed.success) {
@@ -456,6 +481,7 @@ export const buildChatRouter = (
       clientSessionId: sessionRecord.sessionId,
       sessionId: sessionRecord.sessionId,
     });
+    frontendActionCoordinator.cancelSession(sessionRecord.sessionId);
     activeRuns.delete(sessionRecord.sessionId);
     lastRunStatuses.delete(sessionRecord.sessionId);
     await sessionMetadataStore.remove(sessionRecord);
@@ -469,6 +495,7 @@ export const buildChatRouter = (
     sessionBridge,
     sessionMetadataStore,
     sessionUiStateStore,
+    frontendActionCoordinator,
   });
 
   registerChatInteractionRoutes(chatRouter, {
@@ -624,6 +651,10 @@ export const buildChatRouter = (
         traceId,
         userId,
       });
+      setRuntimeSessionContext({
+        ...(getRuntimeSessionContext(binding.sessionId)!),
+        capabilities: parsed.data.capabilities,
+      });
       const { record: ensuredSessionRecord, created: sessionCreated } =
         await sessionMetadataStore.ensure({
           actorKey,
@@ -1023,6 +1054,10 @@ export const buildChatRouter = (
         });
       };
 
+      const unregisterFrontendActionSink = parsed.data.capabilities.includes("frontend-action@1")
+        ? frontendActionCoordinator.registerSink(clientSessionId, (request) => publish("frontend_action", request as unknown as Record<string, unknown>))
+        : undefined;
+
       try {
         if (isPressureTrendDemoPrompt(promptText)) {
           const toolCall = createPressureTrendDemoToolCall();
@@ -1078,6 +1113,7 @@ export const buildChatRouter = (
           traceId: requestContext.traceId,
           projectId: requestContext.projectId,
           signal: abortController.signal,
+          suppressLegacyFrontendActions: parsed.data.capabilities.includes("frontend-action@1"),
           write: (event, data) => {
             publish(event, data);
           },
@@ -1148,6 +1184,7 @@ export const buildChatRouter = (
           }
         }
       } finally {
+        unregisterFrontendActionSink?.();
         if (abortController.signal.aborted) {
           activeRun.messages = updateLastAssistantMessage(activeRun.messages, (message) => ({
             ...message,

src/routes/chatAuxiliaryRoutes.ts

@@ -8,6 +8,7 @@ import { type ResultReferenceResolver } from "../results/resolver.js";
 import { RESULT_REFERENCE_KIND } from "../results/store.js";
 import { type SessionMetadataStore } from "../sessions/metadataStore.js";
 import { type SessionUiStateStore } from "../sessions/uiStateStore.js";
+import { type FrontendActionCoordinator } from "../frontendAction/coordinator.js";
 import { toActorKey, toProjectKey } from "../utils/fileStore.js";
 import {
   type ActiveRun,
@@ -28,6 +29,7 @@ type RegisterAuxiliaryRoutesOptions = {
   sessionBridge: ChatSessionBridge;
   sessionMetadataStore: SessionMetadataStore;
   sessionUiStateStore: SessionUiStateStore;
+  frontendActionCoordinator: FrontendActionCoordinator;
 };
 
 const toSessionUiStateContext = (sessionId: string) => ({
@@ -43,6 +45,7 @@ export const registerChatAuxiliaryRoutes = (
     sessionBridge,
     sessionMetadataStore,
     sessionUiStateStore,
+    frontendActionCoordinator,
   }: RegisterAuxiliaryRoutesOptions,
 ) => {
   chatRouter.get("/render-ref/:render_ref", async (req, res) => {
@@ -93,6 +96,7 @@ export const registerChatAuxiliaryRoutes = (
     }
 
     try {
+      frontendActionCoordinator.cancelSession(parsed.data.session_id);
       const authContext = getLocalAgentContext(req);
       const projectId = authContext.projectId;
       const userId = authContext.userId;

src/routes/chatStream.ts

@@ -74,6 +74,7 @@ type StreamPromptOptions = {
   traceId?: string;
   projectId?: string;
   signal?: AbortSignal;
+  suppressLegacyFrontendActions?: boolean;
   write: (event: string, data: Record<string, unknown>) => void;
 };
 
@@ -141,6 +142,7 @@ export const streamPromptResponse = async ({
   traceId,
   projectId,
   signal,
+  suppressLegacyFrontendActions = false,
   write,
 }: StreamPromptOptions): Promise<{
   aborted: boolean;
@@ -828,7 +830,7 @@ export const streamPromptResponse = async ({
               params: toolParams,
               reason,
             });
-            if (envelope) {
+            if (envelope && !(suppressLegacyFrontendActions && ["zoom_to_map", "locate_features", "view_history", "view_scada"].includes(part.tool))) {
               write("ui_envelope", {
                 session_id: clientSessionId,
                 envelope_id: createEnvelopeId(part.tool),

src/runtime/sessionContext.ts

@@ -9,6 +9,7 @@ export type RuntimeSessionContext = {
   projectKey: string;
   sessionId: string;
   traceId: string;
+  capabilities?: string[];
 };
 
 const contexts = new Map<string, RuntimeSessionContext>();

src/server.ts

@@ -16,6 +16,7 @@ import {
   ResultReferenceStore,
 } from "./results/store.js";
 import { buildChatRouter } from "./routes/chat.js";
+import { FrontendActionCoordinator, FrontendActionError } from "./frontendAction/coordinator.js";
 import { opencodeRuntime } from "./runtime/opencode.js";
 import {
   getRuntimeSessionContext,
@@ -38,6 +39,7 @@ const learningOrchestrator = new LearningOrchestrator(
 const resultReferenceStore = new ResultReferenceStore();
 const resultReferenceResolver = new ResultReferenceResolver(resultReferenceStore);
 const internalToken = config.AGENT_INTERNAL_TOKEN ?? randomUUID();
+const frontendActionCoordinator = new FrontendActionCoordinator();
 
 // 这个 token 只用于仍需服务端上下文的工具桥（store_render_ref）。
 process.env.TJWATER_AGENT_INTERNAL_TOKEN = internalToken;
@@ -45,6 +47,42 @@ process.env.TJWATER_AGENT_INTERNAL_TOKEN = internalToken;
 app.use(cors());
 app.use(express.json({ limit: "1mb" }));
 
+app.post("/internal/frontend-actions/request", async (req, res) => {
+  if (req.header("x-agent-internal-token") !== internalToken) {
+    res.status(403).json({ message: "forbidden" });
+    return;
+  }
+  const runtimeSessionId = typeof req.body?.session_id === "string" ? req.body.session_id.trim() : "";
+  const context = runtimeSessionId ? getRuntimeSessionContext(runtimeSessionId) : null;
+  if (!context) {
+    res.status(404).json({ message: "session context not found" });
+    return;
+  }
+  if (!context.capabilities?.includes("frontend-action@1")) {
+    // Legacy clients still execute the existing UIEnvelope emitted from the tool call.
+    res.json({
+      version: "frontend-action-result@1",
+      status: "succeeded",
+      output: { legacy: true },
+      detail: "legacy client executes the action from its UIEnvelope",
+    });
+    return;
+  }
+  try {
+    const result = await frontendActionCoordinator.request({
+      sessionId: context.clientSessionId,
+      toolCallId: typeof req.body?.call_id === "string" ? req.body.call_id : "",
+      name: typeof req.body?.name === "string" ? req.body.name : "",
+      params: req.body?.params,
+      fallbackText: typeof req.body?.fallback_text === "string" ? req.body.fallback_text : undefined,
+    });
+    res.json(result);
+  } catch (error) {
+    const status = error instanceof FrontendActionError ? error.status : 500;
+    res.status(status).json({ message: error instanceof Error ? error.message : String(error) });
+  }
+});
+
 app.get("/health", async (_req, res) => {
   try {
     const runtime = await opencodeRuntime.health();
@@ -308,6 +346,7 @@ app.use(
     sessionTranscriptStore,
     learningOrchestrator,
     resultReferenceResolver,
+    frontendActionCoordinator,
   ),
 );