feat: upgrade investigation and presentation runtime
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
// Opt-in behavioral evaluation: real inference, synthetic user data, isolated native sessions.
|
||||
import "../../src/config.js";
|
||||
import assert from "node:assert/strict";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { OpenCodeClient } from "@opencode/client";
|
||||
import { getAgentModelOptions, loadAgentModelConfig, parseModelReference } from "../../src/chat/modelConfig.js";
|
||||
import { answerUiAnchors, AnswerUiPublication } from "../../src/chat/answer-ui-publication.js";
|
||||
import { parseAnswerUiInput } from "../../src/chat/answer-ui.js";
|
||||
import { serviceOpencodeConfig } from "../../src/runtime/opencode2-config.js";
|
||||
import { startOpencode2 } from "../../src/runtime/opencode2-server.js";
|
||||
import { createConversationWorkspace } from "../../src/runtime/conversationWorkspace.js";
|
||||
|
||||
if (process.env.TJWATER_ANSWER_UI_EVAL !== "1") {
|
||||
console.log("Skipped. Set TJWATER_ANSWER_UI_EVAL=1 to send three synthetic prompts to a real model.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const cases = [
|
||||
{ id: "comparison", component: "Table", prompt: "以下仅为演练资料：管道甲管径400 mm、长度120 m；乙管径250 mm、长度90 m；丙管径300 mm、长度150 m。请比较这三根管道，并说明哪些结论不能仅凭这些属性判断。只依据这些资料回答，不查询外部数据。" },
|
||||
{ id: "trend", component: "Chart", prompt: "以下仅为演练资料：今天北京时间08:00压力0.42 MPa，09:00压力0.36 MPa，10:00压力0.24 MPa。请分析压力的变化趋势，指出哪个时段下降更明显，并说明判断局限。只依据这些资料回答，不查询外部数据。" },
|
||||
{ id: "short-explanation", component: null, prompt: "请用一句话解释供水管网中的压力是什么意思。" },
|
||||
];
|
||||
const models = loadAgentModelConfig();
|
||||
const model = process.env.TJWATER_ANSWER_UI_EVAL_MODEL ?? models.defaults.chat;
|
||||
assert(getAgentModelOptions(models).some((option) => option.id === model), "Select an enabled chat model from agent-models.json");
|
||||
const { providerID, modelID } = parseModelReference(model);
|
||||
const root = await mkdtemp(join(tmpdir(), "tjwater-answer-ui-selection-"));
|
||||
const controller = new AbortController();
|
||||
const stop = () => controller.abort();
|
||||
process.once("SIGINT", stop);
|
||||
process.once("SIGTERM", stop);
|
||||
let native: Awaited<ReturnType<typeof startOpencode2>> | undefined;
|
||||
let sessionID: string | undefined;
|
||||
let failures = 0;
|
||||
const internalToken = randomBytes(32).toString("hex");
|
||||
// Only the host-question gate is mocked; both presentation tools execute their real code.
|
||||
const bridge = createServer(async (req, res) => {
|
||||
if (req.method !== "POST" || req.headers["x-agent-internal-token"] !== internalToken) { res.writeHead(403).end(); return; }
|
||||
let raw = "";
|
||||
for await (const chunk of req) raw += chunk;
|
||||
const body = JSON.parse(raw) as { session_id?: string; tool?: string };
|
||||
if (!sessionID || body.session_id !== sessionID) { res.writeHead(403).end(); return; }
|
||||
if (req.url === "/internal/chat-interactions/policy") {
|
||||
res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ enabled: false, waiting: false }));
|
||||
} else if (["/internal/chat-interactions/enter", "/internal/chat-interactions/leave"].includes(req.url ?? "")
|
||||
&& ["present_answer", "final_answer"].includes(body.tool ?? "")) {
|
||||
res.setHeader("content-type", "application/json"); res.end("{}");
|
||||
} else res.writeHead(403).end();
|
||||
});
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => { bridge.once("error", reject); bridge.listen(0, "127.0.0.1", resolve); });
|
||||
const bridgeUrl = `http://127.0.0.1:${(bridge.address() as { port: number }).port}`;
|
||||
const source = JSON.parse(await readFile("opencode.json", "utf8"));
|
||||
const permissions = [
|
||||
{ action: "*", resource: "*", effect: "deny" as const },
|
||||
...["present_answer", "final_answer"].map((action) => ({ action, resource: "*", effect: "allow" as const })),
|
||||
];
|
||||
const nativeConfig = serviceOpencodeConfig({ ...source, mcp: { servers: {} }, plugins: [] }, models, root);
|
||||
nativeConfig.permissions = permissions;
|
||||
nativeConfig.skills = [];
|
||||
nativeConfig.agents.instruction.permissions = permissions;
|
||||
native = await startOpencode2({
|
||||
command: process.env.OPENCODE_COMMAND ?? "opencode2", hostname: "127.0.0.1", port: 0,
|
||||
timeout: 30_000, signal: controller.signal, config: nativeConfig,
|
||||
env: {
|
||||
AGENT_INTERNAL_TOKEN: internalToken, TJWATER_AGENT_WORKSPACE_ROOT: root,
|
||||
TJWATER_AGENT_INTERNAL_BASE_URL: bridgeUrl,
|
||||
TJWATER_AGENT_MCP_CONNECTION_PATH: join(root, "absent-query-mcp.json"),
|
||||
OPENCODE_DB: join(root, "data", "opencode.db"), XDG_DATA_HOME: join(root, "data"),
|
||||
XDG_CONFIG_HOME: join(root, "config"), XDG_CACHE_HOME: join(root, "cache"), XDG_STATE_HOME: join(root, "state"),
|
||||
},
|
||||
});
|
||||
console.log(JSON.stringify({ model, cases: cases.length, scope: "native component selection; no business access or browser rendering" }));
|
||||
for (const scenario of cases) {
|
||||
controller.signal.throwIfAborted();
|
||||
const started = Date.now();
|
||||
const directory = await createConversationWorkspace(root);
|
||||
sessionID = (await native.client.session.create({ agent: "instruction", model: { providerID, id: modelID }, location: { directory } })).id;
|
||||
try {
|
||||
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(120_000)]);
|
||||
await native.client.session.prompt({ sessionID, text: scenario.prompt }, { signal });
|
||||
await native.client.session.wait({ sessionID }, { signal });
|
||||
const messages: Awaited<ReturnType<OpenCodeClient["message"]["list"]>>["data"] = (await native.client.message.list({ sessionID, order: "asc", limit: 100 }, { signal })).data;
|
||||
const publication = new AnswerUiPublication();
|
||||
const blocks = new Map<string, NonNullable<ReturnType<typeof parseAnswerUiInput>>>();
|
||||
let final: { answer: string; ui_block_ids: string[] } | undefined;
|
||||
for (const message of messages) {
|
||||
if (message.type !== "assistant") continue;
|
||||
assert(!message.error, "Model response failed");
|
||||
for (const content of message.content) {
|
||||
if (content.type !== "tool") continue;
|
||||
assert(["present_answer", "final_answer"].includes(content.name), "Unexpected tool attempted");
|
||||
assert.equal(content.state.status, "completed", "Presentation tool did not complete");
|
||||
if (content.state.status !== "completed") continue;
|
||||
const input = content.state.input;
|
||||
if (content.name === "present_answer") {
|
||||
const block = parseAnswerUiInput(input);
|
||||
assert(block, "Invalid answer UI payload");
|
||||
assert.equal(publication.publish(block), undefined, "Invalid publication lifecycle");
|
||||
blocks.set(block.block_id, block);
|
||||
} else {
|
||||
assert.equal(final, undefined, "Final answer was submitted more than once");
|
||||
const value = input as { answer: string; ui_block_ids?: string[] };
|
||||
final = { answer: value.answer, ui_block_ids: value.ui_block_ids ?? [] };
|
||||
assert.deepEqual([...new Set(answerUiAnchors(final.answer))], final.ui_block_ids, "Final anchors differ from references");
|
||||
assert.deepEqual(publication.finalize(final.ui_block_ids), final.ui_block_ids, "Final answer references unavailable blocks");
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(!messages.some((message) => message.type === "idle" && message.outcome === "failed"), "Native session failed before completion");
|
||||
assert(final, "Missing final answer");
|
||||
assert(final.answer.trim(), "Empty final answer");
|
||||
const nodes = final.ui_block_ids.flatMap((id) => Object.values(blocks.get(id)!.spec.elements));
|
||||
if (scenario.component) {
|
||||
const matches = nodes.filter((node) => node.type === scenario.component);
|
||||
assert(matches.length > 0, `Final answer did not retain ${scenario.component}`);
|
||||
if (scenario.component === "Chart") {
|
||||
assert(matches.some((node) => {
|
||||
const props = node.props as { x_data: string[]; series: Array<{ data: number[] }> };
|
||||
return props.x_data.length === 3 && ["08:00", "09:00", "10:00"].every((time, i) => props.x_data[i].includes(time))
|
||||
&& props.series.some((series) => JSON.stringify(series.data) === "[0.42,0.36,0.24]")
|
||||
&& JSON.stringify(node.props).includes("MPa");
|
||||
}), "Trend lost input values, times or units");
|
||||
} else {
|
||||
assert(matches.some((node) => {
|
||||
const props = node.props as { rows: string[][] };
|
||||
return props.rows.length === 3 && [["甲", "400", "120"], ["乙", "250", "90"], ["丙", "300", "150"]]
|
||||
.every((values) => props.rows.some((row) => values.every((value) => row.some((cell) => cell.includes(value)))));
|
||||
}), "Comparison lost supplied objects or values");
|
||||
}
|
||||
} else assert.equal(blocks.size, 0, "Short explanation generated unnecessary UI");
|
||||
console.log(JSON.stringify({ case: scenario.id, passed: true, components: nodes.map((node) => node.type), duration_ms: Date.now() - started }));
|
||||
} catch (error) {
|
||||
failures += 1;
|
||||
// Assertion messages are ours; never print provider response bodies or credentials.
|
||||
console.log(JSON.stringify({ case: scenario.id, passed: false, reason: error instanceof assert.AssertionError ? error.message : "Native request failed or timed out", duration_ms: Date.now() - started }));
|
||||
} finally {
|
||||
await native.client.session.interrupt({ sessionID }).catch(() => {});
|
||||
await native.client.session.remove({ sessionID }).catch(() => {});
|
||||
sessionID = undefined;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (native && sessionID) await native.client.session.remove({ sessionID }).catch(() => {});
|
||||
native?.close();
|
||||
if (native) await native.exited;
|
||||
await new Promise<void>((resolve) => bridge.close(() => resolve()));
|
||||
await rm(root, { recursive: true, force: true });
|
||||
process.removeListener("SIGINT", stop);
|
||||
process.removeListener("SIGTERM", stop);
|
||||
}
|
||||
if (failures) process.exitCode = 1;
|
||||
Reference in New Issue
Block a user