157 lines
9.9 KiB
TypeScript
157 lines
9.9 KiB
TypeScript
// Opt-in behavioral evaluation: real inference, synthetic user data, isolated native sessions.
|
||
import "../../src/config.js";
|
||
import assert from "node:assert/strict";
|
||
import { randomBytes } from "node:crypto";
|
||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||
import { createServer } from "node:http";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import type { OpenCodeClient } from "@opencode/client";
|
||
import { getAgentModelOptions, loadAgentModelConfig, parseModelReference } from "../../src/chat/modelConfig.js";
|
||
import { answerUiAnchors, AnswerUiPublication } from "../../src/chat/answer-ui-publication.js";
|
||
import { parseAnswerUiInput } from "../../src/chat/answer-ui.js";
|
||
import { serviceOpencodeConfig } from "../../src/runtime/opencode2-config.js";
|
||
import { startOpencode2 } from "../../src/runtime/opencode2-server.js";
|
||
import { createConversationWorkspace } from "../../src/runtime/conversationWorkspace.js";
|
||
|
||
if (process.env.TJWATER_ANSWER_UI_EVAL !== "1") {
|
||
console.log("Skipped. Set TJWATER_ANSWER_UI_EVAL=1 to send three synthetic prompts to a real model.");
|
||
process.exit(0);
|
||
}
|
||
|
||
const cases = [
|
||
{ id: "comparison", component: "Table", prompt: "以下仅为演练资料：管道甲管径400 mm、长度120 m；乙管径250 mm、长度90 m；丙管径300 mm、长度150 m。请比较这三根管道，并说明哪些结论不能仅凭这些属性判断。只依据这些资料回答，不查询外部数据。" },
|
||
{ id: "trend", component: "Chart", prompt: "以下仅为演练资料：今天北京时间08:00压力0.42 MPa，09:00压力0.36 MPa，10:00压力0.24 MPa。请分析压力的变化趋势，指出哪个时段下降更明显，并说明判断局限。只依据这些资料回答，不查询外部数据。" },
|
||
{ id: "short-explanation", component: null, prompt: "请用一句话解释供水管网中的压力是什么意思。" },
|
||
];
|
||
const models = loadAgentModelConfig();
|
||
const model = process.env.TJWATER_ANSWER_UI_EVAL_MODEL ?? models.defaults.chat;
|
||
assert(getAgentModelOptions(models).some((option) => option.id === model), "Select an enabled chat model from agent-models.json");
|
||
const { providerID, modelID } = parseModelReference(model);
|
||
const root = await mkdtemp(join(tmpdir(), "tjwater-answer-ui-selection-"));
|
||
const controller = new AbortController();
|
||
const stop = () => controller.abort();
|
||
process.once("SIGINT", stop);
|
||
process.once("SIGTERM", stop);
|
||
let native: Awaited<ReturnType<typeof startOpencode2>> | undefined;
|
||
let sessionID: string | undefined;
|
||
let failures = 0;
|
||
const internalToken = randomBytes(32).toString("hex");
|
||
// Only the host-question gate is mocked; both presentation tools execute their real code.
|
||
const bridge = createServer(async (req, res) => {
|
||
if (req.method !== "POST" || req.headers["x-agent-internal-token"] !== internalToken) { res.writeHead(403).end(); return; }
|
||
let raw = "";
|
||
for await (const chunk of req) raw += chunk;
|
||
const body = JSON.parse(raw) as { session_id?: string; tool?: string };
|
||
if (!sessionID || body.session_id !== sessionID) { res.writeHead(403).end(); return; }
|
||
if (req.url === "/internal/chat-interactions/policy") {
|
||
res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ enabled: false, waiting: false }));
|
||
} else if (["/internal/chat-interactions/enter", "/internal/chat-interactions/leave"].includes(req.url ?? "")
|
||
&& ["present_answer", "final_answer"].includes(body.tool ?? "")) {
|
||
res.setHeader("content-type", "application/json"); res.end("{}");
|
||
} else res.writeHead(403).end();
|
||
});
|
||
try {
|
||
await new Promise<void>((resolve, reject) => { bridge.once("error", reject); bridge.listen(0, "127.0.0.1", resolve); });
|
||
const bridgeUrl = `http://127.0.0.1:${(bridge.address() as { port: number }).port}`;
|
||
const source = JSON.parse(await readFile("opencode.json", "utf8"));
|
||
const permissions = [
|
||
{ action: "*", resource: "*", effect: "deny" as const },
|
||
...["present_answer", "final_answer"].map((action) => ({ action, resource: "*", effect: "allow" as const })),
|
||
];
|
||
const nativeConfig = serviceOpencodeConfig({ ...source, mcp: { servers: {} }, plugins: [] }, models, root);
|
||
nativeConfig.permissions = permissions;
|
||
nativeConfig.skills = [];
|
||
nativeConfig.agents.instruction.permissions = permissions;
|
||
native = await startOpencode2({
|
||
command: process.env.OPENCODE_COMMAND ?? "opencode2", hostname: "127.0.0.1", port: 0,
|
||
timeout: 30_000, signal: controller.signal, config: nativeConfig,
|
||
env: {
|
||
AGENT_INTERNAL_TOKEN: internalToken, TJWATER_AGENT_WORKSPACE_ROOT: root,
|
||
TJWATER_AGENT_INTERNAL_BASE_URL: bridgeUrl,
|
||
TJWATER_AGENT_MCP_CONNECTION_PATH: join(root, "absent-query-mcp.json"),
|
||
OPENCODE_DB: join(root, "data", "opencode.db"), XDG_DATA_HOME: join(root, "data"),
|
||
XDG_CONFIG_HOME: join(root, "config"), XDG_CACHE_HOME: join(root, "cache"), XDG_STATE_HOME: join(root, "state"),
|
||
},
|
||
});
|
||
console.log(JSON.stringify({ model, cases: cases.length, scope: "native component selection; no business access or browser rendering" }));
|
||
for (const scenario of cases) {
|
||
controller.signal.throwIfAborted();
|
||
const started = Date.now();
|
||
const directory = await createConversationWorkspace(root);
|
||
sessionID = (await native.client.session.create({ agent: "instruction", model: { providerID, id: modelID }, location: { directory } })).id;
|
||
try {
|
||
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(120_000)]);
|
||
await native.client.session.prompt({ sessionID, text: scenario.prompt }, { signal });
|
||
await native.client.session.wait({ sessionID }, { signal });
|
||
const messages: Awaited<ReturnType<OpenCodeClient["message"]["list"]>>["data"] = (await native.client.message.list({ sessionID, order: "asc", limit: 100 }, { signal })).data;
|
||
const publication = new AnswerUiPublication();
|
||
const blocks = new Map<string, NonNullable<ReturnType<typeof parseAnswerUiInput>>>();
|
||
let final: { answer: string; ui_block_ids: string[] } | undefined;
|
||
for (const message of messages) {
|
||
if (message.type !== "assistant") continue;
|
||
assert(!message.error, "Model response failed");
|
||
for (const content of message.content) {
|
||
if (content.type !== "tool") continue;
|
||
assert(["present_answer", "final_answer"].includes(content.name), "Unexpected tool attempted");
|
||
assert.equal(content.state.status, "completed", "Presentation tool did not complete");
|
||
if (content.state.status !== "completed") continue;
|
||
const input = content.state.input;
|
||
if (content.name === "present_answer") {
|
||
const block = parseAnswerUiInput(input);
|
||
assert(block, "Invalid answer UI payload");
|
||
assert.equal(publication.publish(block), undefined, "Invalid publication lifecycle");
|
||
blocks.set(block.block_id, block);
|
||
} else {
|
||
assert.equal(final, undefined, "Final answer was submitted more than once");
|
||
const value = input as { answer: string; ui_block_ids?: string[] };
|
||
final = { answer: value.answer, ui_block_ids: value.ui_block_ids ?? [] };
|
||
assert.deepEqual([...new Set(answerUiAnchors(final.answer))], final.ui_block_ids, "Final anchors differ from references");
|
||
assert.deepEqual(publication.finalize(final.ui_block_ids), final.ui_block_ids, "Final answer references unavailable blocks");
|
||
}
|
||
}
|
||
}
|
||
assert(!messages.some((message) => message.type === "idle" && message.outcome === "failed"), "Native session failed before completion");
|
||
assert(final, "Missing final answer");
|
||
assert(final.answer.trim(), "Empty final answer");
|
||
const nodes = final.ui_block_ids.flatMap((id) => Object.values(blocks.get(id)!.spec.elements));
|
||
if (scenario.component) {
|
||
const matches = nodes.filter((node) => node.type === scenario.component);
|
||
assert(matches.length > 0, `Final answer did not retain ${scenario.component}`);
|
||
if (scenario.component === "Chart") {
|
||
assert(matches.some((node) => {
|
||
const props = node.props as { x_data: string[]; series: Array<{ data: number[] }> };
|
||
return props.x_data.length === 3 && ["08:00", "09:00", "10:00"].every((time, i) => props.x_data[i].includes(time))
|
||
&& props.series.some((series) => JSON.stringify(series.data) === "[0.42,0.36,0.24]")
|
||
&& JSON.stringify(node.props).includes("MPa");
|
||
}), "Trend lost input values, times or units");
|
||
} else {
|
||
assert(matches.some((node) => {
|
||
const props = node.props as { rows: string[][] };
|
||
return props.rows.length === 3 && [["甲", "400", "120"], ["乙", "250", "90"], ["丙", "300", "150"]]
|
||
.every((values) => props.rows.some((row) => values.every((value) => row.some((cell) => cell.includes(value)))));
|
||
}), "Comparison lost supplied objects or values");
|
||
}
|
||
} else assert.equal(blocks.size, 0, "Short explanation generated unnecessary UI");
|
||
console.log(JSON.stringify({ case: scenario.id, passed: true, components: nodes.map((node) => node.type), duration_ms: Date.now() - started }));
|
||
} catch (error) {
|
||
failures += 1;
|
||
// Assertion messages are ours; never print provider response bodies or credentials.
|
||
console.log(JSON.stringify({ case: scenario.id, passed: false, reason: error instanceof assert.AssertionError ? error.message : "Native request failed or timed out", duration_ms: Date.now() - started }));
|
||
} finally {
|
||
await native.client.session.interrupt({ sessionID }).catch(() => {});
|
||
await native.client.session.remove({ sessionID }).catch(() => {});
|
||
sessionID = undefined;
|
||
}
|
||
}
|
||
} finally {
|
||
if (native && sessionID) await native.client.session.remove({ sessionID }).catch(() => {});
|
||
native?.close();
|
||
if (native) await native.exited;
|
||
await new Promise<void>((resolve) => bridge.close(() => resolve()));
|
||
await rm(root, { recursive: true, force: true });
|
||
process.removeListener("SIGINT", stop);
|
||
process.removeListener("SIGTERM", stop);
|
||
}
|
||
if (failures) process.exitCode = 1;
|