import { expect, test, type Page } from "@playwright/test"; import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { mockAgentApi } from "./support/mock-agent-api"; import { mockWorkbenchContext } from "./support/mock-workbench-context"; import { responsesFixture } from "./support/responses-fixture"; const voiceCatalog = { defaultVoice: "zh-CN-XiaoxiaoNeural", voices: [ { id: "zh-CN-XiaoxiaoNeural", name: "Xiaoxiao", locale: "zh-CN", gender: "Female" }, { id: "zh-CN-YunyangNeural", name: "Yunyang", locale: "zh-CN", gender: "Male" }, { id: "zh-CN-XiaoyiNeural", name: "Xiaoyi", locale: "zh-CN", gender: "Female" } ] }; test("voice loading bypasses a cached catalog from an earlier application version", async ({ page, baseURL }) => { let legacy = true; let catalogRequests = 0; const oldCatalog = { ...voiceCatalog, voices: [ ...voiceCatalog.voices, { id: "en-US-AriaNeural", name: "Aria", locale: "en-US", gender: "Female" } ] }; const server = createServer((request, response) => { if (request.url === "/api/tts/edge/voices") { catalogRequests += 1; response.setHeader("Content-Type", "application/json"); response.setHeader("Cache-Control", "private, max-age=3600"); response.end(JSON.stringify(legacy ? oldCatalog : voiceCatalog)); } else { response.setHeader("Content-Type", "text/html"); response.end("Speech catalog cache regression"); } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const { port } = server.address() as AddressInfo; try { await page.goto(`http://127.0.0.1:${port}`); expect(await page.evaluate(async () => (await (await fetch("/api/tts/edge/voices")).json()).voices.length)).toBe(4); legacy = false; // Confirm that ordinary fetch really reuses the cached legacy response. expect(await page.evaluate(async () => (await (await fetch("/api/tts/edge/voices")).json()).voices.length)).toBe(4); expect(catalogRequests).toBe(1); const catalog = await page.evaluate(async (viteOrigin) => { const { fetchSpeechVoiceCatalog } = await import(`${viteOrigin}/src/features/agent/speech-voice-catalog.ts`); return fetchSpeechVoiceCatalog(`${location.origin}/api/tts/edge/voices`); }, baseURL); expect(catalog.voices).toEqual(voiceCatalog.voices); expect(catalogRequests).toBe(2); } finally { await new Promise((resolve) => server.close(() => resolve())); } }); async function openSpeechMessage(page: Page, options: { liveVoices?: boolean; catalogUnavailable?: boolean; onCatalogRead?: () => void } = {}) { if (!options.liveVoices) await page.route("**/api/tts/edge/voices", (route) => { options.onCatalogRead?.(); return options.catalogUnavailable ? route.fulfill({ status: 503, json: { error: "音色列表暂时不可用" } }) : route.fulfill({ json: voiceCatalog }); }); await mockWorkbenchContext(page); await mockAgentApi(page); await page.route("**/api/v1/responses", (route) => route.fulfill({ headers: { "Access-Control-Allow-Origin": "*", "Content-Type": "text/event-stream" }, body: responsesFixture([ { type: "start", messageId: "speech-example" }, { type: "data-progress", data: { id: "checking", phase: "context", title: "运行情况核对完成", status: "completed" } }, { type: "data-final_answer", data: { content: "当前供水运行平稳。" } }, { type: "finish", finishReason: "stop" } ]) })); await page.goto("/", { waitUntil: "domcontentloaded" }); await page.getByRole("button", { name: (page.viewportSize()?.width ?? 1440) < 1024 ? "打开 Agent 面板" : /展开 Agent 助手面板/ }).click(); const panel = page.getByRole("complementary", { name: "Agent 命令面板" }); await panel.getByRole("textbox", { name: "询问供水问题,或描述需要处理的任务" }).fill("检查供水运行"); await panel.getByRole("button", { name: "发送 Agent 指令", exact: true }).click(); await expect(panel.getByText("当前供水运行平稳。", { exact: true })).toBeVisible(); return panel; } test("speech failure shows an explanation and playback can be retried", async ({ page }) => { await page.addInitScript(() => { Object.defineProperty(window, "Audio", { configurable: true, value: class { preload = ""; onended = null; onerror = null; async play() {} pause() {} load() {} removeAttribute() {} } }); }); let requests = 0; await page.route("**/api/tts/edge", (route) => { requests += 1; return requests === 1 ? route.fulfill({ status: 502, contentType: "application/json", body: JSON.stringify({ error: "语音生成失败" }) }) : route.fulfill({ contentType: "audio/mpeg", body: "mock-audio" }); }); const panel = await openSpeechMessage(page); const play = panel.getByRole("button", { name: "语音播放", exact: true }); await play.click(); const alert = page.getByRole("alert").filter({ hasText: "语音播放失败" }); await expect(alert).toBeVisible(); await expect(alert).toContainText("语音生成失败"); await expect(play).toBeVisible(); await play.click(); await expect(panel.getByRole("button", { name: "暂停语音播放", exact: true })).toBeVisible(); await panel.getByRole("button", { name: "停止语音播放", exact: true }).click(); await expect(play).toBeVisible(); expect(requests).toBe(2); }); for (const width of [1440, 360, 320]) { test(`voice selection supports search, persistence and keyboard dismissal at ${width}px`, async ({ page }, testInfo) => { await page.setViewportSize({ width, height: width === 320 ? 640 : 900 }); await page.emulateMedia({ reducedMotion: "reduce" }); await page.addInitScript(() => { Object.defineProperty(window, "Audio", { configurable: true, value: class { preload = ""; onended = null; onerror = null; async play() {} pause() {} load() {} removeAttribute() {} } }); }); const requests: Array<{ text: string; voice?: string }> = []; await page.route("**/api/tts/edge", (route) => { requests.push(route.request().postDataJSON()); return route.fulfill({ contentType: "audio/mpeg", body: "mock-audio" }); }); let catalogReads = 0; const panel = await openSpeechMessage(page, { onCatalogRead: () => { catalogReads += 1; } }); const picker = panel.getByRole("button", { name: /^选择朗读音色/ }); await expect(picker).toHaveAccessibleName("选择朗读音色,当前晓晓"); await picker.click(); const dialog = page.getByRole("dialog", { name: "选择朗读音色" }); await expect(dialog).toHaveCSS("background-color", "rgb(255, 255, 255)"); await expect(dialog).toHaveCSS("backdrop-filter", "none"); await expect(dialog.getByText("3 种音色,选择后下次播放生效")).toBeVisible(); await expect(dialog.getByRole("option")).toHaveCount(4); await page.screenshot({ path: testInfo.outputPath(`voice-menu-${width}.png`) }); const bounds = await dialog.boundingBox(); expect(bounds!.x).toBeGreaterThanOrEqual(0); expect(bounds!.x + bounds!.width).toBeLessThanOrEqual(width); await dialog.getByRole("combobox", { name: "搜索音色" }).fill("云扬"); await dialog.getByRole("option", { name: /云扬/ }).click(); await expect(picker).toHaveAccessibleName("选择朗读音色,当前云扬"); await panel.getByRole("button", { name: "语音播放", exact: true }).click(); await expect(panel.getByRole("button", { name: "暂停语音播放", exact: true })).toBeVisible(); expect(await panel.locator("[data-agent-message-toolbar]").last().evaluate((element) => element.scrollWidth - element.clientWidth)).toBeLessThanOrEqual(1); expect(requests.at(-1)?.voice).toBe("zh-CN-YunyangNeural"); await panel.getByRole("button", { name: "停止语音播放", exact: true }).click(); await picker.click(); await dialog.getByRole("combobox", { name: "搜索音色" }).fill("晓伊"); await page.keyboard.press("ArrowDown"); await page.keyboard.press("Enter"); await expect(picker).toHaveAccessibleName("选择朗读音色,当前晓伊"); await picker.click(); await page.keyboard.press("Escape"); await expect(dialog).not.toBeVisible(); await expect(panel).toBeVisible(); await expect(picker).toBeFocused(); expect(catalogReads).toBe(1); await openSpeechMessage(page); await expect(picker).toHaveAccessibleName("选择朗读音色,当前晓伊"); await panel.getByRole("button", { name: "语音播放", exact: true }).click(); await expect(panel.getByRole("button", { name: "暂停语音播放", exact: true })).toBeVisible(); expect(requests.at(-1)?.voice).toBe("zh-CN-XiaoyiNeural"); }); } test("voice catalog failures can be retried without blocking the default choice", async ({ page }) => { const panel = await openSpeechMessage(page, { catalogUnavailable: true }); await panel.getByRole("button", { name: /^选择朗读音色/ }).click(); const dialog = page.getByRole("dialog", { name: "选择朗读音色" }); await expect(dialog.getByRole("alert")).toContainText("音色列表加载失败"); await expect(dialog.getByRole("option", { name: /默认音色/ })).toBeVisible(); await page.route("**/api/tts/edge/voices", (route) => route.fulfill({ json: voiceCatalog })); await dialog.getByRole("button", { name: "重新加载音色" }).click(); await expect(dialog.getByRole("option", { name: /云扬/ })).toBeVisible(); }); test("live Edge TTS lists Mandarin voices and plays the chosen voice in the browser", async ({ page }, testInfo) => { test.skip(process.env.TJWATER_TEST_LIVE_TTS !== "true", "Requires the real Edge TTS upstream service"); test.setTimeout(60_000); const panel = await openSpeechMessage(page, { liveVoices: true }); const catalog = await (await page.request.get("/api/tts/edge/voices")).json(); expect(catalog.voices.length).toBeGreaterThan(0); expect(catalog.voices.every((voice: { locale: string }) => voice.locale === "zh-CN")).toBe(true); const picker = panel.getByRole("button", { name: /^选择朗读音色/ }); await expect(picker).toHaveAccessibleName("选择朗读音色,当前晓晓", { timeout: 15_000 }); await picker.click(); const dialog = page.getByRole("dialog", { name: "选择朗读音色" }); await expect(dialog.getByRole("option")).toHaveCount(catalog.voices.length + 1); await dialog.getByRole("combobox", { name: "搜索音色" }).fill("云扬"); await dialog.getByRole("option", { name: /云扬/ }).click(); const responsePromise = page.waitForResponse((response) => response.url().endsWith("/api/tts/edge")); await panel.getByRole("button", { name: "语音播放", exact: true }).click(); const response = await responsePromise; expect(response.status()).toBe(200); expect(response.request().postDataJSON().voice).toBe("zh-CN-YunyangNeural"); expect(response.headers()["content-type"]).toBe("audio/mpeg"); expect(Number(response.headers()["content-length"])).toBeGreaterThan(0); const pause = panel.getByRole("button", { name: "暂停语音播放", exact: true }); await expect(pause).toBeVisible({ timeout: 35_000 }); await pause.click(); await page.screenshot({ path: testInfo.outputPath("speech-paused.png") }); await panel.getByRole("button", { name: "继续语音播放", exact: true }).click(); await expect(pause).toBeVisible(); await panel.getByRole("button", { name: "停止语音播放", exact: true }).click(); await expect(panel.getByRole("button", { name: "语音播放", exact: true })).toBeVisible(); });