diff --git a/package-lock.json b/package-lock.json index 49713fa..f8c2b38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "deck.gl": "^9.1.14", "echarts": "^6.0.0", "echarts-for-react": "^3.0.5", + "edge-tts-ts": "^1.0.0", "framer-motion": "^12.38.0", "js-cookie": "^3.0.5", "next": "^16.1.6", @@ -11457,6 +11458,15 @@ "integrity": "sha512-VtDvQpIJBvBatnONUsPzXYFVKQQAhuf3XTNOAsdBxCNO/QCtUUd8LSgjn0GVarBkCad6aJCZfXgrjYbl/KRr7w==", "license": "MIT" }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -12385,6 +12395,36 @@ "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", "license": "0BSD" }, + "node_modules/edge-tts-ts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/edge-tts-ts/-/edge-tts-ts-1.0.0.tgz", + "integrity": "sha512-327gpuN0VjvMsuvbizqabzqMiYpdHb0Slt8/hyf+ridtSkOGN68oogLpFnm8KznunbCnoo3DWMTAn7j6sd3WrA==", + "license": "MIT", + "dependencies": { + "commander": "^14.0.3", + "isomorphic-ws": "^5.0.0", + "sound-play": "^1.1.0", + "uuid": "^13.0.0", + "ws": "^8.20.0" + }, + "bin": { + "edge-playback": "dist/cli/edge-playback.js", + "edge-tts": "dist/cli/edge-tts.js" + } + }, + "node_modules/edge-tts-ts/node_modules/uuid": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.2.tgz", + "integrity": "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.227", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.227.tgz", @@ -14904,6 +14944,15 @@ "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "license": "MIT" }, + "node_modules/isomorphic-ws": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz", + "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==", + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", @@ -19790,6 +19839,12 @@ "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==", "license": "MIT" }, + "node_modules/sound-play": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/sound-play/-/sound-play-1.1.0.tgz", + "integrity": "sha512-Bd/L0AoCwITFeOnpNLMsfPXrV5GG5NhrC/T6odveahYbhPZkdTnrFXRia9FCC5WBWdUTw1d+yvLBvi4wnD1xOA==", + "license": "MIT" + }, "node_modules/source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -21444,10 +21499,9 @@ } }, "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "dev": true, + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/package.json b/package.json index 8789533..eeff8bf 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "deck.gl": "^9.1.14", "echarts": "^6.0.0", "echarts-for-react": "^3.0.5", + "edge-tts-ts": "^1.0.0", "framer-motion": "^12.38.0", "js-cookie": "^3.0.5", "next": "^16.1.6", diff --git a/src/app/api/tts/edge/route.test.ts b/src/app/api/tts/edge/route.test.ts new file mode 100644 index 0000000..5757b4d --- /dev/null +++ b/src/app/api/tts/edge/route.test.ts @@ -0,0 +1,50 @@ +/** + * @jest-environment node + */ + +import { POST } from "./route"; + +const streamMock = jest.fn(); + +jest.mock("edge-tts-ts", () => ({ + Communicate: jest.fn().mockImplementation(() => ({ + stream: streamMock, + })), +})); + +describe("POST /api/tts/edge", () => { + beforeEach(() => { + streamMock.mockReset(); + }); + + it("returns synthesized mp3 audio", async () => { + streamMock.mockImplementation(async function* () { + yield { type: "audio", data: new Uint8Array([1, 2]) }; + yield { type: "SentenceBoundary", offset: 0, duration: 1, text: "测试" }; + yield { type: "audio", data: new Uint8Array([3]) }; + }); + + const response = await POST( + new Request("http://localhost/api/tts/edge", { + method: "POST", + body: JSON.stringify({ text: "测试文本" }), + }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("audio/mpeg"); + expect(Array.from(new Uint8Array(await response.arrayBuffer()))).toEqual([1, 2, 3]); + }); + + it("rejects empty text", async () => { + const response = await POST( + new Request("http://localhost/api/tts/edge", { + method: "POST", + body: JSON.stringify({ text: " " }), + }), + ); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: "text is required" }); + }); +}); diff --git a/src/app/api/tts/edge/route.ts b/src/app/api/tts/edge/route.ts new file mode 100644 index 0000000..52948d7 --- /dev/null +++ b/src/app/api/tts/edge/route.ts @@ -0,0 +1,76 @@ +import { NextResponse } from "next/server"; +import { Communicate } from "edge-tts-ts"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const DEFAULT_VOICE = process.env.EDGE_TTS_VOICE || "zh-CN-XiaoxiaoNeural"; +const MAX_TEXT_LENGTH = 12000; + +type EdgeTtsRequest = { + text?: unknown; + voice?: unknown; +}; + +const jsonError = (message: string, status: number) => + NextResponse.json({ error: message }, { status }); + +export async function POST(request: Request) { + let payload: EdgeTtsRequest; + try { + payload = (await request.json()) as EdgeTtsRequest; + } catch { + return jsonError("Invalid JSON body", 400); + } + + const text = typeof payload.text === "string" ? payload.text.trim() : ""; + if (!text) { + return jsonError("text is required", 400); + } + if (text.length > MAX_TEXT_LENGTH) { + return jsonError(`text must be ${MAX_TEXT_LENGTH} characters or fewer`, 413); + } + + const voice = + typeof payload.voice === "string" && payload.voice.trim() + ? payload.voice.trim() + : DEFAULT_VOICE; + + try { + const communicate = new Communicate(text, { voice }); + const chunks: Uint8Array[] = []; + let byteLength = 0; + + for await (const chunk of communicate.stream()) { + if (chunk.type !== "audio") continue; + chunks.push(chunk.data); + byteLength += chunk.data.byteLength; + } + + if (byteLength === 0) { + return jsonError("Edge TTS returned empty audio", 502); + } + + const audio = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + audio.set(chunk, offset); + offset += chunk.byteLength; + } + + const audioBuffer = audio.buffer.slice( + audio.byteOffset, + audio.byteOffset + audio.byteLength, + ); + + return new Response(audioBuffer, { + headers: { + "Content-Type": "audio/mpeg", + "Cache-Control": "no-store", + }, + }); + } catch (error) { + console.error("[EdgeTTS] Failed to synthesize speech:", error); + return jsonError("Failed to synthesize speech", 502); + } +} diff --git a/src/components/chat/AgentTurn.tsx b/src/components/chat/AgentTurn.tsx index 54b10aa..0e0965e 100644 --- a/src/components/chat/AgentTurn.tsx +++ b/src/components/chat/AgentTurn.tsx @@ -6,6 +6,7 @@ import { AnimatePresence, motion } from "framer-motion"; import { Avatar, Box, + CircularProgress, IconButton, Paper, Stack, @@ -22,8 +23,12 @@ import { parseContentWithToolCalls, type ContentSegment, } from "./chatMessageSections"; -import type { Message, SpeechState } from "./GlobalChatbox.types"; +import type { + Message, + SpeechState, +} from "./GlobalChatbox.types"; import { stripMarkdown } from "./globalChatboxUtils"; +import { findSpeechSelectionStartOffset } from "./speechStartOptions"; import { AgentProgressTimeline } from "./AgentProgressTimeline"; import { ChartGenerationSkeleton, ChatInlineChart } from "./ChatInlineChart"; import { ChatToolCallBlock } from "./ChatToolCallBlock"; @@ -40,7 +45,11 @@ type AgentTurnProps = { message: Message; isStreaming: boolean; messageSpeechState: SpeechState; - onSpeak: (messageId: string, text: string) => void; + onSpeak: ( + messageId: string, + text: string, + options?: { startOffset?: number }, + ) => void; onPause: () => void; onResume: () => void; onStopSpeech: () => void; @@ -170,6 +179,11 @@ export const AgentTurn = React.memo( const isErrorMessage = Boolean(message.isError); const isStreamingAssistant = !isUser && !isErrorMessage && isStreaming; const [isHovered, setIsHovered] = React.useState(false); + const answerContentRef = React.useRef(null); + const [selectedSpeechStart, setSelectedSpeechStart] = React.useState<{ + offset: number; + preview: string; + } | null>(null); const isProgressComplete = message.progress?.some( (item) => item.phase === "complete" && item.status === "completed", ) ?? false; @@ -185,6 +199,43 @@ export const AgentTurn = React.memo( [isErrorMessage, isUser, message.content], ); const answerContent = parsedAssistantSections?.answer ?? message.content; + const speechText = useMemo( + () => stripMarkdown(answerContent), + [answerContent], + ); + const handleCaptureSpeechSelection = React.useCallback(() => { + const selection = window.getSelection(); + const container = answerContentRef.current; + if (!selection || selection.rangeCount === 0 || selection.isCollapsed || !container) { + return; + } + + const range = selection.getRangeAt(0); + if (!container.contains(range.commonAncestorContainer)) { + return; + } + + const selectedText = selection.toString(); + const startOffset = findSpeechSelectionStartOffset(speechText, selectedText); + if (startOffset === null) { + setSelectedSpeechStart(null); + return; + } + + const preview = selectedText.replace(/\s+/g, " ").trim().slice(0, 24); + setSelectedSpeechStart({ + offset: startOffset, + preview: preview.length === 24 ? `${preview}...` : preview, + }); + }, [speechText]); + React.useEffect(() => { + setSelectedSpeechStart(null); + }, [message.id, speechText]); + const handleSpeakFromCurrentStart = () => { + onSpeak(message.id, speechText, { + startOffset: selectedSpeechStart?.offset ?? 0, + }); + }; const contentSegments: ContentSegment[] = useMemo( () => !isUser && !isErrorMessage @@ -333,6 +384,10 @@ export const AgentTurn = React.memo( ) : null} onSpeak(message.id, stripMarkdown(answerContent))} - aria-label="朗读消息" + onClick={handleSpeakFromCurrentStart} + aria-label={selectedSpeechStart ? "从选中位置朗读" : "朗读消息"} sx={{ color: "text.secondary", opacity: 0.68, p: 0.5 }} > ) : null} + {messageSpeechState === "loading" ? ( + <> + + + + + + + + ) : null} {messageSpeechState === "playing" ? ( <> diff --git a/src/components/chat/AgentWorkspace.tsx b/src/components/chat/AgentWorkspace.tsx index 2833b7e..69c9ee8 100644 --- a/src/components/chat/AgentWorkspace.tsx +++ b/src/components/chat/AgentWorkspace.tsx @@ -25,7 +25,11 @@ type AgentWorkspaceProps = { onScrollStateChange?: (isNearBottom: boolean) => void; speakingMessageId: string | null; speechState: SpeechState; - onSpeak: (messageId: string, text: string) => void; + onSpeak: ( + messageId: string, + text: string, + options?: { startOffset?: number }, + ) => void; onPauseSpeech: () => void; onResumeSpeech: () => void; onStopSpeech: () => void; @@ -42,7 +46,11 @@ type TurnListProps = { streamingMessageId: string | null; speakingMessageId: string | null; speechState: SpeechState; - onSpeak: (messageId: string, text: string) => void; + onSpeak: ( + messageId: string, + text: string, + options?: { startOffset?: number }, + ) => void; onPauseSpeech: () => void; onResumeSpeech: () => void; onStopSpeech: () => void; diff --git a/src/components/chat/GlobalChatbox.types.ts b/src/components/chat/GlobalChatbox.types.ts index 95df637..9bb97fb 100644 --- a/src/components/chat/GlobalChatbox.types.ts +++ b/src/components/chat/GlobalChatbox.types.ts @@ -70,7 +70,7 @@ export type Props = { onClose: () => void; }; -export type SpeechState = "idle" | "playing" | "paused"; +export type SpeechState = "idle" | "loading" | "playing" | "paused"; export type ChatSessionSummary = { id: string; diff --git a/src/components/chat/globalChatboxVoice.ts b/src/components/chat/globalChatboxVoice.ts index 2524c2b..7f41b75 100644 --- a/src/components/chat/globalChatboxVoice.ts +++ b/src/components/chat/globalChatboxVoice.ts @@ -1,32 +1,11 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import config from "@/config/config"; import type { SpeechState } from "./GlobalChatbox.types"; +import { splitSpeechTextIntoChunks } from "./speechStartOptions"; -type AudioStreamStartResponse = { - stream_id?: string; - audio_url?: string; - status_url?: string; - result_url?: string; - sample_rate?: number; - channels?: number; - error?: string; +type SpeakOptions = { + startOffset?: number; }; -type AudioStreamStatusResponse = { - state?: "starting" | "running" | "done" | "failed" | "closed"; - ready?: boolean; - failed?: boolean; - closed?: boolean; - status_text?: string; - error?: string; -}; - -type AudioStreamResultResponse = { - run_status?: string; - error?: string; -}; - -// WebKit Speech Recognition compatibility interface SpeechRecognitionEvent extends Event { readonly resultIndex: number; readonly results: SpeechRecognitionResultList; @@ -54,56 +33,64 @@ declare global { new (): SpeechRecognition; prototype: SpeechRecognition; }; - webkitAudioContext?: typeof AudioContext; } } export function useSpeechSynthesis() { const [speechState, setSpeechState] = useState("idle"); const [speakingMessageId, setSpeakingMessageId] = useState(null); - const audioContextRef = useRef(null); - const streamAbortControllerRef = useRef(null); - const activeSourceNodesRef = useRef>(new Set()); - const streamIdRef = useRef(null); - const closeUrlRef = useRef(null); - const statusUrlRef = useRef(null); - const resultUrlRef = useRef(null); - const statusPollTimeoutRef = useRef(null); + const audioRef = useRef(null); + const currentAudioUrlRef = useRef(null); + const audioObjectUrlsRef = useRef>(new Set()); + const fetchAbortControllersRef = useRef>(new Set()); + const chunkAudioUrlCacheRef = useRef>(new Map()); + const chunkFetchPromisesRef = useRef>>(new Map()); + const chunksRef = useRef([]); + const currentChunkIndexRef = useRef(0); + const playChunkRef = useRef<(chunkIndex: number, playbackToken: number) => Promise>( + async () => {}, + ); const playbackTokenRef = useRef(0); + const activeMessageIdRef = useRef(null); const isSupported = typeof window !== "undefined" && - typeof window.FormData !== "undefined" && - (typeof window.AudioContext !== "undefined" || - typeof window.webkitAudioContext !== "undefined"); + typeof window.Audio !== "undefined" && + typeof window.URL !== "undefined" && + typeof window.fetch !== "undefined"; - const trimTrailingSlash = useCallback((value: string) => value.replace(/\/+$/, ""), []); + const detachCurrentAudio = useCallback((revokeCurrentUrl: boolean) => { + const audio = audioRef.current; + audioRef.current = null; + if (audio) { + audio.pause(); + audio.onended = null; + audio.onerror = null; + audio.removeAttribute("src"); + audio.load(); + } - const buildServiceUrl = useCallback( - (path: string) => `${trimTrailingSlash(config.AUDIO_SERVICE_URL)}${path.startsWith("/") ? path : `/${path}`}`, - [trimTrailingSlash], - ); + const currentUrl = currentAudioUrlRef.current; + currentAudioUrlRef.current = null; + if (revokeCurrentUrl && currentUrl) { + URL.revokeObjectURL(currentUrl); + audioObjectUrlsRef.current.delete(currentUrl); + chunkAudioUrlCacheRef.current.delete(currentChunkIndexRef.current); + } + }, []); - const resolveServiceUrl = useCallback( - (pathOrUrl: string) => { - if (/^https?:\/\//i.test(pathOrUrl)) { - return pathOrUrl; - } - return buildServiceUrl(pathOrUrl); - }, - [buildServiceUrl], - ); - - const withQueryParams = useCallback( - (urlString: string, params: Record) => { - const url = new URL(urlString); - Object.entries(params).forEach(([key, value]) => { - url.searchParams.set(key, value); - }); - return url.toString(); - }, - [], - ); + const releaseAudio = useCallback(() => { + fetchAbortControllersRef.current.forEach((controller) => controller.abort()); + fetchAbortControllersRef.current.clear(); + chunkFetchPromisesRef.current.clear(); + detachCurrentAudio(false); + audioObjectUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)); + audioObjectUrlsRef.current.clear(); + chunkAudioUrlCacheRef.current.clear(); + chunksRef.current = []; + currentChunkIndexRef.current = 0; + activeMessageIdRef.current = null; + }, [detachCurrentAudio]); const readErrorMessage = useCallback(async (response: Response, fallback: string) => { try { @@ -114,402 +101,170 @@ export function useSpeechSynthesis() { } }, []); - const closeStream = useCallback(async (closeUrl: string) => { - const response = await fetch(closeUrl, { - method: "POST", - }); + const fetchChunkAudio = useCallback( + (chunkIndex: number, playbackToken: number) => { + const cachedUrl = chunkAudioUrlCacheRef.current.get(chunkIndex); + if (cachedUrl) return Promise.resolve(cachedUrl); - if (!response.ok) { - console.error("[GlobalChatbox] Failed to close audio stream:", closeUrl); - } - }, []); + const existingPromise = chunkFetchPromisesRef.current.get(chunkIndex); + if (existingPromise) return existingPromise; - const stopStatusPolling = useCallback(() => { - if (statusPollTimeoutRef.current !== null) { - window.clearTimeout(statusPollTimeoutRef.current); - statusPollTimeoutRef.current = null; - } - }, []); - - const fetchStreamResult = useCallback( - async (resultUrl: string) => { - const response = await fetch(resultUrl); - if (response.status === 202) { - return false; - } - if (!response.ok) { - throw new Error( - await readErrorMessage( - response, - `Audio stream result failed with status ${response.status}`, - ), - ); + const chunkText = chunksRef.current[chunkIndex]; + if (!chunkText) { + return Promise.reject(new Error("Speech chunk is missing")); } - const payload = (await response.json()) as AudioStreamResultResponse; - if (payload.error) { - throw new Error(payload.error); - } + const abortController = new AbortController(); + fetchAbortControllersRef.current.add(abortController); - return true; + const promise = fetch("/api/tts/edge", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ text: chunkText }), + signal: abortController.signal, + }) + .then(async (response) => { + if (!response.ok) { + throw new Error( + await readErrorMessage(response, `Edge TTS failed with status ${response.status}`), + ); + } + const audioBlob = await response.blob(); + if (!audioBlob.size) { + throw new Error("Edge TTS returned empty audio"); + } + if (playbackToken !== playbackTokenRef.current) { + throw new DOMException("Edge TTS chunk cancelled", "AbortError"); + } + + const objectUrl = URL.createObjectURL(audioBlob); + audioObjectUrlsRef.current.add(objectUrl); + chunkAudioUrlCacheRef.current.set(chunkIndex, objectUrl); + return objectUrl; + }) + .finally(() => { + fetchAbortControllersRef.current.delete(abortController); + chunkFetchPromisesRef.current.delete(chunkIndex); + }); + + chunkFetchPromisesRef.current.set(chunkIndex, promise); + return promise; }, [readErrorMessage], ); - const clearAudio = useCallback(async () => { - const abortController = streamAbortControllerRef.current; - streamAbortControllerRef.current = null; - abortController?.abort(); - - activeSourceNodesRef.current.forEach((source) => { - try { - source.onended = null; - source.stop(); - } catch { - // ignore stop errors when source already ended - } - source.disconnect(); - }); - activeSourceNodesRef.current.clear(); - - const audioContext = audioContextRef.current; - audioContextRef.current = null; - if (!audioContext) return; - - try { - await audioContext.close(); - } catch { - // ignore close errors when context already closed - } - }, []); - - const playPcmStream = useCallback( - async ({ - audioUrl, - sampleRate, - channels, - playbackToken, - }: { - audioUrl: string; - sampleRate: number; - channels: number; - playbackToken: number; - }) => { - const AudioContextCtor = window.AudioContext ?? window.webkitAudioContext; - if (!AudioContextCtor) { - throw new Error("WebAudio AudioContext is not available in this browser"); - } - - const abortController = new AbortController(); - streamAbortControllerRef.current = abortController; - - const response = await fetch(withQueryParams(audioUrl, { format: "pcm" }), { - signal: abortController.signal, + const prefetchChunk = useCallback( + (chunkIndex: number, playbackToken: number) => { + if (chunkIndex >= chunksRef.current.length) return; + void fetchChunkAudio(chunkIndex, playbackToken).catch((error) => { + if ( + playbackToken === playbackTokenRef.current && + !(error instanceof DOMException && error.name === "AbortError") + ) { + console.error("[GlobalChatbox] Failed to prefetch Edge TTS chunk:", error); + } }); - if (!response.ok) { - throw new Error( - await readErrorMessage(response, `Audio stream failed with status ${response.status}`), - ); - } - if (!response.body) { - throw new Error("Audio stream response body is missing"); - } - - const audioContext = new AudioContextCtor({ - sampleRate, - }); - audioContextRef.current = audioContext; - - const reader = response.body.getReader(); - const bytesPerFrame = Math.max(1, channels) * 2; - let bufferedRemainder = new Uint8Array(0); - let nextStartTime = audioContext.currentTime + 0.05; - let activeSources = 0; - let streamEnded = false; - let resolvePlaybackDrain: (() => void) | null = null; - const playbackDrainPromise = new Promise((resolve) => { - resolvePlaybackDrain = resolve; - }); - - const maybeResolvePlaybackDrain = () => { - if (streamEnded && activeSources === 0) { - resolvePlaybackDrain?.(); - } - }; - - const schedulePcmChunk = (pcmBytes: Uint8Array) => { - const frameCount = pcmBytes.byteLength / bytesPerFrame; - if (frameCount <= 0) return; - - const buffer = audioContext.createBuffer(Math.max(1, channels), frameCount, sampleRate); - const view = new DataView(pcmBytes.buffer, pcmBytes.byteOffset, pcmBytes.byteLength); - for (let frame = 0; frame < frameCount; frame += 1) { - for (let channel = 0; channel < Math.max(1, channels); channel += 1) { - const sampleIndex = frame * Math.max(1, channels) + channel; - const pcm = view.getInt16(sampleIndex * 2, true); - buffer.getChannelData(channel)[frame] = pcm / 32768; - } - } - - const source = audioContext.createBufferSource(); - source.buffer = buffer; - source.connect(audioContext.destination); - const sourceStartTime = Math.max(nextStartTime, audioContext.currentTime + 0.01); - nextStartTime = sourceStartTime + buffer.duration; - - activeSources += 1; - activeSourceNodesRef.current.add(source); - source.onended = () => { - activeSources -= 1; - activeSourceNodesRef.current.delete(source); - source.disconnect(); - maybeResolvePlaybackDrain(); - }; - source.start(sourceStartTime); - }; - - const concatUint8Arrays = (a: Uint8Array, b: Uint8Array) => { - if (a.byteLength === 0) return b; - if (b.byteLength === 0) return a; - const merged = new Uint8Array(a.byteLength + b.byteLength); - merged.set(a); - merged.set(b, a.byteLength); - return merged; - }; - - while (true) { - if (playbackToken !== playbackTokenRef.current) { - throw new DOMException("PCM stream playback cancelled", "AbortError"); - } - - const { done, value } = await reader.read(); - if (done) break; - if (!value || value.byteLength === 0) continue; - - const merged = concatUint8Arrays(bufferedRemainder, value); - const alignedByteLength = merged.byteLength - (merged.byteLength % bytesPerFrame); - if (alignedByteLength === 0) { - bufferedRemainder = new Uint8Array(merged); - continue; - } - - const alignedChunk = merged.slice(0, alignedByteLength); - bufferedRemainder = new Uint8Array(merged.slice(alignedByteLength)); - schedulePcmChunk(alignedChunk); - } - - streamEnded = true; - maybeResolvePlaybackDrain(); - await playbackDrainPromise; }, - [readErrorMessage, withQueryParams], + [fetchChunkAudio], ); - const stopPlayback = useCallback(async () => { - await clearAudio(); - stopStatusPolling(); + const playChunk = useCallback( + async (chunkIndex: number, playbackToken: number) => { + setSpeechState("loading"); + const objectUrl = await fetchChunkAudio(chunkIndex, playbackToken); + if (playbackToken !== playbackTokenRef.current) return; - const closeUrl = closeUrlRef.current; - streamIdRef.current = null; - closeUrlRef.current = null; - statusUrlRef.current = null; - resultUrlRef.current = null; - setSpeechState("idle"); - setSpeakingMessageId(null); + detachCurrentAudio(true); + currentChunkIndexRef.current = chunkIndex; + const audio = new Audio(objectUrl); + audio.preload = "auto"; + audioRef.current = audio; + currentAudioUrlRef.current = objectUrl; - if (closeUrl) { - try { - await closeStream(closeUrl); - } catch (error) { - console.error("[GlobalChatbox] Failed to close audio stream:", error); - } - } - }, [clearAudio, closeStream, stopStatusPolling]); - - const pollStreamStatus = useCallback( - (playbackToken: number, statusUrl: string, resultUrl: string) => { - stopStatusPolling(); - - statusPollTimeoutRef.current = window.setTimeout(async () => { - if ( - playbackToken !== playbackTokenRef.current || - statusUrlRef.current !== statusUrl || - resultUrlRef.current !== resultUrl - ) { + audio.onended = () => { + if (playbackToken !== playbackTokenRef.current) return; + detachCurrentAudio(true); + const nextChunkIndex = chunkIndex + 1; + if (nextChunkIndex >= chunksRef.current.length) { + releaseAudio(); + setSpeechState("idle"); + setSpeakingMessageId(null); return; } - try { - const response = await fetch(statusUrl); - if (!response.ok) { - throw new Error( - await readErrorMessage( - response, - `Audio stream status failed with status ${response.status}`, - ), - ); - } + currentChunkIndexRef.current = nextChunkIndex; + void playChunkRef.current(nextChunkIndex, playbackToken); + }; + audio.onerror = () => { + if (playbackToken !== playbackTokenRef.current) return; + playbackTokenRef.current += 1; + releaseAudio(); + setSpeechState("idle"); + setSpeakingMessageId(null); + console.error("[GlobalChatbox] Edge TTS audio playback failed"); + }; - const payload = (await response.json()) as AudioStreamStatusResponse; - if ( - playbackToken !== playbackTokenRef.current || - statusUrlRef.current !== statusUrl || - resultUrlRef.current !== resultUrl - ) { - return; - } - - if (payload.failed || payload.state === "failed") { - console.error( - "[GlobalChatbox] Audio stream failed:", - payload.error || payload.status_text || statusUrl, - ); - playbackTokenRef.current += 1; - void stopPlayback(); - return; - } - - if (payload.closed || payload.state === "closed") { - stopStatusPolling(); - return; - } - - if (payload.ready || payload.state === "done") { - try { - const isResultReady = await fetchStreamResult(resultUrl); - if (isResultReady) { - stopStatusPolling(); - return; - } - } catch (error) { - console.error("[GlobalChatbox] Failed to fetch audio stream result:", error); - } - } - - pollStreamStatus(playbackToken, statusUrl, resultUrl); - } catch (error) { - if ( - playbackToken === playbackTokenRef.current && - statusUrlRef.current === statusUrl && - resultUrlRef.current === resultUrl - ) { - console.error("[GlobalChatbox] Failed to poll audio stream status:", error); - pollStreamStatus(playbackToken, statusUrl, resultUrl); - } - } - }, 1000); + await audio.play(); + if (playbackToken !== playbackTokenRef.current) return; + setSpeechState("playing"); + prefetchChunk(chunkIndex + 1, playbackToken); }, - [fetchStreamResult, readErrorMessage, stopPlayback, stopStatusPolling], + [detachCurrentAudio, fetchChunkAudio, prefetchChunk, releaseAudio], ); - const stop = useCallback(() => { - playbackTokenRef.current += 1; - void stopPlayback(); - }, [stopPlayback]); + useEffect(() => { + playChunkRef.current = playChunk; + }, [playChunk]); + + const playExistingAudio = useCallback(async () => { + const audio = audioRef.current; + if (!audio) return false; + + setSpeakingMessageId(activeMessageIdRef.current); + setSpeechState("playing"); + try { + await audio.play(); + prefetchChunk(currentChunkIndexRef.current + 1, playbackTokenRef.current); + return true; + } catch (error) { + playbackTokenRef.current += 1; + releaseAudio(); + setSpeechState("idle"); + setSpeakingMessageId(null); + console.error("[GlobalChatbox] Failed to resume Edge TTS playback:", error); + return false; + } + }, [prefetchChunk, releaseAudio]); const speak = useCallback( - async (messageId: string, text: string) => { + async (messageId: string, text: string, options: SpeakOptions = {}) => { const normalizedText = text.trim(); if (!isSupported || !normalizedText) return; + const startOffset = Math.max( + 0, + Math.min(options.startOffset ?? 0, normalizedText.length), + ); + const textToSpeak = normalizedText.slice(startOffset).trim(); + const chunks = splitSpeechTextIntoChunks(textToSpeak); + if (!chunks.length) return; + const playbackToken = playbackTokenRef.current + 1; playbackTokenRef.current = playbackToken; - await stopPlayback(); + releaseAudio(); + + chunksRef.current = chunks; + currentChunkIndexRef.current = 0; + activeMessageIdRef.current = messageId; setSpeakingMessageId(messageId); - setSpeechState("playing"); + setSpeechState("loading"); try { - const formData = new FormData(); - formData.append("text", normalizedText); - formData.append("demo_id", "demo-1"); - - const response = await fetch(buildServiceUrl("/api/generate-stream/start"), { - method: "POST", - body: formData, - }); - - if (!response.ok) { - throw new Error( - await readErrorMessage( - response, - `Audio stream start failed with status ${response.status}`, - ), - ); - } - - const payload = (await response.json()) as AudioStreamStartResponse; - const streamId = payload.stream_id; - const sampleRate = - typeof payload.sample_rate === "number" && payload.sample_rate > 0 - ? payload.sample_rate - : 24000; - const channels = - typeof payload.channels === "number" && payload.channels > 0 - ? payload.channels - : 1; - const audioUrl = payload.audio_url - ? resolveServiceUrl(payload.audio_url) - : buildServiceUrl( - `/api/generate-stream/${encodeURIComponent(streamId ?? "")}/audio?format=pcm`, - ); - const rawStatusUrl = payload.status_url - ? resolveServiceUrl(payload.status_url) - : buildServiceUrl(`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/status`); - const statusUrl = withQueryParams(rawStatusUrl, { compact: "1" }); - const rawResultUrl = payload.result_url - ? resolveServiceUrl(payload.result_url) - : buildServiceUrl(`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/result`); - const resultUrl = withQueryParams(rawResultUrl, { - compact: "1", - include_audio: "0", - }); - const closeUrl = buildServiceUrl( - `/api/generate-stream/${encodeURIComponent(streamId ?? "")}/close`, - ); - - if (!streamId) { - throw new Error(payload.error || "Audio stream start response is missing stream_id"); - } - - if (playbackToken !== playbackTokenRef.current) { - await closeStream(closeUrl); - return; - } - - streamIdRef.current = streamId; - closeUrlRef.current = closeUrl; - statusUrlRef.current = statusUrl; - resultUrlRef.current = resultUrl; - - pollStreamStatus(playbackToken, statusUrl, resultUrl); - await playPcmStream({ - audioUrl, - sampleRate, - channels, - playbackToken, - }); - - if (playbackToken !== playbackTokenRef.current) { - return; - } - - await clearAudio(); - if (streamIdRef.current === streamId) { - streamIdRef.current = null; - closeUrlRef.current = null; - statusUrlRef.current = null; - resultUrlRef.current = null; - setSpeechState("idle"); - setSpeakingMessageId(null); - } - stopStatusPolling(); - await fetchStreamResult(resultUrl).catch((error) => { - console.error("[GlobalChatbox] Failed to fetch audio stream result:", error); - }); - await closeStream(closeUrl); + await playChunk(0, playbackToken); } catch (error) { - await clearAudio(); if ( error instanceof DOMException && error.name === "AbortError" && @@ -517,73 +272,51 @@ export function useSpeechSynthesis() { ) { return; } - const closeUrl = closeUrlRef.current; - streamIdRef.current = null; - closeUrlRef.current = null; - statusUrlRef.current = null; - resultUrlRef.current = null; + + releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); - if (closeUrl) { - try { - await closeStream(closeUrl); - } catch (closeError) { - console.error("[GlobalChatbox] Failed to close audio stream:", closeError); - } - } - console.error("[GlobalChatbox] Failed to play audio stream:", error); + console.error("[GlobalChatbox] Failed to play Edge TTS audio:", error); } }, - [ - buildServiceUrl, - clearAudio, - closeStream, - fetchStreamResult, - isSupported, - playPcmStream, - readErrorMessage, - resolveServiceUrl, - pollStreamStatus, - stopPlayback, - stopStatusPolling, - withQueryParams, - ], + [isSupported, playChunk, releaseAudio], ); const pause = useCallback(() => { - if (!isSupported || !audioContextRef.current) return; - void audioContextRef.current.suspend().then( - () => { - setSpeechState("paused"); - }, - (error) => { - console.error("[GlobalChatbox] Failed to pause PCM playback:", error); - }, - ); - }, [isSupported]); + const audio = audioRef.current; + if (!isSupported || !audio || speechState !== "playing") return; + audio.pause(); + setSpeechState("paused"); + }, [isSupported, speechState]); const resume = useCallback(() => { - if (!isSupported || !audioContextRef.current) return; - void audioContextRef.current.resume().then( - () => { - setSpeechState("playing"); - }, - (error) => { - playbackTokenRef.current += 1; - void stopPlayback(); - console.error("[GlobalChatbox] Failed to resume audio playback:", error); - }, - ); - }, [isSupported, stopPlayback]); + if (!isSupported) return; + void playExistingAudio(); + }, [isSupported, playExistingAudio]); + + const stop = useCallback(() => { + playbackTokenRef.current += 1; + releaseAudio(); + setSpeechState("idle"); + setSpeakingMessageId(null); + }, [releaseAudio]); useEffect(() => { return () => { playbackTokenRef.current += 1; - void stopPlayback(); + releaseAudio(); }; - }, [stopPlayback]); + }, [releaseAudio]); - return { speechState, speakingMessageId, speak, pause, resume, stop, isSupported }; + return { + speechState, + speakingMessageId, + speak, + pause, + resume, + stop, + isSupported, + }; } export function useSpeechRecognition(onResult: (text: string) => void) { @@ -618,7 +351,6 @@ export function useSpeechRecognition(onResult: (text: string) => void) { recognition.onerror = () => { setIsListening(false); - recognitionRef.current = null; }; recognition.onend = () => { @@ -627,8 +359,8 @@ export function useSpeechRecognition(onResult: (text: string) => void) { }; recognitionRef.current = recognition; - recognition.start(); setIsListening(true); + recognition.start(); }, [isSupported]); const stop = useCallback(() => { @@ -639,7 +371,7 @@ export function useSpeechRecognition(onResult: (text: string) => void) { useEffect(() => { return () => { - recognitionRef.current?.stop(); + recognitionRef.current?.abort(); }; }, []); diff --git a/src/components/chat/speechStartOptions.test.ts b/src/components/chat/speechStartOptions.test.ts new file mode 100644 index 0000000..c92806d --- /dev/null +++ b/src/components/chat/speechStartOptions.test.ts @@ -0,0 +1,31 @@ +import { + findSpeechSelectionStartOffset, + splitSpeechTextIntoChunks, +} from "./speechStartOptions"; + +describe("findSpeechSelectionStartOffset", () => { + it("finds the reading start from selected reply text", () => { + const text = "第一段内容。\n\n第二段 包含空格。\n第三段内容。"; + + expect(findSpeechSelectionStartOffset(text, "第二段 包含空格")).toBe( + text.indexOf("第二段"), + ); + expect(findSpeechSelectionStartOffset(text, "第三段")).toBe(text.indexOf("第三段")); + expect(findSpeechSelectionStartOffset(text, "不存在")).toBeNull(); + }); +}); + +describe("splitSpeechTextIntoChunks", () => { + it("splits long text into bounded chunks", () => { + const text = Array.from({ length: 80 }, (_, index) => `第${index}句内容足够长。`).join(""); + const chunks = splitSpeechTextIntoChunks(text); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.every((chunk) => chunk.length <= 520)).toBe(true); + expect(chunks.join("")).toBe(text); + }); + + it("keeps short text as one chunk", () => { + expect(splitSpeechTextIntoChunks("短句。")).toEqual(["短句。"]); + }); +}); diff --git a/src/components/chat/speechStartOptions.ts b/src/components/chat/speechStartOptions.ts new file mode 100644 index 0000000..3c06966 --- /dev/null +++ b/src/components/chat/speechStartOptions.ts @@ -0,0 +1,98 @@ +const compactWhitespace = (value: string) => value.replace(/\s+/g, " ").trim(); +const MAX_SPEECH_CHUNK_LENGTH = 520; +const MIN_SPEECH_CHUNK_LENGTH = 180; +const SPEECH_SENTENCE_PATTERN = /[^。！？!?；;\n]+(?:[。！？!?；;]+|(?=\n|$))/g; + +const normalizeWithOffsetMap = (value: string) => { + let normalized = ""; + const offsetMap: number[] = []; + let isPreviousWhitespace = false; + + Array.from(value).forEach((char, index) => { + if (/\s/u.test(char)) { + if (!isPreviousWhitespace && normalized.length > 0) { + normalized += " "; + offsetMap.push(index); + } + isPreviousWhitespace = true; + return; + } + + normalized += char; + offsetMap.push(index); + isPreviousWhitespace = false; + }); + + return { + normalized: normalized.trimEnd(), + offsetMap, + }; +}; + +export function findSpeechSelectionStartOffset( + text: string, + selectedText: string, +): number | null { + const needle = selectedText.trim(); + if (!needle) return null; + + const exactIndex = text.indexOf(needle); + if (exactIndex >= 0) return exactIndex; + + const normalizedNeedle = compactWhitespace(needle); + if (!normalizedNeedle) return null; + + const haystack = normalizeWithOffsetMap(text); + const normalizedIndex = haystack.normalized.indexOf(normalizedNeedle); + if (normalizedIndex < 0) return null; + + return haystack.offsetMap[normalizedIndex] ?? null; +} + +export function splitSpeechTextIntoChunks(text: string): string[] { + const normalizedText = text.trim(); + if (!normalizedText) return []; + + const segments = Array.from(normalizedText.matchAll(SPEECH_SENTENCE_PATTERN), (match) => + compactWhitespace(match[0]), + ).filter(Boolean); + const sourceSegments = segments.length > 0 ? segments : [normalizedText]; + const chunks: string[] = []; + let currentChunk = ""; + + const flush = () => { + if (!currentChunk) return; + chunks.push(currentChunk); + currentChunk = ""; + }; + + const pushLongSegment = (segment: string) => { + for (let offset = 0; offset < segment.length; offset += MAX_SPEECH_CHUNK_LENGTH) { + chunks.push(segment.slice(offset, offset + MAX_SPEECH_CHUNK_LENGTH)); + } + }; + + sourceSegments.forEach((segment) => { + if (segment.length > MAX_SPEECH_CHUNK_LENGTH) { + flush(); + pushLongSegment(segment); + return; + } + + const candidate = currentChunk ? `${currentChunk}${segment}` : segment; + if ( + currentChunk && + candidate.length > MAX_SPEECH_CHUNK_LENGTH && + currentChunk.length >= MIN_SPEECH_CHUNK_LENGTH + ) { + flush(); + currentChunk = segment; + return; + } + + currentChunk = candidate; + }); + + flush(); + return chunks; +}