import { useCallback, useEffect, useRef, useState } from "react"; import type { SpeechState } from "./GlobalChatbox.types"; import { splitSpeechTextIntoChunks } from "./speechStartOptions"; type SpeakOptions = { startOffset?: number; }; interface SpeechRecognitionEvent extends Event { readonly resultIndex: number; readonly results: SpeechRecognitionResultList; } interface SpeechRecognition extends EventTarget { lang: string; continuous: boolean; interimResults: boolean; onresult: ((event: SpeechRecognitionEvent) => void) | null; onerror: ((event: Event) => void) | null; onend: (() => void) | null; start(): void; stop(): void; abort(): void; } declare global { interface Window { SpeechRecognition?: { new (): SpeechRecognition; prototype: SpeechRecognition; }; webkitSpeechRecognition?: { new (): SpeechRecognition; prototype: SpeechRecognition; }; } } export function useSpeechSynthesis() { const [speechState, setSpeechState] = useState("idle"); const [speakingMessageId, setSpeakingMessageId] = useState(null); const audioRef = useRef(null); const currentAudioUrlRef = useRef(null); const audioObjectUrlsRef = useRef>(new Set()); const fetchAbortControllersRef = useRef>(new Set()); const chunkAudioUrlCacheRef = useRef>(new Map()); const chunkFetchPromisesRef = useRef>>(new Map()); const chunksRef = useRef([]); const currentChunkIndexRef = useRef(0); const playChunkRef = useRef<(chunkIndex: number, playbackToken: number) => Promise>( async () => {}, ); const playbackTokenRef = useRef(0); const activeMessageIdRef = useRef(null); const isSupported = typeof window !== "undefined" && typeof window.Audio !== "undefined" && typeof window.URL !== "undefined" && typeof window.fetch !== "undefined"; const detachCurrentAudio = useCallback((revokeCurrentUrl: boolean) => { const audio = audioRef.current; audioRef.current = null; if (audio) { audio.pause(); audio.onended = null; audio.onerror = null; audio.removeAttribute("src"); audio.load(); } const currentUrl = currentAudioUrlRef.current; currentAudioUrlRef.current = null; if (revokeCurrentUrl && currentUrl) { URL.revokeObjectURL(currentUrl); audioObjectUrlsRef.current.delete(currentUrl); chunkAudioUrlCacheRef.current.delete(currentChunkIndexRef.current); } }, []); const releaseAudio = useCallback(() => { fetchAbortControllersRef.current.forEach((controller) => controller.abort()); fetchAbortControllersRef.current.clear(); chunkFetchPromisesRef.current.clear(); detachCurrentAudio(false); audioObjectUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)); audioObjectUrlsRef.current.clear(); chunkAudioUrlCacheRef.current.clear(); chunksRef.current = []; currentChunkIndexRef.current = 0; activeMessageIdRef.current = null; }, [detachCurrentAudio]); const readErrorMessage = useCallback(async (response: Response, fallback: string) => { try { const payload = (await response.json()) as { error?: string; message?: string }; return payload.error || payload.message || fallback; } catch { return fallback; } }, []); const fetchChunkAudio = useCallback( (chunkIndex: number, playbackToken: number) => { const cachedUrl = chunkAudioUrlCacheRef.current.get(chunkIndex); if (cachedUrl) return Promise.resolve(cachedUrl); const existingPromise = chunkFetchPromisesRef.current.get(chunkIndex); if (existingPromise) return existingPromise; const chunkText = chunksRef.current[chunkIndex]; if (!chunkText) { return Promise.reject(new Error("Speech chunk is missing")); } const abortController = new AbortController(); fetchAbortControllersRef.current.add(abortController); const promise = fetch("/api/tts/edge", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ text: chunkText }), signal: abortController.signal, }) .then(async (response) => { if (!response.ok) { throw new Error( await readErrorMessage(response, `Edge TTS failed with status ${response.status}`), ); } const audioBlob = await response.blob(); if (!audioBlob.size) { throw new Error("Edge TTS returned empty audio"); } if (playbackToken !== playbackTokenRef.current) { throw new DOMException("Edge TTS chunk cancelled", "AbortError"); } const objectUrl = URL.createObjectURL(audioBlob); audioObjectUrlsRef.current.add(objectUrl); chunkAudioUrlCacheRef.current.set(chunkIndex, objectUrl); return objectUrl; }) .finally(() => { fetchAbortControllersRef.current.delete(abortController); chunkFetchPromisesRef.current.delete(chunkIndex); }); chunkFetchPromisesRef.current.set(chunkIndex, promise); return promise; }, [readErrorMessage], ); const prefetchChunk = useCallback( (chunkIndex: number, playbackToken: number) => { if (chunkIndex >= chunksRef.current.length) return; void fetchChunkAudio(chunkIndex, playbackToken).catch((error) => { if ( playbackToken === playbackTokenRef.current && !(error instanceof DOMException && error.name === "AbortError") ) { console.error("[GlobalChatbox] Failed to prefetch Edge TTS chunk:", error); } }); }, [fetchChunkAudio], ); const playChunk = useCallback( async (chunkIndex: number, playbackToken: number) => { setSpeechState("loading"); const objectUrl = await fetchChunkAudio(chunkIndex, playbackToken); if (playbackToken !== playbackTokenRef.current) return; detachCurrentAudio(true); currentChunkIndexRef.current = chunkIndex; const audio = new Audio(objectUrl); audio.preload = "auto"; audioRef.current = audio; currentAudioUrlRef.current = objectUrl; audio.onended = () => { if (playbackToken !== playbackTokenRef.current) return; detachCurrentAudio(true); const nextChunkIndex = chunkIndex + 1; if (nextChunkIndex >= chunksRef.current.length) { releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); return; } currentChunkIndexRef.current = nextChunkIndex; void playChunkRef.current(nextChunkIndex, playbackToken); }; audio.onerror = () => { if (playbackToken !== playbackTokenRef.current) return; playbackTokenRef.current += 1; releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); console.error("[GlobalChatbox] Edge TTS audio playback failed"); }; await audio.play(); if (playbackToken !== playbackTokenRef.current) return; setSpeechState("playing"); prefetchChunk(chunkIndex + 1, playbackToken); }, [detachCurrentAudio, fetchChunkAudio, prefetchChunk, releaseAudio], ); useEffect(() => { playChunkRef.current = playChunk; }, [playChunk]); const playExistingAudio = useCallback(async () => { const audio = audioRef.current; if (!audio) return false; setSpeakingMessageId(activeMessageIdRef.current); setSpeechState("playing"); try { await audio.play(); prefetchChunk(currentChunkIndexRef.current + 1, playbackTokenRef.current); return true; } catch (error) { playbackTokenRef.current += 1; releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); console.error("[GlobalChatbox] Failed to resume Edge TTS playback:", error); return false; } }, [prefetchChunk, releaseAudio]); const speak = useCallback( async (messageId: string, text: string, options: SpeakOptions = {}) => { const normalizedText = text.trim(); if (!isSupported || !normalizedText) return; const startOffset = Math.max( 0, Math.min(options.startOffset ?? 0, normalizedText.length), ); const textToSpeak = normalizedText.slice(startOffset).trim(); const chunks = splitSpeechTextIntoChunks(textToSpeak); if (!chunks.length) return; const playbackToken = playbackTokenRef.current + 1; playbackTokenRef.current = playbackToken; releaseAudio(); chunksRef.current = chunks; currentChunkIndexRef.current = 0; activeMessageIdRef.current = messageId; setSpeakingMessageId(messageId); setSpeechState("loading"); try { await playChunk(0, playbackToken); } catch (error) { if ( error instanceof DOMException && error.name === "AbortError" && playbackToken !== playbackTokenRef.current ) { return; } releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); console.error("[GlobalChatbox] Failed to play Edge TTS audio:", error); } }, [isSupported, playChunk, releaseAudio], ); const pause = useCallback(() => { const audio = audioRef.current; if (!isSupported || !audio || speechState !== "playing") return; audio.pause(); setSpeechState("paused"); }, [isSupported, speechState]); const resume = useCallback(() => { if (!isSupported) return; void playExistingAudio(); }, [isSupported, playExistingAudio]); const stop = useCallback(() => { playbackTokenRef.current += 1; releaseAudio(); setSpeechState("idle"); setSpeakingMessageId(null); }, [releaseAudio]); useEffect(() => { return () => { playbackTokenRef.current += 1; releaseAudio(); }; }, [releaseAudio]); return { speechState, speakingMessageId, speak, pause, resume, stop, isSupported, }; } export function useSpeechRecognition(onResult: (text: string) => void) { const [isListening, setIsListening] = useState(false); const recognitionRef = useRef(null); const onResultRef = useRef(onResult); useEffect(() => { onResultRef.current = onResult; }, [onResult]); const isSupported = typeof window !== "undefined" && ("SpeechRecognition" in window || "webkitSpeechRecognition" in window); const start = useCallback(() => { if (!isSupported || recognitionRef.current) return; const Ctor = window.SpeechRecognition ?? window.webkitSpeechRecognition; if (!Ctor) return; const recognition = new Ctor(); recognition.lang = "zh-CN"; recognition.continuous = true; recognition.interimResults = false; recognition.onresult = (event: SpeechRecognitionEvent) => { for (let i = event.resultIndex; i < event.results.length; i++) { if (event.results[i].isFinal) { onResultRef.current(event.results[i][0].transcript); } } }; recognition.onerror = () => { setIsListening(false); }; recognition.onend = () => { setIsListening(false); recognitionRef.current = null; }; recognitionRef.current = recognition; setIsListening(true); recognition.start(); }, [isSupported]); const stop = useCallback(() => { recognitionRef.current?.stop(); recognitionRef.current = null; setIsListening(false); }, []); useEffect(() => { return () => { recognitionRef.current?.abort(); }; }, []); return { isListening, start, stop, isSupported }; }