380 lines
12 KiB
TypeScript
380 lines
12 KiB
TypeScript
import { useCallback, useEffect, useRef, useState } from "react";
|
|
import type { SpeechState } from "./GlobalChatbox.types";
|
|
import { splitSpeechTextIntoChunks } from "./speechStartOptions";
|
|
|
|
type SpeakOptions = {
|
|
startOffset?: number;
|
|
};
|
|
|
|
interface SpeechRecognitionEvent extends Event {
|
|
readonly resultIndex: number;
|
|
readonly results: SpeechRecognitionResultList;
|
|
}
|
|
|
|
interface SpeechRecognition extends EventTarget {
|
|
lang: string;
|
|
continuous: boolean;
|
|
interimResults: boolean;
|
|
onresult: ((event: SpeechRecognitionEvent) => void) | null;
|
|
onerror: ((event: Event) => void) | null;
|
|
onend: (() => void) | null;
|
|
start(): void;
|
|
stop(): void;
|
|
abort(): void;
|
|
}
|
|
|
|
declare global {
|
|
interface Window {
|
|
SpeechRecognition?: {
|
|
new (): SpeechRecognition;
|
|
prototype: SpeechRecognition;
|
|
};
|
|
webkitSpeechRecognition?: {
|
|
new (): SpeechRecognition;
|
|
prototype: SpeechRecognition;
|
|
};
|
|
}
|
|
}
|
|
|
|
export function useSpeechSynthesis() {
|
|
const [speechState, setSpeechState] = useState<SpeechState>("idle");
|
|
const [speakingMessageId, setSpeakingMessageId] = useState<string | null>(null);
|
|
const audioRef = useRef<HTMLAudioElement | null>(null);
|
|
const currentAudioUrlRef = useRef<string | null>(null);
|
|
const audioObjectUrlsRef = useRef<Set<string>>(new Set());
|
|
const fetchAbortControllersRef = useRef<Set<AbortController>>(new Set());
|
|
const chunkAudioUrlCacheRef = useRef<Map<number, string>>(new Map());
|
|
const chunkFetchPromisesRef = useRef<Map<number, Promise<string>>>(new Map());
|
|
const chunksRef = useRef<string[]>([]);
|
|
const currentChunkIndexRef = useRef(0);
|
|
const playChunkRef = useRef<(chunkIndex: number, playbackToken: number) => Promise<void>>(
|
|
async () => {},
|
|
);
|
|
const playbackTokenRef = useRef(0);
|
|
const activeMessageIdRef = useRef<string | null>(null);
|
|
|
|
const isSupported =
|
|
typeof window !== "undefined" &&
|
|
typeof window.Audio !== "undefined" &&
|
|
typeof window.URL !== "undefined" &&
|
|
typeof window.fetch !== "undefined";
|
|
|
|
const detachCurrentAudio = useCallback((revokeCurrentUrl: boolean) => {
|
|
const audio = audioRef.current;
|
|
audioRef.current = null;
|
|
if (audio) {
|
|
audio.pause();
|
|
audio.onended = null;
|
|
audio.onerror = null;
|
|
audio.removeAttribute("src");
|
|
audio.load();
|
|
}
|
|
|
|
const currentUrl = currentAudioUrlRef.current;
|
|
currentAudioUrlRef.current = null;
|
|
if (revokeCurrentUrl && currentUrl) {
|
|
URL.revokeObjectURL(currentUrl);
|
|
audioObjectUrlsRef.current.delete(currentUrl);
|
|
chunkAudioUrlCacheRef.current.delete(currentChunkIndexRef.current);
|
|
}
|
|
}, []);
|
|
|
|
const releaseAudio = useCallback(() => {
|
|
fetchAbortControllersRef.current.forEach((controller) => controller.abort());
|
|
fetchAbortControllersRef.current.clear();
|
|
chunkFetchPromisesRef.current.clear();
|
|
detachCurrentAudio(false);
|
|
audioObjectUrlsRef.current.forEach((url) => URL.revokeObjectURL(url));
|
|
audioObjectUrlsRef.current.clear();
|
|
chunkAudioUrlCacheRef.current.clear();
|
|
chunksRef.current = [];
|
|
currentChunkIndexRef.current = 0;
|
|
activeMessageIdRef.current = null;
|
|
}, [detachCurrentAudio]);
|
|
|
|
const readErrorMessage = useCallback(async (response: Response, fallback: string) => {
|
|
try {
|
|
const payload = (await response.json()) as { error?: string; message?: string };
|
|
return payload.error || payload.message || fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}, []);
|
|
|
|
const fetchChunkAudio = useCallback(
|
|
(chunkIndex: number, playbackToken: number) => {
|
|
const cachedUrl = chunkAudioUrlCacheRef.current.get(chunkIndex);
|
|
if (cachedUrl) return Promise.resolve(cachedUrl);
|
|
|
|
const existingPromise = chunkFetchPromisesRef.current.get(chunkIndex);
|
|
if (existingPromise) return existingPromise;
|
|
|
|
const chunkText = chunksRef.current[chunkIndex];
|
|
if (!chunkText) {
|
|
return Promise.reject(new Error("Speech chunk is missing"));
|
|
}
|
|
|
|
const abortController = new AbortController();
|
|
fetchAbortControllersRef.current.add(abortController);
|
|
|
|
const promise = fetch("/api/tts/edge", {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({ text: chunkText }),
|
|
signal: abortController.signal,
|
|
})
|
|
.then(async (response) => {
|
|
if (!response.ok) {
|
|
throw new Error(
|
|
await readErrorMessage(response, `Edge TTS failed with status ${response.status}`),
|
|
);
|
|
}
|
|
const audioBlob = await response.blob();
|
|
if (!audioBlob.size) {
|
|
throw new Error("Edge TTS returned empty audio");
|
|
}
|
|
if (playbackToken !== playbackTokenRef.current) {
|
|
throw new DOMException("Edge TTS chunk cancelled", "AbortError");
|
|
}
|
|
|
|
const objectUrl = URL.createObjectURL(audioBlob);
|
|
audioObjectUrlsRef.current.add(objectUrl);
|
|
chunkAudioUrlCacheRef.current.set(chunkIndex, objectUrl);
|
|
return objectUrl;
|
|
})
|
|
.finally(() => {
|
|
fetchAbortControllersRef.current.delete(abortController);
|
|
chunkFetchPromisesRef.current.delete(chunkIndex);
|
|
});
|
|
|
|
chunkFetchPromisesRef.current.set(chunkIndex, promise);
|
|
return promise;
|
|
},
|
|
[readErrorMessage],
|
|
);
|
|
|
|
const prefetchChunk = useCallback(
|
|
(chunkIndex: number, playbackToken: number) => {
|
|
if (chunkIndex >= chunksRef.current.length) return;
|
|
void fetchChunkAudio(chunkIndex, playbackToken).catch((error) => {
|
|
if (
|
|
playbackToken === playbackTokenRef.current &&
|
|
!(error instanceof DOMException && error.name === "AbortError")
|
|
) {
|
|
console.error("[GlobalChatbox] Failed to prefetch Edge TTS chunk:", error);
|
|
}
|
|
});
|
|
},
|
|
[fetchChunkAudio],
|
|
);
|
|
|
|
const playChunk = useCallback(
|
|
async (chunkIndex: number, playbackToken: number) => {
|
|
setSpeechState("loading");
|
|
const objectUrl = await fetchChunkAudio(chunkIndex, playbackToken);
|
|
if (playbackToken !== playbackTokenRef.current) return;
|
|
|
|
detachCurrentAudio(true);
|
|
currentChunkIndexRef.current = chunkIndex;
|
|
const audio = new Audio(objectUrl);
|
|
audio.preload = "auto";
|
|
audioRef.current = audio;
|
|
currentAudioUrlRef.current = objectUrl;
|
|
|
|
audio.onended = () => {
|
|
if (playbackToken !== playbackTokenRef.current) return;
|
|
detachCurrentAudio(true);
|
|
const nextChunkIndex = chunkIndex + 1;
|
|
if (nextChunkIndex >= chunksRef.current.length) {
|
|
releaseAudio();
|
|
setSpeechState("idle");
|
|
setSpeakingMessageId(null);
|
|
return;
|
|
}
|
|
|
|
currentChunkIndexRef.current = nextChunkIndex;
|
|
void playChunkRef.current(nextChunkIndex, playbackToken);
|
|
};
|
|
audio.onerror = () => {
|
|
if (playbackToken !== playbackTokenRef.current) return;
|
|
playbackTokenRef.current += 1;
|
|
releaseAudio();
|
|
setSpeechState("idle");
|
|
setSpeakingMessageId(null);
|
|
console.error("[GlobalChatbox] Edge TTS audio playback failed");
|
|
};
|
|
|
|
await audio.play();
|
|
if (playbackToken !== playbackTokenRef.current) return;
|
|
setSpeechState("playing");
|
|
prefetchChunk(chunkIndex + 1, playbackToken);
|
|
},
|
|
[detachCurrentAudio, fetchChunkAudio, prefetchChunk, releaseAudio],
|
|
);
|
|
|
|
useEffect(() => {
|
|
playChunkRef.current = playChunk;
|
|
}, [playChunk]);
|
|
|
|
const playExistingAudio = useCallback(async () => {
|
|
const audio = audioRef.current;
|
|
if (!audio) return false;
|
|
|
|
setSpeakingMessageId(activeMessageIdRef.current);
|
|
setSpeechState("playing");
|
|
try {
|
|
await audio.play();
|
|
prefetchChunk(currentChunkIndexRef.current + 1, playbackTokenRef.current);
|
|
return true;
|
|
} catch (error) {
|
|
playbackTokenRef.current += 1;
|
|
releaseAudio();
|
|
setSpeechState("idle");
|
|
setSpeakingMessageId(null);
|
|
console.error("[GlobalChatbox] Failed to resume Edge TTS playback:", error);
|
|
return false;
|
|
}
|
|
}, [prefetchChunk, releaseAudio]);
|
|
|
|
const speak = useCallback(
|
|
async (messageId: string, text: string, options: SpeakOptions = {}) => {
|
|
const normalizedText = text.trim();
|
|
if (!isSupported || !normalizedText) return;
|
|
|
|
const startOffset = Math.max(
|
|
0,
|
|
Math.min(options.startOffset ?? 0, normalizedText.length),
|
|
);
|
|
const textToSpeak = normalizedText.slice(startOffset).trim();
|
|
const chunks = splitSpeechTextIntoChunks(textToSpeak);
|
|
if (!chunks.length) return;
|
|
|
|
const playbackToken = playbackTokenRef.current + 1;
|
|
playbackTokenRef.current = playbackToken;
|
|
releaseAudio();
|
|
|
|
chunksRef.current = chunks;
|
|
currentChunkIndexRef.current = 0;
|
|
activeMessageIdRef.current = messageId;
|
|
|
|
setSpeakingMessageId(messageId);
|
|
setSpeechState("loading");
|
|
|
|
try {
|
|
await playChunk(0, playbackToken);
|
|
} catch (error) {
|
|
if (
|
|
error instanceof DOMException &&
|
|
error.name === "AbortError" &&
|
|
playbackToken !== playbackTokenRef.current
|
|
) {
|
|
return;
|
|
}
|
|
|
|
releaseAudio();
|
|
setSpeechState("idle");
|
|
setSpeakingMessageId(null);
|
|
console.error("[GlobalChatbox] Failed to play Edge TTS audio:", error);
|
|
}
|
|
},
|
|
[isSupported, playChunk, releaseAudio],
|
|
);
|
|
|
|
const pause = useCallback(() => {
|
|
const audio = audioRef.current;
|
|
if (!isSupported || !audio || speechState !== "playing") return;
|
|
audio.pause();
|
|
setSpeechState("paused");
|
|
}, [isSupported, speechState]);
|
|
|
|
const resume = useCallback(() => {
|
|
if (!isSupported) return;
|
|
void playExistingAudio();
|
|
}, [isSupported, playExistingAudio]);
|
|
|
|
const stop = useCallback(() => {
|
|
playbackTokenRef.current += 1;
|
|
releaseAudio();
|
|
setSpeechState("idle");
|
|
setSpeakingMessageId(null);
|
|
}, [releaseAudio]);
|
|
|
|
useEffect(() => {
|
|
return () => {
|
|
playbackTokenRef.current += 1;
|
|
releaseAudio();
|
|
};
|
|
}, [releaseAudio]);
|
|
|
|
return {
|
|
speechState,
|
|
speakingMessageId,
|
|
speak,
|
|
pause,
|
|
resume,
|
|
stop,
|
|
isSupported,
|
|
};
|
|
}
|
|
|
|
export function useSpeechRecognition(onResult: (text: string) => void) {
|
|
const [isListening, setIsListening] = useState(false);
|
|
const recognitionRef = useRef<SpeechRecognition | null>(null);
|
|
const onResultRef = useRef(onResult);
|
|
useEffect(() => {
|
|
onResultRef.current = onResult;
|
|
}, [onResult]);
|
|
|
|
const isSupported =
|
|
typeof window !== "undefined" &&
|
|
("SpeechRecognition" in window || "webkitSpeechRecognition" in window);
|
|
|
|
const start = useCallback(() => {
|
|
if (!isSupported || recognitionRef.current) return;
|
|
const Ctor = window.SpeechRecognition ?? window.webkitSpeechRecognition;
|
|
if (!Ctor) return;
|
|
|
|
const recognition = new Ctor();
|
|
recognition.lang = "zh-CN";
|
|
recognition.continuous = true;
|
|
recognition.interimResults = false;
|
|
|
|
recognition.onresult = (event: SpeechRecognitionEvent) => {
|
|
for (let i = event.resultIndex; i < event.results.length; i++) {
|
|
if (event.results[i].isFinal) {
|
|
onResultRef.current(event.results[i][0].transcript);
|
|
}
|
|
}
|
|
};
|
|
|
|
recognition.onerror = () => {
|
|
setIsListening(false);
|
|
};
|
|
|
|
recognition.onend = () => {
|
|
setIsListening(false);
|
|
recognitionRef.current = null;
|
|
};
|
|
|
|
recognitionRef.current = recognition;
|
|
setIsListening(true);
|
|
recognition.start();
|
|
}, [isSupported]);
|
|
|
|
const stop = useCallback(() => {
|
|
recognitionRef.current?.stop();
|
|
recognitionRef.current = null;
|
|
setIsListening(false);
|
|
}, []);
|
|
|
|
useEffect(() => {
|
|
return () => {
|
|
recognitionRef.current?.abort();
|
|
};
|
|
}, []);
|
|
|
|
return { isListening, start, stop, isSupported };
|
|
}
|