feat(chat): add Edge TTS playback
This commit is contained in:
@@ -1,32 +1,11 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import config from "@/config/config";
|
||||
import type { SpeechState } from "./GlobalChatbox.types";
|
||||
import { splitSpeechTextIntoChunks } from "./speechStartOptions";
|
||||
|
||||
type AudioStreamStartResponse = {
|
||||
stream_id?: string;
|
||||
audio_url?: string;
|
||||
status_url?: string;
|
||||
result_url?: string;
|
||||
sample_rate?: number;
|
||||
channels?: number;
|
||||
error?: string;
|
||||
type SpeakOptions = {
|
||||
startOffset?: number;
|
||||
};
|
||||
|
||||
type AudioStreamStatusResponse = {
|
||||
state?: "starting" | "running" | "done" | "failed" | "closed";
|
||||
ready?: boolean;
|
||||
failed?: boolean;
|
||||
closed?: boolean;
|
||||
status_text?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type AudioStreamResultResponse = {
|
||||
run_status?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
// WebKit Speech Recognition compatibility
|
||||
interface SpeechRecognitionEvent extends Event {
|
||||
readonly resultIndex: number;
|
||||
readonly results: SpeechRecognitionResultList;
|
||||
@@ -54,56 +33,64 @@ declare global {
|
||||
new (): SpeechRecognition;
|
||||
prototype: SpeechRecognition;
|
||||
};
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
}
|
||||
}
|
||||
|
||||
export function useSpeechSynthesis() {
|
||||
const [speechState, setSpeechState] = useState<SpeechState>("idle");
|
||||
const [speakingMessageId, setSpeakingMessageId] = useState<string | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const streamAbortControllerRef = useRef<AbortController | null>(null);
|
||||
const activeSourceNodesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
|
||||
const streamIdRef = useRef<string | null>(null);
|
||||
const closeUrlRef = useRef<string | null>(null);
|
||||
const statusUrlRef = useRef<string | null>(null);
|
||||
const resultUrlRef = useRef<string | null>(null);
|
||||
const statusPollTimeoutRef = useRef<number | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const currentAudioUrlRef = useRef<string | null>(null);
|
||||
const audioObjectUrlsRef = useRef<Set<string>>(new Set());
|
||||
const fetchAbortControllersRef = useRef<Set<AbortController>>(new Set());
|
||||
const chunkAudioUrlCacheRef = useRef<Map<number, string>>(new Map());
|
||||
const chunkFetchPromisesRef = useRef<Map<number, Promise<string>>>(new Map());
|
||||
const chunksRef = useRef<string[]>([]);
|
||||
const currentChunkIndexRef = useRef(0);
|
||||
const playChunkRef = useRef<(chunkIndex: number, playbackToken: number) => Promise<void>>(
|
||||
async () => {},
|
||||
);
|
||||
const playbackTokenRef = useRef(0);
|
||||
const activeMessageIdRef = useRef<string | null>(null);
|
||||
|
||||
const isSupported =
|
||||
typeof window !== "undefined" &&
|
||||
typeof window.FormData !== "undefined" &&
|
||||
(typeof window.AudioContext !== "undefined" ||
|
||||
typeof window.webkitAudioContext !== "undefined");
|
||||
typeof window.Audio !== "undefined" &&
|
||||
typeof window.URL !== "undefined" &&
|
||||
typeof window.fetch !== "undefined";
|
||||
|
||||
const trimTrailingSlash = useCallback((value: string) => value.replace(/\/+$/, ""), []);
|
||||
const detachCurrentAudio = useCallback((revokeCurrentUrl: boolean) => {
|
||||
const audio = audioRef.current;
|
||||
audioRef.current = null;
|
||||
if (audio) {
|
||||
audio.pause();
|
||||
audio.onended = null;
|
||||
audio.onerror = null;
|
||||
audio.removeAttribute("src");
|
||||
audio.load();
|
||||
}
|
||||
|
||||
const buildServiceUrl = useCallback(
|
||||
(path: string) => `${trimTrailingSlash(config.AUDIO_SERVICE_URL)}${path.startsWith("/") ? path : `/${path}`}`,
|
||||
[trimTrailingSlash],
|
||||
);
|
||||
const currentUrl = currentAudioUrlRef.current;
|
||||
currentAudioUrlRef.current = null;
|
||||
if (revokeCurrentUrl && currentUrl) {
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
audioObjectUrlsRef.current.delete(currentUrl);
|
||||
chunkAudioUrlCacheRef.current.delete(currentChunkIndexRef.current);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const resolveServiceUrl = useCallback(
|
||||
(pathOrUrl: string) => {
|
||||
if (/^https?:\/\//i.test(pathOrUrl)) {
|
||||
return pathOrUrl;
|
||||
}
|
||||
return buildServiceUrl(pathOrUrl);
|
||||
},
|
||||
[buildServiceUrl],
|
||||
);
|
||||
|
||||
const withQueryParams = useCallback(
|
||||
(urlString: string, params: Record<string, string>) => {
|
||||
const url = new URL(urlString);
|
||||
Object.entries(params).forEach(([key, value]) => {
|
||||
url.searchParams.set(key, value);
|
||||
});
|
||||
return url.toString();
|
||||
},
|
||||
[],
|
||||
);
|
||||
const releaseAudio = useCallback(() => {
|
||||
fetchAbortControllersRef.current.forEach((controller) => controller.abort());
|
||||
fetchAbortControllersRef.current.clear();
|
||||
chunkFetchPromisesRef.current.clear();
|
||||
detachCurrentAudio(false);
|
||||
audioObjectUrlsRef.current.forEach((url) => URL.revokeObjectURL(url));
|
||||
audioObjectUrlsRef.current.clear();
|
||||
chunkAudioUrlCacheRef.current.clear();
|
||||
chunksRef.current = [];
|
||||
currentChunkIndexRef.current = 0;
|
||||
activeMessageIdRef.current = null;
|
||||
}, [detachCurrentAudio]);
|
||||
|
||||
const readErrorMessage = useCallback(async (response: Response, fallback: string) => {
|
||||
try {
|
||||
@@ -114,402 +101,170 @@ export function useSpeechSynthesis() {
|
||||
}
|
||||
}, []);
|
||||
|
||||
const closeStream = useCallback(async (closeUrl: string) => {
|
||||
const response = await fetch(closeUrl, {
|
||||
method: "POST",
|
||||
});
|
||||
const fetchChunkAudio = useCallback(
|
||||
(chunkIndex: number, playbackToken: number) => {
|
||||
const cachedUrl = chunkAudioUrlCacheRef.current.get(chunkIndex);
|
||||
if (cachedUrl) return Promise.resolve(cachedUrl);
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("[GlobalChatbox] Failed to close audio stream:", closeUrl);
|
||||
}
|
||||
}, []);
|
||||
const existingPromise = chunkFetchPromisesRef.current.get(chunkIndex);
|
||||
if (existingPromise) return existingPromise;
|
||||
|
||||
const stopStatusPolling = useCallback(() => {
|
||||
if (statusPollTimeoutRef.current !== null) {
|
||||
window.clearTimeout(statusPollTimeoutRef.current);
|
||||
statusPollTimeoutRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const fetchStreamResult = useCallback(
|
||||
async (resultUrl: string) => {
|
||||
const response = await fetch(resultUrl);
|
||||
if (response.status === 202) {
|
||||
return false;
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
await readErrorMessage(
|
||||
response,
|
||||
`Audio stream result failed with status ${response.status}`,
|
||||
),
|
||||
);
|
||||
const chunkText = chunksRef.current[chunkIndex];
|
||||
if (!chunkText) {
|
||||
return Promise.reject(new Error("Speech chunk is missing"));
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as AudioStreamResultResponse;
|
||||
if (payload.error) {
|
||||
throw new Error(payload.error);
|
||||
}
|
||||
const abortController = new AbortController();
|
||||
fetchAbortControllersRef.current.add(abortController);
|
||||
|
||||
return true;
|
||||
const promise = fetch("/api/tts/edge", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ text: chunkText }),
|
||||
signal: abortController.signal,
|
||||
})
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
await readErrorMessage(response, `Edge TTS failed with status ${response.status}`),
|
||||
);
|
||||
}
|
||||
const audioBlob = await response.blob();
|
||||
if (!audioBlob.size) {
|
||||
throw new Error("Edge TTS returned empty audio");
|
||||
}
|
||||
if (playbackToken !== playbackTokenRef.current) {
|
||||
throw new DOMException("Edge TTS chunk cancelled", "AbortError");
|
||||
}
|
||||
|
||||
const objectUrl = URL.createObjectURL(audioBlob);
|
||||
audioObjectUrlsRef.current.add(objectUrl);
|
||||
chunkAudioUrlCacheRef.current.set(chunkIndex, objectUrl);
|
||||
return objectUrl;
|
||||
})
|
||||
.finally(() => {
|
||||
fetchAbortControllersRef.current.delete(abortController);
|
||||
chunkFetchPromisesRef.current.delete(chunkIndex);
|
||||
});
|
||||
|
||||
chunkFetchPromisesRef.current.set(chunkIndex, promise);
|
||||
return promise;
|
||||
},
|
||||
[readErrorMessage],
|
||||
);
|
||||
|
||||
const clearAudio = useCallback(async () => {
|
||||
const abortController = streamAbortControllerRef.current;
|
||||
streamAbortControllerRef.current = null;
|
||||
abortController?.abort();
|
||||
|
||||
activeSourceNodesRef.current.forEach((source) => {
|
||||
try {
|
||||
source.onended = null;
|
||||
source.stop();
|
||||
} catch {
|
||||
// ignore stop errors when source already ended
|
||||
}
|
||||
source.disconnect();
|
||||
});
|
||||
activeSourceNodesRef.current.clear();
|
||||
|
||||
const audioContext = audioContextRef.current;
|
||||
audioContextRef.current = null;
|
||||
if (!audioContext) return;
|
||||
|
||||
try {
|
||||
await audioContext.close();
|
||||
} catch {
|
||||
// ignore close errors when context already closed
|
||||
}
|
||||
}, []);
|
||||
|
||||
const playPcmStream = useCallback(
|
||||
async ({
|
||||
audioUrl,
|
||||
sampleRate,
|
||||
channels,
|
||||
playbackToken,
|
||||
}: {
|
||||
audioUrl: string;
|
||||
sampleRate: number;
|
||||
channels: number;
|
||||
playbackToken: number;
|
||||
}) => {
|
||||
const AudioContextCtor = window.AudioContext ?? window.webkitAudioContext;
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("WebAudio AudioContext is not available in this browser");
|
||||
}
|
||||
|
||||
const abortController = new AbortController();
|
||||
streamAbortControllerRef.current = abortController;
|
||||
|
||||
const response = await fetch(withQueryParams(audioUrl, { format: "pcm" }), {
|
||||
signal: abortController.signal,
|
||||
const prefetchChunk = useCallback(
|
||||
(chunkIndex: number, playbackToken: number) => {
|
||||
if (chunkIndex >= chunksRef.current.length) return;
|
||||
void fetchChunkAudio(chunkIndex, playbackToken).catch((error) => {
|
||||
if (
|
||||
playbackToken === playbackTokenRef.current &&
|
||||
!(error instanceof DOMException && error.name === "AbortError")
|
||||
) {
|
||||
console.error("[GlobalChatbox] Failed to prefetch Edge TTS chunk:", error);
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
await readErrorMessage(response, `Audio stream failed with status ${response.status}`),
|
||||
);
|
||||
}
|
||||
if (!response.body) {
|
||||
throw new Error("Audio stream response body is missing");
|
||||
}
|
||||
|
||||
const audioContext = new AudioContextCtor({
|
||||
sampleRate,
|
||||
});
|
||||
audioContextRef.current = audioContext;
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const bytesPerFrame = Math.max(1, channels) * 2;
|
||||
let bufferedRemainder = new Uint8Array(0);
|
||||
let nextStartTime = audioContext.currentTime + 0.05;
|
||||
let activeSources = 0;
|
||||
let streamEnded = false;
|
||||
let resolvePlaybackDrain: (() => void) | null = null;
|
||||
const playbackDrainPromise = new Promise<void>((resolve) => {
|
||||
resolvePlaybackDrain = resolve;
|
||||
});
|
||||
|
||||
const maybeResolvePlaybackDrain = () => {
|
||||
if (streamEnded && activeSources === 0) {
|
||||
resolvePlaybackDrain?.();
|
||||
}
|
||||
};
|
||||
|
||||
const schedulePcmChunk = (pcmBytes: Uint8Array) => {
|
||||
const frameCount = pcmBytes.byteLength / bytesPerFrame;
|
||||
if (frameCount <= 0) return;
|
||||
|
||||
const buffer = audioContext.createBuffer(Math.max(1, channels), frameCount, sampleRate);
|
||||
const view = new DataView(pcmBytes.buffer, pcmBytes.byteOffset, pcmBytes.byteLength);
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
for (let channel = 0; channel < Math.max(1, channels); channel += 1) {
|
||||
const sampleIndex = frame * Math.max(1, channels) + channel;
|
||||
const pcm = view.getInt16(sampleIndex * 2, true);
|
||||
buffer.getChannelData(channel)[frame] = pcm / 32768;
|
||||
}
|
||||
}
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(audioContext.destination);
|
||||
const sourceStartTime = Math.max(nextStartTime, audioContext.currentTime + 0.01);
|
||||
nextStartTime = sourceStartTime + buffer.duration;
|
||||
|
||||
activeSources += 1;
|
||||
activeSourceNodesRef.current.add(source);
|
||||
source.onended = () => {
|
||||
activeSources -= 1;
|
||||
activeSourceNodesRef.current.delete(source);
|
||||
source.disconnect();
|
||||
maybeResolvePlaybackDrain();
|
||||
};
|
||||
source.start(sourceStartTime);
|
||||
};
|
||||
|
||||
const concatUint8Arrays = (a: Uint8Array, b: Uint8Array) => {
|
||||
if (a.byteLength === 0) return b;
|
||||
if (b.byteLength === 0) return a;
|
||||
const merged = new Uint8Array(a.byteLength + b.byteLength);
|
||||
merged.set(a);
|
||||
merged.set(b, a.byteLength);
|
||||
return merged;
|
||||
};
|
||||
|
||||
while (true) {
|
||||
if (playbackToken !== playbackTokenRef.current) {
|
||||
throw new DOMException("PCM stream playback cancelled", "AbortError");
|
||||
}
|
||||
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value || value.byteLength === 0) continue;
|
||||
|
||||
const merged = concatUint8Arrays(bufferedRemainder, value);
|
||||
const alignedByteLength = merged.byteLength - (merged.byteLength % bytesPerFrame);
|
||||
if (alignedByteLength === 0) {
|
||||
bufferedRemainder = new Uint8Array(merged);
|
||||
continue;
|
||||
}
|
||||
|
||||
const alignedChunk = merged.slice(0, alignedByteLength);
|
||||
bufferedRemainder = new Uint8Array(merged.slice(alignedByteLength));
|
||||
schedulePcmChunk(alignedChunk);
|
||||
}
|
||||
|
||||
streamEnded = true;
|
||||
maybeResolvePlaybackDrain();
|
||||
await playbackDrainPromise;
|
||||
},
|
||||
[readErrorMessage, withQueryParams],
|
||||
[fetchChunkAudio],
|
||||
);
|
||||
|
||||
const stopPlayback = useCallback(async () => {
|
||||
await clearAudio();
|
||||
stopStatusPolling();
|
||||
const playChunk = useCallback(
|
||||
async (chunkIndex: number, playbackToken: number) => {
|
||||
setSpeechState("loading");
|
||||
const objectUrl = await fetchChunkAudio(chunkIndex, playbackToken);
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
|
||||
const closeUrl = closeUrlRef.current;
|
||||
streamIdRef.current = null;
|
||||
closeUrlRef.current = null;
|
||||
statusUrlRef.current = null;
|
||||
resultUrlRef.current = null;
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
detachCurrentAudio(true);
|
||||
currentChunkIndexRef.current = chunkIndex;
|
||||
const audio = new Audio(objectUrl);
|
||||
audio.preload = "auto";
|
||||
audioRef.current = audio;
|
||||
currentAudioUrlRef.current = objectUrl;
|
||||
|
||||
if (closeUrl) {
|
||||
try {
|
||||
await closeStream(closeUrl);
|
||||
} catch (error) {
|
||||
console.error("[GlobalChatbox] Failed to close audio stream:", error);
|
||||
}
|
||||
}
|
||||
}, [clearAudio, closeStream, stopStatusPolling]);
|
||||
|
||||
const pollStreamStatus = useCallback(
|
||||
(playbackToken: number, statusUrl: string, resultUrl: string) => {
|
||||
stopStatusPolling();
|
||||
|
||||
statusPollTimeoutRef.current = window.setTimeout(async () => {
|
||||
if (
|
||||
playbackToken !== playbackTokenRef.current ||
|
||||
statusUrlRef.current !== statusUrl ||
|
||||
resultUrlRef.current !== resultUrl
|
||||
) {
|
||||
audio.onended = () => {
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
detachCurrentAudio(true);
|
||||
const nextChunkIndex = chunkIndex + 1;
|
||||
if (nextChunkIndex >= chunksRef.current.length) {
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(statusUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
await readErrorMessage(
|
||||
response,
|
||||
`Audio stream status failed with status ${response.status}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
currentChunkIndexRef.current = nextChunkIndex;
|
||||
void playChunkRef.current(nextChunkIndex, playbackToken);
|
||||
};
|
||||
audio.onerror = () => {
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
console.error("[GlobalChatbox] Edge TTS audio playback failed");
|
||||
};
|
||||
|
||||
const payload = (await response.json()) as AudioStreamStatusResponse;
|
||||
if (
|
||||
playbackToken !== playbackTokenRef.current ||
|
||||
statusUrlRef.current !== statusUrl ||
|
||||
resultUrlRef.current !== resultUrl
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.failed || payload.state === "failed") {
|
||||
console.error(
|
||||
"[GlobalChatbox] Audio stream failed:",
|
||||
payload.error || payload.status_text || statusUrl,
|
||||
);
|
||||
playbackTokenRef.current += 1;
|
||||
void stopPlayback();
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.closed || payload.state === "closed") {
|
||||
stopStatusPolling();
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.ready || payload.state === "done") {
|
||||
try {
|
||||
const isResultReady = await fetchStreamResult(resultUrl);
|
||||
if (isResultReady) {
|
||||
stopStatusPolling();
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[GlobalChatbox] Failed to fetch audio stream result:", error);
|
||||
}
|
||||
}
|
||||
|
||||
pollStreamStatus(playbackToken, statusUrl, resultUrl);
|
||||
} catch (error) {
|
||||
if (
|
||||
playbackToken === playbackTokenRef.current &&
|
||||
statusUrlRef.current === statusUrl &&
|
||||
resultUrlRef.current === resultUrl
|
||||
) {
|
||||
console.error("[GlobalChatbox] Failed to poll audio stream status:", error);
|
||||
pollStreamStatus(playbackToken, statusUrl, resultUrl);
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
await audio.play();
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
setSpeechState("playing");
|
||||
prefetchChunk(chunkIndex + 1, playbackToken);
|
||||
},
|
||||
[fetchStreamResult, readErrorMessage, stopPlayback, stopStatusPolling],
|
||||
[detachCurrentAudio, fetchChunkAudio, prefetchChunk, releaseAudio],
|
||||
);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
playbackTokenRef.current += 1;
|
||||
void stopPlayback();
|
||||
}, [stopPlayback]);
|
||||
useEffect(() => {
|
||||
playChunkRef.current = playChunk;
|
||||
}, [playChunk]);
|
||||
|
||||
const playExistingAudio = useCallback(async () => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return false;
|
||||
|
||||
setSpeakingMessageId(activeMessageIdRef.current);
|
||||
setSpeechState("playing");
|
||||
try {
|
||||
await audio.play();
|
||||
prefetchChunk(currentChunkIndexRef.current + 1, playbackTokenRef.current);
|
||||
return true;
|
||||
} catch (error) {
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
console.error("[GlobalChatbox] Failed to resume Edge TTS playback:", error);
|
||||
return false;
|
||||
}
|
||||
}, [prefetchChunk, releaseAudio]);
|
||||
|
||||
const speak = useCallback(
|
||||
async (messageId: string, text: string) => {
|
||||
async (messageId: string, text: string, options: SpeakOptions = {}) => {
|
||||
const normalizedText = text.trim();
|
||||
if (!isSupported || !normalizedText) return;
|
||||
|
||||
const startOffset = Math.max(
|
||||
0,
|
||||
Math.min(options.startOffset ?? 0, normalizedText.length),
|
||||
);
|
||||
const textToSpeak = normalizedText.slice(startOffset).trim();
|
||||
const chunks = splitSpeechTextIntoChunks(textToSpeak);
|
||||
if (!chunks.length) return;
|
||||
|
||||
const playbackToken = playbackTokenRef.current + 1;
|
||||
playbackTokenRef.current = playbackToken;
|
||||
await stopPlayback();
|
||||
releaseAudio();
|
||||
|
||||
chunksRef.current = chunks;
|
||||
currentChunkIndexRef.current = 0;
|
||||
activeMessageIdRef.current = messageId;
|
||||
|
||||
setSpeakingMessageId(messageId);
|
||||
setSpeechState("playing");
|
||||
setSpeechState("loading");
|
||||
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append("text", normalizedText);
|
||||
formData.append("demo_id", "demo-1");
|
||||
|
||||
const response = await fetch(buildServiceUrl("/api/generate-stream/start"), {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
await readErrorMessage(
|
||||
response,
|
||||
`Audio stream start failed with status ${response.status}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as AudioStreamStartResponse;
|
||||
const streamId = payload.stream_id;
|
||||
const sampleRate =
|
||||
typeof payload.sample_rate === "number" && payload.sample_rate > 0
|
||||
? payload.sample_rate
|
||||
: 24000;
|
||||
const channels =
|
||||
typeof payload.channels === "number" && payload.channels > 0
|
||||
? payload.channels
|
||||
: 1;
|
||||
const audioUrl = payload.audio_url
|
||||
? resolveServiceUrl(payload.audio_url)
|
||||
: buildServiceUrl(
|
||||
`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/audio?format=pcm`,
|
||||
);
|
||||
const rawStatusUrl = payload.status_url
|
||||
? resolveServiceUrl(payload.status_url)
|
||||
: buildServiceUrl(`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/status`);
|
||||
const statusUrl = withQueryParams(rawStatusUrl, { compact: "1" });
|
||||
const rawResultUrl = payload.result_url
|
||||
? resolveServiceUrl(payload.result_url)
|
||||
: buildServiceUrl(`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/result`);
|
||||
const resultUrl = withQueryParams(rawResultUrl, {
|
||||
compact: "1",
|
||||
include_audio: "0",
|
||||
});
|
||||
const closeUrl = buildServiceUrl(
|
||||
`/api/generate-stream/${encodeURIComponent(streamId ?? "")}/close`,
|
||||
);
|
||||
|
||||
if (!streamId) {
|
||||
throw new Error(payload.error || "Audio stream start response is missing stream_id");
|
||||
}
|
||||
|
||||
if (playbackToken !== playbackTokenRef.current) {
|
||||
await closeStream(closeUrl);
|
||||
return;
|
||||
}
|
||||
|
||||
streamIdRef.current = streamId;
|
||||
closeUrlRef.current = closeUrl;
|
||||
statusUrlRef.current = statusUrl;
|
||||
resultUrlRef.current = resultUrl;
|
||||
|
||||
pollStreamStatus(playbackToken, statusUrl, resultUrl);
|
||||
await playPcmStream({
|
||||
audioUrl,
|
||||
sampleRate,
|
||||
channels,
|
||||
playbackToken,
|
||||
});
|
||||
|
||||
if (playbackToken !== playbackTokenRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
await clearAudio();
|
||||
if (streamIdRef.current === streamId) {
|
||||
streamIdRef.current = null;
|
||||
closeUrlRef.current = null;
|
||||
statusUrlRef.current = null;
|
||||
resultUrlRef.current = null;
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
}
|
||||
stopStatusPolling();
|
||||
await fetchStreamResult(resultUrl).catch((error) => {
|
||||
console.error("[GlobalChatbox] Failed to fetch audio stream result:", error);
|
||||
});
|
||||
await closeStream(closeUrl);
|
||||
await playChunk(0, playbackToken);
|
||||
} catch (error) {
|
||||
await clearAudio();
|
||||
if (
|
||||
error instanceof DOMException &&
|
||||
error.name === "AbortError" &&
|
||||
@@ -517,73 +272,51 @@ export function useSpeechSynthesis() {
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const closeUrl = closeUrlRef.current;
|
||||
streamIdRef.current = null;
|
||||
closeUrlRef.current = null;
|
||||
statusUrlRef.current = null;
|
||||
resultUrlRef.current = null;
|
||||
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
if (closeUrl) {
|
||||
try {
|
||||
await closeStream(closeUrl);
|
||||
} catch (closeError) {
|
||||
console.error("[GlobalChatbox] Failed to close audio stream:", closeError);
|
||||
}
|
||||
}
|
||||
console.error("[GlobalChatbox] Failed to play audio stream:", error);
|
||||
console.error("[GlobalChatbox] Failed to play Edge TTS audio:", error);
|
||||
}
|
||||
},
|
||||
[
|
||||
buildServiceUrl,
|
||||
clearAudio,
|
||||
closeStream,
|
||||
fetchStreamResult,
|
||||
isSupported,
|
||||
playPcmStream,
|
||||
readErrorMessage,
|
||||
resolveServiceUrl,
|
||||
pollStreamStatus,
|
||||
stopPlayback,
|
||||
stopStatusPolling,
|
||||
withQueryParams,
|
||||
],
|
||||
[isSupported, playChunk, releaseAudio],
|
||||
);
|
||||
|
||||
const pause = useCallback(() => {
|
||||
if (!isSupported || !audioContextRef.current) return;
|
||||
void audioContextRef.current.suspend().then(
|
||||
() => {
|
||||
setSpeechState("paused");
|
||||
},
|
||||
(error) => {
|
||||
console.error("[GlobalChatbox] Failed to pause PCM playback:", error);
|
||||
},
|
||||
);
|
||||
}, [isSupported]);
|
||||
const audio = audioRef.current;
|
||||
if (!isSupported || !audio || speechState !== "playing") return;
|
||||
audio.pause();
|
||||
setSpeechState("paused");
|
||||
}, [isSupported, speechState]);
|
||||
|
||||
const resume = useCallback(() => {
|
||||
if (!isSupported || !audioContextRef.current) return;
|
||||
void audioContextRef.current.resume().then(
|
||||
() => {
|
||||
setSpeechState("playing");
|
||||
},
|
||||
(error) => {
|
||||
playbackTokenRef.current += 1;
|
||||
void stopPlayback();
|
||||
console.error("[GlobalChatbox] Failed to resume audio playback:", error);
|
||||
},
|
||||
);
|
||||
}, [isSupported, stopPlayback]);
|
||||
if (!isSupported) return;
|
||||
void playExistingAudio();
|
||||
}, [isSupported, playExistingAudio]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
}, [releaseAudio]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
playbackTokenRef.current += 1;
|
||||
void stopPlayback();
|
||||
releaseAudio();
|
||||
};
|
||||
}, [stopPlayback]);
|
||||
}, [releaseAudio]);
|
||||
|
||||
return { speechState, speakingMessageId, speak, pause, resume, stop, isSupported };
|
||||
return {
|
||||
speechState,
|
||||
speakingMessageId,
|
||||
speak,
|
||||
pause,
|
||||
resume,
|
||||
stop,
|
||||
isSupported,
|
||||
};
|
||||
}
|
||||
|
||||
export function useSpeechRecognition(onResult: (text: string) => void) {
|
||||
@@ -618,7 +351,6 @@ export function useSpeechRecognition(onResult: (text: string) => void) {
|
||||
|
||||
recognition.onerror = () => {
|
||||
setIsListening(false);
|
||||
recognitionRef.current = null;
|
||||
};
|
||||
|
||||
recognition.onend = () => {
|
||||
@@ -627,8 +359,8 @@ export function useSpeechRecognition(onResult: (text: string) => void) {
|
||||
};
|
||||
|
||||
recognitionRef.current = recognition;
|
||||
recognition.start();
|
||||
setIsListening(true);
|
||||
recognition.start();
|
||||
}, [isSupported]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
@@ -639,7 +371,7 @@ export function useSpeechRecognition(onResult: (text: string) => void) {
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
recognitionRef.current?.stop();
|
||||
recognitionRef.current?.abort();
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user