feat: extend agent-driven map workbench
This commit is contained in:
@@ -8,13 +8,23 @@ import {
|
||||
DatabaseZap,
|
||||
GitMerge,
|
||||
History,
|
||||
LoaderCircle,
|
||||
MapPinned,
|
||||
Mic,
|
||||
Pause,
|
||||
Play,
|
||||
Plus,
|
||||
SendHorizontal
|
||||
SendHorizontal,
|
||||
Shield,
|
||||
ShieldCheck,
|
||||
Square,
|
||||
Volume2
|
||||
} from "lucide-react";
|
||||
import { AnimatePresence, MotionConfig, motion } from "motion/react";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { AnimatePresence, MotionConfig, motion, useReducedMotion } from "motion/react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { useStickToBottomContext } from "use-stick-to-bottom";
|
||||
import { showMapNotice } from "@/features/map/core/components/notice";
|
||||
import { Agent, AgentContent } from "@/shared/ai-elements/agent";
|
||||
import {
|
||||
Conversation,
|
||||
@@ -52,7 +62,15 @@ import { AgentMessageDetails, AgentMessageProgress } from "./agent-message-detai
|
||||
import { AgentOperationalBrief } from "./agent-operational-brief";
|
||||
import { AgentUiEnvelopeRenderer } from "./agent-ui-envelope-renderer";
|
||||
import { StreamingTokenResponse } from "./streaming-token-response";
|
||||
import { useAgentSpeechRecognition, useAgentSpeechSynthesis } from "../hooks/use-agent-voice";
|
||||
import {
|
||||
findSpeechSelectionStartOffset,
|
||||
stripSpeechMarkdown,
|
||||
type AgentSpeakOptions,
|
||||
type AgentSpeechState
|
||||
} from "../speech";
|
||||
import type {
|
||||
AgentApprovalMode,
|
||||
AgentChatMessage,
|
||||
AgentModelOption,
|
||||
AgentPermissionReply,
|
||||
@@ -75,6 +93,7 @@ type AgentCommandPanelProps = {
|
||||
streamRenderState?: AgentStreamRenderState;
|
||||
modelOptions?: AgentModelOption[];
|
||||
selectedModel?: string;
|
||||
approvalMode?: AgentApprovalMode;
|
||||
uiResults?: AgentUiResult[];
|
||||
onCollapse: () => void;
|
||||
onRefreshHistory?: () => Promise<void> | void;
|
||||
@@ -83,6 +102,7 @@ type AgentCommandPanelProps = {
|
||||
onRenameHistorySession?: (sessionId: string, title: string) => Promise<void> | void;
|
||||
onDeleteHistorySession?: (sessionId: string) => Promise<void> | void;
|
||||
onSelectModel?: (model: string) => void;
|
||||
onApprovalModeChange?: (mode: AgentApprovalMode) => void;
|
||||
onSubmitPrompt: (prompt: string) => Promise<void> | void;
|
||||
onStopPrompt?: () => void;
|
||||
onReplyPermission?: (permission: AgentPermissionRequest, reply: AgentPermissionReply) => Promise<void> | void;
|
||||
@@ -116,6 +136,7 @@ export function AgentCommandPanel({
|
||||
streamRenderState = {},
|
||||
modelOptions = [],
|
||||
selectedModel = "",
|
||||
approvalMode = "request",
|
||||
uiResults = [],
|
||||
onCollapse,
|
||||
onRefreshHistory,
|
||||
@@ -124,6 +145,7 @@ export function AgentCommandPanel({
|
||||
onRenameHistorySession,
|
||||
onDeleteHistorySession,
|
||||
onSelectModel,
|
||||
onApprovalModeChange,
|
||||
onSubmitPrompt,
|
||||
onStopPrompt,
|
||||
onReplyPermission,
|
||||
@@ -135,6 +157,33 @@ export function AgentCommandPanel({
|
||||
const [loadingSessionId, setLoadingSessionId] = useState<string | null>(null);
|
||||
const [scrollRequestId, setScrollRequestId] = useState(0);
|
||||
const trimmedPrompt = prompt.trim();
|
||||
const {
|
||||
speechState,
|
||||
speakingMessageId,
|
||||
speak,
|
||||
pause: pauseSpeech,
|
||||
resume: resumeSpeech,
|
||||
stop: stopSpeech,
|
||||
isSupported: isSpeechSupported
|
||||
} = useAgentSpeechSynthesis();
|
||||
const appendRecognitionResult = useCallback((text: string) => {
|
||||
setPrompt((current) => `${current}${current.trim() ? " " : ""}${text}`);
|
||||
}, []);
|
||||
const showRecognitionError = useCallback((message: string) => {
|
||||
showMapNotice({
|
||||
id: "agent-speech-recognition-error",
|
||||
tone: "error",
|
||||
title: "语音输入不可用",
|
||||
message,
|
||||
duration: 5000
|
||||
});
|
||||
}, []);
|
||||
const {
|
||||
isListening,
|
||||
start: startListening,
|
||||
stop: stopListening,
|
||||
isSupported: isRecognitionSupported
|
||||
} = useAgentSpeechRecognition(appendRecognitionResult);
|
||||
|
||||
useEffect(() => {
|
||||
if (historyOpen) {
|
||||
@@ -142,11 +191,20 @@ export function AgentCommandPanel({
|
||||
}
|
||||
}, [historyOpen, onRefreshHistory]);
|
||||
|
||||
useEffect(() => {
|
||||
stopSpeech();
|
||||
}, [activeSessionId, stopSpeech]);
|
||||
|
||||
useEffect(() => {
|
||||
if (streaming) stopListening();
|
||||
}, [stopListening, streaming]);
|
||||
|
||||
const submitPrompt = (nextPrompt: string) => {
|
||||
if (!nextPrompt || streaming) {
|
||||
return;
|
||||
}
|
||||
setPrompt("");
|
||||
stopListening();
|
||||
setScrollRequestId((requestId) => requestId + 1);
|
||||
void Promise.resolve(onSubmitPrompt(nextPrompt)).catch(() => {
|
||||
setPrompt(nextPrompt);
|
||||
@@ -166,7 +224,7 @@ export function AgentCommandPanel({
|
||||
)}
|
||||
>
|
||||
<Agent className="flex h-full min-h-0 flex-col border-0 bg-transparent">
|
||||
<header className="agent-panel-header acrylic-panel">
|
||||
<header className="agent-panel-header">
|
||||
<div className="flex min-h-16 items-center justify-between gap-2 px-3 py-2">
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2.5 text-slate-900">
|
||||
<AgentPersona
|
||||
@@ -289,6 +347,13 @@ export function AgentCommandPanel({
|
||||
onReplyPermission={onReplyPermission}
|
||||
onReplyQuestion={onReplyQuestion}
|
||||
onRejectQuestion={onRejectQuestion}
|
||||
isSpeechSupported={isSpeechSupported}
|
||||
speakingMessageId={speakingMessageId}
|
||||
speechState={speechState}
|
||||
onSpeak={speak}
|
||||
onPauseSpeech={pauseSpeech}
|
||||
onResumeSpeech={resumeSpeech}
|
||||
onStopSpeech={stopSpeech}
|
||||
/>
|
||||
) : (
|
||||
<motion.div
|
||||
@@ -349,13 +414,27 @@ export function AgentCommandPanel({
|
||||
placeholder="输入调度问题，Agent 将通过后端会话流式响应"
|
||||
/>
|
||||
</PromptInputBody>
|
||||
<PromptInputFooter className="justify-end">
|
||||
<PromptInputFooter className="justify-between gap-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<ApprovalModeSelect
|
||||
value={approvalMode}
|
||||
disabled={streaming}
|
||||
onValueChange={onApprovalModeChange}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-1.5">
|
||||
<AgentModelSelect
|
||||
models={modelOptions}
|
||||
value={selectedModel}
|
||||
onValueChange={onSelectModel}
|
||||
/>
|
||||
{isRecognitionSupported ? (
|
||||
<VoiceInputButton
|
||||
isListening={isListening}
|
||||
disabled={streaming}
|
||||
onClick={isListening ? stopListening : () => startListening(showRecognitionError)}
|
||||
/>
|
||||
) : null}
|
||||
<PromptInputSubmit
|
||||
aria-label={streaming ? "停止 Agent 指令" : "发送 Agent 指令"}
|
||||
title={streaming ? "停止 Agent 指令" : "发送 Agent 指令"}
|
||||
@@ -376,6 +455,177 @@ export function AgentCommandPanel({
|
||||
);
|
||||
}
|
||||
|
||||
const VOICE_WAVE_BARS = [
|
||||
{ height: 5, duration: 0.72, delay: 0.08 },
|
||||
{ height: 10, duration: 0.58, delay: 0 },
|
||||
{ height: 15, duration: 0.66, delay: 0.12 },
|
||||
{ height: 9, duration: 0.62, delay: 0.04 },
|
||||
{ height: 4, duration: 0.7, delay: 0.16 }
|
||||
] as const;
|
||||
|
||||
function VoiceInputButton({
|
||||
isListening,
|
||||
disabled,
|
||||
onClick
|
||||
}: {
|
||||
isListening: boolean;
|
||||
disabled: boolean;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
const reduceMotion = useReducedMotion();
|
||||
const label = isListening ? "停止语音输入" : "语音输入";
|
||||
|
||||
return (
|
||||
<motion.button
|
||||
type="button"
|
||||
aria-label={label}
|
||||
aria-pressed={isListening}
|
||||
title={label}
|
||||
className={cn(
|
||||
"group relative grid h-8 w-8 shrink-0 place-items-center overflow-visible rounded-lg border",
|
||||
"border-transparent bg-transparent text-blue-600 shadow-none",
|
||||
"transition-[color,background-color,border-color,box-shadow,transform] duration-200",
|
||||
"hover:bg-slate-50 hover:text-blue-700",
|
||||
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-blue-600/25 focus-visible:ring-offset-1",
|
||||
"disabled:cursor-not-allowed disabled:opacity-45",
|
||||
isListening &&
|
||||
"bg-red-50 text-red-600 hover:bg-red-100 hover:text-red-700"
|
||||
)}
|
||||
disabled={disabled}
|
||||
onClick={onClick}
|
||||
whileTap={reduceMotion ? undefined : { scale: 0.94 }}
|
||||
>
|
||||
{isListening ? (
|
||||
<motion.span
|
||||
data-slot="voice-input-pulse"
|
||||
className="pointer-events-none absolute inset-0 rounded-lg bg-red-400/20"
|
||||
aria-hidden="true"
|
||||
initial={{ opacity: 0, scale: 0.9 }}
|
||||
animate={reduceMotion ? { opacity: 0.38, scale: 1 } : { opacity: [0.4, 0], scale: [1, 1.28] }}
|
||||
transition={reduceMotion ? { duration: 0.16 } : { duration: 1.7, ease: "easeOut", repeat: Infinity }}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
<span className="relative grid h-5 w-5 place-items-center overflow-hidden">
|
||||
<AnimatePresence initial={false} mode="wait">
|
||||
{isListening ? (
|
||||
<motion.span
|
||||
key="voice-wave"
|
||||
className="flex h-4 items-center gap-0.5"
|
||||
aria-hidden="true"
|
||||
initial={{ opacity: 0, scale: 0.72 }}
|
||||
animate={{ opacity: 1, scale: 1 }}
|
||||
exit={{ opacity: 0, scale: 0.72 }}
|
||||
transition={{ duration: 0.16, ease: "easeOut" }}
|
||||
>
|
||||
{VOICE_WAVE_BARS.map((bar) => (
|
||||
<motion.span
|
||||
key={bar.height}
|
||||
className="w-px rounded-full bg-red-500"
|
||||
style={{ height: bar.height }}
|
||||
animate={
|
||||
reduceMotion
|
||||
? { scaleY: 0.72 }
|
||||
: { scaleY: [0.38, 1, 0.52, 0.82, 0.38] }
|
||||
}
|
||||
transition={
|
||||
reduceMotion
|
||||
? { duration: 0.16 }
|
||||
: { duration: bar.duration, delay: bar.delay, ease: "easeInOut", repeat: Infinity }
|
||||
}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
))}
|
||||
</motion.span>
|
||||
) : (
|
||||
<motion.span
|
||||
key="voice-microphone"
|
||||
className="grid place-items-center"
|
||||
aria-hidden="true"
|
||||
initial={{ opacity: 0, scale: 0.72, y: 2 }}
|
||||
animate={{ opacity: 1, scale: 1, y: 0 }}
|
||||
exit={{ opacity: 0, scale: 0.72, y: -2 }}
|
||||
transition={{ duration: 0.16, ease: "easeOut" }}
|
||||
>
|
||||
<Mic
|
||||
size={15}
|
||||
strokeWidth={2.15}
|
||||
className="transition-transform duration-200 group-hover:scale-105"
|
||||
/>
|
||||
</motion.span>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
</span>
|
||||
</motion.button>
|
||||
);
|
||||
}
|
||||
|
||||
function ApprovalModeSelect({
|
||||
value,
|
||||
disabled,
|
||||
onValueChange
|
||||
}: {
|
||||
value: AgentApprovalMode;
|
||||
disabled: boolean;
|
||||
onValueChange?: (mode: AgentApprovalMode) => void;
|
||||
}) {
|
||||
const requestApproval = value === "request";
|
||||
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
className="h-8 shrink-0 gap-1.5 rounded-md border-transparent bg-transparent px-1.5 text-xs text-slate-600 shadow-none hover:bg-slate-50"
|
||||
disabled={disabled || !onValueChange}
|
||||
aria-label="权限批准模式"
|
||||
>
|
||||
{requestApproval ? <Shield size={14} aria-hidden="true" /> : <ShieldCheck size={14} aria-hidden="true" />}
|
||||
<span>{requestApproval ? "请求批准" : "始终允许"}</span>
|
||||
<ChevronsUpDown size={12} className="opacity-50" aria-hidden="true" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent
|
||||
side="top"
|
||||
align="start"
|
||||
sideOffset={8}
|
||||
className="agent-panel-control w-[220px] rounded-xl p-2 shadow-lg"
|
||||
>
|
||||
<DropdownMenuLabel className="px-2 pb-2 pt-1 text-xs text-slate-500">权限批准模式</DropdownMenuLabel>
|
||||
<DropdownMenuItem
|
||||
className={cn(
|
||||
"items-start rounded-lg border border-transparent px-2.5 py-2 focus:bg-slate-50",
|
||||
requestApproval && "border-blue-200 bg-blue-50 focus:bg-blue-50"
|
||||
)}
|
||||
onSelect={() => onValueChange?.("request")}
|
||||
>
|
||||
<Shield className="mt-0.5 text-blue-600" aria-hidden="true" />
|
||||
<span>
|
||||
<span className="block text-sm font-semibold text-slate-800">请求批准</span>
|
||||
<span className="block text-xs leading-5 text-slate-500">工具权限逐次确认</span>
|
||||
</span>
|
||||
{requestApproval ? <CheckCircle2 className="ml-auto mt-0.5 text-blue-600" aria-hidden="true" /> : null}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
className={cn(
|
||||
"mt-1 items-start rounded-lg border border-transparent px-2.5 py-2 focus:bg-slate-50",
|
||||
!requestApproval && "border-emerald-200 bg-emerald-50 focus:bg-emerald-50"
|
||||
)}
|
||||
onSelect={() => onValueChange?.("always")}
|
||||
>
|
||||
<ShieldCheck className="mt-0.5 text-emerald-600" aria-hidden="true" />
|
||||
<span>
|
||||
<span className="block text-sm font-semibold text-slate-800">始终允许</span>
|
||||
<span className="block text-xs leading-5 text-slate-500">自动允许本轮权限请求</span>
|
||||
</span>
|
||||
{!requestApproval ? <CheckCircle2 className="ml-auto mt-0.5 text-emerald-600" aria-hidden="true" /> : null}
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
}
|
||||
|
||||
type AgentConversationScrollSnapshot = {
|
||||
activeSessionId?: string | null;
|
||||
lastUserMessageId: string | null;
|
||||
@@ -593,27 +843,46 @@ function AgentEmptyState() {
|
||||
return (
|
||||
<section
|
||||
aria-labelledby="agent-empty-state-title"
|
||||
className="surface-reading mx-auto w-full max-w-sm rounded-2xl border border-slate-200/80 px-4 pb-4 pt-5 text-center shadow-sm"
|
||||
className="surface-reading mx-auto w-full max-w-[360px] overflow-hidden rounded-2xl border border-slate-200/80 shadow-[0_1px_2px_rgba(15,23,42,0.06),0_12px_32px_rgba(15,23,42,0.07)]"
|
||||
lang="zh-CN"
|
||||
>
|
||||
<div className="mx-auto grid h-16 w-16 place-items-center rounded-full border border-violet-100 bg-white shadow-[0_10px_28px_rgba(126,34,206,0.12)]">
|
||||
<AgentReadyMark />
|
||||
<div className="px-4 py-5">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="grid h-[72px] w-[72px] shrink-0 place-items-center rounded-2xl bg-[oklch(0.97_0.018_292)] shadow-[inset_0_0_0_1px_oklch(0.91_0.03_292),0_12px_30px_rgba(88,71,120,0.10)]">
|
||||
<AgentReadyMark />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1 text-left">
|
||||
<h2
|
||||
id="agent-empty-state-title"
|
||||
className="text-balance text-lg font-semibold leading-7 text-slate-900"
|
||||
>
|
||||
我已就绪，请描述任务
|
||||
</h2>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="mt-5 text-pretty text-left text-sm leading-6 text-slate-500">
|
||||
直接说出你想调查的问题，我会整理监测与空间证据，并将分析结果带回地图。
|
||||
</p>
|
||||
</div>
|
||||
<h2 id="agent-empty-state-title" className="mt-4 text-lg font-semibold leading-7 text-slate-900">
|
||||
我已就绪，请描述任务
|
||||
</h2>
|
||||
<p className="mx-auto mt-2 max-w-[30ch] text-sm leading-6 text-slate-500">
|
||||
使用自然语言下达指令，我会整理监测与空间证据，并在地图上呈现分析结果。
|
||||
</p>
|
||||
<ul className="mt-5 grid grid-cols-2 gap-2" aria-label="Agent 分析能力">
|
||||
{AGENT_CAPABILITIES.map(({ icon: Icon, label }) => (
|
||||
|
||||
<ul
|
||||
className="grid grid-cols-2 border-t border-slate-200/70 bg-slate-50/55"
|
||||
aria-label="Agent 分析能力"
|
||||
>
|
||||
{AGENT_CAPABILITIES.map(({ icon: Icon, label }, index) => (
|
||||
<li
|
||||
key={label}
|
||||
className="surface-control flex min-h-16 flex-col items-center justify-center gap-1.5 rounded-xl border border-slate-200/70 px-2 py-2.5 text-center shadow-[0_6px_18px_rgba(15,23,42,0.04)] sm:flex-row sm:gap-2 sm:px-2.5 sm:text-left"
|
||||
className={cn(
|
||||
"flex min-h-[72px] items-center gap-3 px-3.5 py-3 text-left",
|
||||
index % 2 === 0 && "border-r border-slate-200/70",
|
||||
index < 2 && "border-b border-slate-200/70"
|
||||
)}
|
||||
>
|
||||
<span className="grid h-8 w-8 shrink-0 place-items-center rounded-lg bg-blue-50 text-blue-700">
|
||||
<Icon size={17} strokeWidth={1.9} aria-hidden="true" />
|
||||
<span className="grid h-8 w-8 shrink-0 place-items-center rounded-xl bg-slate-100 text-slate-600">
|
||||
<Icon size={16} strokeWidth={1.8} aria-hidden="true" />
|
||||
</span>
|
||||
<span className="whitespace-nowrap text-xs font-semibold leading-5 text-slate-800">{label}</span>
|
||||
<span className="text-xs font-medium leading-5 text-slate-700">{label}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
@@ -627,7 +896,14 @@ function BackendMessageList({
|
||||
streamRenderState,
|
||||
onReplyPermission,
|
||||
onReplyQuestion,
|
||||
onRejectQuestion
|
||||
onRejectQuestion,
|
||||
isSpeechSupported,
|
||||
speakingMessageId,
|
||||
speechState,
|
||||
onSpeak,
|
||||
onPauseSpeech,
|
||||
onResumeSpeech,
|
||||
onStopSpeech
|
||||
}: {
|
||||
messages: AgentChatMessage[];
|
||||
streaming: boolean;
|
||||
@@ -635,6 +911,13 @@ function BackendMessageList({
|
||||
onReplyPermission?: (permission: AgentPermissionRequest, reply: AgentPermissionReply) => Promise<void> | void;
|
||||
onReplyQuestion?: (request: AgentQuestionRequest, answers: string[][]) => Promise<void> | void;
|
||||
onRejectQuestion?: (request: AgentQuestionRequest) => Promise<void> | void;
|
||||
isSpeechSupported: boolean;
|
||||
speakingMessageId: string | null;
|
||||
speechState: AgentSpeechState;
|
||||
onSpeak: (messageId: string, text: string, options?: AgentSpeakOptions) => Promise<void> | void;
|
||||
onPauseSpeech: () => void;
|
||||
onResumeSpeech: () => void;
|
||||
onStopSpeech: () => void;
|
||||
}) {
|
||||
return (
|
||||
<motion.div className="flex w-full flex-col gap-4" variants={agentPanelListVariants} animate="animate">
|
||||
@@ -667,13 +950,25 @@ function BackendMessageList({
|
||||
) : null}
|
||||
{message.content ? (
|
||||
message.role === "assistant" ? (
|
||||
<StreamingTokenResponse
|
||||
className="leading-6"
|
||||
content={message.content}
|
||||
<AgentSpeechMessage
|
||||
messageId={message.id}
|
||||
streamDone={streamRenderState[message.id]?.done ?? !streaming}
|
||||
streaming={streaming && index === messages.length - 1}
|
||||
/>
|
||||
content={message.content}
|
||||
disabled={streaming && index === messages.length - 1}
|
||||
isSupported={isSpeechSupported}
|
||||
speechState={speakingMessageId === message.id ? speechState : "idle"}
|
||||
onSpeak={onSpeak}
|
||||
onPause={onPauseSpeech}
|
||||
onResume={onResumeSpeech}
|
||||
onStop={onStopSpeech}
|
||||
>
|
||||
<StreamingTokenResponse
|
||||
className="leading-6"
|
||||
content={message.content}
|
||||
messageId={message.id}
|
||||
streamDone={streamRenderState[message.id]?.done ?? !streaming}
|
||||
streaming={streaming && index === messages.length - 1}
|
||||
/>
|
||||
</AgentSpeechMessage>
|
||||
) : (
|
||||
<p className="whitespace-pre-wrap">{message.content}</p>
|
||||
)
|
||||
@@ -697,3 +992,235 @@ function BackendMessageList({
|
||||
</motion.div>
|
||||
);
|
||||
}
|
||||
|
||||
type SpeechSelection = {
|
||||
startOffset: number;
|
||||
left: number;
|
||||
top: number;
|
||||
};
|
||||
|
||||
function AgentSpeechMessage({
|
||||
messageId,
|
||||
content,
|
||||
disabled,
|
||||
isSupported,
|
||||
speechState,
|
||||
onSpeak,
|
||||
onPause,
|
||||
onResume,
|
||||
onStop,
|
||||
children
|
||||
}: {
|
||||
messageId: string;
|
||||
content: string;
|
||||
disabled: boolean;
|
||||
isSupported: boolean;
|
||||
speechState: AgentSpeechState;
|
||||
onSpeak: (messageId: string, text: string, options?: AgentSpeakOptions) => Promise<void> | void;
|
||||
onPause: () => void;
|
||||
onResume: () => void;
|
||||
onStop: () => void;
|
||||
children: ReactNode;
|
||||
}) {
|
||||
const contentRef = useRef<HTMLDivElement | null>(null);
|
||||
const lastSelectionRef = useRef<SpeechSelection | null>(null);
|
||||
const [selection, setSelection] = useState<SpeechSelection | null>(null);
|
||||
const speechText = useMemo(() => stripSpeechMarkdown(content), [content]);
|
||||
const selectionAnchor = selection ?? lastSelectionRef.current;
|
||||
|
||||
if (selection) {
|
||||
lastSelectionRef.current = selection;
|
||||
}
|
||||
|
||||
const captureSelection = useCallback(() => {
|
||||
if (!isSupported || disabled) {
|
||||
setSelection(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const browserSelection = window.getSelection();
|
||||
const container = contentRef.current;
|
||||
if (!browserSelection || browserSelection.rangeCount === 0 || browserSelection.isCollapsed || !container) {
|
||||
setSelection(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const range = browserSelection.getRangeAt(0);
|
||||
if (!container.contains(range.commonAncestorContainer)) {
|
||||
setSelection(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const startOffset = findSpeechSelectionStartOffset(speechText, browserSelection.toString());
|
||||
const rect = range.getBoundingClientRect();
|
||||
if (startOffset === null || (!rect.width && !rect.height)) {
|
||||
setSelection(null);
|
||||
return;
|
||||
}
|
||||
|
||||
setSelection({
|
||||
startOffset,
|
||||
left: Math.min(Math.max(rect.left + rect.width / 2, 88), window.innerWidth - 88),
|
||||
top: Math.max(rect.top - 10, 48)
|
||||
});
|
||||
}, [disabled, isSupported, speechText]);
|
||||
|
||||
useEffect(() => {
|
||||
setSelection(null);
|
||||
}, [messageId, speechText]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selection) return;
|
||||
const close = () => setSelection(null);
|
||||
const closeCollapsedSelection = () => {
|
||||
if (window.getSelection()?.isCollapsed) close();
|
||||
};
|
||||
const closeOnEscape = (event: KeyboardEvent) => {
|
||||
if (event.key === "Escape") close();
|
||||
};
|
||||
|
||||
document.addEventListener("selectionchange", closeCollapsedSelection);
|
||||
document.addEventListener("keydown", closeOnEscape);
|
||||
window.addEventListener("resize", close);
|
||||
window.addEventListener("scroll", close, true);
|
||||
return () => {
|
||||
document.removeEventListener("selectionchange", closeCollapsedSelection);
|
||||
document.removeEventListener("keydown", closeOnEscape);
|
||||
window.removeEventListener("resize", close);
|
||||
window.removeEventListener("scroll", close, true);
|
||||
};
|
||||
}, [selection]);
|
||||
|
||||
const speakFromSelection = () => {
|
||||
if (!selection) return;
|
||||
void Promise.resolve(onSpeak(messageId, speechText, { startOffset: selection.startOffset }));
|
||||
window.getSelection()?.removeAllRanges();
|
||||
setSelection(null);
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div
|
||||
ref={contentRef}
|
||||
onPointerUp={() => window.requestAnimationFrame(captureSelection)}
|
||||
onKeyUp={() => window.requestAnimationFrame(captureSelection)}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
|
||||
{selectionAnchor && typeof document !== "undefined" ? createPortal(
|
||||
<div
|
||||
className="pointer-events-none fixed z-[80] -translate-x-1/2 -translate-y-full"
|
||||
style={{ left: selectionAnchor.left, top: selectionAnchor.top }}
|
||||
>
|
||||
<AnimatePresence>
|
||||
{selection ? (
|
||||
<motion.button
|
||||
key="speech-selection-action"
|
||||
type="button"
|
||||
className="glass-transient pointer-events-auto inline-flex h-10 origin-bottom items-center gap-2 rounded-xl border px-2.5 pr-3 text-xs font-semibold text-slate-800 transition-colors hover:text-blue-700 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-blue-600/25"
|
||||
initial={{ opacity: 0, y: 8, scale: 0.95 }}
|
||||
animate={{
|
||||
opacity: 1,
|
||||
y: 0,
|
||||
scale: 1,
|
||||
transition: { duration: 0.15, ease: [0.16, 1, 0.3, 1] }
|
||||
}}
|
||||
exit={{
|
||||
opacity: 0,
|
||||
y: 0,
|
||||
scale: 0.95,
|
||||
transition: { duration: 0.1, ease: [0.4, 0, 1, 1] }
|
||||
}}
|
||||
whileTap={{ scale: 0.96 }}
|
||||
onMouseDown={(event) => event.preventDefault()}
|
||||
onClick={speakFromSelection}
|
||||
>
|
||||
<span className="grid h-6 w-6 shrink-0 place-items-center rounded-lg bg-blue-50/75 text-blue-600" aria-hidden="true">
|
||||
<Volume2 size={15} />
|
||||
</span>
|
||||
从这里开始朗读
|
||||
</motion.button>
|
||||
) : null}
|
||||
</AnimatePresence>
|
||||
</div>,
|
||||
document.body
|
||||
) : null}
|
||||
|
||||
{isSupported && !disabled ? (
|
||||
<div className="mt-2 flex min-h-8 items-center gap-1 border-t border-slate-100 pt-1.5">
|
||||
{speechState === "idle" ? (
|
||||
<SpeechIconButton label="语音播放" onClick={() => void Promise.resolve(onSpeak(messageId, speechText))}>
|
||||
<Volume2 size={15} aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
) : null}
|
||||
{speechState === "loading" ? (
|
||||
<>
|
||||
<SpeechIconButton label="正在生成语音" disabled>
|
||||
<LoaderCircle size={15} className="animate-spin text-blue-600" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
<SpeechIconButton label="停止语音播放" tone="danger" onClick={onStop}>
|
||||
<Square size={14} fill="currentColor" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
</>
|
||||
) : null}
|
||||
{speechState === "playing" ? (
|
||||
<>
|
||||
<SpeechIconButton label="暂停语音播放" tone="primary" onClick={onPause}>
|
||||
<Pause size={15} fill="currentColor" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
<SpeechIconButton label="停止语音播放" tone="danger" onClick={onStop}>
|
||||
<Square size={14} fill="currentColor" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
</>
|
||||
) : null}
|
||||
{speechState === "paused" ? (
|
||||
<>
|
||||
<SpeechIconButton label="继续语音播放" tone="primary" onClick={onResume}>
|
||||
<Play size={15} fill="currentColor" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
<SpeechIconButton label="停止语音播放" tone="danger" onClick={onStop}>
|
||||
<Square size={14} fill="currentColor" aria-hidden="true" />
|
||||
</SpeechIconButton>
|
||||
</>
|
||||
) : null}
|
||||
<span className="text-[11px] text-slate-400">
|
||||
{speechState === "idle" ? "语音播放" : speechState === "loading" ? "正在生成" : speechState === "playing" ? "播放中" : "已暂停"}
|
||||
</span>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SpeechIconButton({
|
||||
label,
|
||||
tone = "default",
|
||||
disabled = false,
|
||||
onClick,
|
||||
children
|
||||
}: {
|
||||
label: string;
|
||||
tone?: "default" | "primary" | "danger";
|
||||
disabled?: boolean;
|
||||
onClick?: () => void;
|
||||
children: ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
aria-label={label}
|
||||
title={label}
|
||||
disabled={disabled}
|
||||
onClick={onClick}
|
||||
className={cn(
|
||||
"grid h-8 w-8 shrink-0 place-items-center rounded-lg text-slate-500",
|
||||
"transition-[color,background-color,transform] hover:bg-slate-100 active:scale-95 disabled:cursor-wait",
|
||||
tone === "primary" && "text-blue-600",
|
||||
tone === "danger" && "text-red-600"
|
||||
)}
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -41,7 +41,11 @@ export function AgentPersona({
|
||||
|
||||
return (
|
||||
<Persona
|
||||
className={cn("shrink-0", className)}
|
||||
className={cn(
|
||||
"shrink-0",
|
||||
variant === "obsidian" && "scale-110",
|
||||
className
|
||||
)}
|
||||
onLoadError={() => setFailed(true)}
|
||||
state={personaState}
|
||||
variant={variant}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { FrontendActionExecutor } from "./executor";
|
||||
import { parseFrontendActionRegistry, parseFrontendActionRequest, type FrontendActionResult } from ".";
|
||||
|
||||
const storage = new Map<string, string>();
|
||||
|
||||
beforeEach(() => {
|
||||
storage.clear();
|
||||
vi.stubGlobal("sessionStorage", {
|
||||
getItem: (key: string) => storage.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => storage.set(key, value)
|
||||
});
|
||||
});
|
||||
|
||||
describe("SCADA frontend actions", () => {
|
||||
it("accepts only the two registered SCADA action names", () => {
|
||||
expect(parseFrontendActionRegistry({
|
||||
schema_version: "frontend-action-registry@1",
|
||||
actions: [
|
||||
{ id: "render_scada_analysis", version: "frontend-action@1" },
|
||||
{ id: "clear_scada_analysis", version: "frontend-action@1" },
|
||||
{ id: "zoom_to_map", version: "frontend-action@1" }
|
||||
]
|
||||
})?.actions.map((action) => action.id)).toEqual([
|
||||
"render_scada_analysis",
|
||||
"clear_scada_analysis"
|
||||
]);
|
||||
expect(parseFrontendActionRequest(createRequest("zoom_to_map"))).toBeNull();
|
||||
expect(parseFrontendActionRequest(createRequest("render_scada_analysis"))).not.toBeNull();
|
||||
});
|
||||
|
||||
it("submits the real browser output and replays the terminal receipt idempotently", async () => {
|
||||
const execute = vi.fn(async () => ({
|
||||
rendered_ids: ["MP01"],
|
||||
missing_ids: [],
|
||||
level_counts: { high: 1, medium: 0, low: 0, unrated: 0 },
|
||||
fitted: true
|
||||
}));
|
||||
const submit = vi.fn(async (
|
||||
_sessionId: string,
|
||||
_actionId: string,
|
||||
_result: FrontendActionResult
|
||||
) => undefined);
|
||||
const executor = new FrontendActionExecutor(execute, submit);
|
||||
const request = createRequest("render_scada_analysis");
|
||||
await executor.handle(request, "session-1");
|
||||
await executor.handle(request, "session-1");
|
||||
expect(execute).toHaveBeenCalledTimes(1);
|
||||
expect(submit).toHaveBeenCalledTimes(2);
|
||||
expect(submit.mock.calls[0][2]).toMatchObject({
|
||||
status: "succeeded",
|
||||
output: { rendered_ids: ["MP01"], fitted: true }
|
||||
});
|
||||
expect(submit.mock.calls[1][2]).toEqual(submit.mock.calls[0][2]);
|
||||
});
|
||||
});
|
||||
|
||||
function createRequest(name: string) {
|
||||
return {
|
||||
version: "frontend-action@1",
|
||||
actionId: "action-1",
|
||||
toolCallId: "call-1",
|
||||
sessionId: "session-1",
|
||||
name,
|
||||
params: { items: [{ sensor_id: "MP01", level: "high" }] },
|
||||
issuedAt: Date.now(),
|
||||
expiresAt: Date.now() + 15_000
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
export const FRONTEND_ACTION_NAMES = ["zoom_to_map", "locate_features", "view_history"] as const;
|
||||
export const FRONTEND_ACTION_NAMES = ["render_scada_analysis", "clear_scada_analysis"] as const;
|
||||
export type FrontendActionName = (typeof FRONTEND_ACTION_NAMES)[number];
|
||||
export type FrontendActionRegistry = { schema_version: "frontend-action-registry@1"; actions: Array<{ id: FrontendActionName; version: "frontend-action@1" }> };
|
||||
export type FrontendActionRequest = { version: "frontend-action@1"; actionId: string; toolCallId: string; sessionId: string; name: FrontendActionName; params: Record<string, unknown>; issuedAt: number; expiresAt: number };
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react";
|
||||
import {
|
||||
splitSpeechTextIntoChunks,
|
||||
type AgentSpeakOptions,
|
||||
type AgentSpeechState
|
||||
} from "../speech";
|
||||
|
||||
interface BrowserSpeechRecognitionEvent extends Event {
|
||||
readonly resultIndex: number;
|
||||
readonly results: SpeechRecognitionResultList;
|
||||
}
|
||||
|
||||
interface BrowserSpeechRecognitionErrorEvent extends Event {
|
||||
readonly error: string;
|
||||
readonly message?: string;
|
||||
}
|
||||
|
||||
interface BrowserSpeechRecognition extends EventTarget {
|
||||
lang: string;
|
||||
continuous: boolean;
|
||||
interimResults: boolean;
|
||||
onresult: ((event: BrowserSpeechRecognitionEvent) => void) | null;
|
||||
onerror: ((event: BrowserSpeechRecognitionErrorEvent) => void) | null;
|
||||
onend: (() => void) | null;
|
||||
start(): void;
|
||||
stop(): void;
|
||||
abort(): void;
|
||||
}
|
||||
|
||||
type BrowserSpeechRecognitionConstructor = new () => BrowserSpeechRecognition;
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
SpeechRecognition?: BrowserSpeechRecognitionConstructor;
|
||||
webkitSpeechRecognition?: BrowserSpeechRecognitionConstructor;
|
||||
}
|
||||
}
|
||||
|
||||
const subscribeToBrowserCapabilities = () => () => undefined;
|
||||
|
||||
function getSpeechSynthesisSupport() {
|
||||
return (
|
||||
typeof window.Audio !== "undefined" &&
|
||||
typeof window.URL !== "undefined" &&
|
||||
typeof window.fetch !== "undefined"
|
||||
);
|
||||
}
|
||||
|
||||
function getSpeechRecognitionSupport() {
|
||||
return "SpeechRecognition" in window || "webkitSpeechRecognition" in window;
|
||||
}
|
||||
|
||||
function getServerBrowserCapability() {
|
||||
return false;
|
||||
}
|
||||
|
||||
export function useAgentSpeechSynthesis() {
|
||||
const [speechState, setSpeechState] = useState<AgentSpeechState>("idle");
|
||||
const [speakingMessageId, setSpeakingMessageId] = useState<string | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const currentAudioUrlRef = useRef<string | null>(null);
|
||||
const audioUrlsRef = useRef<Set<string>>(new Set());
|
||||
const fetchControllersRef = useRef<Set<AbortController>>(new Set());
|
||||
const chunkUrlCacheRef = useRef<Map<number, string>>(new Map());
|
||||
const chunkPromisesRef = useRef<Map<number, Promise<string>>>(new Map());
|
||||
const chunksRef = useRef<string[]>([]);
|
||||
const currentChunkIndexRef = useRef(0);
|
||||
const playbackTokenRef = useRef(0);
|
||||
const activeMessageIdRef = useRef<string | null>(null);
|
||||
const playChunkRef = useRef<(chunkIndex: number, playbackToken: number) => Promise<void>>(async () => undefined);
|
||||
|
||||
const isSupported = useSyncExternalStore(
|
||||
subscribeToBrowserCapabilities,
|
||||
getSpeechSynthesisSupport,
|
||||
getServerBrowserCapability
|
||||
);
|
||||
|
||||
const detachAudio = useCallback((revokeUrl: boolean) => {
|
||||
const audio = audioRef.current;
|
||||
audioRef.current = null;
|
||||
if (audio) {
|
||||
audio.pause();
|
||||
audio.onended = null;
|
||||
audio.onerror = null;
|
||||
audio.removeAttribute("src");
|
||||
audio.load();
|
||||
}
|
||||
|
||||
const currentUrl = currentAudioUrlRef.current;
|
||||
currentAudioUrlRef.current = null;
|
||||
if (revokeUrl && currentUrl) {
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
audioUrlsRef.current.delete(currentUrl);
|
||||
chunkUrlCacheRef.current.delete(currentChunkIndexRef.current);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const releaseAudio = useCallback(() => {
|
||||
fetchControllersRef.current.forEach((controller) => controller.abort());
|
||||
fetchControllersRef.current.clear();
|
||||
chunkPromisesRef.current.clear();
|
||||
detachAudio(false);
|
||||
audioUrlsRef.current.forEach((url) => URL.revokeObjectURL(url));
|
||||
audioUrlsRef.current.clear();
|
||||
chunkUrlCacheRef.current.clear();
|
||||
chunksRef.current = [];
|
||||
currentChunkIndexRef.current = 0;
|
||||
activeMessageIdRef.current = null;
|
||||
}, [detachAudio]);
|
||||
|
||||
const fetchChunkAudio = useCallback((chunkIndex: number, playbackToken: number) => {
|
||||
const cachedUrl = chunkUrlCacheRef.current.get(chunkIndex);
|
||||
if (cachedUrl) return Promise.resolve(cachedUrl);
|
||||
|
||||
const currentPromise = chunkPromisesRef.current.get(chunkIndex);
|
||||
if (currentPromise) return currentPromise;
|
||||
|
||||
const text = chunksRef.current[chunkIndex];
|
||||
if (!text) return Promise.reject(new Error("语音片段不存在"));
|
||||
|
||||
const controller = new AbortController();
|
||||
fetchControllersRef.current.add(controller);
|
||||
const promise = fetch("/api/tts/edge", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text }),
|
||||
signal: controller.signal
|
||||
})
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
const payload = await response.json().catch(() => null) as { error?: string } | null;
|
||||
throw new Error(payload?.error ?? `语音生成失败 (${response.status})`);
|
||||
}
|
||||
const audioBlob = await response.blob();
|
||||
if (!audioBlob.size) throw new Error("语音服务返回了空音频");
|
||||
if (playbackToken !== playbackTokenRef.current) {
|
||||
throw new DOMException("语音播放已取消", "AbortError");
|
||||
}
|
||||
|
||||
const objectUrl = URL.createObjectURL(audioBlob);
|
||||
audioUrlsRef.current.add(objectUrl);
|
||||
chunkUrlCacheRef.current.set(chunkIndex, objectUrl);
|
||||
return objectUrl;
|
||||
})
|
||||
.finally(() => {
|
||||
fetchControllersRef.current.delete(controller);
|
||||
chunkPromisesRef.current.delete(chunkIndex);
|
||||
});
|
||||
|
||||
chunkPromisesRef.current.set(chunkIndex, promise);
|
||||
return promise;
|
||||
}, []);
|
||||
|
||||
const prefetchChunk = useCallback((chunkIndex: number, playbackToken: number) => {
|
||||
if (chunkIndex >= chunksRef.current.length) return;
|
||||
void fetchChunkAudio(chunkIndex, playbackToken).catch(() => undefined);
|
||||
}, [fetchChunkAudio]);
|
||||
|
||||
const playChunk = useCallback(async (chunkIndex: number, playbackToken: number) => {
|
||||
setSpeechState("loading");
|
||||
const objectUrl = await fetchChunkAudio(chunkIndex, playbackToken);
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
|
||||
detachAudio(true);
|
||||
currentChunkIndexRef.current = chunkIndex;
|
||||
const audio = new Audio(objectUrl);
|
||||
audio.preload = "auto";
|
||||
audioRef.current = audio;
|
||||
currentAudioUrlRef.current = objectUrl;
|
||||
|
||||
audio.onended = () => {
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
detachAudio(true);
|
||||
const nextChunkIndex = chunkIndex + 1;
|
||||
if (nextChunkIndex >= chunksRef.current.length) {
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
return;
|
||||
}
|
||||
currentChunkIndexRef.current = nextChunkIndex;
|
||||
void playChunkRef.current(nextChunkIndex, playbackToken);
|
||||
};
|
||||
audio.onerror = () => {
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
};
|
||||
|
||||
await audio.play();
|
||||
if (playbackToken !== playbackTokenRef.current) return;
|
||||
setSpeechState("playing");
|
||||
prefetchChunk(chunkIndex + 1, playbackToken);
|
||||
}, [detachAudio, fetchChunkAudio, prefetchChunk, releaseAudio]);
|
||||
|
||||
useEffect(() => {
|
||||
playChunkRef.current = playChunk;
|
||||
}, [playChunk]);
|
||||
|
||||
const speak = useCallback(async (messageId: string, text: string, options: AgentSpeakOptions = {}) => {
|
||||
const normalizedText = text.trim();
|
||||
if (!isSupported || !normalizedText) return;
|
||||
|
||||
const startOffset = Math.max(0, Math.min(options.startOffset ?? 0, normalizedText.length));
|
||||
const chunks = splitSpeechTextIntoChunks(normalizedText.slice(startOffset));
|
||||
if (!chunks.length) return;
|
||||
|
||||
const playbackToken = playbackTokenRef.current + 1;
|
||||
playbackTokenRef.current = playbackToken;
|
||||
releaseAudio();
|
||||
chunksRef.current = chunks;
|
||||
activeMessageIdRef.current = messageId;
|
||||
setSpeakingMessageId(messageId);
|
||||
setSpeechState("loading");
|
||||
|
||||
try {
|
||||
await playChunk(0, playbackToken);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") return;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
}
|
||||
}, [isSupported, playChunk, releaseAudio]);
|
||||
|
||||
const pause = useCallback(() => {
|
||||
if (!audioRef.current || speechState !== "playing") return;
|
||||
audioRef.current.pause();
|
||||
setSpeechState("paused");
|
||||
}, [speechState]);
|
||||
|
||||
const resume = useCallback(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
setSpeakingMessageId(activeMessageIdRef.current);
|
||||
setSpeechState("playing");
|
||||
void audio.play().catch(() => {
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
});
|
||||
}, [releaseAudio]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
setSpeechState("idle");
|
||||
setSpeakingMessageId(null);
|
||||
}, [releaseAudio]);
|
||||
|
||||
useEffect(() => () => {
|
||||
playbackTokenRef.current += 1;
|
||||
releaseAudio();
|
||||
}, [releaseAudio]);
|
||||
|
||||
return { speechState, speakingMessageId, speak, pause, resume, stop, isSupported };
|
||||
}
|
||||
|
||||
export function useAgentSpeechRecognition(onResult: (text: string) => void) {
|
||||
const [isListening, setIsListening] = useState(false);
|
||||
const recognitionRef = useRef<BrowserSpeechRecognition | null>(null);
|
||||
const onResultRef = useRef(onResult);
|
||||
const onErrorRef = useRef<(message: string) => void>(() => undefined);
|
||||
|
||||
useEffect(() => {
|
||||
onResultRef.current = onResult;
|
||||
}, [onResult]);
|
||||
|
||||
const isSupported = useSyncExternalStore(
|
||||
subscribeToBrowserCapabilities,
|
||||
getSpeechRecognitionSupport,
|
||||
getServerBrowserCapability
|
||||
);
|
||||
|
||||
const start = useCallback((onError?: (message: string) => void) => {
|
||||
if (!isSupported || recognitionRef.current) return;
|
||||
const Recognition = window.SpeechRecognition ?? window.webkitSpeechRecognition;
|
||||
if (!Recognition) return;
|
||||
|
||||
onErrorRef.current = onError ?? (() => undefined);
|
||||
|
||||
const recognition = new Recognition();
|
||||
recognition.lang = "zh-CN";
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = false;
|
||||
recognition.onresult = (event) => {
|
||||
for (let index = event.resultIndex; index < event.results.length; index += 1) {
|
||||
if (event.results[index].isFinal) {
|
||||
onResultRef.current(event.results[index][0].transcript);
|
||||
}
|
||||
}
|
||||
};
|
||||
recognition.onerror = (event) => {
|
||||
setIsListening(false);
|
||||
if (event.error !== "aborted") {
|
||||
onErrorRef.current(getSpeechRecognitionErrorMessage(event.error));
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
if (recognitionRef.current === recognition) {
|
||||
recognitionRef.current = null;
|
||||
}
|
||||
setIsListening(false);
|
||||
};
|
||||
|
||||
recognitionRef.current = recognition;
|
||||
try {
|
||||
recognition.start();
|
||||
setIsListening(true);
|
||||
} catch {
|
||||
recognitionRef.current = null;
|
||||
setIsListening(false);
|
||||
onErrorRef.current("无法启动语音输入，请检查浏览器麦克风权限后重试");
|
||||
}
|
||||
}, [isSupported]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
recognitionRef.current?.stop();
|
||||
recognitionRef.current = null;
|
||||
setIsListening(false);
|
||||
}, []);
|
||||
|
||||
useEffect(() => () => recognitionRef.current?.abort(), []);
|
||||
|
||||
return { isListening, start, stop, isSupported };
|
||||
}
|
||||
|
||||
function getSpeechRecognitionErrorMessage(error: string) {
|
||||
switch (error) {
|
||||
case "not-allowed":
|
||||
case "service-not-allowed":
|
||||
return "无法使用麦克风，请在浏览器地址栏允许麦克风权限后重试";
|
||||
case "audio-capture":
|
||||
return "未检测到可用麦克风，请检查设备连接";
|
||||
case "network":
|
||||
return "语音识别服务连接失败，请检查网络后重试";
|
||||
case "no-speech":
|
||||
return "未检测到语音，请靠近麦克风后重试";
|
||||
default:
|
||||
return "语音输入暂时不可用，请稍后重试";
|
||||
}
|
||||
}
|
||||
+1
-12
@@ -2,20 +2,16 @@ export { AgentCollapsedRail } from "./components/agent-collapsed-rail";
|
||||
export { AgentCommandPanel } from "./components/agent-command-panel";
|
||||
export { AgentPersona } from "./components/agent-persona";
|
||||
export { normalizeSafeChartData, parseChartSpec, pointToLabelValue } from "./chart-data";
|
||||
export { parseAssistantMessageSections, parseContentWithToolCalls } from "./message-sections";
|
||||
export {
|
||||
applyPermissionResponse,
|
||||
applyQuestionResponse,
|
||||
cancelRunningTodos,
|
||||
completeRunningProgress,
|
||||
finalizeAssistantMessageAfterAbort,
|
||||
toTodoUpdate,
|
||||
updateMessageById,
|
||||
upsertPermission,
|
||||
upsertProgress,
|
||||
upsertQuestion
|
||||
} from "./session-state";
|
||||
export type {
|
||||
AgentApprovalMode,
|
||||
AgentChatMessage,
|
||||
AgentChatProgress,
|
||||
AgentInteractionToolRef,
|
||||
@@ -26,7 +22,6 @@ export type {
|
||||
AgentQuestionInfo,
|
||||
AgentQuestionOption,
|
||||
AgentQuestionRequest,
|
||||
AgentStreamRenderChunk,
|
||||
AgentStreamRenderMessageState,
|
||||
AgentStreamRenderState,
|
||||
AgentTodoItem,
|
||||
@@ -38,9 +33,3 @@ export type {
|
||||
SafeChartSeries,
|
||||
SafeChartSpec
|
||||
} from "./chart-data";
|
||||
export type {
|
||||
AssistantMessageSections,
|
||||
ContentSegment,
|
||||
ParsedToolContent,
|
||||
ToolCall
|
||||
} from "./message-sections";
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseAssistantMessageSections, parseContentWithToolCalls } from "./message-sections";
|
||||
|
||||
describe("parseAssistantMessageSections", () => {
|
||||
it("returns plain assistant content when there is no thought block", () => {
|
||||
expect(parseAssistantMessageSections("直接回答")).toEqual({
|
||||
answer: "直接回答",
|
||||
thought: null,
|
||||
thoughtComplete: false
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts a completed thought block and keeps the final answer visible", () => {
|
||||
expect(parseAssistantMessageSections("<think>先分析需求</think>\n\n最终回答")).toEqual({
|
||||
answer: "最终回答",
|
||||
thought: "先分析需求",
|
||||
thoughtComplete: true
|
||||
});
|
||||
});
|
||||
|
||||
it("supports streaming thought content before the closing tag arrives", () => {
|
||||
expect(parseAssistantMessageSections("准备中...\n<think>继续推理中")).toEqual({
|
||||
answer: "准备中...",
|
||||
thought: "继续推理中",
|
||||
thoughtComplete: false
|
||||
});
|
||||
});
|
||||
|
||||
it("merges multiple thought blocks into a single collapsed section", () => {
|
||||
expect(
|
||||
parseAssistantMessageSections(
|
||||
"<think>第一段思考</think>\n答案开头\n<think>第二段思考</think>\n答案结尾"
|
||||
)
|
||||
).toEqual({
|
||||
answer: "答案开头\n\n答案结尾",
|
||||
thought: "第一段思考\n\n第二段思考",
|
||||
thoughtComplete: true
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseContentWithToolCalls", () => {
|
||||
it("returns a single text segment when there are no tool calls", () => {
|
||||
const result = parseContentWithToolCalls("普通文本回答");
|
||||
|
||||
expect(result.segments).toEqual([{ type: "text", content: "普通文本回答" }]);
|
||||
expect(result.toolCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("parses a complete tool_call block", () => {
|
||||
const content =
|
||||
'分析完成。\n<tool_call>{"tool":"locate_junctions","params":{"ids":["J1","J2"]}}</tool_call>\n以上是结果。';
|
||||
const result = parseContentWithToolCalls(content);
|
||||
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls[0].tool).toBe("locate_junctions");
|
||||
expect(result.toolCalls[0].params).toEqual({ ids: ["J1", "J2"] });
|
||||
expect(result.segments).toHaveLength(3);
|
||||
expect(result.segments[0]).toEqual({ type: "text", content: "分析完成。" });
|
||||
expect(result.segments[1]).toMatchObject({
|
||||
type: "tool_call",
|
||||
toolCall: { tool: "locate_junctions" }
|
||||
});
|
||||
expect(result.segments[2]).toEqual({ type: "text", content: "以上是结果。" });
|
||||
});
|
||||
|
||||
it("parses multiple tool_call blocks", () => {
|
||||
const content =
|
||||
'文本1\n<tool_call>{"tool":"locate_pipes","params":{"ids":["P1"]}}</tool_call>\n文本2\n<tool_call>{"tool":"chart","params":{"title":"图"}}</tool_call>';
|
||||
const result = parseContentWithToolCalls(content);
|
||||
|
||||
expect(result.toolCalls).toHaveLength(2);
|
||||
expect(result.toolCalls[0].tool).toBe("locate_pipes");
|
||||
expect(result.toolCalls[1].tool).toBe("chart");
|
||||
expect(result.segments).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("detects an unclosed tool_call tag as pending while streaming", () => {
|
||||
const result = parseContentWithToolCalls('正在分析...\n<tool_call>{"tool":"locate_no');
|
||||
|
||||
expect(result.segments).toEqual([
|
||||
{ type: "text", content: "正在分析..." },
|
||||
{ type: "tool_call_pending" }
|
||||
]);
|
||||
expect(result.toolCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("strips partial opening tags during streaming", () => {
|
||||
const result = parseContentWithToolCalls("正在分析...\n<tool_c");
|
||||
|
||||
expect(result.segments).toEqual([{ type: "text", content: "正在分析..." }]);
|
||||
});
|
||||
|
||||
it("handles malformed JSON gracefully", () => {
|
||||
const result = parseContentWithToolCalls("前文\n<tool_call>{invalid json}</tool_call>\n后文");
|
||||
|
||||
expect(result.toolCalls).toHaveLength(0);
|
||||
expect(result.segments.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("returns empty segments for empty content", () => {
|
||||
const result = parseContentWithToolCalls("");
|
||||
|
||||
expect(result.segments).toHaveLength(0);
|
||||
expect(result.toolCalls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,129 +0,0 @@
|
||||
export type AssistantMessageSections = {
|
||||
answer: string;
|
||||
thought: string | null;
|
||||
thoughtComplete: boolean;
|
||||
};
|
||||
|
||||
export type ToolCall = {
|
||||
id: string;
|
||||
tool: string;
|
||||
params: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type ContentSegment =
|
||||
| { type: "text"; content: string }
|
||||
| { type: "tool_call"; toolCall: ToolCall }
|
||||
| { type: "tool_call_pending" };
|
||||
|
||||
export type ParsedToolContent = {
|
||||
segments: ContentSegment[];
|
||||
toolCalls: ToolCall[];
|
||||
};
|
||||
|
||||
const THINK_BLOCK_PATTERN = /<think>([\s\S]*?)<\/think>/gi;
|
||||
const THINK_OPEN_TAG = "<think>";
|
||||
const THINK_CLOSE_TAG = "</think>";
|
||||
|
||||
export function parseAssistantMessageSections(content: string): AssistantMessageSections {
|
||||
if (!content) {
|
||||
return { answer: "", thought: null, thoughtComplete: false };
|
||||
}
|
||||
|
||||
const thoughtParts: string[] = [];
|
||||
let answer = content;
|
||||
|
||||
answer = answer.replace(THINK_BLOCK_PATTERN, (_, thoughtContent: string) => {
|
||||
const trimmedThought = thoughtContent.trim();
|
||||
if (trimmedThought) {
|
||||
thoughtParts.push(trimmedThought);
|
||||
}
|
||||
|
||||
return "\n";
|
||||
});
|
||||
|
||||
const lastOpenIndex = answer.lastIndexOf(THINK_OPEN_TAG);
|
||||
const lastCloseIndex = answer.lastIndexOf(THINK_CLOSE_TAG);
|
||||
const hasUnclosedThought = lastOpenIndex !== -1 && lastOpenIndex > lastCloseIndex;
|
||||
|
||||
if (hasUnclosedThought) {
|
||||
const streamingThought = answer.slice(lastOpenIndex + THINK_OPEN_TAG.length).trim();
|
||||
|
||||
if (streamingThought) {
|
||||
thoughtParts.push(streamingThought);
|
||||
}
|
||||
|
||||
answer = answer.slice(0, lastOpenIndex);
|
||||
}
|
||||
|
||||
const normalizedAnswer = answer.replace(/\n{3,}/g, "\n\n").trim();
|
||||
const normalizedThought = thoughtParts.join("\n\n").trim();
|
||||
|
||||
return {
|
||||
answer: normalizedAnswer,
|
||||
thought: normalizedThought || null,
|
||||
thoughtComplete: Boolean(normalizedThought) && !hasUnclosedThought
|
||||
};
|
||||
}
|
||||
|
||||
const TOOL_CALL_OPEN_TAG = "<tool_call>";
|
||||
const TOOL_CALL_PATTERN = /<tool_call>([\s\S]*?)<\/tool_call>/gi;
|
||||
const PARTIAL_TOOL_TAG_TAIL = /<(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l(?:l)?)?)?)?)?)?)?)?)?$/;
|
||||
|
||||
export function parseContentWithToolCalls(content: string): ParsedToolContent {
|
||||
if (!content) {
|
||||
return { segments: [], toolCalls: [] };
|
||||
}
|
||||
|
||||
const segments: ContentSegment[] = [];
|
||||
const toolCalls: ToolCall[] = [];
|
||||
let lastIndex = 0;
|
||||
let toolCallIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = TOOL_CALL_PATTERN.exec(content)) !== null) {
|
||||
const textBefore = content.slice(lastIndex, match.index);
|
||||
if (textBefore.trim()) {
|
||||
segments.push({ type: "text", content: textBefore.trim() });
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(match[1].trim()) as {
|
||||
tool?: string;
|
||||
params?: Record<string, unknown>;
|
||||
};
|
||||
const toolCall: ToolCall = {
|
||||
id: `tc-${toolCallIndex++}`,
|
||||
tool: parsed.tool ?? "unknown",
|
||||
params: parsed.params ?? {}
|
||||
};
|
||||
segments.push({ type: "tool_call", toolCall });
|
||||
toolCalls.push(toolCall);
|
||||
} catch {
|
||||
segments.push({ type: "text", content: match[0] });
|
||||
}
|
||||
|
||||
lastIndex = match.index + match[0].length;
|
||||
}
|
||||
|
||||
const remaining = content.slice(lastIndex);
|
||||
const unclosedIndex = remaining.lastIndexOf(TOOL_CALL_OPEN_TAG);
|
||||
|
||||
if (unclosedIndex !== -1) {
|
||||
const textBefore = remaining.slice(0, unclosedIndex);
|
||||
if (textBefore.trim()) {
|
||||
segments.push({ type: "text", content: textBefore.trim() });
|
||||
}
|
||||
segments.push({ type: "tool_call_pending" });
|
||||
} else {
|
||||
const cleaned = remaining.replace(PARTIAL_TOOL_TAG_TAIL, "").trim();
|
||||
if (cleaned) {
|
||||
segments.push({ type: "text", content: cleaned });
|
||||
}
|
||||
}
|
||||
|
||||
if (segments.length === 0) {
|
||||
segments.push({ type: "text", content });
|
||||
}
|
||||
|
||||
return { segments, toolCalls };
|
||||
}
|
||||
@@ -12,14 +12,6 @@ import type {
|
||||
|
||||
type RecordValue = Record<string, unknown>;
|
||||
|
||||
export function updateMessageById(
|
||||
messages: AgentChatMessage[],
|
||||
messageId: string,
|
||||
updater: (message: AgentChatMessage) => AgentChatMessage
|
||||
) {
|
||||
return messages.map((message) => (message.id === messageId ? updater(message) : message));
|
||||
}
|
||||
|
||||
export function upsertProgress(progress: AgentChatProgress[] | undefined, data: unknown) {
|
||||
const payload = asRecord(data);
|
||||
const id = readString(payload.id) ?? `progress-${Date.now().toString(36)}`;
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
findSpeechSelectionStartOffset,
|
||||
splitSpeechTextIntoChunks,
|
||||
stripSpeechMarkdown
|
||||
} from "./speech";
|
||||
|
||||
describe("Agent 语音文本处理", () => {
|
||||
it("定位经过空白压缩的选择文本", () => {
|
||||
const text = "第一段内容。\n第二段 包含空格。";
|
||||
expect(findSpeechSelectionStartOffset(text, "第二段 包含空格")).toBe(text.indexOf("第二段"));
|
||||
expect(findSpeechSelectionStartOffset(text, "不存在")).toBeNull();
|
||||
});
|
||||
|
||||
it("按句子拆分长文本", () => {
|
||||
const chunks = splitSpeechTextIntoChunks(`${"甲".repeat(260)}。${"乙".repeat(260)}。`);
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks.join("")).toContain("乙");
|
||||
});
|
||||
|
||||
it("移除会干扰朗读的 Markdown 标记", () => {
|
||||
expect(stripSpeechMarkdown("## 分析结果\n**水位** [详情](https://example.com)")).toBe("分析结果\n水位 详情");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
const compactWhitespace = (value: string) => value.replace(/\s+/g, " ").trim();
|
||||
const MAX_SPEECH_CHUNK_LENGTH = 520;
|
||||
const MIN_SPEECH_CHUNK_LENGTH = 180;
|
||||
const SPEECH_SENTENCE_PATTERN = /[^。！？!?；;\n]+(?:[。！？!?；;]+|(?=\n|$))/g;
|
||||
|
||||
export type AgentSpeechState = "idle" | "loading" | "playing" | "paused";
|
||||
|
||||
export type AgentSpeakOptions = {
|
||||
startOffset?: number;
|
||||
};
|
||||
|
||||
const normalizeWithOffsetMap = (value: string) => {
|
||||
let normalized = "";
|
||||
const offsetMap: number[] = [];
|
||||
let previousWasWhitespace = false;
|
||||
|
||||
Array.from(value).forEach((character, index) => {
|
||||
if (/\s/u.test(character)) {
|
||||
if (!previousWasWhitespace && normalized.length > 0) {
|
||||
normalized += " ";
|
||||
offsetMap.push(index);
|
||||
}
|
||||
previousWasWhitespace = true;
|
||||
return;
|
||||
}
|
||||
|
||||
normalized += character;
|
||||
offsetMap.push(index);
|
||||
previousWasWhitespace = false;
|
||||
});
|
||||
|
||||
return { normalized: normalized.trimEnd(), offsetMap };
|
||||
};
|
||||
|
||||
export function stripSpeechMarkdown(markdown: string) {
|
||||
return markdown
|
||||
.replace(/```[\s\S]*?```/g, "")
|
||||
.replace(/`([^`]+)`/g, "$1")
|
||||
.replace(/!\[.*?\]\(.*?\)/g, "")
|
||||
.replace(/\[([^\]]+)\]\(.*?\)/g, "$1")
|
||||
.replace(/#{1,6}\s+/g, "")
|
||||
.replace(/\*{1,3}(.+?)\*{1,3}/g, "$1")
|
||||
.replace(/~~(.+?)~~/g, "$1")
|
||||
.replace(/^>\s+/gm, "")
|
||||
.replace(/^[-*+]\s+/gm, "")
|
||||
.replace(/^\d+\.\s+/gm, "")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/\n{2,}/g, "\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function findSpeechSelectionStartOffset(text: string, selectedText: string): number | null {
|
||||
const needle = selectedText.trim();
|
||||
if (!needle) return null;
|
||||
|
||||
const exactIndex = text.indexOf(needle);
|
||||
if (exactIndex >= 0) return exactIndex;
|
||||
|
||||
const normalizedNeedle = compactWhitespace(needle);
|
||||
if (!normalizedNeedle) return null;
|
||||
|
||||
const haystack = normalizeWithOffsetMap(text);
|
||||
const normalizedIndex = haystack.normalized.indexOf(normalizedNeedle);
|
||||
if (normalizedIndex < 0) return null;
|
||||
|
||||
return haystack.offsetMap[normalizedIndex] ?? null;
|
||||
}
|
||||
|
||||
export function splitSpeechTextIntoChunks(text: string): string[] {
|
||||
const normalizedText = text.trim();
|
||||
if (!normalizedText) return [];
|
||||
|
||||
const segments = Array.from(normalizedText.matchAll(SPEECH_SENTENCE_PATTERN), (match) =>
|
||||
compactWhitespace(match[0])
|
||||
).filter(Boolean);
|
||||
const sourceSegments = segments.length > 0 ? segments : [normalizedText];
|
||||
const chunks: string[] = [];
|
||||
let currentChunk = "";
|
||||
|
||||
const flush = () => {
|
||||
if (!currentChunk) return;
|
||||
chunks.push(currentChunk);
|
||||
currentChunk = "";
|
||||
};
|
||||
|
||||
for (const segment of sourceSegments) {
|
||||
if (segment.length > MAX_SPEECH_CHUNK_LENGTH) {
|
||||
flush();
|
||||
for (let offset = 0; offset < segment.length; offset += MAX_SPEECH_CHUNK_LENGTH) {
|
||||
chunks.push(segment.slice(offset, offset + MAX_SPEECH_CHUNK_LENGTH));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidate = currentChunk ? `${currentChunk}${segment}` : segment;
|
||||
if (
|
||||
currentChunk &&
|
||||
candidate.length > MAX_SPEECH_CHUNK_LENGTH &&
|
||||
currentChunk.length >= MIN_SPEECH_CHUNK_LENGTH
|
||||
) {
|
||||
flush();
|
||||
currentChunk = segment;
|
||||
continue;
|
||||
}
|
||||
|
||||
currentChunk = candidate;
|
||||
}
|
||||
|
||||
flush();
|
||||
return chunks;
|
||||
}
|
||||
@@ -8,13 +8,7 @@ export type AgentChatMessage = {
|
||||
todos?: AgentTodoUpdate;
|
||||
};
|
||||
|
||||
export type AgentStreamRenderChunk = {
|
||||
id: number;
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type AgentStreamRenderMessageState = {
|
||||
chunks: AgentStreamRenderChunk[];
|
||||
done: boolean;
|
||||
};
|
||||
|
||||
@@ -44,6 +38,8 @@ export type AgentPermissionStatus =
|
||||
|
||||
export type AgentPermissionReply = "once" | "always" | "reject";
|
||||
|
||||
export type AgentApprovalMode = "request" | "always";
|
||||
|
||||
export type AgentModelOption = {
|
||||
id: string;
|
||||
label: string;
|
||||
|
||||
Reference in New Issue
Block a user