fix(agent): stabilize stream token output

src/routes/chatStream.ts

@@ -99,8 +99,6 @@ type SmoothTextEndPart = Extract<SmoothTextStreamPart, { type: "text-end" }>;
 const segmenters = new Map<string, Intl.Segmenter | null>();
 const CJK_SCRIPT_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
 const CJK_PUNCTUATION_PATTERN = /[。！？!?，、；：,;:]/u;
-const CJK_SENTENCE_END_PATTERN = /[。！？!?]+(?:["'”’』」）)\]\s]+)?/u;
-const CJK_PAUSE_PATTERN = /[，、；：,;:]+(?:\s+)?/u;
 const CJK_FALLBACK_TRIGGER_CHARS = 24;
 const CJK_FALLBACK_MIN_CHARS = 12;
 const CJK_FALLBACK_TARGET_CHARS = 18;
@@ -133,20 +131,6 @@ function isCjkSmoothingContent(content: string, locale: string) {
   );
 }
 
-function detectCjkPunctuationChunk(content: string) {
-  const sentenceMatch = CJK_SENTENCE_END_PATTERN.exec(content);
-  if (sentenceMatch) {
-    return content.slice(0, sentenceMatch.index + sentenceMatch[0].length);
-  }
-
-  const pauseMatch = CJK_PAUSE_PATTERN.exec(content);
-  if (pauseMatch) {
-    return content.slice(0, pauseMatch.index + pauseMatch[0].length);
-  }
-
-  return null;
-}
-
 function detectCjkFallbackChunk(content: string, locale: string) {
   if (codePointLength(content) < CJK_FALLBACK_TRIGGER_CHARS) {
     return null;
@@ -179,10 +163,7 @@ export function detectTokenSmoothingChunk(content: string, locale = "zh") {
   }
 
   if (isCjkSmoothingContent(content, locale)) {
-    return (
-      detectCjkPunctuationChunk(content) ??
-      detectCjkFallbackChunk(content, locale)
-    );
+    return detectCjkFallbackChunk(content, locale);
   }
 
   const wordChunk = content.match(/^\S+\s*/u)?.[0];