fix(agent): stabilize stream token output

src/routes/chat.ts

@@ -764,6 +764,15 @@ export const buildChatRouter = (
           textPart.state = "streaming";
         }
         writeUiChunk({ type: "text-delta", id: textPartId, delta });
+        writeUiChunk({
+          type: "data-stream_token",
+          data: {
+            session_id: clientSessionId,
+            message_id: assistantUiMessageId,
+            content: delta,
+          },
+          transient: true,
+        } as UIMessageChunk);
         syncLatestUiMessages();
       };
       const writeDataPart = (event: string, data: Record<string, unknown>) => {
@@ -879,6 +888,16 @@ export const buildChatRouter = (
       res.on("close", handleClientClose);
 
       const publish = (event: string, data: Record<string, unknown>) => {
+        const subscriberData =
+          event === "token"
+            ? {
+                ...data,
+                message_id:
+                  typeof data.message_id === "string"
+                    ? data.message_id
+                    : assistantUiMessageId,
+              }
+            : data;
         if (event === "token") {
           activeRun.messages = updateLastAssistantMessage(activeRun.messages, (message) => ({
             ...message,
@@ -1011,7 +1030,7 @@ export const buildChatRouter = (
         }
 
         for (const subscriber of activeRun.subscribers) {
-          subscriber.write(event, data);
+          subscriber.write(event, subscriberData);
         }
         void queueSessionUiStatePersist().catch((error) => {
           logger.warn({ err: error, sessionId: clientSessionId }, "failed to persist chat stream state");

src/routes/chatStream.ts

@@ -99,8 +99,6 @@ type SmoothTextEndPart = Extract<SmoothTextStreamPart, { type: "text-end" }>;
 const segmenters = new Map<string, Intl.Segmenter | null>();
 const CJK_SCRIPT_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
 const CJK_PUNCTUATION_PATTERN = /[。！？!?，、；：,;:]/u;
-const CJK_SENTENCE_END_PATTERN = /[。！？!?]+(?:["'”’』」）)\]\s]+)?/u;
-const CJK_PAUSE_PATTERN = /[，、；：,;:]+(?:\s+)?/u;
 const CJK_FALLBACK_TRIGGER_CHARS = 24;
 const CJK_FALLBACK_MIN_CHARS = 12;
 const CJK_FALLBACK_TARGET_CHARS = 18;
@@ -133,20 +131,6 @@ function isCjkSmoothingContent(content: string, locale: string) {
   );
 }
 
-function detectCjkPunctuationChunk(content: string) {
-  const sentenceMatch = CJK_SENTENCE_END_PATTERN.exec(content);
-  if (sentenceMatch) {
-    return content.slice(0, sentenceMatch.index + sentenceMatch[0].length);
-  }
-
-  const pauseMatch = CJK_PAUSE_PATTERN.exec(content);
-  if (pauseMatch) {
-    return content.slice(0, pauseMatch.index + pauseMatch[0].length);
-  }
-
-  return null;
-}
-
 function detectCjkFallbackChunk(content: string, locale: string) {
   if (codePointLength(content) < CJK_FALLBACK_TRIGGER_CHARS) {
     return null;
@@ -179,10 +163,7 @@ export function detectTokenSmoothingChunk(content: string, locale = "zh") {
   }
 
   if (isCjkSmoothingContent(content, locale)) {
-    return (
-      detectCjkPunctuationChunk(content) ??
-      detectCjkFallbackChunk(content, locale)
-    );
+    return detectCjkFallbackChunk(content, locale);
   }
 
   const wordChunk = content.match(/^\S+\s*/u)?.[0];