import { describe, expect, it } from "vitest"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { createStreamingMarkdownBufferState, isMarkdownOpenerBoundary, resolveStreamingMarkdownDisplayContent, splitStreamingText, splitStreamingTextFallback, takeNextStreamingMarkdownChunk, } from "./components/streaming-token-response-state"; describe("streaming token response state", () => { it("splits streaming text without dropping content", () => { const content = "管网压力异常需要继续分析"; const chunks = splitStreamingText(content, "zh"); expect(chunks.length).toBeGreaterThan(0); expect(chunks.join("")).toBe(content); }); it("uses bounded code point chunks for fallback segmentation", () => { expect(splitStreamingTextFallback("管网压力")).toEqual(["管网", "压力"]); }); it("emits continuous Chinese text without waiting for markdown block closure", () => { let state = createStreamingMarkdownBufferState("正在分析管网压力异常需要继续分析"); const chunks: string[] = []; for (let index = 0; index < 20; index += 1) { const next = takeNextStreamingMarkdownChunk(state); state = next.state; if (!next.chunk) { break; } chunks.push(next.chunk); } expect(chunks.length).toBeGreaterThan(0); expect(chunks.join("")).toBe("正在分析管网压力异常需要继续分析"); }); it("flushes a heading opener atomically", () => { const next = takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("### 压力分析") ); expect(next.chunk).toBe("### 压"); expect(isMarkdownOpenerBoundary("### 压力分析")).toBe(true); }); it("waits for a heading opener to include text", () => { const next = takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("### ") ); expect(next.chunk).toBe(""); expect(next.state.displayContent).toBe(""); }); it("flushes list and quote openers atomically", () => { expect( takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("- 节点 A") ).chunk ).toBe("- 节"); expect( takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("> 注意压力") ).chunk ).toBe("> 注"); }); it("flushes code and mermaid fence openers as complete lines", () => { expect( takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("```ts\nconst pressure = 1;") ).chunk ).toBe("```ts\n"); expect( takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("```mermaid\ngraph TD") ).chunk ).toBe("```mermaid\n"); }); it("waits for fenced code openers to reach the line ending", () => { const next = takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState("```ts") ); expect(next.chunk).toBe(""); expect(next.state.displayContent).toBe(""); }); it("flushes table header and separator rows atomically", () => { const content = "| 节点 | 压力 |\n| --- | --- |\n| J1 | 0.32 |\n\n后续"; expect( takeNextStreamingMarkdownChunk( createStreamingMarkdownBufferState(content) ).chunk ).toBe("| 节点 | 压力 |\n| --- | --- |\n"); }); it("grows table rows after the table opener is visible", () => { const tableOpener = "| 节点 | 压力 |\n| --- | --- |\n"; const state = createStreamingMarkdownBufferState( `${tableOpener}| J1 | 0.32 |`, tableOpener ); expect(takeNextStreamingMarkdownChunk(state).chunk).not.toBe(""); }); it("flushes all display content after streaming is done", () => { const content = "前文\n\n```ts\nconst pressure = 1;"; expect(resolveStreamingMarkdownDisplayContent(content, "前文", true)).toBe(content); }); it("StreamingTokenResponse does not use closed-block markdown gating", () => { const source = readFileSync( join(process.cwd(), "features/agent/components/streaming-token-response.tsx"), "utf8" ); expect(source).not.toContain("splitStreamingMarkdownContent"); }); });