import { createStreamingFileParser } from "../responseParsing/streamingFileParser.js"; import { StreamingFileParseResult } from "../types.js"; const FILE_PATH_PREFIX = "File path: "; describe("streamingFileParser", () => { let results: StreamingFileParseResult[]; let processChunk: (chunk: string) => void; const xmlElement = "code"; beforeEach(() => { results = []; const streamingParser = createStreamingFileParser( (result) => { results.push(result); }, FILE_PATH_PREFIX, xmlElement ); processChunk = streamingParser.processChunk; }); describe("streamingFileParser with XML", () => { test("parses a single complete file block", () => { const input = `Some markdown text File path: src/test.ts const x = 1; More markdown`; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, { type: "text-block", content: "Some markdown text\n" }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "const x = 1;" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, { type: "text", content: "More markdown" }, ]; expect(results).toEqual(expected); }); test("handles streaming chunks correctly", () => { processChunk("File path: sr"); processChunk("c/test.ts\n\nconst x"); processChunk(" = 1;\n"); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: sr" }, { type: "text", content: "c/test.ts\n\nconst x" }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "const x" }, { type: "text", content: " = 1;\n" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("handles multiple file blocks", () => { const input = `File path: src/first.ts const first = true; Some markdown in between File path: src/second.ts const second = false; `; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, { type: "start-file-block", path: "src/first.ts" }, { type: "text", content: "const first = true;", }, { type: "end-file-block", file: { path: "src/first.ts", content: "const first = true;" }, }, { type: "text", content: "Some markdown in between\n", }, { type: "text-block", content: "Some markdown in between\n" }, { type: "start-file-block", path: "src/second.ts" }, { type: "text", content: "const second = false;" }, { type: "end-file-block", file: { path: "src/second.ts", content: "const second = false;" }, }, ]; expect(results).toEqual(expected); }); test("handles empty file content", () => { const input = `File path: src/empty.ts `; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, { type: "start-file-block", path: "src/empty.ts" }, { type: "text", content: "" }, { type: "end-file-block", file: { path: "src/empty.ts", content: "" }, }, ]; expect(results).toEqual(expected); }); test("ignores regular code blocks without file paths", () => { const input = `Regular markdown const x = 1; `; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, ]; expect(results).toEqual(expected); }); test("handles markdown content between file blocks split across chunks", () => { processChunk("File path: src/first.ts\n\nconst x = 1;\n"); processChunk("Some markdown *with* "); processChunk("**formatting** and a [link](http://example.com)\n"); processChunk("File path: src/second.ts\n\nconst y = 2;"); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: src/first.ts\n\nconst x = 1;\n", }, { type: "start-file-block", path: "src/first.ts" }, { type: "text", content: "const x = 1;" }, { type: "end-file-block", file: { path: "src/first.ts", content: "const x = 1;" }, }, { type: "text", content: "Some markdown *with* " }, { type: "text", content: "**formatting** and a [link](http://example.com)\n", }, { type: "text", content: "File path: src/second.ts\n\nconst y = 2;", }, { type: "text-block", content: "Some markdown *with* **formatting** and a [link](http://example.com)\n", }, { type: "start-file-block", path: "src/second.ts" }, { type: "text", content: "const y = 2;" }, { type: "end-file-block", file: { path: "src/second.ts", content: "const y = 2;" }, }, ]; expect(results).toEqual(expected); }); test("handles consecutive file blocks without separator", () => { const input = "File path: src/first.ts\n\nconst x = 1;File path: src/second.ts\n\nconst y = 2;"; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, { type: "start-file-block", path: "src/first.ts" }, { type: "text", content: "const x = 1;", }, { type: "end-file-block", file: { path: "src/first.ts", content: "const x = 1;" }, }, { type: "start-file-block", path: "src/second.ts" }, { type: "text", content: "const y = 2;" }, { type: "end-file-block", file: { path: "src/second.ts", content: "const y = 2;" }, }, ]; expect(results).toEqual(expected); }); test("handles XML tags split across chunks", () => { processChunk("File path: src/test.ts\nconst x = 1;"); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: src/test.ts\nconst x = 1;" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("handles XML with different element names", () => { results = []; const { processChunk } = createStreamingFileParser( (result) => { results.push(result); }, FILE_PATH_PREFIX, "source" ); const input = `File path: src/test.ts const x = 1; `; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "const x = 1;" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("ignores non-matching XML tags", () => { const input = `File path: src/test.ts const x = 1; `; processChunk(input); const expected: StreamingFileParseResult[] = [ { type: "text", content: input }, ]; expect(results).toEqual(expected); }); test("handles File path: keyword split across chunks", () => { processChunk("File "); processChunk("path:"); processChunk(" src/test.ts\n\nconst x = 1;"); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File " }, { type: "text", content: "path:" }, { type: "text", content: " src/test.ts\n\nconst x = 1;", }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "const x = 1;" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("handles extreme chunking of start sequence", () => { processChunk("F"); processChunk("ile "); processChunk("pat"); processChunk("h: "); processChunk("sr"); processChunk("c/te"); processChunk("st.ts"); processChunk("\n"); processChunk(""); processChunk("\n"); processChunk("const x = 1;"); processChunk(""); const expected: StreamingFileParseResult[] = [ { type: "text", content: "F" }, { type: "text", content: "ile " }, { type: "text", content: "pat" }, { type: "text", content: "h: " }, { type: "text", content: "sr" }, { type: "text", content: "c/te" }, { type: "text", content: "st.ts" }, { type: "text", content: "\n" }, { type: "text", content: "" }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "\n" }, { type: "text", content: "const x = 1;" }, { type: "text", content: "" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("handles newlines split in various positions", () => { processChunk("File path: src/test.ts"); processChunk("\n"); processChunk(""); processChunk("\n"); processChunk("const x = 1;"); processChunk("\n"); processChunk(""); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: src/test.ts" }, { type: "text", content: "\n" }, { type: "text", content: "" }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "\n" }, { type: "text", content: "const x = 1;" }, { type: "text", content: "\n" }, { type: "text", content: "" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); test("handles multiple files with various split points", () => { processChunk("File path: src/a.ts\n\nconst x = 1;\nFi"); processChunk("le path: src/b.ts"); processChunk("\n\n"); processChunk("const y = 2;"); processChunk(""); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: src/a.ts\n\nconst x = 1;\nFi" }, { type: "end-file-block", file: { path: "src/a.ts", content: "const x = 1;" }, }, { type: "text", content: "Fi" }, { type: "text", content: "le path: src/b.ts" }, { type: "text", content: "\n\n" }, { type: "start-file-block", path: "src/b.ts" }, { type: "text", content: "const y = 2;" }, { type: "text", content: "" }, { type: "end-file-block", file: { path: "src/b.ts", content: "const y = 2;" }, }, ]; expect(results).toEqual(expected); }); test("handles partial file marker followed by different file", () => { processChunk("File pa"); processChunk("th: src/a.ts\n\nconst x = 1;\n"); processChunk("File path: src/b.ts\n\nconst y = 2;"); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File pa" }, { type: "text", content: "th: src/a.ts\n\nconst x = 1;\n", }, { type: "start-file-block", path: "src/a.ts" }, { type: "text", content: "const x = 1;" }, { type: "end-file-block", file: { path: "src/a.ts", content: "const x = 1;" }, }, { type: "text", content: "File path: src/b.ts\n\nconst y = 2;" }, { type: "start-file-block", path: "src/b.ts" }, { type: "text", content: "const y = 2;" }, { type: "end-file-block", file: { path: "src/b.ts", content: "const y = 2;" }, }, ]; expect(results).toEqual(expected); }); }); describe("finish functionality with XML", () => { let results: StreamingFileParseResult[]; let processChunk: (chunk: string) => void; let finish: () => void; const xmlElement = "code"; beforeEach(() => { results = []; const streamingParser = createStreamingFileParser( (result) => { results.push(result); }, FILE_PATH_PREFIX, xmlElement ); processChunk = streamingParser.processChunk; finish = streamingParser.finish; }); test("emits remaining content as text-block when finish is called", () => { processChunk("Some unfinished markdown content"); finish(); expect(results).toEqual([ { type: "text", content: "Some unfinished markdown content" }, { type: "text-block", content: "Some unfinished markdown content" }, ]); }); test("does nothing when finish is called with empty buffer", () => { finish(); expect(results).toEqual([]); }); test("does nothing when finish is called with whitespace-only buffer", () => { processChunk(" \n \t "); finish(); expect(results).toEqual([{ type: "text", content: " \n \t " }]); }); test("handles multiple chunks followed by finish", () => { processChunk("Some initial "); processChunk("content followed "); processChunk("by more text"); finish(); expect(results).toEqual([ { type: "text", content: "Some initial " }, { type: "text", content: "content followed " }, { type: "text", content: "by more text" }, { type: "text-block", content: "Some initial content followed by more text", }, ]); }); test("handles input ending with XML closing tag", () => { processChunk("File path: src/test.ts\n\nconst x = 1;"); processChunk(""); const expected: StreamingFileParseResult[] = [ { type: "text", content: "File path: src/test.ts\n\nconst x = 1;", }, { type: "start-file-block", path: "src/test.ts" }, { type: "text", content: "const x = 1;" }, { type: "text", content: "" }, { type: "end-file-block", file: { path: "src/test.ts", content: "const x = 1;" }, }, ]; expect(results).toEqual(expected); }); }); });