import { createStreamingFileParser } from "../responseParsing/streamingFileParser.js";
import { StreamingFileParseResult } from "../types.js";
const FILE_PATH_PREFIX = "File path: ";
describe("streamingFileParser", () => {
let results: StreamingFileParseResult[];
let processChunk: (chunk: string) => void;
const xmlElement = "code";
beforeEach(() => {
results = [];
const streamingParser = createStreamingFileParser(
(result) => {
results.push(result);
},
FILE_PATH_PREFIX,
xmlElement
);
processChunk = streamingParser.processChunk;
});
describe("streamingFileParser with XML", () => {
test("parses a single complete file block", () => {
const input = `Some markdown text
File path: src/test.ts
const x = 1;
More markdown`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
{ type: "text-block", content: "Some markdown text\n" },
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "const x = 1;" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
{ type: "text", content: "More markdown" },
];
expect(results).toEqual(expected);
});
test("handles streaming chunks correctly", () => {
processChunk("File path: sr");
processChunk("c/test.ts\n\nconst x");
processChunk(" = 1;\n");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File path: sr" },
{ type: "text", content: "c/test.ts\n\nconst x" },
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "const x" },
{ type: "text", content: " = 1;\n" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("handles multiple file blocks", () => {
const input = `File path: src/first.ts
const first = true;
Some markdown in between
File path: src/second.ts
const second = false;
`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
{ type: "start-file-block", path: "src/first.ts" },
{
type: "text",
content: "const first = true;",
},
{
type: "end-file-block",
file: { path: "src/first.ts", content: "const first = true;" },
},
{
type: "text",
content: "Some markdown in between\n",
},
{ type: "text-block", content: "Some markdown in between\n" },
{ type: "start-file-block", path: "src/second.ts" },
{ type: "text", content: "const second = false;" },
{
type: "end-file-block",
file: { path: "src/second.ts", content: "const second = false;" },
},
];
expect(results).toEqual(expected);
});
test("handles empty file content", () => {
const input = `File path: src/empty.ts
`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
{ type: "start-file-block", path: "src/empty.ts" },
{ type: "text", content: "" },
{
type: "end-file-block",
file: { path: "src/empty.ts", content: "" },
},
];
expect(results).toEqual(expected);
});
test("ignores regular code blocks without file paths", () => {
const input = `Regular markdown
const x = 1;
`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
];
expect(results).toEqual(expected);
});
test("handles markdown content between file blocks split across chunks", () => {
processChunk("File path: src/first.ts\n\nconst x = 1;\n");
processChunk("Some markdown *with* ");
processChunk("**formatting** and a [link](http://example.com)\n");
processChunk("File path: src/second.ts\n\nconst y = 2;");
const expected: StreamingFileParseResult[] = [
{
type: "text",
content: "File path: src/first.ts\n\nconst x = 1;\n",
},
{ type: "start-file-block", path: "src/first.ts" },
{ type: "text", content: "const x = 1;" },
{
type: "end-file-block",
file: { path: "src/first.ts", content: "const x = 1;" },
},
{ type: "text", content: "Some markdown *with* " },
{
type: "text",
content: "**formatting** and a [link](http://example.com)\n",
},
{
type: "text",
content: "File path: src/second.ts\n\nconst y = 2;",
},
{
type: "text-block",
content:
"Some markdown *with* **formatting** and a [link](http://example.com)\n",
},
{ type: "start-file-block", path: "src/second.ts" },
{ type: "text", content: "const y = 2;" },
{
type: "end-file-block",
file: { path: "src/second.ts", content: "const y = 2;" },
},
];
expect(results).toEqual(expected);
});
test("handles consecutive file blocks without separator", () => {
const input =
"File path: src/first.ts\n\nconst x = 1;File path: src/second.ts\n\nconst y = 2;";
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
{ type: "start-file-block", path: "src/first.ts" },
{
type: "text",
content: "const x = 1;",
},
{
type: "end-file-block",
file: { path: "src/first.ts", content: "const x = 1;" },
},
{ type: "start-file-block", path: "src/second.ts" },
{ type: "text", content: "const y = 2;" },
{
type: "end-file-block",
file: { path: "src/second.ts", content: "const y = 2;" },
},
];
expect(results).toEqual(expected);
});
test("handles XML tags split across chunks", () => {
processChunk("File path: src/test.ts\nconst x = 1;");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File path: src/test.ts\nconst x = 1;" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("handles XML with different element names", () => {
results = [];
const { processChunk } = createStreamingFileParser(
(result) => {
results.push(result);
},
FILE_PATH_PREFIX,
"source"
);
const input = `File path: src/test.ts
const x = 1;
`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "const x = 1;" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("ignores non-matching XML tags", () => {
const input = `File path: src/test.ts
const x = 1;
`;
processChunk(input);
const expected: StreamingFileParseResult[] = [
{ type: "text", content: input },
];
expect(results).toEqual(expected);
});
test("handles File path: keyword split across chunks", () => {
processChunk("File ");
processChunk("path:");
processChunk(" src/test.ts\n\nconst x = 1;");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File " },
{ type: "text", content: "path:" },
{
type: "text",
content: " src/test.ts\n\nconst x = 1;",
},
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "const x = 1;" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("handles extreme chunking of start sequence", () => {
processChunk("F");
processChunk("ile ");
processChunk("pat");
processChunk("h: ");
processChunk("sr");
processChunk("c/te");
processChunk("st.ts");
processChunk("\n");
processChunk("");
processChunk("\n");
processChunk("const x = 1;");
processChunk("");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "F" },
{ type: "text", content: "ile " },
{ type: "text", content: "pat" },
{ type: "text", content: "h: " },
{ type: "text", content: "sr" },
{ type: "text", content: "c/te" },
{ type: "text", content: "st.ts" },
{ type: "text", content: "\n" },
{ type: "text", content: "" },
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "\n" },
{ type: "text", content: "const x = 1;" },
{ type: "text", content: "" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("handles newlines split in various positions", () => {
processChunk("File path: src/test.ts");
processChunk("\n");
processChunk("");
processChunk("\n");
processChunk("const x = 1;");
processChunk("\n");
processChunk("");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File path: src/test.ts" },
{ type: "text", content: "\n" },
{ type: "text", content: "" },
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "\n" },
{ type: "text", content: "const x = 1;" },
{ type: "text", content: "\n" },
{ type: "text", content: "" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
test("handles multiple files with various split points", () => {
processChunk("File path: src/a.ts\n\nconst x = 1;\nFi");
processChunk("le path: src/b.ts");
processChunk("\n\n");
processChunk("const y = 2;");
processChunk("");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File path: src/a.ts\n\nconst x = 1;\nFi" },
{
type: "end-file-block",
file: { path: "src/a.ts", content: "const x = 1;" },
},
{ type: "text", content: "Fi" },
{ type: "text", content: "le path: src/b.ts" },
{ type: "text", content: "\n\n" },
{ type: "start-file-block", path: "src/b.ts" },
{ type: "text", content: "const y = 2;" },
{ type: "text", content: "" },
{
type: "end-file-block",
file: { path: "src/b.ts", content: "const y = 2;" },
},
];
expect(results).toEqual(expected);
});
test("handles partial file marker followed by different file", () => {
processChunk("File pa");
processChunk("th: src/a.ts\n\nconst x = 1;\n");
processChunk("File path: src/b.ts\n\nconst y = 2;");
const expected: StreamingFileParseResult[] = [
{ type: "text", content: "File pa" },
{
type: "text",
content: "th: src/a.ts\n\nconst x = 1;\n",
},
{ type: "start-file-block", path: "src/a.ts" },
{ type: "text", content: "const x = 1;" },
{
type: "end-file-block",
file: { path: "src/a.ts", content: "const x = 1;" },
},
{ type: "text", content: "File path: src/b.ts\n\nconst y = 2;" },
{ type: "start-file-block", path: "src/b.ts" },
{ type: "text", content: "const y = 2;" },
{
type: "end-file-block",
file: { path: "src/b.ts", content: "const y = 2;" },
},
];
expect(results).toEqual(expected);
});
});
describe("finish functionality with XML", () => {
let results: StreamingFileParseResult[];
let processChunk: (chunk: string) => void;
let finish: () => void;
const xmlElement = "code";
beforeEach(() => {
results = [];
const streamingParser = createStreamingFileParser(
(result) => {
results.push(result);
},
FILE_PATH_PREFIX,
xmlElement
);
processChunk = streamingParser.processChunk;
finish = streamingParser.finish;
});
test("emits remaining content as text-block when finish is called", () => {
processChunk("Some unfinished markdown content");
finish();
expect(results).toEqual([
{ type: "text", content: "Some unfinished markdown content" },
{ type: "text-block", content: "Some unfinished markdown content" },
]);
});
test("does nothing when finish is called with empty buffer", () => {
finish();
expect(results).toEqual([]);
});
test("does nothing when finish is called with whitespace-only buffer", () => {
processChunk(" \n \t ");
finish();
expect(results).toEqual([{ type: "text", content: " \n \t " }]);
});
test("handles multiple chunks followed by finish", () => {
processChunk("Some initial ");
processChunk("content followed ");
processChunk("by more text");
finish();
expect(results).toEqual([
{ type: "text", content: "Some initial " },
{ type: "text", content: "content followed " },
{ type: "text", content: "by more text" },
{
type: "text-block",
content: "Some initial content followed by more text",
},
]);
});
test("handles input ending with XML closing tag", () => {
processChunk("File path: src/test.ts\n\nconst x = 1;");
processChunk("");
const expected: StreamingFileParseResult[] = [
{
type: "text",
content: "File path: src/test.ts\n\nconst x = 1;",
},
{ type: "start-file-block", path: "src/test.ts" },
{ type: "text", content: "const x = 1;" },
{ type: "text", content: "" },
{
type: "end-file-block",
file: { path: "src/test.ts", content: "const x = 1;" },
},
];
expect(results).toEqual(expected);
});
});
});