/** * Self-healing behavior tests for createCodeExecutionTool(). * * Covers two automatic recovery signals appended to formattedOutput: * 1. [OUTPUT_TOO_LARGE] — stdout > 16384 chars is capped (head 8192 + tail 4096) * 2. [CODE_TRUNCATION_LIKELY] — syntax error on code > 1500 chars suggests LLM truncation * * node-fetch is mocked via jest.mock so no real HTTP traffic is made. * The mock is structured to match node-fetch's CJS export shape: * require('node-fetch') === the fetch function itself (not a wrapper object). */ import { DynamicStructuredTool } from '@langchain/core/tools'; // --------------------------------------------------------------------------- // Constants mirroring the implementation thresholds // --------------------------------------------------------------------------- /** Minimum stdout length that triggers capping. */ const STDOUT_MAX_CHARS = 16384; /** Characters kept from the beginning of oversized stdout. */ const STDOUT_HEAD_CHARS = 8192; /** Characters kept from the end of oversized stdout. */ const STDOUT_TAIL_CHARS = 4096; /** Minimum code length required for truncation detection to fire. */ const CODE_TRUNCATION_MIN_CHARS = 1500; // --------------------------------------------------------------------------- // node-fetch mock // // node-fetch@2 is a CJS module. When required, the module itself IS the fetch // function. ts-jest hoists jest.mock() calls, so this factory runs before any // imports resolve — giving us a controlled mock from the first import. // --------------------------------------------------------------------------- const mockFetchFn = jest.fn(); jest.mock('node-fetch', () => { // node-fetch@2 is CJS: require('node-fetch') IS the fetch function. // Return mockFetchFn directly so the module value equals our mock. return mockFetchFn; }); // Import the tool factory AFTER jest.mock() so it picks up the mock. import { createCodeExecutionTool } from './CodeExecutor'; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- /** * Builds a minimal mock Response object that satisfies the tool's usage of * `response.ok` and `response.json()`. * * @param body - The JSON-serialisable response body. * @param status - HTTP status code (default 200). */ function mockResponse(body: object, status = 200) { return { ok: status >= 200 && status < 300, status, json: async () => body, }; } /** * Generates a string of exactly `length` characters by repeating a single * character. Avoids allocating enormous string literals in source. * * @param length - Desired string length. * @param char - Character to repeat (default 'x'). */ function repeat(length: number, char = 'x'): string { return char.repeat(length); } /** * Generates syntactically valid Python code that is at least `minLength` * characters long by padding with comment lines. Truncation detection * relies solely on `stderr` content, not actual code validity. * * @param minLength - Minimum character count for the returned code string. */ function longCode(minLength: number): string { let code = '# Auto-generated padding code\nprint("hello")\n'; while (code.length < minLength) { code += `# padding line ${code.length}\n`; } return code; } // --------------------------------------------------------------------------- // Test environment setup // --------------------------------------------------------------------------- beforeAll(() => { process.env.CODE_EXECUTOR_BASEURL = 'http://localhost:8088'; process.env.CODE_EXECUTOR_API_KEY = 'test-key'; }); afterAll(() => { delete process.env.CODE_EXECUTOR_BASEURL; delete process.env.CODE_EXECUTOR_API_KEY; }); beforeEach(() => { mockFetchFn.mockReset(); }); // --------------------------------------------------------------------------- // Tool factory and invocation helpers // --------------------------------------------------------------------------- /** * Creates a fresh tool instance using the test API key. */ function makeTool(): DynamicStructuredTool { return createCodeExecutionTool({ apiKey: 'test-key' }); } /** * Invokes the tool and returns only the string content part of the result. * The tool returns `[formattedOutput, artifact]` when responseFormat is * `content_and_artifact`; we surface only the string for assertion. * * @param execTool - Tool instance to invoke. * @param input - Execution input (lang, code, optional args). */ async function invoke( execTool: DynamicStructuredTool, input: { lang: string; code: string; args?: string[] } ): Promise { const result = await execTool.invoke(input, { toolCall: {} } as never); if (Array.isArray(result)) { return result[0] as string; } return result as string; } // =========================================================================== // Suite 1 — OUTPUT_TOO_LARGE tag // =========================================================================== describe('[OUTPUT_TOO_LARGE] stdout capping', () => { it('appends [OUTPUT_TOO_LARGE] when stdout exceeds 16384 chars', async () => { const largeStdout = repeat(STDOUT_MAX_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's1', stdout: largeStdout, stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("hi")', }); expect(output).toContain('[OUTPUT_TOO_LARGE]'); }); it('does NOT append [OUTPUT_TOO_LARGE] for stdout at exactly 16384 chars', async () => { const normalStdout = repeat(STDOUT_MAX_CHARS); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's2', stdout: normalStdout, stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("hi")', }); expect(output).not.toContain('[OUTPUT_TOO_LARGE]'); }); it('does NOT append [OUTPUT_TOO_LARGE] for short stdout', async () => { mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's3', stdout: 'hello world', stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("hello world")', }); expect(output).not.toContain('[OUTPUT_TOO_LARGE]'); }); it('preserves head content from oversized stdout', async () => { // First STDOUT_HEAD_CHARS chars are 'a', the remainder are 'b' const head = repeat(STDOUT_HEAD_CHARS, 'a'); const rest = repeat(STDOUT_MAX_CHARS + 1 - STDOUT_HEAD_CHARS, 'b'); const largeStdout = head + rest; mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's4', stdout: largeStdout, stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("x")' }); expect(output).toContain(repeat(STDOUT_HEAD_CHARS, 'a')); }); it('preserves tail content from oversized stdout', async () => { // Last STDOUT_TAIL_CHARS chars are 'z', the leading portion is 'y' const leading = repeat(STDOUT_MAX_CHARS + 1 - STDOUT_TAIL_CHARS, 'y'); const tail = repeat(STDOUT_TAIL_CHARS, 'z'); const largeStdout = leading + tail; mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's5', stdout: largeStdout, stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("x")' }); expect(output).toContain(repeat(STDOUT_TAIL_CHARS, 'z')); }); it('inserts [...N chars omitted...] between head and tail when capping', async () => { const totalLen = STDOUT_MAX_CHARS + 500; const largeStdout = repeat(totalLen); const expectedOmitted = totalLen - STDOUT_HEAD_CHARS - STDOUT_TAIL_CHARS; mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 's6', stdout: largeStdout, stderr: '' }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("x")' }); expect(output).toContain(`[...${expectedOmitted} chars omitted...]`); }); }); // =========================================================================== // Suite 2 — CODE_TRUNCATION_LIKELY tag // =========================================================================== describe('[CODE_TRUNCATION_LIKELY] truncation detection', () => { it('appends [CODE_TRUNCATION_LIKELY] for long code + SyntaxError in stderr', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't1', stdout: '', stderr: 'SyntaxError: invalid syntax (line 42)', }) ); const output = await invoke(makeTool(), { lang: 'py', code }); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); it('does NOT append [CODE_TRUNCATION_LIKELY] for short code + SyntaxError (likely a real bug)', async () => { // Deliberately broken but well under 1500 chars — this is a genuine syntax error const shortCode = 'print(x'; mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't2', stdout: '', stderr: 'SyntaxError: unexpected EOF while parsing', }) ); const output = await invoke(makeTool(), { lang: 'py', code: shortCode }); expect(output).not.toContain('[CODE_TRUNCATION_LIKELY]'); }); it('does NOT append [CODE_TRUNCATION_LIKELY] for long code + non-syntax error (NameError)', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't3', stdout: '', stderr: "NameError: name 'undefined_var' is not defined", }) ); const output = await invoke(makeTool(), { lang: 'py', code }); expect(output).not.toContain('[CODE_TRUNCATION_LIKELY]'); }); it('detects "unexpected end" in stderr for long code', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't4', stdout: '', stderr: 'unexpected end of input at line 200', }) ); const output = await invoke(makeTool(), { lang: 'js', code }); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); it('detects "unexpected eof" in stderr for long code', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't5', stdout: '', stderr: 'Unexpected EOF — did you forget a closing bracket?', }) ); const output = await invoke(makeTool(), { lang: 'js', code }); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); it('detects "unterminated" in stderr for long code', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't6', stdout: '', stderr: 'unterminated string literal at line 50', }) ); const output = await invoke(makeTool(), { lang: 'py', code }); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); it('is case-insensitive when matching stderr keywords', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 't7', stdout: '', // All-caps — the implementation uses .toLowerCase() before matching stderr: 'SYNTAXERROR: INVALID SYNTAX ON LINE 99', }) ); const output = await invoke(makeTool(), { lang: 'py', code }); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); }); // =========================================================================== // Suite 3 — Normal execution (no self-healing tags) // =========================================================================== describe('normal execution — no self-healing tags', () => { it('produces no self-healing tags on successful execution with short stdout', async () => { mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 'n1', stdout: 'Hello, world!\n', stderr: '', }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print("Hello, world!")', }); expect(output).not.toContain('[OUTPUT_TOO_LARGE]'); expect(output).not.toContain('[CODE_TRUNCATION_LIKELY]'); expect(output).toContain('Hello, world!'); }); it('produces no self-healing tags when stderr contains a runtime error on short code', async () => { mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 'n2', stdout: '', stderr: 'ZeroDivisionError: division by zero', }) ); const output = await invoke(makeTool(), { lang: 'py', code: 'print(1/0)' }); expect(output).not.toContain('[OUTPUT_TOO_LARGE]'); expect(output).not.toContain('[CODE_TRUNCATION_LIKELY]'); expect(output).toContain('ZeroDivisionError'); }); }); // =========================================================================== // Suite 4 — Both tags appear simultaneously // =========================================================================== describe('both self-healing tags can appear together', () => { it('appends both [OUTPUT_TOO_LARGE] and [CODE_TRUNCATION_LIKELY] when both conditions are met', async () => { const code = longCode(CODE_TRUNCATION_MIN_CHARS + 1); const largeStdout = repeat(STDOUT_MAX_CHARS + 1); mockFetchFn.mockResolvedValueOnce( mockResponse({ session_id: 'b1', stdout: largeStdout, stderr: 'SyntaxError: invalid syntax', }) ); const output = await invoke(makeTool(), { lang: 'py', code }); expect(output).toContain('[OUTPUT_TOO_LARGE]'); expect(output).toContain('[CODE_TRUNCATION_LIKELY]'); }); }); // =========================================================================== // Suite 5 — Tool construction guard // =========================================================================== describe('createCodeExecutionTool() construction', () => { it('throws when no API key is available', () => { const savedKey = process.env.CODE_EXECUTOR_API_KEY; delete process.env.CODE_EXECUTOR_API_KEY; expect(() => createCodeExecutionTool({})).toThrow( 'No API key provided for code execution tool.' ); process.env.CODE_EXECUTOR_API_KEY = savedKey; }); it('accepts apiKey directly from params without requiring env var', () => { const savedKey = process.env.CODE_EXECUTOR_API_KEY; delete process.env.CODE_EXECUTOR_API_KEY; expect(() => createCodeExecutionTool({ apiKey: 'direct-key' }) ).not.toThrow(); process.env.CODE_EXECUTOR_API_KEY = savedKey; }); });