import process from "node:process"; import { readFileSync } from "node:fs"; import { writeFile } from "node:fs/promises"; import { OpenAI } from "openai"; import { SupportTicket } from "../../lib/support"; import { type SupportTicketContext, type SupportRequestResult, } from "../../lib/schemas"; import { MultiAttemptStrategy } from "../../lib/support/offer-detection"; const testCasesPath = "./data/prompt/require-support/test.json"; const outputFilePath = "./out/support-ticket/results.txt"; type TestCase = { id: number; chatHistory: Array<{ role: "user" | "assistant"; content: string }>; userRequestedSupportTicket: string; userAcceptsOfferedSupportTicket: string; generateSupportTicket: string; }; function readFromFile(path: string): TestCase[] { try { const fileContent = readFileSync(path, "utf8"); return JSON.parse(fileContent) as TestCase[]; } catch (error) { if (error instanceof Error) { throw new TypeError(`Unable to read file: ${path}, ${error.message}`); } throw new Error(`Unable to read file: ${path}`); } } async function testSupportTicket( testCase: TestCase, apiKey: string ): Promise<{ id: number; chatHistory: Array<{ role: "user" | "assistant"; content: string }>; expected: { userRequestedSupportTicket: string; userAcceptsOfferedSupportTicket: string; generateSupportTicket: string; }; actual: SupportRequestResult; passed: boolean; }> { const supportTicketContext: SupportTicketContext = { chatHistory: testCase.chatHistory, environmentContext: "", environmentMetadata: {}, }; const openai = new OpenAI({ apiKey }); const supportTicket = new SupportTicket( new MultiAttemptStrategy(supportTicketContext, openai), supportTicketContext, openai ); const result: SupportRequestResult = await supportTicket.requireSupport(); const isEqual = result.userRequestedSupportTicket === testCase.userRequestedSupportTicket && result.userAcceptsOfferedSupportTicket === testCase.userAcceptsOfferedSupportTicket && result.generateSupportTicket === testCase.generateSupportTicket; if (!isEqual) { console.log( `Failed tests: ID: ${testCase.id},\nExpected: ${JSON.stringify( { userRequestedSupportTicket: testCase.userRequestedSupportTicket, userAcceptsOfferedSupportTicket: testCase.userAcceptsOfferedSupportTicket, generateSupportTicket: testCase.generateSupportTicket, }, null, 2 )}\nReceived: ${JSON.stringify(result, null, 2)}` ); } expect(result).toStrictEqual({ userRequestedSupportTicket: testCase.userRequestedSupportTicket, userAcceptsOfferedSupportTicket: testCase.userAcceptsOfferedSupportTicket, generateSupportTicket: testCase.generateSupportTicket, }); return { id: testCase.id, chatHistory: testCase.chatHistory, expected: { userRequestedSupportTicket: testCase.userRequestedSupportTicket, userAcceptsOfferedSupportTicket: testCase.userAcceptsOfferedSupportTicket, generateSupportTicket: testCase.generateSupportTicket, }, actual: result, passed: isEqual, }; } async function runTestCases(testCases: TestCase[], apiKey: string) { const results = await Promise.allSettled( testCases.map(async (testCase) => testSupportTicket(testCase, apiKey)) ); const formattedResults = formatResults(results); await appendToFile(formattedResults.join("\n"), outputFilePath); } async function appendToFile(content: string, filePath: string) { const timestamp = new Date().toISOString(); const formattedContent = `${timestamp}:\n${content}\n`; try { const fs = require("node:fs").promises; // Using promises for async file operations await fs.appendFile(filePath, formattedContent); } catch (error) { console.error(`Error appending to file: ${filePath}`, error); } } function formatResults(results: any[]) { return results.map((result) => { if (result.status === "fulfilled") { return `Test ID: ${result.value.id}\nPassed: ${result.value.passed}\nExpected: ${JSON.stringify( result.value.expected, null, 2 )}\nActual: ${JSON.stringify(result.value.actual, null, 2)}\n\n`; } return `Test ID: ${result.reason.id}\nError: ${result.reason.message}\n\n`; }); } test("Test require support cases", async () => { const testCases = readFromFile(testCasesPath); const apiKey = process.env.OPENAI_API_KEY; if (!apiKey) { throw new Error("OPENAI_API_KEY is not set"); } await runTestCases(testCases, apiKey); });