// src/schemas/validate.ts import { z } from 'zod'; import type * as t from '@/types'; import { Providers } from '@/common'; /** * Validation result from structured output */ export interface ValidationResult> { success: boolean; data?: T; error?: string; raw?: unknown; } /** * Validates structured output against a JSON Schema. * * @param output - The output to validate * @param schema - JSON Schema to validate against * @returns Validation result with success flag and data or error */ export function validateStructuredOutput( output: unknown, schema: Record ): ValidationResult { try { // Parse output if it's a string const data = typeof output === 'string' ? JSON.parse(output) : output; // Basic JSON Schema validation const errors = validateAgainstJsonSchema(data, schema); if (errors.length > 0) { return { success: false, error: `Validation failed: ${errors.join('; ')}`, raw: data, }; } return { success: true, data: data as Record, }; } catch (e) { return { success: false, error: `Failed to parse output: ${(e as Error).message}`, raw: output, }; } } /** * Validates data against a JSON Schema (simplified implementation). * For complex schemas, consider using ajv or similar library. */ function validateAgainstJsonSchema( data: unknown, schema: Record, path = '' ): string[] { const errors: string[] = []; if (schema.type === 'object' && typeof data !== 'object') { errors.push(`${path || 'root'}: expected object, got ${typeof data}`); return errors; } if (schema.type === 'array' && !Array.isArray(data)) { errors.push(`${path || 'root'}: expected array, got ${typeof data}`); return errors; } if (schema.type === 'string' && typeof data !== 'string') { errors.push(`${path || 'root'}: expected string, got ${typeof data}`); return errors; } if (schema.type === 'number' && typeof data !== 'number') { errors.push(`${path || 'root'}: expected number, got ${typeof data}`); return errors; } if (schema.type === 'boolean' && typeof data !== 'boolean') { errors.push(`${path || 'root'}: expected boolean, got ${typeof data}`); return errors; } // Validate required properties if ( schema.type === 'object' && Array.isArray(schema.required) && typeof data === 'object' && data !== null ) { for (const requiredProp of schema.required as string[]) { if (!(requiredProp in data)) { errors.push( `${path || 'root'}: missing required property '${requiredProp}'` ); } } } // Validate nested properties if ( schema.type === 'object' && schema.properties && typeof data === 'object' && data !== null ) { const properties = schema.properties as Record< string, Record >; for (const [key, propSchema] of Object.entries(properties)) { if (key in data) { const nestedErrors = validateAgainstJsonSchema( (data as Record)[key], propSchema, path ? `${path}.${key}` : key ); errors.push(...nestedErrors); } } } // Validate array items if (schema.type === 'array' && schema.items && Array.isArray(data)) { const itemSchema = schema.items as Record; for (let i = 0; i < data.length; i++) { const nestedErrors = validateAgainstJsonSchema( data[i], itemSchema, `${path || 'root'}[${i}]` ); errors.push(...nestedErrors); } } // Validate enum values if (schema.enum && Array.isArray(schema.enum)) { if (!schema.enum.includes(data)) { errors.push( `${path || 'root'}: value '${data}' not in enum [${(schema.enum as unknown[]).join(', ')}]` ); } } return errors; } /** * Converts a Zod schema to JSON Schema format. * This is a simplified converter for common types. */ export function zodToJsonSchema(zodSchema: z.ZodType): Record { // Use the zod-to-json-schema library if available // For now, provide a basic implementation try { // eslint-disable-next-line @typescript-eslint/no-require-imports const { zodToJsonSchema: convert } = require('zod-to-json-schema'); return convert(zodSchema) as Record; } catch { // Fallback: return a generic object schema return { type: 'object', additionalProperties: true, }; } } /** * Creates a structured output error message for retry. */ export function createValidationErrorMessage( result: ValidationResult, customMessage?: string ): string { if (customMessage) { return customMessage; } return `The response did not match the expected schema. Error: ${result.error}. Please try again and ensure your response exactly matches the required format.`; } /** * Checks if a value is a valid JSON Schema object. */ export function isValidJsonSchema( value: unknown ): value is Record { if (typeof value !== 'object' || value === null) { return false; } const schema = value as Record; // Basic check: must have a type or properties return ( typeof schema.type === 'string' || typeof schema.properties === 'object' || typeof schema.$schema === 'string' ); } /** * Normalizes a JSON Schema by adding defaults and cleaning up. */ export function normalizeJsonSchema( schema: Record, config?: t.StructuredOutputConfig ): Record { const normalized = { ...schema }; // Ensure type is set if (!normalized.type && normalized.properties) { normalized.type = 'object'; } // Add title from config name if (config?.name && !normalized.title) { normalized.title = config.name; } // Add description from config if (config?.description && !normalized.description) { normalized.description = config.description; } // Enable additionalProperties: false for strict mode if (config?.strict !== false && normalized.type === 'object') { normalized.additionalProperties = normalized.additionalProperties ?? false; } return normalized; } /** * Keywords unsupported by native structured output providers. * These are stripped from schemas and moved to description fields when strict mode is enabled. */ const UNSUPPORTED_NUMERIC_KEYWORDS = ['minimum', 'maximum', 'exclusiveMinimum', 'exclusiveMaximum', 'multipleOf'] as const; const UNSUPPORTED_STRING_KEYWORDS = ['minLength', 'maxLength', 'pattern', 'format'] as const; const UNSUPPORTED_ARRAY_KEYWORDS = ['minItems', 'maxItems', 'uniqueItems'] as const; const UNSUPPORTED_OBJECT_KEYWORDS = ['minProperties', 'maxProperties', 'patternProperties'] as const; /** * Result from schema preparation, includes the prepared schema and any warnings. */ export interface SchemaPreparationResult { schema: Record; warnings: string[]; } /** * Prepares a JSON Schema for a specific provider's native structured output API. * * This function normalizes the schema to comply with provider-specific requirements: * - Adds `additionalProperties: false` recursively to all objects (required by all providers in strict mode) * - Ensures all properties are listed in `required` (required by OpenAI and Anthropic) * - Strips unsupported constraint keywords (minimum, maxLength, etc.) and moves them to description * - Returns warnings for any modifications made * * @param schema - The original JSON Schema * @param provider - The LLM provider * @param strict - Whether strict mode is enabled (default: true) * @returns The prepared schema and any warnings */ export function prepareSchemaForProvider( schema: Record, provider: Providers | string, strict = true, ): SchemaPreparationResult { const warnings: string[] = []; if (!strict) { return { schema: { ...schema }, warnings }; } const prepared = prepareObjectRecursive(schema, '', provider, warnings); return { schema: prepared, warnings }; } /** * Recursively prepares a schema node for native structured output. */ function prepareObjectRecursive( schema: Record, path: string, provider: Providers | string, warnings: string[], depth = 0, ): Record { const result = { ...schema }; const currentPath = path || 'root'; // Warn on deep nesting (OpenAI limit is 5) if (depth > 5 && (provider === Providers.OPENAI || provider === Providers.AZURE)) { warnings.push(`${currentPath}: nesting depth ${depth} exceeds OpenAI's limit of 5 levels`); } // Handle object type if (result.type === 'object' || (result.properties && !result.type)) { if (!result.type) { result.type = 'object'; } // Add additionalProperties: false if (result.additionalProperties !== false) { result.additionalProperties = false; if (path) { warnings.push(`${currentPath}: set additionalProperties to false for strict mode`); } } // Ensure all properties are in required if (result.properties && typeof result.properties === 'object') { const properties = result.properties as Record>; const propertyNames = Object.keys(properties); const currentRequired = Array.isArray(result.required) ? result.required as string[] : []; const missingRequired = propertyNames.filter(name => !currentRequired.includes(name)); if (missingRequired.length > 0) { result.required = [...currentRequired, ...missingRequired]; warnings.push(`${currentPath}: added ${missingRequired.join(', ')} to required (all properties must be required for strict mode; use type union with null for optional fields)`); } // Recursively prepare nested properties const preparedProperties: Record> = {}; for (const [key, propSchema] of Object.entries(properties)) { preparedProperties[key] = preparePropertySchema( propSchema, path ? `${path}.${key}` : key, provider, warnings, depth + 1, ); } result.properties = preparedProperties; } } // Handle array type if (result.type === 'array' && result.items && typeof result.items === 'object') { result.items = preparePropertySchema( result.items as Record, `${currentPath}[]`, provider, warnings, depth + 1, ); // Strip unsupported array keywords for (const keyword of UNSUPPORTED_ARRAY_KEYWORDS) { if (keyword in result) { appendToDescription(result, `${keyword}: ${result[keyword]}`); delete result[keyword]; warnings.push(`${currentPath}: moved unsupported keyword '${keyword}' to description`); } } } // Handle anyOf if (Array.isArray(result.anyOf)) { result.anyOf = (result.anyOf as Record[]).map((variant, i) => preparePropertySchema(variant, `${currentPath}.anyOf[${i}]`, provider, warnings, depth + 1) ); } // Handle $defs / definitions for (const defsKey of ['$defs', 'definitions']) { if (result[defsKey] && typeof result[defsKey] === 'object') { const defs = result[defsKey] as Record>; const preparedDefs: Record> = {}; for (const [key, defSchema] of Object.entries(defs)) { preparedDefs[key] = preparePropertySchema( defSchema, `${defsKey}.${key}`, provider, warnings, depth, ); } result[defsKey] = preparedDefs; } } return result; } /** * Prepares a single property schema, stripping unsupported keywords based on type. */ function preparePropertySchema( schema: Record, path: string, provider: Providers | string, warnings: string[], depth: number, ): Record { const result = { ...schema }; // Strip unsupported numeric keywords if (result.type === 'number' || result.type === 'integer') { for (const keyword of UNSUPPORTED_NUMERIC_KEYWORDS) { if (keyword in result) { appendToDescription(result, `${keyword}: ${result[keyword]}`); delete result[keyword]; warnings.push(`${path}: moved unsupported keyword '${keyword}' to description`); } } } // Strip unsupported string keywords if (result.type === 'string') { for (const keyword of UNSUPPORTED_STRING_KEYWORDS) { if (keyword in result) { appendToDescription(result, `${keyword}: ${result[keyword]}`); delete result[keyword]; warnings.push(`${path}: moved unsupported keyword '${keyword}' to description`); } } } // Strip unsupported object keywords if (result.type === 'object') { for (const keyword of UNSUPPORTED_OBJECT_KEYWORDS) { if (keyword in result) { appendToDescription(result, `${keyword}: ${result[keyword]}`); delete result[keyword]; warnings.push(`${path}: moved unsupported keyword '${keyword}' to description`); } } } // Recursively handle nested objects and arrays if (result.type === 'object' || result.properties || result.type === 'array' || result.anyOf || result.$defs || result.definitions) { return prepareObjectRecursive(result, path, provider, warnings, depth); } return result; } /** * Appends a constraint description to the schema's description field. */ function appendToDescription(schema: Record, constraint: string): void { const existing = typeof schema.description === 'string' ? schema.description : ''; const constraintNote = `[Constraint: ${constraint}]`; schema.description = existing ? `${existing} ${constraintNote}` : constraintNote; }