import { CLIExecutionResult } from './cli-executor'; export enum OutputType { JSON = 'json', YAML = 'yaml', TABLE = 'table', PLAIN_TEXT = 'plain_text' } export interface ValidationResult { isValid: boolean; errors: string[]; warnings: string[]; outputType: OutputType; metadata?: { lineCount?: number; columnCount?: number; jsonKeys?: string[]; yamlKeys?: string[]; tableHeaders?: string[]; detectedPatterns?: string[]; exitCode?: number; executionTime?: number; timedOut?: boolean; }; } export interface SchemaValidationRule { path: string; type: 'string' | 'number' | 'boolean' | 'array' | 'object'; required?: boolean; pattern?: RegExp; minLength?: number; maxLength?: number; min?: number; max?: number; enum?: any[]; } export interface TextPattern { name: string; pattern: RegExp; required?: boolean; description?: string; } export interface TableValidationOptions { expectedColumns?: string[]; minRows?: number; maxRows?: number; columnSeparator?: string | RegExp; headerRequired?: boolean; allowEmptyRows?: boolean; } export interface JsonValidationOptions { schema?: SchemaValidationRule[]; strictMode?: boolean; allowAdditionalProperties?: boolean; } export interface YamlValidationOptions { schema?: SchemaValidationRule[]; strictMode?: boolean; allowMultipleDocuments?: boolean; } export interface TextValidationOptions { patterns?: TextPattern[]; minLines?: number; maxLines?: number; encoding?: string; allowEmptyOutput?: boolean; } export class OutputValidator { /** * Detect the type of output with improved accuracy */ static detectOutputType(output: string): OutputType { const trimmed = output.trim(); if (!trimmed) { return OutputType.PLAIN_TEXT; } // Check for JSON first (most specific) if (this.isValidJson(trimmed)) { return OutputType.JSON; } // Check for YAML (improved detection) if (this.looksLikeYaml(trimmed)) { return OutputType.YAML; } // Check for table format (improved detection) if (this.looksLikeTable(trimmed)) { return OutputType.TABLE; } return OutputType.PLAIN_TEXT; } /** * Comprehensive output validation with detailed results */ static validateOutput( output: string, expectedType?: OutputType, options?: JsonValidationOptions | YamlValidationOptions | TableValidationOptions | TextValidationOptions ): ValidationResult { const detectedType = this.detectOutputType(output); // Check if detected type matches expected type if (expectedType && detectedType !== expectedType) { return { isValid: false, errors: [`Expected ${expectedType} output, but detected ${detectedType}`], warnings: [], outputType: detectedType }; } // Validate based on detected type switch (detectedType) { case OutputType.JSON: return this.validateJson(output, options as JsonValidationOptions); case OutputType.YAML: return this.validateYaml(output, options as YamlValidationOptions); case OutputType.TABLE: return this.validateTable(output, options as TableValidationOptions); case OutputType.PLAIN_TEXT: return this.validatePlainText(output, options as TextValidationOptions); default: return { isValid: false, errors: [`Unknown output type: ${detectedType}`], warnings: [], outputType: detectedType }; } } /** * Enhanced JSON validation with schema support */ static validateJson(output: string, options: JsonValidationOptions = {}): ValidationResult { const errors: string[] = []; const warnings: string[] = []; const metadata: any = {}; try { const parsed = JSON.parse(output); // Extract JSON keys for metadata if (typeof parsed === 'object' && parsed !== null) { metadata.jsonKeys = this.extractJsonKeys(parsed); } // Schema validation if provided if (options.schema) { const schemaErrors = this.validateJsonSchema(parsed, options.schema); errors.push(...schemaErrors); } // Check for additional properties if strict mode if (options.strictMode && !options.allowAdditionalProperties && options.schema) { const additionalProps = this.findAdditionalProperties(parsed, options.schema); if (additionalProps.length > 0) { warnings.push(`Additional properties found: ${additionalProps.join(', ')}`); } } return { isValid: errors.length === 0, errors, warnings, outputType: OutputType.JSON, metadata }; } catch (error) { return { isValid: false, errors: [`Invalid JSON: ${error instanceof Error ? error.message : String(error)}`], warnings: [], outputType: OutputType.JSON }; } } /** * YAML validation with schema support */ static validateYaml(output: string, options: YamlValidationOptions = {}): ValidationResult { const errors: string[] = []; const warnings: string[] = []; const metadata: any = {}; try { // Basic YAML parsing (simplified - would need yaml library for full support) const yamlKeys = this.extractYamlKeys(output); metadata.yamlKeys = yamlKeys; // Check for multiple documents const documentCount = (output.match(/^---/gm) || []).length; if (documentCount > 1 && !options.allowMultipleDocuments) { warnings.push(`Multiple YAML documents found (${documentCount})`); } // Additional YAML-specific warnings could be added here if (options.strictMode && yamlKeys.length === 0) { warnings.push('No YAML keys detected in strict mode'); } // Basic structure validation if (yamlKeys.length === 0) { errors.push('No YAML keys found in output'); } return { isValid: errors.length === 0, errors, warnings, outputType: OutputType.YAML, metadata }; } catch (error) { return { isValid: false, errors: [`Invalid YAML: ${error instanceof Error ? error.message : String(error)}`], warnings: [], outputType: OutputType.YAML }; } } /** * Enhanced table validation with comprehensive options */ static validateTable(output: string, options: TableValidationOptions = {}): ValidationResult { const errors: string[] = []; const warnings: string[] = []; const metadata: any = {}; const lines = output.split('\n'); const nonEmptyLines = lines.filter(line => line.trim()); metadata.lineCount = lines.length; if (nonEmptyLines.length === 0) { errors.push('Empty table output'); return { isValid: false, errors, warnings, outputType: OutputType.TABLE, metadata }; } // Detect table structure const tableInfo = this.analyzeTableStructure(output, options.columnSeparator); metadata.columnCount = tableInfo.columnCount; metadata.tableHeaders = tableInfo.headers; // Validate row count if (options.minRows && tableInfo.dataRows < options.minRows) { errors.push(`Expected at least ${options.minRows} data rows, found ${tableInfo.dataRows}`); } if (options.maxRows && tableInfo.dataRows > options.maxRows) { errors.push(`Expected at most ${options.maxRows} data rows, found ${tableInfo.dataRows}`); } // Validate headers if (options.headerRequired !== false && !tableInfo.hasHeader) { errors.push('Table header is required but not found'); } // Check for expected columns if (options.expectedColumns && tableInfo.headers) { const missingColumns = options.expectedColumns.filter(col => !tableInfo.headers!.some(header => header.toLowerCase().includes(col.toLowerCase())) ); if (missingColumns.length > 0) { errors.push(`Missing expected columns: ${missingColumns.join(', ')}`); } } // Check for empty rows if (!options.allowEmptyRows && tableInfo.emptyRows > 0) { warnings.push(`Found ${tableInfo.emptyRows} empty rows in table`); } return { isValid: errors.length === 0, errors, warnings, outputType: OutputType.TABLE, metadata }; } /** * Plain text validation with pattern matching */ static validatePlainText(output: string, options: TextValidationOptions = {}): ValidationResult { const errors: string[] = []; const warnings: string[] = []; const metadata: any = {}; const lines = output.split('\n'); metadata.lineCount = lines.length; // Check if empty output is allowed if (!output.trim() && !options.allowEmptyOutput) { errors.push('Empty output not allowed'); } // Validate line count if (options.minLines && lines.length < options.minLines) { errors.push(`Expected at least ${options.minLines} lines, found ${lines.length}`); } if (options.maxLines && lines.length > options.maxLines) { errors.push(`Expected at most ${options.maxLines} lines, found ${lines.length}`); } // Pattern matching if (options.patterns) { const detectedPatterns: string[] = []; for (const pattern of options.patterns) { const matches = output.match(pattern.pattern); if (matches) { detectedPatterns.push(pattern.name); } else if (pattern.required) { errors.push(`Required pattern '${pattern.name}' not found: ${pattern.description || pattern.pattern.toString()}`); } } metadata.detectedPatterns = detectedPatterns; } return { isValid: errors.length === 0, errors, warnings, outputType: OutputType.PLAIN_TEXT, metadata }; } /** * Validate command execution result with comprehensive options */ static validateResult( result: CLIExecutionResult, expectedType?: OutputType, options?: JsonValidationOptions | YamlValidationOptions | TableValidationOptions | TextValidationOptions ): ValidationResult { // Check for command execution failure if (result.exitCode !== 0) { return { isValid: false, errors: [`Command failed with exit code ${result.exitCode}: ${result.stderr}`], warnings: [], outputType: OutputType.PLAIN_TEXT, metadata: { exitCode: result.exitCode, executionTime: result.executionTime, timedOut: result.timedOut } }; } // Validate the output return this.validateOutput(result.stdout, expectedType, options); } /** * Create a detailed validation report */ static createValidationReport(results: ValidationResult[]): string { const report: string[] = []; report.push('=== Output Validation Report ===\n'); let totalTests = results.length; let passedTests = results.filter(r => r.isValid).length; let failedTests = totalTests - passedTests; report.push(`Total Tests: ${totalTests}`); report.push(`Passed: ${passedTests}`); report.push(`Failed: ${failedTests}`); report.push(`Success Rate: ${((passedTests / totalTests) * 100).toFixed(1)}%\n`); // Group by output type const byType = results.reduce((acc, result, index) => { if (!acc[result.outputType]) { acc[result.outputType] = []; } acc[result.outputType].push({ result, index }); return acc; }, {} as Record>); for (const [type, typeResults] of Object.entries(byType)) { report.push(`\n--- ${type.toUpperCase()} Results ---`); typeResults.forEach(({ result, index }) => { const status = result.isValid ? '✓ PASS' : '✗ FAIL'; report.push(`Test ${index + 1}: ${status}`); if (result.errors.length > 0) { report.push(` Errors:`); result.errors.forEach(error => report.push(` - ${error}`)); } if (result.warnings.length > 0) { report.push(` Warnings:`); result.warnings.forEach(warning => report.push(` - ${warning}`)); } if (result.metadata) { report.push(` Metadata: ${JSON.stringify(result.metadata, null, 2)}`); } }); } return report.join('\n'); } // Private helper methods private static isValidJson(str: string): boolean { try { JSON.parse(str); return true; } catch { return false; } } private static looksLikeYaml(str: string): boolean { const lines = str.split('\n'); // YAML characteristics const hasColons = str.includes(':'); const hasIndentation = lines.some(line => line.match(/^\s+/)); const hasListItems = lines.some(line => line.trim().startsWith('-')); const hasDocumentSeparator = str.includes('---'); // Must have colons and either indentation, list items, or document separators return hasColons && (hasIndentation || hasListItems || hasDocumentSeparator); } private static looksLikeTable(str: string): boolean { const lines = str.split('\n').filter(line => line.trim()); if (lines.length < 2) return false; // Check for common table patterns const hasPipes = lines.some(line => line.includes('|')); const hasTabs = lines.some(line => line.includes('\t')); const hasMultipleSpaces = lines.some(line => /\s{2,}/.test(line)); const hasConsistentColumns = this.hasConsistentColumnStructure(lines); return hasPipes || hasTabs || (hasMultipleSpaces && hasConsistentColumns); } private static hasConsistentColumnStructure(lines: string[]): boolean { if (lines.length < 2) return false; // Check if lines have similar structure (similar number of words/columns) const wordCounts = lines.map(line => line.trim().split(/\s+/).length); const avgWordCount = wordCounts.reduce((a, b) => a + b, 0) / wordCounts.length; // Allow some variance but expect consistency return wordCounts.every(count => Math.abs(count - avgWordCount) <= 2); } private static extractJsonKeys(obj: any, prefix = ''): string[] { const keys: string[] = []; if (typeof obj === 'object' && obj !== null) { if (Array.isArray(obj)) { obj.forEach((item, index) => { const itemKeys = this.extractJsonKeys(item, `${prefix}[${index}]`); keys.push(...itemKeys); }); } else { Object.keys(obj).forEach(key => { const fullKey = prefix ? `${prefix}.${key}` : key; keys.push(fullKey); const nestedKeys = this.extractJsonKeys(obj[key], fullKey); keys.push(...nestedKeys); }); } } return keys; } private static extractYamlKeys(yamlStr: string): string[] { const keys: string[] = []; const lines = yamlStr.split('\n'); for (const line of lines) { const trimmed = line.trim(); if (trimmed && trimmed.includes(':') && !trimmed.startsWith('#')) { const colonIndex = trimmed.indexOf(':'); const key = trimmed.substring(0, colonIndex).trim(); if (key && !key.startsWith('-')) { keys.push(key); } } } return keys; } private static analyzeTableStructure(output: string, separator?: string | RegExp) { const lines = output.split('\n'); const nonEmptyLines = lines.filter(line => line.trim()); let headers: string[] | null = null; let columnCount = 0; let dataRows = 0; let emptyRows = 0; let hasHeader = false; if (nonEmptyLines.length > 0) { // Assume first non-empty line is header const firstLine = nonEmptyLines[0]; if (separator) { headers = firstLine.split(separator).map(h => h.trim()); } else { // Auto-detect separator if (firstLine.includes('|')) { headers = firstLine.split('|').map(h => h.trim()).filter(h => h); } else if (firstLine.includes('\t')) { headers = firstLine.split('\t').map(h => h.trim()); } else { headers = firstLine.split(/\s{2,}/).map(h => h.trim()); } } columnCount = headers.length; hasHeader = true; // Count actual data rows, excluding header and separator lines dataRows = nonEmptyLines.slice(1).filter(line => { // Skip markdown-style separator lines like |------|-----|------| const trimmed = line.trim(); return !this.isTableSeparatorLine(trimmed); }).length; } emptyRows = lines.length - nonEmptyLines.length; return { headers, columnCount, dataRows, emptyRows, hasHeader }; } private static isTableSeparatorLine(line: string): boolean { // Check if line is a markdown-style table separator (e.g., |------|-----|------|) const trimmed = line.trim(); if (trimmed.includes('|')) { // Remove pipes and check if remaining content is mostly dashes and spaces const withoutPipes = trimmed.replace(/\|/g, ''); return /^[\s\-:]*$/.test(withoutPipes); } return false; } private static validateJsonSchema(data: any, schema: SchemaValidationRule[]): string[] { const errors: string[] = []; for (const rule of schema) { const value = this.getValueByPath(data, rule.path); // Check if required field is missing if (rule.required && (value === undefined || value === null)) { errors.push(`Required field '${rule.path}' is missing`); continue; } // Skip validation if field is not required and missing if (!rule.required && (value === undefined || value === null)) { continue; } // Type validation if (!this.validateType(value, rule.type)) { errors.push(`Field '${rule.path}' expected type ${rule.type}, got ${typeof value}`); continue; } // Additional validations based on type if (rule.type === 'string' && typeof value === 'string') { if (rule.pattern && !rule.pattern.test(value)) { errors.push(`Field '${rule.path}' does not match pattern ${rule.pattern}`); } if (rule.minLength && value.length < rule.minLength) { errors.push(`Field '${rule.path}' length ${value.length} is less than minimum ${rule.minLength}`); } if (rule.maxLength && value.length > rule.maxLength) { errors.push(`Field '${rule.path}' length ${value.length} exceeds maximum ${rule.maxLength}`); } } if (rule.type === 'number' && typeof value === 'number') { if (rule.min !== undefined && value < rule.min) { errors.push(`Field '${rule.path}' value ${value} is less than minimum ${rule.min}`); } if (rule.max !== undefined && value > rule.max) { errors.push(`Field '${rule.path}' value ${value} exceeds maximum ${rule.max}`); } } if (rule.enum && !rule.enum.includes(value)) { errors.push(`Field '${rule.path}' value '${value}' is not in allowed values: ${rule.enum.join(', ')}`); } } return errors; } private static getValueByPath(obj: any, path: string): any { return path.split('.').reduce((current, key) => { return current && current[key] !== undefined ? current[key] : undefined; }, obj); } private static validateType(value: any, expectedType: string): boolean { switch (expectedType) { case 'string': return typeof value === 'string'; case 'number': return typeof value === 'number' && !isNaN(value); case 'boolean': return typeof value === 'boolean'; case 'array': return Array.isArray(value); case 'object': return typeof value === 'object' && value !== null && !Array.isArray(value); default: return false; } } private static findAdditionalProperties(data: any, schema: SchemaValidationRule[]): string[] { const definedPaths = new Set(schema.map(rule => rule.path)); const actualPaths = this.extractJsonKeys(data); return actualPaths.filter(path => !definedPaths.has(path)); } }