#!/usr/bin/env node /** * OAP Conformance Test Runner * * Validates OAP implementations against the specification by: * - Validating JSON against schemas * - Evaluating policy pack logic * - Verifying Ed25519 signatures over JCS payloads * - Producing PASS/FAIL reports */ import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from "fs"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import { SchemaValidator } from "./validators.js"; import { JCS } from "./jcs.js"; import { Ed25519 } from "./ed25519.js"; import { TestCase, TestResult, ConformanceReport } from "./cases.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); interface RunnerOptions { pack?: string; verbose?: boolean; report?: boolean; } class ConformanceRunner { private validator: SchemaValidator; private jcs: JCS; private ed25519: Ed25519; private options: RunnerOptions; constructor(options: RunnerOptions = {}) { this.options = options; this.validator = new SchemaValidator(); this.jcs = new JCS(); this.ed25519 = new Ed25519(); } async run(): Promise { console.log(chalk.blue.bold("\nšŸ” OAP Conformance Test Runner v1.0.0\n")); const spinner = ora("Loading test cases...").start(); try { // Load test cases const testCases = await this.loadTestCases(); spinner.succeed(`Loaded ${testCases.length} test cases`); // Run tests const results = await this.runTests(testCases); // Generate report const report = this.generateReport(results); // Display results this.displayResults(report); // Save report if requested if (this.options.report) { await this.saveReport(report); } // Exit with appropriate code process.exit(report.summary.failed > 0 ? 1 : 0); } catch (error) { spinner.fail("Test execution failed"); console.error(chalk.red("Error:"), error); process.exit(1); } } private async loadTestCases(): Promise { const casesDir = join(__dirname, "..", "cases"); const testCases: TestCase[] = []; // Load policy pack test cases const packDirs = this.getPackDirectories(casesDir); for (const packDir of packDirs) { const packId = packDir.split("/").pop()!; // Skip if specific pack requested and doesn't match if (this.options.pack && !packId.includes(this.options.pack)) { continue; } const packCases = await this.loadPackTestCases(packDir, packId); testCases.push(...packCases); } return testCases; } private getPackDirectories(casesDir: string): string[] { if (!existsSync(casesDir)) { return []; } return readdirSync(casesDir, { withFileTypes: true }) .filter((dirent: any) => dirent.isDirectory()) .map((dirent: any) => join(casesDir, dirent.name)); } private async loadPackTestCases( packDir: string, packId: string ): Promise { const testCases: TestCase[] = []; try { // Load passports const passportsDir = join(packDir, "passports"); const contextsDir = join(packDir, "contexts"); const expectedDir = join(packDir, "expected"); const receiptsDir = join(packDir, "receipts"); if ( !existsSync(passportsDir) || !existsSync(contextsDir) || !existsSync(expectedDir) ) { console.warn( chalk.yellow(`āš ļø Skipping ${packId}: missing required directories`) ); return testCases; } // Load passport files const passportFiles = this.getJsonFiles(passportsDir); const contextFiles = this.getJsonFiles(contextsDir); const expectedFiles = this.getJsonFiles(expectedDir); const receiptFiles = existsSync(receiptsDir) ? this.getJsonFiles(receiptsDir) : []; // Create test cases for (const contextFile of contextFiles) { const contextName = contextFile.replace(".json", ""); const expectedFile = expectedFiles.find((f) => f.includes(contextName)); if (!expectedFile) { console.warn( chalk.yellow(`āš ļø No expected result for context: ${contextName}`) ); continue; } const testCase: TestCase = { id: `${packId}:${contextName}`, packId, contextName, passport: this.loadJsonFile(join(passportsDir, passportFiles[0])), // Use first passport context: this.loadJsonFile(join(contextsDir, contextFile)), expected: this.loadJsonFile(join(expectedDir, expectedFile)), receipt: receiptFiles.find((f) => f.includes(contextName)) ? this.loadJsonFile( join( receiptsDir, receiptFiles.find((f) => f.includes(contextName))! ) ) : undefined, }; testCases.push(testCase); } } catch (error) { console.warn(chalk.yellow(`āš ļø Error loading ${packId}: ${error}`)); } return testCases; } private getJsonFiles(dir: string): string[] { return readdirSync(dir).filter((file: string) => file.endsWith(".json")); } private loadJsonFile(filePath: string): any { return JSON.parse(readFileSync(filePath, "utf-8")); } private async runTests(testCases: TestCase[]): Promise { const results: TestResult[] = []; for (const testCase of testCases) { const spinner = ora(`Running ${testCase.id}...`).start(); try { const result = await this.runTestCase(testCase); results.push(result); if (result.passed) { spinner.succeed(`${testCase.id}: PASS`); } else { spinner.fail(`${testCase.id}: FAIL`); } if (this.options.verbose && !result.passed) { console.log(chalk.red(" Errors:"), result.errors); } } catch (error) { spinner.fail(`${testCase.id}: ERROR`); results.push({ testCase, passed: false, errors: [`Test execution failed: ${error}`], warnings: [], }); } } return results; } private async runTestCase(testCase: TestCase): Promise { const errors: string[] = []; const warnings: string[] = []; // 1. Validate passport against schema const passportValidation = await this.validator.validatePassport( testCase.passport ); if (!passportValidation.valid) { errors.push( `Passport validation failed: ${passportValidation.errors.join(", ")}` ); } // 2. Validate context against policy requirements const contextValidation = await this.validator.validateContext( testCase.packId, testCase.context ); if (!contextValidation.valid) { errors.push( `Context validation failed: ${contextValidation.errors.join(", ")}` ); } // 3. Evaluate policy logic (simplified - in real implementation this would call the policy evaluator) const policyResult = await this.evaluatePolicy(testCase); if (!policyResult.valid) { errors.push( `Policy evaluation failed: ${policyResult.errors.join(", ")}` ); } // 4. Validate expected decision against schema const decisionValidation = await this.validator.validateDecision( testCase.expected ); if (!decisionValidation.valid) { errors.push( `Expected decision validation failed: ${decisionValidation.errors.join( ", " )}` ); } // 5. Verify signature if receipt provided if (testCase.receipt) { const signatureValidation = await this.verifySignature(testCase.receipt); if (!signatureValidation.valid) { errors.push( `Signature verification failed: ${signatureValidation.errors.join( ", " )}` ); } } // 6. Compare actual vs expected (simplified) const comparison = this.compareResults( policyResult.decision, testCase.expected ); if (!comparison.matches) { errors.push(`Result mismatch: ${comparison.differences.join(", ")}`); } return { testCase, passed: errors.length === 0, errors, warnings, }; } private async evaluatePolicy( testCase: TestCase ): Promise<{ valid: boolean; errors: string[]; decision: any }> { // This is a simplified policy evaluation // In a real implementation, this would call the actual policy evaluator const { packId, passport, context } = testCase; // Basic policy logic based on pack ID let allow = true; const reasons: any[] = []; if (packId === "finance.payment.refund.v1") { const amount = context.amount || 0; const currency = context.currency || "USD"; // Check currency limits const limits = passport.limits?.["finance.payment.refund"]?.currency_limits?.[ currency ]; if (limits) { if (amount > limits.max_per_tx) { allow = false; reasons.push({ code: "oap.limit_exceeded", message: `Amount ${amount} exceeds max per transaction ${limits.max_per_tx}`, }); } } else { // Currency not supported allow = false; reasons.push({ code: "oap.currency_unsupported", message: `Currency ${currency} not supported for this passport`, }); } } if (packId === "data.export.create.v1") { const includePii = context.include_pii || false; const allowPii = passport.limits?.["data.export"]?.allow_pii || false; if (includePii && !allowPii) { allow = false; reasons.push({ code: "oap.pii_blocked", message: "PII export not allowed for this passport", }); } } // If no reasons and allow is true, add success reason if (allow && reasons.length === 0) { reasons.push({ code: "oap.allowed", message: "Transaction within limits and policy requirements", }); } const decision = { decision_id: `test_${Date.now()}`, policy_id: packId, agent_id: passport.agent_id || passport.passport_id, owner_id: passport.owner_id, assurance_level: passport.assurance_level, allow, reasons, created_at: new Date().toISOString(), expires_in: 3600, passport_digest: await this.computePassportDigest(passport), signature: "ed25519:test_signature_placeholder_64_chars_long_for_conformance_testing", kid: "oap:registry:test-key", }; return { valid: true, errors: [], decision, }; } private async computePassportDigest(passport: any): Promise { const canonical = this.jcs.canonicalize(passport); const hash = await crypto.subtle.digest( "SHA-256", new TextEncoder().encode(canonical) ); const hashArray = Array.from(new Uint8Array(hash)); const hashHex = hashArray .map((b) => b.toString(16).padStart(2, "0")) .join(""); return `sha256:${hashHex}`; } private async verifySignature( receipt: any ): Promise<{ valid: boolean; errors: string[] }> { // Simplified signature verification for conformance testing // In a real implementation, this would verify Ed25519 signatures if (!receipt.signature) { return { valid: false, errors: ["No signature provided"] }; } // For conformance testing, we'll just check that signature exists and has correct format if (!receipt.signature.startsWith("ed25519:")) { return { valid: false, errors: ["Invalid signature format"] }; } return { valid: true, errors: [] }; } private compareResults( actual: any, expected: any ): { matches: boolean; differences: string[] } { const differences: string[] = []; // Compare key fields if (actual.allow !== expected.allow) { differences.push( `allow: expected ${expected.allow}, got ${actual.allow}` ); } if (actual.assurance_level !== expected.assurance_level) { differences.push( `assurance_level: expected ${expected.assurance_level}, got ${actual.assurance_level}` ); } // Compare reasons (simplified) if (actual.reasons?.length !== expected.reasons?.length) { differences.push( `reasons: expected ${expected.reasons?.length} reasons, got ${actual.reasons?.length}` ); } return { matches: differences.length === 0, differences, }; } private generateReport(results: TestResult[]): ConformanceReport { const passed = results.filter((r) => r.passed).length; const failed = results.filter((r) => !r.passed).length; const total = results.length; const summary = { total, passed, failed, successRate: total > 0 ? (passed / total) * 100 : 0, }; const details = results.map((result) => ({ testCase: result.testCase.id, passed: result.passed, errors: result.errors, warnings: result.warnings, })); return { timestamp: new Date().toISOString(), summary, details, }; } private displayResults(report: ConformanceReport): void { console.log("\n" + chalk.blue.bold("šŸ“Š Conformance Test Results\n")); const { summary } = report; console.log(chalk.green(`āœ… Passed: ${summary.passed}`)); console.log(chalk.red(`āŒ Failed: ${summary.failed}`)); console.log( chalk.blue(`šŸ“ˆ Success Rate: ${summary.successRate.toFixed(1)}%`) ); if (summary.failed > 0) { console.log("\n" + chalk.red.bold("Failed Tests:")); report.details .filter((d: any) => !d.passed) .forEach((detail: any) => { console.log(chalk.red(` • ${detail.testCase}`)); detail.errors.forEach((error: any) => { console.log(chalk.gray(` - ${error}`)); }); }); } console.log("\n" + chalk.blue("šŸŽÆ Conformance testing complete!\n")); } private async saveReport(report: ConformanceReport): Promise { const reportsDir = join(__dirname, "..", "reports"); if (!existsSync(reportsDir)) { mkdirSync(reportsDir, { recursive: true }); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const reportFile = join(reportsDir, `conformance-${timestamp}.json`); writeFileSync(reportFile, JSON.stringify(report, null, 2)); console.log(chalk.green(`šŸ“„ Report saved to: ${reportFile}`)); } } // CLI setup const program = new Command(); program .name("oap-conformance") .description("Open Agent Passport conformance test runner") .version("1.0.0"); program .option("-p, --pack ", "Run tests for specific policy pack") .option("-v, --verbose", "Verbose output") .option("-r, --report", "Generate detailed report") .action(async (options: any) => { const runner = new ConformanceRunner(options); await runner.run(); }); program.parse();