/** * Architecture Fitness Tests * * Scans the codebase for forbidden anti-patterns that violate the KCode * Harness architecture. Each rule checks a specific structural constraint * and reports violations with file, line, and severity. * * These tests are deterministic — no LLM evaluation required. * They run as part of the release gate (Phase 5). */ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { MODULE_MIGRATION_MAP } from "../gates/migration-switch.ts"; /* ------------------------------------------------------------------ */ /* Types */ /* ------------------------------------------------------------------ */ /** Severity of an architecture fitness rule. */ export type FitnessSeverity = "block" | "warn"; /** * A single architecture fitness rule. * * `check` receives the project root and returns zero or more violations. */ export interface ArchitectureFitnessRule { /** Unique rule identifier, e.g. "PROMPT_NO_POLICY". */ id: string; /** Human-readable description of what the rule checks. */ description: string; /** Whether violations of this rule block the release or just warn. */ severity: FitnessSeverity; /** Execute the rule against the project root and return violations. */ check: (projectRoot: string) => ArchitectureFitnessViolation[]; } /** * A single violation found by a fitness rule. */ export interface ArchitectureFitnessViolation { /** Rule that was violated. */ ruleId: string; /** Human-readable description of the violation. */ description: string; /** Relative path to the offending file. */ file: string; /** 1-based line number where the violation was found (if applicable). */ line?: number; /** Severity inherited from the rule. */ severity: FitnessSeverity; } /** * Result of running all fitness rules. */ export interface ArchitectureFitnessReport { /** True if no "block" violations were found. */ passed: boolean; /** All violations found, sorted by severity (block first). */ violations: ArchitectureFitnessViolation[]; /** Score from 0–100. Starts at 100, each block = -15, each warn = -5. */ score: number; /** Human-readable summary of the fitness check. */ summary: string; } /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ /** * Read a file relative to projectRoot. Returns undefined if the file * does not exist or cannot be read. */ function readProjectFile(projectRoot: string, relativePath: string): string | undefined { const abs = join(projectRoot, relativePath); if (!existsSync(abs)) return undefined; try { return readFileSync(abs, "utf8"); } catch { return undefined; } } /** * Scan file content for regex matches and return violations. * `pattern` must have the `g` flag for multi-match. */ function scanForMatches( ruleId: string, severity: FitnessSeverity, filePath: string, content: string, pattern: RegExp, describe: (match: RegExpExecArray) => string, ): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; const lines = content.split("\n"); for (let i = 0; i < lines.length; i++) { const line = lines[i]; // Reset lastIndex for each line pattern.lastIndex = 0; let m: RegExpExecArray | null; while ((m = pattern.exec(line)) !== null) { violations.push({ ruleId, description: describe(m), file: filePath, line: i + 1, severity, }); // Prevent infinite loop on zero-length matches if (m.index === pattern.lastIndex) { pattern.lastIndex++; } } } return violations; } /** * List of relative paths to scan for a given rule. */ type FileList = string[]; /* ------------------------------------------------------------------ */ /* Rule 1: PROMPT_NO_POLICY */ /* prompt.ts must not contain policy rule logic. */ /* Scan for block/deny/reject patterns that indicate inline policy. */ /* ------------------------------------------------------------------ */ const PROMPT_POLICY_PATTERN = /\b(block|deny|reject|hard-deny|soft-deny)\b/gi; const PROMPT_FILES: FileList = [ "src/harness/prompt.ts", ]; function checkPromptNoPolicy(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; for (const relPath of PROMPT_FILES) { const content = readProjectFile(projectRoot, relPath); if (!content) continue; const matches = scanForMatches( "PROMPT_NO_POLICY", "block", relPath, content, PROMPT_POLICY_PATTERN, (m) => `Prompt renderer contains policy keyword "${m[0]}". Policy rules must live in PolicyRegistry, not in prompt renderer.`, ); violations.push(...matches); } return violations; } const rulePromptNoPolicy: ArchitectureFitnessRule = { id: "PROMPT_NO_POLICY", description: "prompt.ts must not contain policy rule logic (block/deny/reject). Policy rules must live in PolicyRegistry.", severity: "block", check: checkPromptNoPolicy, }; /* ------------------------------------------------------------------ */ /* Rule 2: TOOL_THROUGH_GATE */ /* Tool calls must go through ToolPipeline. */ /* Direct evaluateToolGateway usage outside of adapter/pipeline */ /* indicates a bypass. */ /* ------------------------------------------------------------------ */ const TOOL_GATEWAY_PATTERN = /evaluateToolGateway\s*\(/g; const TOOL_GATEWAY_ALLOWED_FILES: Set = new Set([ "src/harness/tool-gateway.ts", // definition "src/harness/tools/tool-pipeline.ts", // pipeline adapter "src/harness/gates/gate-runner.ts", // gate runner integration "src/harness/gates/policies/tool-gateway-policy.ts", // policy adapter "src/harness/write-transaction.ts", // rewrite target (delegates to WritePipeline in V2) ]); function checkToolThroughGate(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; const harnessDir = join(projectRoot, "src", "harness"); if (!existsSync(harnessDir)) return violations; const filesToScan = [ "src/harness/action-policy.ts", "src/harness/repair.ts", "src/harness/goal-loop.ts", "src/harness/demand-loop.ts", "src/harness/delegation.ts", "src/harness/gates.ts", "src/harness/state.ts", "src/harness/observation.ts", "src/harness/control-frame.ts", "src/harness/next-action.ts", "src/harness/action-router.ts", "src/harness/context-compiler.ts", ]; for (const relPath of filesToScan) { if (TOOL_GATEWAY_ALLOWED_FILES.has(relPath)) continue; // Skip legacy files documented as rewrite/adapter in migration map const migration = MODULE_MIGRATION_MAP.find( (m) => m.oldPath === relPath && (m.status === "rewrite" || m.status === "adapter") ); if (migration) continue; const content = readProjectFile(projectRoot, relPath); if (!content) continue; const matches = scanForMatches( "TOOL_THROUGH_GATE", "block", relPath, content, TOOL_GATEWAY_PATTERN, () => `Direct evaluateToolGateway call bypasses ToolPipeline. All tool calls must go through ToolPipeline or GateRunner.`, ); violations.push(...matches); } return violations; } const ruleToolThroughGate: ArchitectureFitnessRule = { id: "TOOL_THROUGH_GATE", description: "Tool calls must go through ToolPipeline. Direct evaluateToolGateway usage outside pipeline is a bypass.", severity: "block", check: checkToolThroughGate, }; /* ------------------------------------------------------------------ */ /* Rule 3: LOOP_NO_SHELL */ /* Loop modules must not spawn shells directly. */ /* Scan for spawnSync, exec, execSync in loop-related files. */ /* ------------------------------------------------------------------ */ const SHELL_SPAWN_PATTERN = /\b(spawnSync|execSync|exec|child_process)\b/g; const LOOP_FILES: FileList = [ "src/harness/goal-loop.ts", "src/harness/demand-loop.ts", "src/harness/repair.ts", "src/harness/loop/loop-kernel.ts", "src/harness/loop/repair-loop.ts", "src/harness/loop/stuck-detector.ts", ]; function checkLoopNoShell(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; for (const relPath of LOOP_FILES) { const content = readProjectFile(projectRoot, relPath); if (!content) continue; const matches = scanForMatches( "LOOP_NO_SHELL", "block", relPath, content, SHELL_SPAWN_PATTERN, (m) => `Loop module contains shell execution pattern "${m[0]}". Loops must use VerifyPlan through GateRunner, not spawn shells directly.`, ); violations.push(...matches); } return violations; } const ruleLoopNoShell: ArchitectureFitnessRule = { id: "LOOP_NO_SHELL", description: "Loop modules must not spawn shells directly. Use VerifyPlan through GateRunner.", severity: "block", check: checkLoopNoShell, }; /* ------------------------------------------------------------------ */ /* Rule 4: EVIDENCE_HAS_SOURCE_TRUST */ /* Evidence records must have sourceTrust field. */ /* Scan evidence.ts for entries created without sourceTrust. */ /* ------------------------------------------------------------------ */ const EVIDENCE_NO_TRUST_PATTERN = /\bsource:\s*["'](?:local-command|project-metadata|user-provided|external-system|manual-note)["']/g; const EVIDENCE_WITH_TRUST_PATTERN = /\bsourceTrust\b/g; const EVIDENCE_FILES: FileList = [ "src/harness/evidence.ts", "src/harness/evidence/evidence-store.ts", "src/harness/evidence/evidence-source.ts", ]; function checkEvidenceHasSourceTrust(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; // Check old evidence.ts — if it uses `source` but not `sourceTrust`, flag it const oldEvidence = readProjectFile(projectRoot, "src/harness/evidence.ts"); if (oldEvidence) { const hasSource = EVIDENCE_NO_TRUST_PATTERN.test(oldEvidence); EVIDENCE_NO_TRUST_PATTERN.lastIndex = 0; const hasTrust = EVIDENCE_WITH_TRUST_PATTERN.test(oldEvidence); EVIDENCE_WITH_TRUST_PATTERN.lastIndex = 0; if (hasSource && !hasTrust) { violations.push({ ruleId: "EVIDENCE_HAS_SOURCE_TRUST", description: "Old evidence.ts uses `source` field but not `sourceTrust`. Evidence records must use sourceTrust for grading.", file: "src/harness/evidence.ts", severity: "warn", }); } } // Check new evidence-store.ts — ensure it references sourceTrust const newStore = readProjectFile(projectRoot, "src/harness/evidence/evidence-store.ts"); if (newStore) { const hasTrust = EVIDENCE_WITH_TRUST_PATTERN.test(newStore); EVIDENCE_WITH_TRUST_PATTERN.lastIndex = 0; if (!hasTrust) { violations.push({ ruleId: "EVIDENCE_HAS_SOURCE_TRUST", description: "EvidenceStore does not reference sourceTrust. All evidence records must have sourceTrust grading.", file: "src/harness/evidence/evidence-store.ts", severity: "block", }); } } return violations; } const ruleEvidenceHasSourceTrust: ArchitectureFitnessRule = { id: "EVIDENCE_HAS_SOURCE_TRUST", description: "Evidence records must have sourceTrust field for trust grading.", severity: "warn", check: checkEvidenceHasSourceTrust, }; /* ------------------------------------------------------------------ */ /* Rule 5: DELEGATION_NO_FACTS */ /* Delegation output must not write to main facts/evidence. */ /* Scan delegation.ts for direct write to facts/evidence index. */ /* ------------------------------------------------------------------ */ const DELEGATION_FACTS_WRITE_PATTERN = /\b(writeActiveRun|writeEvidenceFile|recordEvidence|writeFact|appendFact)\s*\(/g; const DELEGATION_FILES: FileList = [ "src/harness/delegation.ts", ]; function checkDelegationNoFacts(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; for (const relPath of DELEGATION_FILES) { const content = readProjectFile(projectRoot, relPath); if (!content) continue; const matches = scanForMatches( "DELEGATION_NO_FACTS", "block", relPath, content, DELEGATION_FACTS_WRITE_PATTERN, (m) => `Delegation module directly calls "${m[0]}". Delegation output must not write to main facts/evidence — only main run can write facts/evidence.`, ); violations.push(...matches); } return violations; } const ruleDelegationNoFacts: ArchitectureFitnessRule = { id: "DELEGATION_NO_FACTS", description: "Delegation output must not write to main facts/evidence. Only main run can write facts/evidence.", severity: "block", check: checkDelegationNoFacts, }; /* ------------------------------------------------------------------ */ /* Rule 6: OLD_RULE_DEPRECATED */ /* Old rule source files must have @deprecated annotation or be */ /* listed in migration-switch.ts as deprecated/deleted. */ /* ------------------------------------------------------------------ */ const DEPRECATED_ANNOTATION_PATTERN = /@deprecated|deprecated|ARCHITECTURE BREAK|migration-switch/i; const OLD_RULE_FILES: FileList = [ "src/harness/tool-gateway.ts", "src/harness/action-policy.ts", "src/harness/gates.ts", "src/harness/prompt-policy.ts", "src/harness/evidence.ts", "src/harness/write-transaction.ts", ]; function checkOldRuleDeprecated(projectRoot: string): ArchitectureFitnessViolation[] { const violations: ArchitectureFitnessViolation[] = []; // First check migration-switch.ts to see which files are listed const migrationContent = readProjectFile(projectRoot, "src/harness/gates/migration-switch.ts"); const listedPaths = new Set(); if (migrationContent) { const pathPattern = /oldPath:\s*["']([^"']+)["']/g; let m: RegExpExecArray | null; while ((m = pathPattern.exec(migrationContent)) !== null) { listedPaths.add(m[1]); } } for (const relPath of OLD_RULE_FILES) { const content = readProjectFile(projectRoot, relPath); if (!content) continue; // Check if the file itself has a deprecation annotation const hasAnnotation = DEPRECATED_ANNOTATION_PATTERN.test(content); DEPRECATED_ANNOTATION_PATTERN.lastIndex = 0; // Check if the file is listed in migration-switch.ts const isListed = listedPaths.has(relPath); if (!hasAnnotation && !isListed) { violations.push({ ruleId: "OLD_RULE_DEPRECATED", description: `Old rule source "${relPath}" has no @deprecated annotation and is not listed in migration-switch.ts. Old rule sources must be deprecated or listed for migration.`, file: relPath, severity: "warn", }); } } return violations; } const ruleOldRuleDeprecated: ArchitectureFitnessRule = { id: "OLD_RULE_DEPRECATED", description: "Old rule source files must have @deprecated annotation or be listed in migration-switch.ts.", severity: "warn", check: checkOldRuleDeprecated, }; /* ------------------------------------------------------------------ */ /* Rule Registry */ /* ------------------------------------------------------------------ */ /** * All architecture fitness rules. * Add new rules here to include them in fitness checks. */ export const ARCHITECTURE_FITNESS_RULES: readonly ArchitectureFitnessRule[] = [ rulePromptNoPolicy, ruleToolThroughGate, ruleLoopNoShell, ruleEvidenceHasSourceTrust, ruleDelegationNoFacts, ruleOldRuleDeprecated, ]; /* ------------------------------------------------------------------ */ /* Score Calculation */ /* ------------------------------------------------------------------ */ /** Points deducted per block violation. */ const BLOCK_DEDUCTION = 15; /** Points deducted per warn violation. */ const WARN_DEDUCTION = 5; /** Starting score. */ const MAX_SCORE = 100; function calculateScore(violations: ArchitectureFitnessViolation[]): number { let score = MAX_SCORE; for (const v of violations) { score -= v.severity === "block" ? BLOCK_DEDUCTION : WARN_DEDUCTION; } return Math.max(0, score); } /* ------------------------------------------------------------------ */ /* Main Entry Point */ /* ------------------------------------------------------------------ */ /** * Run all architecture fitness rules against the project. * * @param projectRoot Absolute path to the project root directory. * @returns A report with all violations, score, and pass/fail status. */ export function runFitnessCheck(projectRoot: string): ArchitectureFitnessReport { const allViolations: ArchitectureFitnessViolation[] = []; for (const rule of ARCHITECTURE_FITNESS_RULES) { try { const violations = rule.check(projectRoot); allViolations.push(...violations); } catch (err) { // If a rule itself fails, report it as a block violation allViolations.push({ ruleId: rule.id, description: `Rule execution failed: ${err instanceof Error ? err.message : String(err)}`, file: "(rule execution)", severity: "block", }); } } // Sort: block violations first, then warn allViolations.sort((a, b) => { if (a.severity !== b.severity) { return a.severity === "block" ? -1 : 1; } return a.ruleId.localeCompare(b.ruleId); }); const blockCount = allViolations.filter((v) => v.severity === "block").length; const warnCount = allViolations.filter((v) => v.severity === "warn").length; const score = calculateScore(allViolations); const passed = blockCount === 0; const summary = passed ? `Architecture fitness passed (score: ${score}/100). ${warnCount} warning(s).` : `Architecture fitness FAILED (score: ${score}/100). ${blockCount} block violation(s), ${warnCount} warning(s).`; return { passed, violations: allViolations, score, summary, }; } /* ------------------------------------------------------------------ */ /* Utility: Format report for console output */ /* ------------------------------------------------------------------ */ /** * Format an ArchitectureFitnessReport as a human-readable string * suitable for console output or CI logs. */ export function formatFitnessReport(report: ArchitectureFitnessReport): string { const lines: string[] = [ "=== Architecture Fitness Report ===", "", `Status: ${report.passed ? "PASSED" : "FAILED"}`, `Score: ${report.score}/100`, "", ]; if (report.violations.length === 0) { lines.push("No violations found."); return lines.join("\n"); } const blockViolations = report.violations.filter((v) => v.severity === "block"); const warnViolations = report.violations.filter((v) => v.severity === "warn"); if (blockViolations.length > 0) { lines.push(`--- Block Violations (${blockViolations.length}) ---`); for (const v of blockViolations) { const loc = v.line ? `${v.file}:${v.line}` : v.file; lines.push(` [${v.ruleId}] ${loc}`); lines.push(` ${v.description}`); } lines.push(""); } if (warnViolations.length > 0) { lines.push(`--- Warnings (${warnViolations.length}) ---`); for (const v of warnViolations) { const loc = v.line ? `${v.file}:${v.line}` : v.file; lines.push(` [${v.ruleId}] ${loc}`); lines.push(` ${v.description}`); } lines.push(""); } lines.push(report.summary); return lines.join("\n"); } // ── CLI entry point (npx tsx architecture-fitness.ts) ───────────────────────── const isMain = process.argv[1]?.endsWith("architecture-fitness.ts") || process.argv[1]?.endsWith("architecture-fitness.js"); if (isMain) { const projectRoot = join(import.meta.dirname ?? __dirname, "..", "..", ".."); const report = runFitnessCheck(projectRoot); console.log(formatFitnessReport(report)); process.exit(report.passed ? 0 : 1); }