/** * A module's e2e fixtures must deploy (at least import) a provider for every * capability its manifest requires. * * The failure this exists to catch is silent in the worst way: the manifest * requirement makes stage 1 of the module's own suite fail at * `module import ` with "Required capability 'X' not found", and every * later stage cascade-skips on requireStage. Nothing else goes red — the * suite is quarantined, or CI never runs it, so a fixture can sit * structurally incapable of passing for six weeks (celilo#1260: * registry-pipeline required idp@1.2.0 from 7e2d27b9 onward and never * deployed an idp provider). * * Both sides of the comparison are computed sets. The required set comes * from each module's manifest.yml parsed by the production validator; the * provided set comes from the manifests of whatever modules the module's * fixtures actually import (`module import ` statements, scanned in * source). A hand-written list of "modules that need an idp" would rot the * moment the next module declares the requirement. * * The check runs the real `validateCapabilityAccess` against an in-memory * database seeded with the fixtures' providers — the same code path the CLI * executes at import time, including the secret-allowlist gate — so the * assertion is over the actual contract stage 1 exercises, not a re-derivation * of it. * * ## What this gate does NOT prove. Read before trusting green. * * - Import is necessary, not sufficient. A fixture can import the provider * and still deploy the consumer before it, or never deploy the provider * at all (the consumer's on_install would then fail against a dead idp). * Only running the suite proves deploy order. * - The import scan is a source scan. An import built dynamically in a way * this scanner cannot parse is invisible here (the scanner requires a * literal module id, a `${CONST}` reference to a string literal in the * same file, or a trailing path segment). * - Fixtures living outside `modules//e2e/*.test.ts` (shared harness * files that deploy on the suite's behalf) are not scanned. */ import { Database, type Database as DatabaseType } from 'bun:sqlite'; import { describe, expect, test } from 'bun:test'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { validateCapabilityAccess } from '../capabilities/validation'; import type { ModuleManifest } from '../manifest/schema'; import { validateManifest } from '../manifest/validate'; import { repoRoot } from './capability-shape'; /** Every module manifest under `modulesDir`, parsed by the production validator. */ function loadModuleManifests(modulesDir: string): Map { const manifests = new Map(); for (const id of readdirSync(modulesDir)) { const path = join(modulesDir, id, 'manifest.yml'); if (!existsSync(path)) continue; const result = validateManifest(readFileSync(path, 'utf8')); if (result.success) manifests.set(id, result.data); // Manifests that fail validation are another gate's job (validate.test.ts // and scripts/check-modules.sh); scanning them here would double-report. } return manifests; } /** * Module ids a fixture source imports, via `module import ` statements. * * Handles the three shapes fixtures actually use: a bare literal * (`'module import authentik'`), a path (`'module import /celilo/modules/authentik'`), * and a `${CONST}` reference to a string literal defined in the same file * (`module import ${PRIMARY}`). Trailing CLI flags (`--accept-aspects`) and * timeouts are stripped. Anything that does not resolve to a plausible * module id is skipped rather than guessed. */ function scanFixtureImports(fixtureSource: string): string[] { const constants = new Map(); for (const match of fixtureSource.matchAll(/const\s+(\w+)\s*=\s*'([^']+)'/g)) { constants.set(match[1], match[2]); } const imports: string[] = []; for (const match of fixtureSource.matchAll(/module import ([^'"\n]+?)['"`]/g)) { let ref = match[1].trim(); const constRef = ref.match(/^\$\{(\w+)\}/); if (constRef) ref = constants.get(constRef[1]) ?? ''; const id = ref.split(/\s+/)[0]?.split('/').pop() ?? ''; if (id && /^[a-z0-9-]+$/.test(id)) imports.push(id); } return imports; } /** The union of modules any `*.test.ts` fixture in `moduleId`'s e2e/ imports. */ function fixtureImportedModules(modulesDir: string, moduleId: string): string[] { const e2eDir = join(modulesDir, moduleId, 'e2e'); if (!existsSync(e2eDir)) return []; const imported = new Set(); for (const file of readdirSync(e2eDir)) { if (!file.endsWith('.test.ts')) continue; for (const id of scanFixtureImports(readFileSync(join(e2eDir, file), 'utf8'))) { imported.add(id); } } return [...imported]; } /** * An in-memory database shaped like the CLI's module registry, seeded with * every capability `modules` provide — what `getProviderManifest` queries. */ function providerDatabase(manifests: Map): DatabaseType { const db = new Database(':memory:'); db.exec('CREATE TABLE modules (id TEXT PRIMARY KEY, manifest_data TEXT)'); db.exec('CREATE TABLE capabilities (module_id TEXT, capability_name TEXT)'); const insertModule = db.prepare('INSERT INTO modules (id, manifest_data) VALUES (?, ?)'); const insertCapability = db.prepare( 'INSERT INTO capabilities (module_id, capability_name) VALUES (?, ?)', ); for (const [id, manifest] of manifests) { insertModule.run(id, JSON.stringify(manifest)); for (const capability of manifest.provides?.capabilities ?? []) { insertCapability.run(id, capability.name); } } return db; } /** * Modules with e2e fixtures whose required capabilities no imported module * provides. The provider union is seeded into a real database and each * consumer is checked with the production `validateCapabilityAccess`. */ async function capabilityCoverageViolations(modulesDir: string): Promise { const manifests = loadModuleManifests(modulesDir); const violations: string[] = []; for (const [id, manifest] of manifests) { if (fixtureImportedModules(modulesDir, id).length === 0) continue; const imported = fixtureImportedModules(modulesDir, id) .map((dep) => manifests.get(dep)) .filter((m): m is ModuleManifest => m !== undefined); const db = providerDatabase(new Map(imported.map((m) => [m.id, m]))); try { const result = await validateCapabilityAccess(manifest, db); if (!result.success) { violations.push(`${id}: ${result.error ?? 'unknown error'}`); } } finally { db.close(); } } return violations; } describe('fixture capability coverage', () => { test('every module with e2e fixtures imports a provider for each required capability', async () => { const violations = await capabilityCoverageViolations(join(repoRoot(), 'modules')); expect(violations).toEqual([]); }); // Reach measurement (Rule 2 of the lane plan): the real scanner and checker // run against a mirrored module tree with planted markers, so the pass and // fail cases are both proven to fire — not reasoned about. This is the // mutation test for the gate itself: the real-tree test above can only go // green, so the mirror is where the gate is shown to fail. describe('reach: the gate fires on a mirrored tree (synthetic modules)', () => { function mirrorTree( dir: string, modules: Record, ): string { mkdirSync(dir, { recursive: true }); for (const [id, spec] of Object.entries(modules)) { const moduleDir = join(dir, id); mkdirSync(join(moduleDir, 'e2e'), { recursive: true }); writeFileSync(join(moduleDir, 'manifest.yml'), spec.manifest); for (const [i, fixture] of (spec.fixtures ?? []).entries()) { writeFileSync(join(moduleDir, 'e2e', `fixture-${i}.test.ts`), fixture); } } return dir; } const providerManifest = [ 'celilo_contract: "1.0"', 'id: marker-provider', 'name: Marker Provider', 'version: 1.0.0', 'description: synthetic provider', 'provides:', ' capabilities:', ' - name: marker_cap', ' version: 1.0.0', '', ].join('\n'); const consumerFixture = "await net.celilo('module import marker-provider');\n"; test('a fixture importing the provider satisfies the requirement', async () => { const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-ok-')); try { mirrorTree(dir, { 'marker-provider': { manifest: providerManifest }, 'marker-consumer': { manifest: [ 'celilo_contract: "1.0"', 'id: marker-consumer', 'name: Marker Consumer', 'version: 1.0.0', 'description: synthetic consumer', 'requires:', ' capabilities:', ' - name: marker_cap', ' version: 1.0.0', '', ].join('\n'), fixtures: [consumerFixture], }, }); expect(await capabilityCoverageViolations(dir)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('a fixture missing the provider fails, naming the module and capability', async () => { const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-fail-')); try { mirrorTree(dir, { 'marker-provider': { manifest: providerManifest }, 'marker-consumer': { manifest: [ 'celilo_contract: "1.0"', 'id: marker-consumer', 'name: Marker Consumer', 'version: 1.0.0', 'description: synthetic consumer', 'requires:', ' capabilities:', ' - name: marker_cap', ' version: 1.0.0', ' - name: marker_other', ' version: 1.0.0', '', ].join('\n'), // provider-b covers marker_cap; nothing covers marker_other fixtures: [consumerFixture], }, 'marker-nonprovider': { manifest: providerManifest.replace('marker_cap', 'marker_other'), }, }); const violations = await capabilityCoverageViolations(dir); expect(violations).toEqual([ expect.stringContaining("marker-consumer: Required capability 'marker_other' not found"), ]); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('a privilege the framework grants needs no provider', async () => { const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-priv-')); try { mirrorTree(dir, { 'marker-consumer': { manifest: [ 'celilo_contract: "1.0"', 'id: marker-consumer', 'name: Marker Consumer', 'version: 1.0.0', 'description: synthetic consumer', 'requires:', ' capabilities:', ' - name: cross_module_read', ' version: 1.0.0', '', ].join('\n'), fixtures: ["await net.celilo('module import nothing-here');\n"], }, }); expect(await capabilityCoverageViolations(dir)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('a ${CONST} import resolves through the same-file string literal', async () => { const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-const-')); try { mirrorTree(dir, { 'marker-provider': { manifest: providerManifest }, 'marker-consumer': { manifest: [ 'celilo_contract: "1.0"', 'id: marker-consumer', 'name: Marker Consumer', 'version: 1.0.0', 'description: synthetic consumer', 'requires:', ' capabilities:', ' - name: marker_cap', ' version: 1.0.0', '', ].join('\n'), fixtures: [ "const PROVIDER = 'marker-provider';\nawait net.celilo(`module import ${PROVIDER} --accept-aspects`);\n", ], }, }); expect(await capabilityCoverageViolations(dir)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); }); });