import { createHmac } from 'node:crypto'; import type { InfraControlPlaneCredentialRef, InfraCredentialPort, InfraLedger, InfraManifest, InfraResourceIdentity, InfraResult, } from '@ankhorage/contracts/infra'; import { expect, test } from 'bun:test'; import { createNodeInfraAdapterPackageResolver } from './features/environment-lifecycle/adapters/outbound/createNodeInfraAdapterPackageResolver.js'; import { destroyInfraEnvironmentAsync } from './features/environment-lifecycle/application/use-cases/destroyInfraEnvironmentAsync.js'; import { upInfraEnvironmentAsync } from './features/environment-lifecycle/application/use-cases/upInfraEnvironmentAsync.js'; import type { InfraOrchestrationDependencies } from './types/infraOrchestration.js'; const projectId = 'infra145-minikube-supabase-recovery'; const profile = projectId; const namespace = `${projectId}-local`; const baseUrl = 'http://127.0.0.1:54322'; const logicalBucket = 'phase10-objects'; const physicalBucket = 'phase10-persistence'; const minioContainer = 'infra145-minio-recovery'; const minioHostEndpoint = 'http://127.0.0.1:19000'; const minioPodEndpoint = 'http://host.minikube.internal:19000'; const minioImage = 'quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z'; const s3AccessKey = 'phase10-access'; const s3SecretKey = 'phase10-secret-key'; const databaseMarker = 'database-survived-from-s3-backup'; const objectMarker = 'storage-object-survived-off-cluster'; const jwtSecret = 'phase10-jwt-secret-that-is-at-least-thirty-two-characters'; const serviceRoleKey = createJwt('service_role'); const persistenceTarget = { endpoint: minioPodEndpoint, region: 'us-east-1', bucket: physicalBucket, credentials: { source: 'control-plane', name: 'S3_PERSISTENCE' }, forcePathStyle: true, } as const; const manifest = { environments: { local: { deployment: { compute: { provider: 'local' }, runtime: { provider: 'minikube', profile, driver: 'docker', cpus: 2, memoryMiB: 5_120, }, }, database: { provider: 'supabase', tier: 'prod', backup: { mode: 'scheduled', target: persistenceTarget, intervalHours: 24 }, }, auth: { provider: 'supabase' }, objectStorage: { provider: 'supabase', buckets: [logicalBucket], backend: persistenceTarget, }, networking: { publicBaseUrl: baseUrl }, }, }, modules: [], } as const satisfies InfraManifest; test.skipIf(process.env.ANKH_INFRA_MINIKUBE_SUPABASE_RECOVERY_E2E !== '1')( 'recovers a fresh Minikube Supabase database while S3-backed Storage remains off-cluster', async () => { await stopMinioAsync(); await startMinioAsync(); try { const dependencies = createDependencies(); const firstUp = requireSuccess( await upInfraEnvironmentAsync({ projectId, manifest }, dependencies), ); await writeDatabaseMarkerAsync(); await uploadStorageObjectAsync(); await forceFreshBackupAsync(); expect(await readLatestBackupPointerAsync()).toMatch(/^database\/\d{8}T\d{6}Z$/); const destroyed = requireSuccess( await destroyInfraEnvironmentAsync( createDestroyRequest(firstUp.ledger, clusterPersistentIdentities(firstUp.ledger)), dependencies, ), ); expectRetainedStorageBucket(destroyed.ledger); const recovered = requireSuccess( await upInfraEnvironmentAsync({ projectId, manifest }, dependencies), ); expect(await readDatabaseMarkerAsync()).toBe(databaseMarker); expect(await downloadStorageObjectAsync()).toBe(objectMarker); const finalDestroy = requireSuccess( await destroyInfraEnvironmentAsync( createDestroyRequest(recovered.ledger, clusterPersistentIdentities(recovered.ledger)), dependencies, ), ); expectRetainedStorageBucket(finalDestroy.ledger); } catch (error) { const diagnostics = await collectFailureDiagnosticsAsync(); const message = error instanceof Error ? error.message : String(error); throw new Error(`${message}\nRecovery diagnostics: ${diagnostics}`, { cause: error }); } finally { await runAllowFailureAsync(['minikube', 'delete', '-p', profile]); await stopMinioAsync(); } }, 1_200_000, ); async function startMinioAsync(): Promise { await runAsync( [ 'docker', 'run', '--detach', '--rm', '--name', minioContainer, '--publish', '19000:9000', '--env', `MINIO_ROOT_USER=${s3AccessKey}`, '--env', `MINIO_ROOT_PASSWORD=${s3SecretKey}`, minioImage, 'server', '/data', '--address', ':9000', ], 'start MinIO', ); await waitForMinioAsync(); await runAsync( [ 'curl', '--fail', '--silent', '--show-error', '--request', 'PUT', '--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', `${s3AccessKey}:${s3SecretKey}`, `${minioHostEndpoint}/${physicalBucket}`, ], 'create MinIO bucket', ); } async function waitForMinioAsync(): Promise { for (const _attempt of Array.from({ length: 60 }, (_, index) => index)) { const response = await fetch(`${minioHostEndpoint}/minio/health/live`).catch(() => undefined); if (response?.ok === true) return; await Bun.sleep(500); } throw new Error('MinIO did not become ready.'); } async function stopMinioAsync(): Promise { await runAllowFailureAsync(['docker', 'rm', '--force', minioContainer]); } async function writeDatabaseMarkerAsync(): Promise { const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-'); await runAsync( [ 'kubectl', 'exec', '--namespace', namespace, pod, '--', 'psql', '-U', 'postgres', '-d', 'postgres', '-v', 'ON_ERROR_STOP=1', '-c', `CREATE TABLE public.phase10_recovery_marker(value text NOT NULL); INSERT INTO public.phase10_recovery_marker(value) VALUES ('${databaseMarker}');`, ], 'write database recovery marker', ); } async function readDatabaseMarkerAsync(): Promise { const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-'); return runAsync( [ 'kubectl', 'exec', '--namespace', namespace, pod, '--', 'psql', '-U', 'postgres', '-d', 'postgres', '-Atc', 'SELECT value FROM public.phase10_recovery_marker LIMIT 1;', ], 'read database recovery marker', ); } async function uploadStorageObjectAsync(): Promise { const response = await fetch(`${baseUrl}/storage/v1/object/${logicalBucket}/recovery.txt`, { method: 'POST', headers: { Authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey, 'Content-Type': 'text/plain', }, body: objectMarker, }); if (!response.ok) throw new Error(`Storage upload failed with HTTP ${response.status}.`); } async function downloadStorageObjectAsync(): Promise { const response = await fetch(`${baseUrl}/storage/v1/object/${logicalBucket}/recovery.txt`, { headers: { Authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey }, }); if (!response.ok) throw new Error(`Storage download failed with HTTP ${response.status}.`); return response.text(); } async function forceFreshBackupAsync(): Promise { await runAsync( ['kubectl', 'rollout', 'restart', 'deployment/supabase-db-backup', '--namespace', namespace], 'restart database backup workload', ); await runAsync( [ 'kubectl', 'rollout', 'status', 'deployment/supabase-db-backup', '--namespace', namespace, '--timeout=180s', ], 'wait for database backup workload', ); } async function readLatestBackupPointerAsync(): Promise { return runAsync( [ 'curl', '--fail', '--silent', '--show-error', '--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', `${s3AccessKey}:${s3SecretKey}`, `${minioHostEndpoint}/${physicalBucket}/database/latest`, ], 'read latest database backup pointer', ); } async function findPodAsync(prefix: string, excludedPrefix?: string): Promise { const names = ( await runAsync( [ 'kubectl', 'get', 'pods', '--namespace', namespace, '-o', 'jsonpath={.items[*].metadata.name}', ], 'list Supabase pods', ) ).split(' '); const name = names.find( (candidate) => candidate.startsWith(prefix) && !candidate.startsWith(excludedPrefix ?? '\0'), ); if (name === undefined) throw new Error(`No pod matched ${prefix}.`); return name; } async function collectFailureDiagnosticsAsync(): Promise { const [ auth, storageStatus, buckets, databaseState, databaseLogs, previousDatabaseLogs, dataRestoreState, dataRestoreLogs, previousDataRestoreLogs, ] = await Promise.all([ readEndpointDiagnosticAsync('/auth/v1/health', false), readEndpointDiagnosticAsync('/storage/v1/status', false), readEndpointDiagnosticAsync('/storage/v1/bucket', true), readDatabaseStateAsync(), readDatabaseLogsAsync(false), readDatabaseLogsAsync(true), readDataRestoreStateAsync(), readDataRestoreLogsAsync(false), readDataRestoreLogsAsync(true), ]); return [ auth, storageStatus, buckets, databaseState, databaseLogs, previousDatabaseLogs, dataRestoreState, dataRestoreLogs, previousDataRestoreLogs, ].join(' | '); } async function readDatabaseStateAsync(): Promise { const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-').catch(() => undefined); if (pod === undefined) return 'database-state=unavailable'; const state = await runAsync( [ 'kubectl', 'get', 'pod', pod, '--namespace', namespace, '-o', 'jsonpath={.status.containerStatuses[?(@.name=="supabase-db")].restartCount}', ], 'read database restart state', ).catch((error: unknown) => (error instanceof Error ? error.message : String(error))); return `database-restarts=${redactDiagnosticText(state)}`; } async function readDatabaseLogsAsync(previous: boolean): Promise { const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-').catch(() => undefined); const label = previous ? 'database-previous-logs' : 'database-logs'; if (pod === undefined) return `${label}=unavailable`; const logs = await runAsync( [ 'kubectl', 'logs', '--namespace', namespace, pod, '--container', 'supabase-db', ...(previous ? ['--previous'] : []), '--tail=240', ], `read ${label}`, ).catch((error: unknown) => (error instanceof Error ? error.message : String(error))); return `${label}=${redactDiagnosticText(logs).slice(-6_000)}`; } async function readDataRestoreStateAsync(): Promise { const pod = await findPodAsync('supabase-db-data-restore-').catch(() => undefined); if (pod === undefined) return 'data-restore-state=unavailable'; const state = await runAsync( [ 'kubectl', 'get', 'pod', pod, '--namespace', namespace, '-o', 'jsonpath={.status.containerStatuses[?(@.name=="supabase-db-data-restore")].restartCount}', ], 'read data restore restart state', ).catch((error: unknown) => (error instanceof Error ? error.message : String(error))); return `data-restore-restarts=${redactDiagnosticText(state)}`; } async function readDataRestoreLogsAsync(previous: boolean): Promise { const pod = await findPodAsync('supabase-db-data-restore-').catch(() => undefined); const label = previous ? 'data-restore-previous-logs' : 'data-restore-logs'; if (pod === undefined) return `${label}=unavailable`; const logs = await runAsync( [ 'kubectl', 'logs', '--namespace', namespace, pod, '--container', 'supabase-db-data-restore', ...(previous ? ['--previous'] : []), '--tail=240', ], `read ${label}`, ).catch((error: unknown) => (error instanceof Error ? error.message : String(error))); return `${label}=${redactDiagnosticText(logs).slice(-6_000)}`; } async function readEndpointDiagnosticAsync(path: string, authenticated: boolean): Promise { const response = await fetch(`${baseUrl}${path}`, { headers: authenticated ? { Authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey } : undefined, }).catch(() => undefined); if (response === undefined) return `${path}=unreachable`; const body = redactDiagnosticText((await response.text()).slice(0, 500)); return `${path}=${response.status}:${body}`; } function redactDiagnosticText(value: string): string { return value .replaceAll(s3SecretKey, '[redacted]') .replaceAll(serviceRoleKey, '[redacted]') .replaceAll(jwtSecret, '[redacted]'); } function createDependencies(): InfraOrchestrationDependencies { return { adapterResolver: createNodeInfraAdapterPackageResolver(), credentials: createCredentialPort(), secrets: { resolveAsync: ({ ref, key }) => Promise.resolve(failure('unexpected-secret', `Unexpected managed secret ${ref}/${key}.`)), }, }; } function createCredentialPort(): InfraCredentialPort { return { findAsync: (reference) => Promise.resolve(findCredential(reference)), resolveAsync: (reference) => Promise.resolve(resolveCredential(reference)), persistAsync: (reference) => Promise.resolve( failure( 'unexpected-credential-persist', `Recovery acceptance must not persist credential ${reference.name}.`, ), ), }; } function findCredential( reference: InfraControlPlaneCredentialRef, ): InfraResult> | null> { if (reference.name === 'SUPABASE_BOOTSTRAP') return success(supabaseCredentials()); if (reference.name === 'S3_PERSISTENCE') { return success({ accessKeyId: s3AccessKey, secretAccessKey: s3SecretKey }); } return success(null); } function resolveCredential( reference: InfraControlPlaneCredentialRef, ): InfraResult>> { if (reference.name === 'SUPABASE_BOOTSTRAP') return success(supabaseCredentials()); if (reference.name === 'S3_PERSISTENCE') { return success({ accessKeyId: s3AccessKey, secretAccessKey: s3SecretKey }); } return failure('unexpected-credential', `Unexpected credential reference: ${reference.name}`); } function createDestroyRequest( previous: InfraLedger, confirmedResources: readonly InfraResourceIdentity[], ) { return { projectId, manifest, environment: 'local' as const, previous, confirmation: { projectId, environment: 'local' as const }, persistence: { policy: 'delete' as const, confirmedResources }, }; } function clusterPersistentIdentities(ledger: InfraLedger): readonly InfraResourceIdentity[] { return ledger.resources .filter(({ identity, persistent }) => persistent && identity.adapter !== 'supabase') .map(({ identity }) => identity); } function expectRetainedStorageBucket(ledger: InfraLedger | null): void { expect( ledger?.resources.map(({ identity }) => `${identity.adapter}:${identity.resourceId}`), ).toEqual([`supabase:bucket/${logicalBucket}`]); } function supabaseCredentials() { return { postgresPassword: 'phase10-postgres-password', jwtSecret, anonKey: createJwt('anon'), serviceRoleKey, realtimeSecretKeyBase: 'r'.repeat(64), realtimeDatabaseEncryptionKey: '0123456789abcdef', pgMetaCryptoKey: 'phase10-meta-crypto-key-that-is-at-least-32-characters', } as const; } function createJwt(role: 'anon' | 'service_role'): string { const header = encodeJwtPart({ alg: 'HS256', typ: 'JWT' }); const payload = encodeJwtPart({ role, iss: 'supabase', iat: 1_700_000_000, exp: 4_102_444_800 }); const unsigned = `${header}.${payload}`; const signature = createHmac('sha256', jwtSecret).update(unsigned).digest('base64url'); return `${unsigned}.${signature}`; } function encodeJwtPart(value: Readonly>): string { return Buffer.from(JSON.stringify(value)).toString('base64url'); } async function runAsync(command: readonly string[], label: string): Promise { const process = Bun.spawn(command, { stdout: 'pipe', stderr: 'pipe' }); const [exitCode, stdout, stderr] = await Promise.all([ process.exited, new Response(process.stdout).text(), new Response(process.stderr).text(), ]); if (exitCode !== 0) throw new Error(`${label} failed: ${stderr.trim()}`); return stdout.trim(); } async function runAllowFailureAsync(command: readonly string[]): Promise { const process = Bun.spawn(command, { stdout: 'ignore', stderr: 'ignore' }); await process.exited; } function requireSuccess(result: InfraResult): T { if (!result.ok) throw new Error(result.diagnostics.map(({ message }) => message).join(' ')); return result.value; } function success(value: T): InfraResult { return { ok: true, value, diagnostics: [] }; } function failure(code: string, message: string): InfraResult { return { ok: false, diagnostics: [{ severity: 'error', code, message }] }; }