import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { log } from '../cli/prompts'; import { executeBuildWithProgress } from './build-stream'; import { planAfterImportVerdict, validateTerraformPlanSafety } from './terraform-safety'; export interface TerraformResult { success: boolean; output: string; error?: string; exitCode?: number; } /** * Execute Terraform workflow with safety validation and streaming progress * Execution function - performs infrastructure provisioning * * @param generatedPath - Path to generated module artifacts * @param phases - Phase tracking object to update during execution * @param extraEnvVars - Optional environment variables (e.g., TF_VAR_* for credentials) * @returns Terraform execution result */ export async function executeTerraform( generatedPath: string, phases?: { terraformInit?: boolean; terraformPlan?: boolean; terraformApply?: boolean; }, extraEnvVars?: Record, options?: { noInteractive?: boolean }, ): Promise { const terraformDir = join(generatedPath, 'terraform'); // Build environment variables for Terraform const terraformEnv: Record = { TF_IN_AUTOMATION: '1', // Disable interactive prompts ...extraEnvVars, // Merge in any additional env vars (e.g., TF_VAR_* credentials) }; const noInteractive = options?.noInteractive; // 1. Always run terraform init -upgrade // This ensures provider versions are updated when templates change // It's idempotent and safe to run on every deployment const initResult = await executeTerraformCommand( 'init', ['-upgrade'], terraformDir, 'Initializing Terraform', terraformEnv, noInteractive, ); if (!initResult.success) { if (phases) phases.terraformInit = false; const errorMessage = parseTerraformError(initResult.output); return { ...initResult, error: errorMessage }; } if (phases) phases.terraformInit = true; // 2. Run terraform plan with streaming progress let planResult = await executeBuildWithProgress({ command: 'terraform', args: ['plan', '-detailed-exitcode', '-no-color'], cwd: terraformDir, title: 'Planning infrastructure', env: terraformEnv, noInteractive, }); // Auto-recover from stale state lock (e.g., interrupted previous deploy) if (!planResult.success && planResult.output.includes('Error acquiring the state lock')) { const unlocked = await autoForceUnlock( terraformDir, planResult.output, terraformEnv, noInteractive, ); if (unlocked) { log.info('Retrying plan after lock recovery...'); planResult = await executeBuildWithProgress({ command: 'terraform', args: ['plan', '-detailed-exitcode', '-no-color'], cwd: terraformDir, title: 'Planning infrastructure', env: terraformEnv, noInteractive, }); } } // Terraform plan exit codes: // 0 = no changes // 1 = error // 2 = changes present (success!) const planExitCode = planResult.exitCode; if (planExitCode !== 0 && planExitCode !== 2) { if (phases) phases.terraformPlan = false; // Parse common errors into clear messages const errorMessage = parseTerraformError(planResult.output); return { success: false, output: planResult.output, error: errorMessage, exitCode: planExitCode, }; } if (phases) phases.terraformPlan = true; // Check exit code: 0 = no changes, 2 = changes present if (planExitCode === 0) { log.message(' No changes needed'); if (phases) phases.terraformApply = true; return { success: true, output: planResult.output, exitCode: 0 }; } // Exit code 2 = changes present, validate safety const safetyCheck = validateTerraformPlanSafety(planResult.output); if (!safetyCheck.safe) { if (phases) phases.terraformApply = false; return { success: false, output: planResult.output, error: safetyCheck.error, exitCode: 2, }; } // 3. Run terraform apply with streaming progress let applyResult = await executeTerraformCommand( 'apply', ['-auto-approve'], terraformDir, 'Applying infrastructure', terraformEnv, noInteractive, ); // Auto-recover from stale state lock during apply if (!applyResult.success && applyResult.output.includes('Error acquiring the state lock')) { const unlocked = await autoForceUnlock( terraformDir, applyResult.output, terraformEnv, noInteractive, ); if (unlocked) { log.info('Retrying apply after lock recovery...'); applyResult = await executeTerraformCommand( 'apply', ['-auto-approve'], terraformDir, 'Applying infrastructure', terraformEnv, noInteractive, ); } } if (!applyResult.success) { // Check if failure is due to "already exists" - this happens when Terraform // loses connection during creation but the resource was actually created if (applyResult.output.includes('already exists')) { log.warn('Resource already exists - attempting automatic recovery...'); // Try to import existing resource into state const importResult = await attemptAutoImport(terraformDir, applyResult.output, terraformEnv); if (importResult.success) { log.success('Resource imported into Terraform state'); // The import REPLACED the plan validated above. That plan described a // create; the plan terraform computes now can describe a REPLACE, // because `ostemplate` and `rootfs[0].storage` do not round-trip // through an import and each forces one. Re-plan and re-check, or the // retry applies a destroy the safety gate exists to forbid — which is // how a redeploy destroyed signal's LXC on a converged fleet // (celilo#1374). const rePlan = await executeBuildWithProgress({ command: 'terraform', args: ['plan', '-detailed-exitcode', '-no-color'], cwd: terraformDir, title: 'Re-planning after import', env: terraformEnv, noInteractive, }); const verdict = planAfterImportVerdict(rePlan.exitCode, rePlan.output); if (verdict.action === 'done') { // The import alone reconciled state with reality: adopt and stop. if (phases) phases.terraformApply = true; log.success('Infrastructure adopted by import (no changes needed)'); return { success: true, output: rePlan.output, exitCode: 0 }; } if (verdict.action === 'refuse') { if (phases) phases.terraformApply = false; log.error('Refusing to apply the plan computed after import'); return { success: false, output: rePlan.output, error: verdict.error, exitCode: 2, }; } log.message(' Retrying deployment...'); // Retry apply after successful import const retryResult = await executeTerraformCommand( 'apply', ['-auto-approve'], terraformDir, 'Applying infrastructure', terraformEnv, noInteractive, ); if (!retryResult.success) { if (phases) phases.terraformApply = false; return retryResult; } if (phases) phases.terraformApply = true; log.success('Infrastructure deployed (recovered from state drift)'); return retryResult; } log.error('Automatic recovery failed - manual import required'); if (phases) phases.terraformApply = false; return { ...applyResult, error: `${applyResult.error}\n\nAutomatic recovery failed. Manual fix:\ncd "${terraformDir}"\nterraform import ${importResult.suggestion || ''}`, }; } if (phases) phases.terraformApply = false; return applyResult; } if (phases) phases.terraformApply = true; log.success('Infrastructure deployed'); return applyResult; } /** * Attempt to automatically import a resource that already exists * Parses Terraform error output to determine resource type and ID, then imports it * * @param terraformDir - Terraform working directory * @param errorOutput - Terraform error output containing "already exists" * @param terraformEnv - Environment variables including provider credentials * @returns Import result with success status and suggestion */ async function attemptAutoImport( terraformDir: string, errorOutput: string, terraformEnv: Record, ): Promise<{ success: boolean; suggestion?: string }> { // Parse error message to extract resource info // Example: "CT 200 already exists on node 'node2'" // Example: "with proxmox_lxc.homebridge" const resourceMatch = errorOutput.match(/with\s+([\w_]+\.\w+)/); if (!resourceMatch) { return { success: false, suggestion: ' ' }; } const resourceName = resourceMatch[1]; // e.g., "proxmox_lxc.homebridge" // Try to extract resource ID from error message let resourceId: string | null = null; // LXC container: "CT 200 already exists on node 'node2'" -> "node2/lxc/200" const lxcMatch = errorOutput.match(/CT (\d+) already exists on node '(\w+)'/); if (lxcMatch) { const vmid = lxcMatch[1]; const node = lxcMatch[2]; resourceId = `${node}/lxc/${vmid}`; } // VM: "VM 100 already exists" -> need to determine format // Add more patterns as needed for other resource types if (!resourceId) { return { success: false, suggestion: `${resourceName} `, }; } // Attempt the import log.info(` Importing ${resourceName} as ${resourceId}...`); const result = await executeBuildWithProgress({ command: 'terraform', args: ['import', resourceName, resourceId], cwd: terraformDir, title: 'Importing existing resource', env: { ...terraformEnv, TF_IN_AUTOMATION: '1', }, }); return { success: result.success, suggestion: `${resourceName} ${resourceId}`, }; } /** * Execute a single Terraform command with streaming output * Execution function - runs terraform command with fuel-gauge progress * * @param command - Terraform subcommand (init, apply, etc.) * @param args - Additional arguments * @param cwd - Working directory * @param title - Progress indicator title * @param env - Environment variables * @returns Execution result */ /** * Detect a stale Terraform state lock and surface actionable guidance. * * We intentionally do NOT auto-delete: if Terraform crashed mid-apply the * state file may be inconsistent, and blindly retrying could double-apply * partial infrastructure changes. The user should verify state before * unlocking. */ async function autoForceUnlock( terraformDir: string, errorOutput: string, _terraformEnv: Record, _noInteractive?: boolean, ): Promise { const lockIdMatch = errorOutput.match(/ID:\s+([0-9a-f-]+)/); const lockId = lockIdMatch?.[1]; const lockFile = join(terraformDir, '.terraform.tfstate.lock.info'); const isLocalLock = existsSync(lockFile); // Try to surface who holds the lock and when it was created. let lockInfo = ''; if (isLocalLock) { try { const raw = JSON.parse(readFileSync(lockFile, 'utf-8')) as Record; const who = raw.Who ?? 'unknown'; const created = raw.Created ? new Date(raw.Created).toLocaleString() : 'unknown'; const op = raw.Operation ?? 'unknown'; lockInfo = ` Held by: ${who}\n Operation: ${op}\n Created: ${created}`; } catch { lockInfo = ` Lock file: ${lockFile}`; } } else if (lockId) { lockInfo = ` Lock ID: ${lockId}`; } log.error( `Terraform state is locked — another deploy may be running, or a previous one crashed.\n${lockInfo}\n\nIf no other deploy is running, unlock with:\n celilo module terraform-unlock `, ); return false; } /** * Parse Terraform error output into a clear, actionable message */ function parseTerraformError(output: string): string { // Proxmox unreachable if (output.includes('dial tcp') && output.includes('connect: operation timed out')) { const ipMatch = output.match(/dial tcp ([^:]+:\d+)/); const target = ipMatch ? ipMatch[1] : 'unknown'; return `Proxmox server unreachable at ${target}\n\nCheck:\n - Is the Proxmox server running?\n - Can this machine reach ${target}?\n - Is a firewall blocking the connection?`; } // Proxmox auth error if (output.includes('401') && output.includes('proxmox')) { return 'Proxmox authentication failed\n\nCheck:\n - API token ID and secret in service config\n - Token permissions in Proxmox'; } // Digital Ocean auth error if (output.includes('401') && output.includes('digitalocean')) { return 'Digital Ocean authentication failed\n\nCheck:\n - API token in service config\n - Token has write permissions'; } // Connection refused if (output.includes('connection refused')) { const ipMatch = output.match(/dial tcp ([^:]+:\d+)/); const target = ipMatch ? ipMatch[1] : 'unknown'; return `Connection refused to ${target}\n\nThe server is not accepting connections on that port.`; } // DNS resolution failure if (output.includes('no such host') || output.includes('could not resolve')) { return 'DNS resolution failed for the infrastructure provider\n\nCheck your network connection and DNS settings.'; } // Proxmox permission denied (e.g., API token can't set certain LXC feature flags) if (output.includes('Permission check failed') && output.includes('proxmox')) { const reasonMatch = output.match(/Permission check failed \(([^)]+)\)/); const reason = reasonMatch ? reasonMatch[1] : 'insufficient permissions'; return `Proxmox permission denied: ${reason}\n\nYour API token lacks the required privileges.\n\nCheck:\n - Token permissions in Proxmox (Datacenter → Permissions → API Tokens)\n - Some operations (e.g., LXC feature flags like keyctl) require root@pam\n - Consider removing unsupported features from the module's Terraform template`; } // Terraform registry unreachable (provider download failure during init) if ( output.includes('registry.terraform.io') && (output.includes('request canceled') || output.includes('Timeout exceeded') || output.includes('could not connect')) ) { return 'Cannot reach Terraform registry (registry.terraform.io)\n\nTerraform needs internet access to download provider plugins.\n\nCheck:\n - Network connectivity from the machine running Celilo\n - DNS resolution: nslookup registry.terraform.io\n - Firewall rules allowing outbound HTTPS (port 443)'; } // Generic: extract the Error: line const errorLineMatch = output.match(/Error: (.+?)(?:\n|$)/); if (errorLineMatch) { return errorLineMatch[1].trim(); } return `Terraform failed:\n${output.substring(0, 300)}`; } async function executeTerraformCommand( command: string, args: string[], cwd: string, title: string, env: Record, noInteractive?: boolean, ): Promise { // Execute with streaming progress const result = await executeBuildWithProgress({ command: 'terraform', args: [command, ...args], cwd, title, env, noInteractive, }); return { success: result.success, output: result.output, error: result.error, exitCode: result.exitCode, }; } /** * Parse Terraform outputs after successful apply * Execution function - queries Terraform state for outputs * * @param terraformDir - Terraform working directory * @returns Parsed Terraform outputs or null if none exist */ export async function parseTerraformOutputs( terraformDir: string, ): Promise | null> { const result = await executeBuildWithProgress({ command: 'terraform', args: ['output', '-json'], cwd: terraformDir, title: 'Reading Terraform outputs', env: { TF_IN_AUTOMATION: '1' }, // The raw JSON is for parsing into a Record, not for display. // Suppress every line from the gauge/display while still capturing // result.output for JSON.parse below. filterOutput: () => null, }); if (!result.success) { // No outputs defined is not an error - return null if (result.output.includes('no outputs')) { return null; } throw new Error(`Failed to read Terraform outputs: ${result.error}`); } try { return JSON.parse(result.output) as Record; } catch (error) { throw new Error( `Failed to parse Terraform outputs JSON: ${error instanceof Error ? error.message : 'Unknown error'}`, ); } }