/** * Proxmox API Client * Validates connection to Proxmox Virtual Environment */ import https from 'node:https'; import type { TestResult } from '../types/infrastructure'; export interface ProxmoxCredentials { api_url: string; api_token_id: string; api_token_secret: string; } export interface ProxmoxProviderConfig { default_target_node: string; lxc_template: string; storage: string; } interface ProxmoxApiResponse { data: T; } interface ProxmoxError { success: false; message: string; details?: Record; } export type ProxmoxResult = { success: true; data: T } | ProxmoxError; /** * Make an authenticated API request to Proxmox * Helper function for all Proxmox API calls */ async function makeProxmoxRequest( credentials: ProxmoxCredentials, path: string, ): Promise> { let timeout: ReturnType | undefined; return new Promise>((resolve) => { try { const { api_url, api_token_id, api_token_secret } = credentials; const authHeader = `PVEAPIToken=${api_token_id}=${api_token_secret}`; const fullUrl = `${api_url}${path}`; const url = new URL(fullUrl); if (process.env.DEBUG) { console.log(`[Proxmox] Request: ${fullUrl}`); } const agent = new https.Agent({ rejectUnauthorized: false, }); const req = https.request( { hostname: url.hostname, port: url.port || 443, path: url.pathname, method: 'GET', headers: { Authorization: authHeader, }, agent, }, (res) => { let body = ''; res.on('data', (chunk) => { body += chunk; }); res.on('end', () => { const statusCode = res.statusCode || 0; if (statusCode === 401) { resolve({ success: false, message: 'Authentication failed - check API token credentials', details: { status: statusCode }, }); return; } if (statusCode < 200 || statusCode >= 300) { resolve({ success: false, message: `API request failed with status ${statusCode}`, details: { status: statusCode, response: body }, }); return; } try { const data = JSON.parse(body) as ProxmoxApiResponse; resolve({ success: true, data: data.data }); } catch (error) { resolve({ success: false, message: 'Failed to parse API response', details: { error: String(error), body }, }); } }); }, ); req.on('error', (error) => { resolve({ success: false, message: `Connection error: ${error.message}`, details: { error: String(error) }, }); }); // Bound the entire request, including connection establishment. On Bun // 1.4, req.setTimeout does not bound a connection to an unreachable host. // Callers need a failed result so they can fall back instead of hanging. timeout = setTimeout(() => { resolve({ success: false, message: 'Request timed out', details: { timeoutMs: 15_000 }, }); req.destroy(); }, 15_000); req.end(); } catch (error) { resolve({ success: false, message: `Request failed: ${error instanceof Error ? error.message : String(error)}`, details: { error: String(error) }, }); } }).finally(() => { if (timeout !== undefined) clearTimeout(timeout); }); } /** * Check if a Proxmox node exists and is online */ async function checkNodeStatus( credentials: ProxmoxCredentials, nodeName: string, ): Promise> { return makeProxmoxRequest(credentials, `/nodes/${nodeName}/status`); } /** * List all nodes in the Proxmox cluster/standalone */ async function listNodes( credentials: ProxmoxCredentials, ): Promise>> { return makeProxmoxRequest(credentials, '/nodes'); } /** * Verify node name matches actual Proxmox configuration * Returns actual node name and warns if mismatch */ async function verifyNodeName( credentials: ProxmoxCredentials, expectedNodeName: string, ): Promise> { const nodesResult = await listNodes(credentials); if (!nodesResult.success) { return nodesResult; } const nodes = nodesResult.data; const nodeNames = nodes.map((n) => n.node); // Check if expected node exists if (!nodeNames.includes(expectedNodeName)) { return { success: false, message: `Node '${expectedNodeName}' not found in Proxmox. Available nodes: ${nodeNames.join(', ')}.\n\nPossible causes:\n 1. Wrong node name configured in Celilo\n 2. Node hostname doesn't match Proxmox registration\n 3. Node not part of this cluster\n\nTo fix:\n 1. Check actual node name: ssh root@ 'hostname'\n 2. Update Celilo service to use correct node name\n 3. Or update Proxmox node hostname:\n - hostnamectl set-hostname ${expectedNodeName}\n - Update /etc/hosts: echo ' ${expectedNodeName}' >> /etc/hosts\n - Restart: systemctl restart pvedaemon pveproxy`, details: { expected: expectedNodeName, available: nodeNames, }, }; } return { success: true, data: { actual_node: expectedNodeName, matches: true, }, }; } /** * Check if storage exists on a node */ async function checkStorageStatus( credentials: ProxmoxCredentials, nodeName: string, storageName: string, ): Promise> { return makeProxmoxRequest(credentials, `/nodes/${nodeName}/storage/${storageName}/status`); } /** * Check if an LXC template exists in storage */ async function checkTemplateExists( credentials: ProxmoxCredentials, nodeName: string, storageName: string, templateName: string, ): Promise> { const result = await makeProxmoxRequest>( credentials, `/nodes/${nodeName}/storage/${storageName}/content`, ); if (!result.success) { return result; } // Check if any volume matches the template name const templateExists = result.data.some((item) => item.volid.includes(templateName)); return { success: true, data: templateExists }; } /** * Required permissions for Celilo to deploy containers * These are the minimum permissions needed for Terraform/Ansible deployment * * Note: VM.Monitor was removed in Proxmox 9.0+ * VM.Audit now covers monitoring functionality */ const REQUIRED_PERMISSIONS = [ 'VM.Allocate', // Create VMs/containers 'VM.Audit', // View VM status (includes monitoring in Proxmox 9+) 'VM.Config.CDROM', 'VM.Config.CPU', 'VM.Config.Disk', 'VM.Config.Memory', 'VM.Config.Network', 'VM.Config.Options', 'VM.PowerMgmt', // Start/stop VMs 'Datastore.AllocateSpace', // Allocate disk space 'SDN.Use', // Use software-defined networking ]; /** * Check if API token has required permissions * Returns list of missing permissions */ async function checkTokenPermissions( credentials: ProxmoxCredentials, ): Promise> { // Get permissions for the token // Proxmox API returns permissions as nested object: { "/path": { "permission": 1 } } const result = await makeProxmoxRequest>>( credentials, '/access/permissions', ); if (!result.success) { return result; } if (process.env.DEBUG) { console.log('[Proxmox] Permissions response:', JSON.stringify(result.data, null, 2)); } // Collect all granted permissions from all paths const grantedPermissions = new Set(); for (const path of Object.keys(result.data)) { const pathPermissions = result.data[path]; if (typeof pathPermissions === 'object' && pathPermissions !== null) { for (const permission of Object.keys(pathPermissions)) { grantedPermissions.add(permission); } } } if (process.env.DEBUG) { console.log('[Proxmox] Granted permissions:', Array.from(grantedPermissions).sort()); } // Check which required permissions are missing const missing = REQUIRED_PERMISSIONS.filter((perm) => !grantedPermissions.has(perm)); const granted = REQUIRED_PERMISSIONS.filter((perm) => grantedPermissions.has(perm)); // Detect if token likely has privilege separation (if it has very few permissions) // Tokens without privilege separation typically have 40-50+ permissions // Tokens WITH separation typically have < 20 permissions (only what's explicitly granted) const privilegeSeparation = grantedPermissions.size < 20; return { success: true, data: { missing, granted, privilegeSeparation, }, }; } /** * Test connection to Proxmox API * Verifies credentials, connectivity, and optionally checks node/storage/template configuration */ export async function testProxmoxConnection( credentials: ProxmoxCredentials, providerConfig?: ProxmoxProviderConfig, ): Promise { const checks: string[] = []; // Step 1: Check version (basic connectivity and authentication) const versionResult = await makeProxmoxRequest<{ version: string; release?: string }>( credentials, '/version', ); if (!versionResult.success) { return { success: false, message: versionResult.message, details: versionResult.details, }; } checks.push(`Connected to Proxmox VE ${versionResult.data.version}`); // Step 2: Check API token permissions const permissionResult = await checkTokenPermissions(credentials); if (!permissionResult.success) { return { success: false, message: `Failed to check API token permissions: ${permissionResult.message}`, details: permissionResult.details, }; } if (permissionResult.data.missing.length > 0) { const missingList = permissionResult.data.missing.join(', '); const hasSeparation = permissionResult.data.privilegeSeparation; // Build fix instructions based on whether privilege separation is enabled let fixInstructions: string; if (hasSeparation) { fixInstructions = 'To fix:\n 1. In Proxmox UI: Datacenter → Permissions → API Tokens\n 2. Delete the existing token\n 3. Create a new token with "Privilege Separation" UNCHECKED\n 4. Update Celilo with the new token credentials'; } else { fixInstructions = `To fix:\n 1. In Proxmox UI: Datacenter → Permissions → Users\n 2. Edit the 'root@pam' user\n 3. Grant the missing permissions to the user\n 4. OR check if the token has explicit permission overrides that need updating`; } return { success: false, message: `API token is missing required permissions: ${missingList}\n\nRequired permissions:\n${REQUIRED_PERMISSIONS.map((p) => ` - ${p}`).join('\n')}\n\n${fixInstructions}`, details: { missing: permissionResult.data.missing, granted: permissionResult.data.granted, privilegeSeparation: hasSeparation, }, }; } checks.push('API token has all required permissions'); // If no providerConfig, return basic connectivity check if (!providerConfig) { return { success: true, message: checks.join('\n✓ '), details: { version: versionResult.data.version, release: versionResult.data.release, permissions: { granted: permissionResult.data.granted, privilegeSeparation: permissionResult.data.privilegeSeparation, }, }, }; } // Step 3: Verify node name matches Proxmox configuration const nodeNameResult = await verifyNodeName(credentials, providerConfig.default_target_node); if (!nodeNameResult.success) { return { success: false, message: nodeNameResult.message, details: nodeNameResult.details, }; } checks.push(`Node '${providerConfig.default_target_node}' found in cluster`); // Step 4: Check node is online const nodeResult = await checkNodeStatus(credentials, providerConfig.default_target_node); if (!nodeResult.success) { return { success: false, message: `Node '${providerConfig.default_target_node}' not found or not accessible`, details: nodeResult.details, }; } checks.push(`Node '${providerConfig.default_target_node}' is online`); // Step 5: Check storage is available const storageResult = await checkStorageStatus( credentials, providerConfig.default_target_node, providerConfig.storage, ); if (!storageResult.success) { return { success: false, message: `Storage '${providerConfig.storage}' not found or not accessible on node '${providerConfig.default_target_node}'`, details: storageResult.details, }; } if (!storageResult.data.active || !storageResult.data.enabled) { return { success: false, message: `Storage '${providerConfig.storage}' is not active or enabled`, details: { active: storageResult.data.active, enabled: storageResult.data.enabled }, }; } checks.push(`Storage '${providerConfig.storage}' is available`); // Step 6: Check template exists // Extract storage name and template filename from the full template path // Format: "local:vztmpl/ubuntu-22.04-standard_22.04-1_amd64.tar.zst" const templateParts = providerConfig.lxc_template.split(':'); const templateStorage = templateParts[0] || providerConfig.storage; const templateFilename = templateParts[1]?.split('/').pop() || providerConfig.lxc_template; const templateResult = await checkTemplateExists( credentials, providerConfig.default_target_node, templateStorage, templateFilename, ); if (!templateResult.success) { return { success: false, message: `Failed to check template availability: ${templateResult.message}`, details: templateResult.details, }; } if (!templateResult.data) { return { success: false, message: `Template '${templateFilename}' not found in storage '${templateStorage}'`, details: { template: providerConfig.lxc_template }, }; } checks.push(`Template '${templateFilename}' found`); // All checks passed return { success: true, message: checks.join('\n✓ '), details: { version: versionResult.data.version, node: providerConfig.default_target_node, storage: providerConfig.storage, template: templateFilename, permissions: { granted: permissionResult.data.granted, privilegeSeparation: permissionResult.data.privilegeSeparation, }, }, }; } /** * List all storage on a node */ export async function listNodeStorage( credentials: ProxmoxCredentials, nodeName: string, ): Promise< ProxmoxResult> > { return makeProxmoxRequest(credentials, `/nodes/${nodeName}/storage`); } /** * Find which Proxmox node a given VMID currently lives on. * * Queries the cluster resource inventory (`/cluster/resources`), which lists * every guest across all nodes with its current node, and matches by VMID * (unique cluster-wide). Returns the node name, or `null` if the VMID isn't * present — i.e. the container hasn't been created yet (a first deploy). * * ISS-0090: this is celilo's source of truth for WHERE a system currently is. * A redeploy must target the node Proxmox reports here, NOT re-derive placement * from the service's `default_target_node` (which only governs new placement) — * otherwise a changed default tries to relocate every running container. */ export async function getNodeForVmid( credentials: ProxmoxCredentials, vmid: number, ): Promise> { // makeProxmoxRequest sends only url.pathname, so a `?type=vm` filter would be // dropped — fetch the full inventory and match by vmid client-side instead. const result = await makeProxmoxRequest>( credentials, '/cluster/resources', ); if (!result.success) { return result; } return { success: true, data: findNodeForVmid(result.data, vmid) }; } /** * Find the node a VMID lives on within a Proxmox cluster-resource list. Pure * matching logic, split out from the network call for testability (Rule 10). * Non-guest entries (storage/node rows) have no `vmid` and are skipped. Returns * the node name, or `null` when the VMID isn't present. */ export function findNodeForVmid( resources: Array<{ vmid?: number; node?: string }>, vmid: number, ): string | null { const match = resources.find((r) => typeof r.vmid === 'number' && r.vmid === vmid); return match?.node ?? null; } /** * One row from `GET /cluster/resources`. The list is heterogeneous — `type` * discriminates node / storage / guest rows. For `type: 'node'`, `status` is * 'online'/'offline' and the mem/cpu/disk fields describe the node's capacity; * for guests it's 'running'/'stopped' and `vmid` is set. */ export interface ProxmoxClusterResource { type: string; node?: string; status?: string; vmid?: number; maxmem?: number; // bytes (node total RAM) mem?: number; // bytes (node RAM in use) maxcpu?: number; // cores cpu?: number; // load fraction 0..1 maxdisk?: number; // bytes (node total disk on the relevant storage) disk?: number; // bytes (disk in use) uptime?: number; // seconds } /** Per-node capacity reconciled from Proxmox (reality, never a cached DB value). */ export interface ProxmoxNodeCapacity { node: string; online: boolean; memTotalMb: number; memFreeMb: number; cpuCores: number; cpuUsedPct: number; // 0..100 diskTotalGb: number; diskFreeGb: number; uptimeSec: number; } const BYTES_PER_MB = 1024 * 1024; const BYTES_PER_GB = 1024 * 1024 * 1024; /** * Summarize per-node capacity from a `/cluster/resources` list. Pure (Rule 10) — * split from the network call for unit testing. Only `type: 'node'` rows count; * guest and storage rows are ignored. Sorted by node name for stable output. */ export function summarizeNodeCapacities( resources: ProxmoxClusterResource[], ): ProxmoxNodeCapacity[] { return resources .filter((r): r is ProxmoxClusterResource & { node: string } => r.type === 'node' && !!r.node) .map((r) => { const maxmem = r.maxmem ?? 0; const mem = r.mem ?? 0; const maxdisk = r.maxdisk ?? 0; const disk = r.disk ?? 0; return { node: r.node, online: r.status === 'online', memTotalMb: Math.round(maxmem / BYTES_PER_MB), memFreeMb: Math.round((maxmem - mem) / BYTES_PER_MB), cpuCores: r.maxcpu ?? 0, cpuUsedPct: Math.round((r.cpu ?? 0) * 100), diskTotalGb: Math.round(maxdisk / BYTES_PER_GB), diskFreeGb: Math.round((maxdisk - disk) / BYTES_PER_GB), uptimeSec: r.uptime ?? 0, }; }) .sort((a, b) => a.node.localeCompare(b.node)); } /** * Cohesive Proxmox introspection client (ISS-0060). Wraps the credentials so * callers don't thread them through every call, and centralizes the live reads * that let celilo treat Proxmox — not a cached DB row — as the source of truth * for where containers live and how much room each node has. */ export class ProxmoxClient { constructor(private readonly credentials: ProxmoxCredentials) {} /** Raw cluster resource inventory (node + guest + storage rows). */ async clusterResources(): Promise> { return makeProxmoxRequest(this.credentials, '/cluster/resources'); } /** Live per-node capacity (RAM/CPU/disk/online), reconciled from Proxmox. */ async nodeCapacities(): Promise> { const result = await this.clusterResources(); if (!result.success) return result; return { success: true, data: summarizeNodeCapacities(result.data) }; } /** The node a VMID currently lives on, or null if it isn't created yet. */ async nodeForVmid(vmid: number): Promise> { const result = await this.clusterResources(); if (!result.success) return result; return { success: true, data: findNodeForVmid(result.data, vmid) }; } /** * Power a guest down or up. Used by `module pause --stop-infra` (design D2), * where stopping the box is opt-in rather than what a pause means. * * `shutdown` rather than `stop` on the way down: it asks the guest to go * quietly instead of pulling its power, and a pause is a planned operation, * not a fault. Proxmox returns a task id; callers that need to know it * finished poll with `pollTaskUntilDone`. */ async setGuestPower( vmid: number, kind: 'lxc' | 'qemu', power: 'shutdown' | 'start', ): Promise> { const node = await this.nodeForVmid(vmid); if (!node.success) return node; if (!node.data) { return { success: false, message: `No Proxmox node hosts vmid ${vmid}` }; } return makeProxmoxPost( this.credentials, `/nodes/${node.data}/${kind}/${vmid}/status/${power}`, {}, ); } /** * Grow a guest's disk, online. This is `pct resize` / `qm resize` (celilo#1133). * * Deliberately NOT routed through Terraform like a cpu/memory resize is. Two * reasons, and the first is the one that matters: a Terraform reconcile means * a full module redeploy — Ansible, package installs, service restarts — and * the box being resized is by definition the one that has run out of room to * do any of that. The cure would need the disk space it exists to provide. * Second, the resize API is additive and online, so celilo can promise the * guest is not power-cycled; delegating that to a provider's update path * cannot promise it. * * `disk` is the config key of the volume to grow — `rootfs` for lxc, `scsi0` * for a cloud-init VM. `size` is absolute (Proxmox also accepts `+NG`, but * canonical state is an absolute size, so we send one). * * ⚠️ lxc grows the filesystem too; qemu grows only the block device and the * guest must extend its own partition. Callers say so rather than implying * the space is usable. */ async resizeGuestDisk( vmid: number, kind: 'lxc' | 'qemu', disk: string, sizeGb: number, ): Promise> { const node = await this.nodeForVmid(vmid); if (!node.success) return node; if (!node.data) { return { success: false, message: `No Proxmox node hosts vmid ${vmid}` }; } return makeProxmoxPut(this.credentials, `/nodes/${node.data}/${kind}/${vmid}/resize`, { disk, size: `${sizeGb}G`, }); } /** Current run state of a guest (`running`, `stopped`, …). */ async guestStatus(vmid: number): Promise> { const result = await this.clusterResources(); if (!result.success) return result; const guest = result.data.find((r) => r.vmid === vmid); return { success: true, data: guest?.status ?? null }; } } /** * List available LXC templates in storage */ export async function listAvailableTemplates( credentials: ProxmoxCredentials, nodeName: string, storageName: string, ): Promise>> { return makeProxmoxRequest( credentials, `/nodes/${nodeName}/storage/${storageName}/content?content=vztmpl`, ); } /** * Make an authenticated form-encoded request (POST or PUT) to the Proxmox API. */ async function makeProxmoxFormRequest( credentials: ProxmoxCredentials, method: 'POST' | 'PUT', path: string, params: Record, ): Promise> { return new Promise((resolve) => { try { const { api_url, api_token_id, api_token_secret } = credentials; const authHeader = `PVEAPIToken=${api_token_id}=${api_token_secret}`; const fullUrl = `${api_url}${path}`; const url = new URL(fullUrl); const postData = new URLSearchParams(params).toString(); if (process.env.DEBUG) { console.log(`[Proxmox] ${method}: ${fullUrl}`); console.log(`[Proxmox] Body: ${postData}`); } const agent = new https.Agent({ rejectUnauthorized: false }); const req = https.request( { hostname: url.hostname, port: url.port || 443, path: url.pathname, method, headers: { Authorization: authHeader, 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(postData), }, agent, }, (res) => { let body = ''; res.on('data', (chunk) => { body += chunk; }); res.on('end', () => { const statusCode = res.statusCode || 0; if (statusCode < 200 || statusCode >= 300) { if (process.env.DEBUG || statusCode >= 400) { console.error(`[Proxmox] ${method} ${path} failed (${statusCode}): ${body}`); } resolve({ success: false, message: `Request failed with status ${statusCode}: ${body}`, details: { status: statusCode, response: body }, }); return; } try { const data = JSON.parse(body) as ProxmoxApiResponse; resolve({ success: true, data: data.data }); } catch (error) { resolve({ success: false, message: 'Failed to parse response', details: { error: String(error), body }, }); } }); }, ); req.on('error', (error) => { resolve({ success: false, message: `Request failed: ${error.message}`, details: { error: String(error) }, }); }); req.write(postData); req.end(); } catch (error) { resolve({ success: false, message: `Request failed: ${error instanceof Error ? error.message : String(error)}`, details: { error: String(error) }, }); } }); } async function makeProxmoxPost( credentials: ProxmoxCredentials, path: string, params: Record, ): Promise> { return makeProxmoxFormRequest(credentials, 'POST', path, params); } async function makeProxmoxPut( credentials: ProxmoxCredentials, path: string, params: Record, ): Promise> { return makeProxmoxFormRequest(credentials, 'PUT', path, params); } /** * Entry from Proxmox's appliance catalog (`pveam available`). The `template` * field is the canonical filename (revision included) that should be passed to * downloadAppliance; constructing it ourselves is a known foot-gun because * Proxmox refreshes revisions over time. */ export interface ProxmoxAppliance { /** Full canonical filename, e.g. "ubuntu-24.04-standard_24.04-2_amd64.tar.zst" */ template: string; /** Package family, e.g. "ubuntu-24.04-standard" */ package: string; /** Version including revision, e.g. "24.04-2" */ version: string; /** Template type, typically "lxc" */ type?: string; /** Operating system, e.g. "ubuntu" */ os?: string; /** Section, e.g. "system" — what `pveam available --section system` filters on */ section?: string; /** Display headline */ headline?: string; } /** * List the LXC templates Proxmox knows are downloadable from its mirror. * Wraps `GET /nodes/{node}/aplinfo` — the same data source `pveam available` * uses. Use this rather than building URLs against download.proxmox.com so * that revision bumps (e.g. ubuntu-24.04 -1 → -2) are picked up automatically. */ export async function listAvailableAppliances( credentials: ProxmoxCredentials, nodeName: string, ): Promise> { return makeProxmoxRequest(credentials, `/nodes/${nodeName}/aplinfo`); } /** * Start a download of an appliance template from Proxmox's mirror. * `templateName` must be the exact `template` field from listAvailableAppliances * (the same string `pveam download