/** * `celilo proxmox vm resize [--memory ] [--cpu ] [--disk ]` * (and `ct resize`) — the canonical resize for a celilo-provisioned instance * (ISS-0150 P2). Updates the SYSTEM's canonical size in module_systems. * * Two apply paths, because the two kinds of change are not alike: * * cpu / memory → declarative. Record the size, redeploy the owning module, * and its Terraform writes the new size to the guest config. * The provider stop/starts the guest to apply it, so this * path costs a reboot and is gated on the operator saying so. * disk (growth) → direct. Proxmox's resize API grows the volume online, so * celilo calls it and skips the redeploy entirely. No reboot, * no Ansible. This is not a shortcut: a full module redeploy * needs disk space to run, and the box being grown is the one * that has run out of it (celilo#1133). * * Both paths leave module_systems canonical, so `proxmox ct list` shows DESIRED * matching ACTUAL and drift clears either way. The instance Terraform reads * `$self:disk` (not `requires.system.disk`, which is only the floor), so the * next deploy agrees with a resize instead of trying to undo it. * * Guardrails (proxmox-resize-guards.ts): floor = max(requires.system) across * co-hosted modules (hard); disk shrink (hard); node capacity (overridable with * --force). Reboot + pre-resize backup are confirmed via the event-bus * interview (CLAUDE.md: no direct CLI prompts) — flags win: --allow-reboot / * --skip-backup / --yes. */ import { and, eq } from 'drizzle-orm'; import { ProxmoxClient, type ProxmoxCredentials } from '../../api-clients/proxmox'; import { getDb } from '../../db/client'; import { moduleSystems, modules } from '../../db/schema'; import { type ModuleManifest, getSingularSystemSpec } from '../../manifest/schema'; import { createModuleBackup } from '../../services/backup-create'; import { askConfirm, withInterviewSession } from '../../services/bus-interview'; import { getServiceCredentials, listContainerServices } from '../../services/container-service'; import { getProvisionedSystems } from '../../services/deployed-systems'; import { deployModule } from '../../services/module-deploy'; import { celiloIntro } from '../prompts'; import type { CommandResult } from '../types'; import type { InstanceKind } from './proxmox-instance-list'; import { computeFloor, validateResize } from './proxmox-resize-guards'; import { resolveProxmoxService } from './proxmox-service'; const PROXMOX_TYPE: Record = { vm: 'qemu', ct: 'lxc' }; const BYTES_PER_GB = 1024 * 1024 * 1024; function numFlag(v: string | boolean | undefined): number | undefined { if (typeof v !== 'string') return undefined; const n = Number.parseInt(v, 10); return Number.isNaN(n) ? undefined : n; } export async function handleProxmoxInstanceResize( kind: InstanceKind, args: string[], flags: Record, ): Promise { celiloIntro(`Resize ${kind === 'vm' ? 'VM' : 'container'}`); const name = args[0]; if (!name) { const e = `Instance name required: celilo proxmox ${kind} resize [--memory ] [--cpu ] [--disk ]`; console.log(`✗ ${e}`); return { success: false, error: e }; } const reqCpu = numFlag(flags.cpu); const reqMemory = numFlag(flags.memory); const reqDisk = numFlag(flags.disk); if (reqCpu == null && reqMemory == null && reqDisk == null) { const e = 'Specify at least one of --memory , --cpu or --disk .'; console.log(`✗ ${e}`); return { success: false, error: e }; } const resolved = resolveProxmoxService(await listContainerServices(), undefined); if ('error' in resolved) { console.log(`✗ ${resolved.error}`); return { success: false, error: resolved.error }; } const { service } = resolved; const creds = (await getServiceCredentials(service.id)) as ProxmoxCredentials; const client = new ProxmoxClient(creds); const cluster = await client.clusterResources(); if (!cluster.success) { console.log(`✗ Could not reach Proxmox: ${cluster.message}`); return { success: false, error: cluster.message }; } const wantType = PROXMOX_TYPE[kind]; const guestByVmid = new Map(); for (const r of cluster.data) { if (r.type === wantType && typeof r.vmid === 'number') guestByVmid.set(r.vmid, r); } const db = getDb(); const systems = getProvisionedSystems(db); const matches = systems.filter( (s) => s.vmid != null && guestByVmid.has(s.vmid) && (s.hostname === name || s.name === name || s.moduleId === name), ); if (matches.length === 0) { const e = `No celilo-provisioned ${kind} named '${name}'. Try: celilo proxmox ${kind} list`; console.log(`✗ ${e}`); return { success: false, error: e }; } if (matches.length > 1) { const e = `Ambiguous '${name}' — matches ${matches.map((m) => m.moduleId).join(', ')}`; console.log(`✗ ${e}`); return { success: false, error: e }; } const target = matches[0]; const vmid = target.vmid as number; const guest = guestByVmid.get(vmid); const node = guest?.node; // Co-hosted = every module recorded on the same physical instance (vmid). const coHosted = systems.filter((s) => s.vmid === vmid); const mins = coHosted.map((s) => { const mod = db.select().from(modules).where(eq(modules.id, s.moduleId)).get(); const spec = mod ? getSingularSystemSpec(mod.manifestData as ModuleManifest) : undefined; return { cpu: spec?.cpu, memory: spec?.memory, disk: spec?.disk }; }); const floor = computeFloor(mins); const nodeCaps = await client.nodeCapacities(); const nodeCap = nodeCaps.success ? nodeCaps.data.find((n) => n.node === node) : undefined; // Disk compares against the LIVE volume, not celilo's record of it. "Can // Proxmox do this" is a question about the real disk: canonical state is // seeded from the manifest floor and goes stale the moment anyone runs `pct // resize` by hand — which, until this command existed, was the only way to // grow one. Comparing a grow request against a stale-low record would wave // through a shrink that Proxmox then refuses. cpu/memory keep using canonical // state, which the reconcile path owns end to end. const liveDiskGb = guest?.maxdisk != null ? Math.round(guest.maxdisk / BYTES_PER_GB) : target.disk; const decision = validateResize( { cpu: reqCpu, memory: reqMemory, disk: reqDisk }, { current: { cpu: target.cpu, memory: target.memory, disk: liveDiskGb }, floor, nodeFreeMemMb: nodeCap?.memFreeMb ?? Number.POSITIVE_INFINITY, nodeTotalCores: nodeCap?.cpuCores ?? Number.POSITIVE_INFINITY, nodeFreeDiskGb: nodeCap?.diskFreeGb ?? Number.POSITIVE_INFINITY, }, { force: flags.force === true }, ); if (decision.errors.length > 0) { console.log('✗ Resize rejected:'); for (const err of decision.errors) console.log(` - ${err}`); if (decision.capacityOnly) console.log(' (pass --force to override the capacity check)'); return { success: false, error: decision.errors.join('; ') }; } const yes = flags.yes === true; const coHostedIds = coHosted.map((s) => s.moduleId); // One decision, made in the guard layer where it is unit-tested, consumed // twice below. `needsReboot` is false exactly when nothing but disk changed: // disk growth applies online through the resize API, so it neither // power-cycles the guest nor needs Terraform to run. // // Both skips that follow hang on it. Growing a disk is purely additive — // nothing is destroyed, nothing is rewritten — so there is no state a backup // would protect, and staging one would be actively harmful: the operator // reaching for --disk has a box that has run out of room, and a backup needs // the very resource it lacks. A cpu/memory change still backs up and still // redeploys, because that one reboots. const diskOnly = !decision.needsReboot; // Pre-resize backup — ON by default; --skip-backup opts out; otherwise the // skip decision is an event-bus interview question (never a direct CLI prompt). let doBackup: boolean; if (diskOnly) doBackup = false; else if (flags['skip-backup'] === true) doBackup = false; else if (yes) doBackup = true; else doBackup = await withInterviewSession(() => askConfirm({ scope: 'proxmox-resize', key: 'backup', message: `Take a pre-resize backup of ${coHostedIds.join(', ')} first?`, defaultValue: true, }), ); if (doBackup) { for (const s of coHosted) { const mod = db.select().from(modules).where(eq(modules.id, s.moduleId)).get(); const manifest = mod?.manifestData as ModuleManifest | undefined; if (!manifest?.hooks?.on_backup) { console.log(` ⚠ ${s.moduleId} has no on_backup hook — skipping its backup`); continue; } const backup = await createModuleBackup(s.moduleId); if (!backup.success) { const e = `Pre-resize backup failed for ${s.moduleId}: ${backup.error}`; console.log(`✗ ${e}`); return { success: false, error: e }; } console.log(` ✓ Backed up ${s.moduleId}`); } } // Reboot confirmation — a cpu/memory change needs a Proxmox stop/start. if (decision.needsReboot) { let allowReboot: boolean; if (flags['allow-reboot'] === true || yes) allowReboot = true; else allowReboot = await withInterviewSession(() => askConfirm({ scope: 'proxmox-resize', key: 'reboot', message: `Resizing ${name} (vmid ${vmid}) requires a stop/start — brief downtime. Proceed?`, defaultValue: false, }), ); if (!allowReboot) { const e = 'Resize aborted — reboot not approved (re-run with --allow-reboot).'; console.log(`✗ ${e}`); return { success: false, error: e }; } } // Disk goes straight to Proxmox's resize API — online, additive, no power // change. Do it BEFORE recording canonical state so a failure leaves celilo's // record matching reality rather than claiming a size the guest doesn't have. if (reqDisk != null) { // rootfs for a container, scsi0 for a cloud-init VM — the volumes celilo's // own Terraform declares (modules/*/terraform/main.tf.tpl). const volume = kind === 'ct' ? 'rootfs' : 'scsi0'; console.log(`▸ Growing ${volume} to ${reqDisk}GB (online — no restart)…`); const resized = await client.resizeGuestDisk(vmid, PROXMOX_TYPE[kind], volume, reqDisk); if (!resized.success) { const e = `Disk resize failed: ${resized.message}`; console.log(`✗ ${e}`); return { success: false, error: e }; } console.log(` ✓ ${volume} is now ${reqDisk}GB`); if (kind === 'vm') { // qemu grows the block device only. Saying "resized" without this would // send the operator back to a box whose df output has not moved. console.log( ` ⚠ The VM's block device grew, but the guest must extend its own partition and filesystem (e.g. growpart /dev/sda 1 && resize2fs /dev/sda1). A container does this for itself; a VM does not.`, ); } } // Update the canonical size on every module_systems row for this instance. for (const s of coHosted) { const set: { cpu?: number; memory?: number; disk?: number; updatedAt: Date } = { updatedAt: new Date(), }; if (reqCpu != null) set.cpu = reqCpu; if (reqMemory != null) set.memory = reqMemory; if (reqDisk != null) set.disk = reqDisk; db.update(moduleSystems) .set(set) .where(and(eq(moduleSystems.moduleId, s.moduleId), eq(moduleSystems.name, s.name))) .run(); } const effCpu = reqCpu ?? target.cpu; const effMem = reqMemory ?? target.memory; const effDisk = reqDisk ?? liveDiskGb; console.log( `✓ Canonical size updated: ${name} → ${effCpu ?? '?'}c / ${effMem ?? '?'}MB / ${effDisk ?? '?'}GB`, ); // Disk-only: the guest already carries the new size and the canonical record // now agrees, so Terraform has nothing to reconcile. Redeploying would run a // full Ansible pass for no change — on a box that just filled up, that is the // expensive, risky half of a cure whose cheap half already landed. if (diskOnly) { console.log(`✓ Resized ${name} to ${reqDisk}GB. No restart, no redeploy.`); return { success: true, message: `resized ${name}` }; } // Reconcile declaratively: redeploy the owning module — its Terraform now reads // the updated system size and writes it to the Proxmox VM config. The proxmox // provider itself stop/starts the VM to apply a cpu/memory change (the reboot // the operator approved above), so no explicit power-cycle is needed here. console.log(`▸ Reconciling ${target.moduleId} (Terraform apply + VM restart)…`); const deployed = await deployModule(target.moduleId, db, {}); if (!deployed.success) { // The provider's restart-on-resize can surface a non-fatal "VM already // running" and exit non-zero even though the deploy reached VERIFIED (the // same deploy-over-SSH pattern noted in CLAUDE.md). Trust the module STATE // over the exit code: VERIFIED means the resize landed. const state = db .select({ state: modules.state }) .from(modules) .where(eq(modules.id, target.moduleId)) .get()?.state; if (state !== 'VERIFIED') { const e = `Size updated but reconcile failed: ${deployed.error}. Re-run: celilo module deploy ${target.moduleId}`; console.log(`✗ ${e}`); return { success: false, error: e }; } console.log( ` ⚠ reconcile exited non-zero but ${target.moduleId} is VERIFIED — treating as success`, ); } console.log(`✓ Resized ${name} and reconciled ${target.moduleId} (new size live).`); return { success: true, message: `resized ${name}` }; }