/** * Makes a fake Proxmox LXC into a real, SSH-able Docker container. * * This is the callback `@celilo/terraform-fake` calls on create and destroy. * It lives here rather than in the package because everything it knows — * zone networks, the `target-machine` image, the compose project name — is * rig-specific (D9). * * The container is the SAME image and the SAME boot contract the machine pool * already uses, so a container-service deploy and a machine-pool deploy differ * only in how the host came to exist. `target-setup.service` inside the image * already installs the fleet key from the mounted `ssh-keys` volume, sets the * default route from `GATEWAY`, and points DNS at the resolver — so this * reuses that rather than reimplementing any of it, and readiness is the same * `systemctl is-active target-setup` the harness waits on for machines. */ import { execFileSync } from 'node:child_process'; import type { GuestRecord } from '@celilo/terraform-fake'; import { ZONE_GATEWAYS, type Zone } from './types'; /** * Every container this creates carries the prefix, and it refuses to touch one * that does not. * * Both halves matter. The prefix is what `cele2e doctor` and the by-name * cleanup sweep use to find debris, so an unprefixed container leaks silently * between runs. The refusal is the containment that made mounting the host * Docker socket acceptable (D2): the simulator can reach every container on * the developer's machine, and must only ever act on its own. */ export const CONTAINER_PREFIX = 'celilo-e2e-'; /** * Marks a sim-created guest as belonging to a compose project. * * The name prefix alone is not enough for cleanup: a guest is named * `celilo-e2e-lxc-` with no timestamp in it, so every by-name filter the * sweeps use (`celilo-e2e-1`, `celilo-e2e-`) misses it. A guest left * behind keeps its zone network's endpoint alive, the network rm fails with * "has active endpoints", and the subnet stays allocated — every later suite * in the run then dies creating that network (celilo#1247). The label is what * `projectTeardownCommands` filters on; it carries the project value, so * teardown finds exactly the guests of the project it is tearing down. */ export const GUEST_PROJECT_LABEL = 'celilo-e2e.project'; /** Shells out to `docker`. Injected so the logic is testable without a daemon. */ export type DockerRunner = (args: string[]) => string; export const realDocker: DockerRunner = (args) => execFileSync('docker', args, { encoding: 'utf-8', timeout: 60_000 }); export interface DockerProvisionerOptions { /** Compose project name — networks and volumes are prefixed with it. */ project: string; docker?: DockerRunner; /** Poll budget for `target-setup` to go active. Matches the machine-pool wait. */ readyTimeoutMs?: number; sleep?: (ms: number) => Promise; } export const containerNameFor = (vmid: number): string => `${CONTAINER_PREFIX}lxc-${vmid}`; /** Parse Proxmox's comma-packed `net0` into its parts. */ export function parseNet(net0: string): Record { const parts: Record = {}; for (const pair of net0.split(',')) { const [key, ...rest] = pair.split('='); if (key && rest.length > 0) parts[key] = rest.join('='); } return parts; } /** * Which rig zone a guest belongs to, from the gateway Terraform was given. * * Deliberately keyed off the gateway rather than the VLAN tag: `ZONE_GATEWAYS` * is already the rig's source of truth for zone addressing, so this cannot * drift from it the way a second tag→zone table would. */ export function zoneForGateway(gateway: string): Zone | undefined { return (Object.keys(ZONE_GATEWAYS) as Zone[]).find((zone) => ZONE_GATEWAYS[zone] === gateway); } export function createDockerProvisioner(options: DockerProvisionerOptions) { const docker = options.docker ?? realDocker; const readyTimeoutMs = options.readyTimeoutMs ?? 60_000; // e2e-sleep-ok: injectable test seam; real callers poll a condition between sleeps. const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); const assertOurs = (name: string): void => { if (!name.startsWith(CONTAINER_PREFIX)) { throw new Error(`refusing to act on ${name}: not a ${CONTAINER_PREFIX} container`); } }; return { async createGuest(guest: GuestRecord): Promise { const net = parseNet(guest.config.net0 ?? ''); const gateway = net.gw; const cidr = net.ip; if (!gateway || !cidr) { throw new Error(`guest ${guest.vmid} has no gw/ip in net0: ${guest.config.net0}`); } const zone = zoneForGateway(gateway); if (!zone) { throw new Error(`no rig zone has gateway ${gateway} (guest ${guest.vmid})`); } const name = containerNameFor(guest.vmid); assertOurs(name); // Project membership must be discoverable without the name: teardown // matches on this label because the guest's name carries no timestamp. // Added before the readiness wait, so even a guest that never finishes // booting is still findable and removable. const projectLabel = ['--label', `${GUEST_PROJECT_LABEL}=${options.project}`] as const; // Same image selection rule the compose generator uses for machines: the // app zone needs a dockerd-capable box. const image = zone === 'app' ? 'celilo-e2e/target-machine-docker' : 'celilo-e2e/target-machine'; docker([ 'run', '-d', '--name', name, ...projectLabel, '--hostname', guest.hostname, '--network', `${options.project}_${zone}`, '--ip', cidr.split('/')[0] ?? '', '--privileged', '--tmpfs', '/run', '--tmpfs', '/run/lock', '--tmpfs', '/tmp', '-v', `${options.project}_ssh-keys:/ssh-keys:ro`, '-e', `GATEWAY=${gateway}`, image, ]); await this.waitUntilReady(name); // Proxmox really does honour `ssh_public_keys`, and a module may pass a // key that is not the fleet key in the mounted volume. Appended AFTER // target-setup has run, because that script *copies* authorized_keys // over and would otherwise clobber this. const key = guest.config.ssh_public_keys?.trim(); if (key) { docker([ 'exec', name, 'bash', '-c', `mkdir -p /root/.ssh && printf '%s\\n' ${JSON.stringify(key)} >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys`, ]); } }, /** * Block until the container's own boot contract says it is ready. * * Reporting the create task OK before this would let Ansible race sshd, * which surfaces as an intermittent connection failure several steps later. */ async waitUntilReady(name: string): Promise { const deadline = Date.now() + readyTimeoutMs; while (Date.now() < deadline) { try { const out = docker(['exec', name, 'systemctl', 'is-active', 'target-setup']); if (out.trim() === 'active') return; } catch { // Still booting: `docker exec` fails until systemd is up. } await sleep(500); } throw new Error(`${name} target-setup did not go active within ${readyTimeoutMs}ms`); }, destroyGuest(guest: GuestRecord): void { const name = containerNameFor(guest.vmid); assertOurs(name); docker(['rm', '-f', name]); }, }; }