import { execFile, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { link, lstat, mkdir, mkdtemp, readdir, readFile, readlink, rename, rm, symlink, writeFile, } from "node:fs/promises"; import { arch, homedir, hostname, platform, release, tmpdir } from "node:os"; import { delimiter, dirname, join, resolve } from "node:path"; import { extractPreparedArchive, fetchEnvironmentBytes, MAX_PREPARED_ARCHIVE_BYTES, sha256, } from "./environment-archive"; import type { EnvironmentRevision, EnvironmentSourceBundle, } from "./environment-contract"; import { environmentPrefix, MAX_ENVIRONMENT_SOURCE_BYTES, parseEnvironmentManifest, validateEnvironmentPath, } from "./environment-contract"; import { materializeSource, validateSourceBundle } from "./environment-source"; export type ResolvedEnvironment = EnvironmentRevision & { organizationId: string; sourceUrl: string; sourcePrefix: string; bundleUrl?: string | null; credentialEnv: Record< string, { value: string; secretName: string; version: number } >; credentialFiles: { path: string; value: string; secretName: string; version: number; mutable?: boolean; }[]; state?: { key: string; version: number; url?: string | null; sha256?: string; prefix?: string; }; }; export type PreparedEnvironment = { prefix: string; workspace: string; cacheHit: boolean; preparation: "snapshot" | "bundle" | "local"; platform: string; }; const DEPENDENCY_DIRECTORIES = new Set(["node_modules", ".venv", "venv"]); const MAX_CAPTURE_BYTES = 256 * 1024; export function currentEnvironmentPlatform(): string { return `${platform()}/${arch() === "x64" ? "amd64" : arch()}`; } export function environmentLocalCachePath( environment: ResolvedEnvironment, root = join(homedir(), ".cache", "dench", "environments"), runtimeAbi = `node${process.versions.node.split(".")[0]}`, ): string { return join( root, sha256(environment.organizationId).slice(0, 24), environment.revision, `${currentEnvironmentPlatform().replace("/", "-")}-${runtimeAbi}`, ); } async function localRuntimeAbi(): Promise { const python = await new Promise((done) => { execFile( "python3", [ "-I", "-c", "import sys,sysconfig; print(sys.version); print(sys.executable); print(sysconfig.get_platform())", ], { timeout: 3000, env: { PATH: process.env.PATH, HOME: tmpdir(), NODE_ENV: "production" }, maxBuffer: 8192, }, (error, stdout) => done(error ? "unavailable" : stdout.trim()), ); }); return `local-${sha256(JSON.stringify({ node: process.versions.modules, platform: currentEnvironmentPlatform(), os: release(), python })).slice(0, 24)}`; } async function exists(path: string) { try { await lstat(path); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } } /** A job-local filesystem lock, only for mutable setup state and credential files. */ export async function acquireEnvironmentWorkspaceLock( workspace: string, purpose: "execution" | "seed" | "prepare" = "execution", ): Promise<() => Promise> { const lock = `${resolve(workspace)}.dench-environment-${purpose}-lock`; await mkdir(dirname(lock), { recursive: true, mode: 0o700 }); const owner = { pid: process.pid, host: hostname(), token: randomUUID() }; const candidate = `${lock}.${owner.token}`; await writeFile(candidate, JSON.stringify(owner), { flag: "wx", mode: 0o600, }); try { for (;;) { try { // Publish a fully written owner record atomically; crashes cannot leave an // empty lock with no process identity to reconcile. await link(candidate, lock); return async () => { try { const current = JSON.parse(await readFile(lock, "utf8")); if (current.token === owner.token) await rm(lock, { force: true }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } }; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; // PID liveness, never wall-clock age: a legitimate command may run 12h+. try { const before = await lstat(lock); if (!before.isFile() || before.isSymbolicLink()) throw new Error( "environment_workspace_lock_invalid: Inspect the workspace lock before continuing.", ); const prior = JSON.parse(await readFile(lock, "utf8")); if ( prior.host === owner.host && Number.isSafeInteger(prior.pid) && prior.pid > 0 ) { try { process.kill(prior.pid, 0); } catch (failure) { if ( (failure as NodeJS.ErrnoException).code === "ESRCH" && (await lstat(lock)).ino === before.ino ) await rm(lock, { force: true }); } } } catch (failure) { if ((failure as NodeJS.ErrnoException).code === "ENOENT") continue; throw new Error( "environment_workspace_lock_invalid: Inspect the workspace lock before continuing.", ); } await new Promise((done) => setTimeout(done, 250)); } } } finally { await rm(candidate, { force: true }); } } async function seedPreparedEntry( source: string, target: string, ): Promise { const sourceStat = await lstat(source); let targetStat: Awaited> | undefined; try { targetStat = await lstat(target); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } const conflict = () => { throw new Error( "environment_workspace_conflict: Saved setup conflicts with an existing workspace file; existing files were preserved.", ); }; if (sourceStat.isDirectory()) { if ( targetStat && (!targetStat.isDirectory() || targetStat.isSymbolicLink()) ) conflict(); await mkdir(target, { recursive: true, mode: sourceStat.mode & 0o777 }); for (const entry of await readdir(source)) await seedPreparedEntry(join(source, entry), join(target, entry)); } else if (sourceStat.isSymbolicLink()) { const destination = await readlink(source); if (targetStat) { if ( !targetStat.isSymbolicLink() || (await readlink(target)) !== destination ) conflict(); } else await symlink(destination, target); } else if (sourceStat.isFile()) { const data = await readFile(source); if (targetStat) { if ( !targetStat.isFile() || sha256(await readFile(target)) !== sha256(data) ) conflict(); return; } const temporary = `${target}.dench-seed-${randomUUID()}`; try { await writeFile(temporary, data, { mode: sourceStat.mode & 0o777, flag: "wx", }); await link(temporary, target); } finally { await rm(temporary, { force: true }); } } else conflict(); } async function sourceFor( environment: ResolvedEnvironment, ): Promise { const bytes = await fetchEnvironmentBytes( environment.sourceUrl, MAX_ENVIRONMENT_SOURCE_BYTES + 512, ); const prefix = Buffer.from(environment.sourcePrefix, "utf8"); if (!bytes.subarray(0, prefix.length).equals(prefix)) throw new Error( "invalid_environment_artifact: Source ownership prefix does not match.", ); const data = bytes.subarray(prefix.length); if (sha256(data) !== environment.source.sha256) throw new Error( "invalid_environment_artifact: Source checksum does not match.", ); return validateSourceBundle(JSON.parse(data.toString("utf8"))); } async function readyAt( prefix: string, environment: ResolvedEnvironment, currentPlatform: string, runtimeAbi: string, ): Promise { try { const marker = JSON.parse( await readFile(join(prefix, ".dench-ready.json"), "utf8"), ); return ( marker.revision === environment.revision && marker.platform === currentPlatform && marker.runtimeAbi === runtimeAbi ); } catch { return false; } } function publicBuildEnvironment( directory: string, variables: Record, ): NodeJS.ProcessEnv { const clean: NodeJS.ProcessEnv = { NODE_ENV: process.env.NODE_ENV ?? "production", }; for (const name of ["PATH", "LANG", "LC_ALL", "TERM", "TMPDIR", "SystemRoot"]) if (process.env[name]) clean[name] = process.env[name]; return { ...clean, ...variables, HOME: directory, XDG_CONFIG_HOME: join(directory, ".config"), }; } export async function prepareEnvironment( environment: ResolvedEnvironment, opts: { workspace: string; prepareLocal?: boolean; cacheRoot?: string; report?: (message: string) => void; }, ): Promise { if (environment.status !== "ready" || !environment.artifacts) throw new Error( "environment_not_ready: Use dench env show to inspect the preparation receipt.", ); environment.manifest = parseEnvironmentManifest(environment.manifest); if ( !environment.organizationId || environment.artifacts.prefix !== environmentPrefix(environment.revision) ) throw new Error( "invalid_environment_artifact: Environment ownership or prefix does not match.", ); const targetPlatform = currentEnvironmentPlatform(); let installedAbi: string | undefined; try { installedAbi = (await readFile("/opt/dench/runtime-abi", "utf8")).trim(); } catch { /* Native clients require their own explicit preparation. */ } const matchesArtifact = targetPlatform === environment.artifacts.platform && installedAbi === environment.artifacts.runtimeAbi; const useLocal = Boolean(opts.prepareLocal) || !matchesArtifact; const localAbi = useLocal ? await localRuntimeAbi() : ""; const localPrefix = environmentLocalCachePath( environment, opts.cacheRoot, localAbi, ); const localReady = useLocal && (await readyAt(localPrefix, environment, targetPlatform, localAbi)); if (useLocal && !opts.prepareLocal && !localReady) throw new Error( `environment_platform_mismatch: This revision targets ${environment.artifacts.platform}. Run once with --prepare-local to build a separate ${targetPlatform} cache.`, ); const prefix = useLocal ? localPrefix : environment.artifacts.prefix; const runtimeAbi = useLocal ? localAbi : environment.artifacts.runtimeAbi; let cacheHit = await readyAt(prefix, environment, targetPlatform, runtimeAbi); let preparation: PreparedEnvironment["preparation"] = useLocal ? "local" : "snapshot"; if (!cacheHit) { const releasePreparation = await acquireEnvironmentWorkspaceLock( prefix, "prepare", ); try { if (await readyAt(prefix, environment, targetPlatform, runtimeAbi)) cacheHit = true; else if (!useLocal) { if ( !environment.manifest.portable || !environment.bundleUrl || !environment.artifacts.bundleSha256 ) throw new Error( "environment_requires_new_sandbox: Start a new job bound to this environment; it cannot be activated in the current sandbox.", ); opts.report?.("Preparing saved environment from its verified bundle."); const bytes = await fetchEnvironmentBytes( environment.bundleUrl, MAX_PREPARED_ARCHIVE_BYTES, environment.artifacts.bundleSha256, ); if ( environment.artifacts.bundleBytes !== undefined && bytes.length !== environment.artifacts.bundleBytes ) throw new Error( "invalid_environment_artifact: Prepared bundle size does not match.", ); const stage = `${prefix}.extract-${randomUUID()}`; try { await extractPreparedArchive(bytes, stage); if (!(await readyAt(stage, environment, targetPlatform, runtimeAbi))) throw new Error( "invalid_environment_artifact: Prepared bundle is missing its verified readiness marker.", ); if (await exists(prefix)) throw new Error( "environment_cache_incomplete: An incomplete environment already occupies the fixed prefix; start a new sandbox.", ); await rename(stage, prefix); } finally { await rm(stage, { recursive: true, force: true }); } preparation = "bundle"; } else { opts.report?.( "Preparing this environment for the local platform once. Business execution starts after verification.", ); // Build at its final prefix because Python environments and native launchers can embed it. if (await exists(prefix)) await rm(prefix, { recursive: true, force: true }); await materializeSource(await sourceFor(environment), prefix); const buildHome = await mkdtemp(join(tmpdir(), "dench-env-build-")); try { const buildEnv = publicBuildEnvironment( buildHome, environment.manifest.env, ); for (const command of [ ...environment.manifest.prepare, ...environment.manifest.verify, ]) { const result = await runEnvironmentProcess( [ process.platform === "win32" ? "cmd.exe" : "/bin/sh", process.platform === "win32" ? "/c" : "-c", command, ], { cwd: prefix, env: buildEnv, secretValues: [], timeoutMs: 20 * 60_000, }, ); if (result.exitCode !== 0) throw new Error( `environment_preparation_failed: Local preparation or verification exited ${result.exitCode}. ${result.stderr.slice(-2000)}`, ); } await writeFile( join(prefix, ".dench-ready.json"), JSON.stringify({ revision: environment.revision, platform: targetPlatform, runtimeAbi: localAbi, }), { mode: 0o600 }, ); } finally { await rm(buildHome, { recursive: true, force: true }); } } } finally { await releasePreparation(); } } const workspace = resolve(opts.workspace); if ( workspace === prefix || workspace.startsWith(`${prefix}/`) || prefix.startsWith(`${workspace}/`) ) throw new Error( "validation_failed: Writable workspace must be separate from the prepared dependency prefix.", ); await mkdir(workspace, { recursive: true, mode: 0o700 }); const markerPath = join(workspace, ".dench-workspace.json"); const marker = { revision: environment.revision, organizationId: environment.organizationId, }; const releaseSeed = await acquireEnvironmentWorkspaceLock(workspace, "seed"); try { if (await exists(markerPath)) { const stored = JSON.parse(await readFile(markerPath, "utf8")); if ( stored.revision !== marker.revision || stored.organizationId !== marker.organizationId ) throw new Error( "environment_workspace_mismatch: This workspace belongs to another environment or organization.", ); } else { for (const item of await readdir(prefix)) { if (item === ".dench-ready.json") continue; const target = join(workspace, item); if (DEPENDENCY_DIRECTORIES.has(item)) { if (await exists(target)) { const stat = await lstat(target); if ( !stat.isSymbolicLink() || (await readlink(target)) !== join(prefix, item) ) throw new Error( "environment_workspace_conflict: A dependency path already exists; existing files were preserved.", ); } else await symlink(join(prefix, item), target, "dir"); } else await seedPreparedEntry(join(prefix, item), target); } await writeFile(markerPath, JSON.stringify(marker), { mode: 0o600, flag: "wx", }); } } finally { await releaseSeed(); } return { prefix, workspace, cacheHit, preparation, platform: targetPlatform }; } /** Keep persisted state and output guarded by the same known credential set. */ export function environmentSecretMatches(values: readonly string[]): string[] { const expanded = values.flatMap((value) => { const strings: string[] = [value]; // Credential files often wrap the actual token inside JSON. Match their // string values too, rather than only matching the complete file bytes. try { const visit = (item: unknown, depth: number): void => { if (depth > 12) return; if (typeof item === "string") { if (item.length >= 4) strings.push(item); } else if (Array.isArray(item)) item.forEach((child) => { visit(child, depth + 1); }); else if (item && typeof item === "object") Object.values(item).forEach((child) => { visit(child, depth + 1); }); }; visit(JSON.parse(value), 0); } catch { /* A plain env value is not JSON. */ } return strings; }); return [ ...new Set( expanded .filter(Boolean) .flatMap((value) => [ value, encodeURIComponent(value), Buffer.from(value).toString("base64"), ]), ), ].sort((a, b) => b.length - a.length); } /** A bounded-delay redactor preserves matches split across stdout/stderr chunks. */ export class EnvironmentOutputRedactor { private pending = ""; private readonly values: string[]; private readonly overlap: number; constructor(values: string[]) { this.values = environmentSecretMatches(values); this.overlap = Math.max(1, ...this.values.map((value) => value.length)); } push(chunk: string, final = false): string { this.pending += chunk; let boundary = final ? this.pending.length : Math.max(0, this.pending.length - this.overlap); // Do not cut through a complete match straddling the flush boundary. for (const value of this.values) { for ( let at = this.pending.indexOf(value); at >= 0; at = this.pending.indexOf(value, at + value.length) ) if (at < boundary && at + value.length > boundary) boundary = at; } let output = this.pending.slice(0, boundary); this.pending = this.pending.slice(boundary); for (const value of this.values) output = output.split(value).join("[REDACTED]"); return output; } } export async function runEnvironmentProcess( command: string[], opts: { cwd: string; env: NodeJS.ProcessEnv; secretValues: string[]; timeoutMs?: number; stdout?: (text: string) => void; stderr?: (text: string) => void; }, ): Promise<{ exitCode: number; signal: string | null; stdout: string; stderr: string; outputTruncated: boolean; }> { if (!command.length || command.some((part) => part.includes("\0"))) throw new Error( "validation_failed: A nonempty command without NUL is required.", ); if ( opts.secretValues.some( (value) => value && command.some((part) => part.includes(value)), ) ) throw new Error( "validation_failed: Credentials must be passed through bindings, never command arguments.", ); const child = spawn(command[0], command.slice(1), { cwd: opts.cwd, env: opts.env, stdio: ["inherit", "pipe", "pipe"], shell: false, detached: process.platform !== "win32", }); const stdout = new EnvironmentOutputRedactor(opts.secretValues); const stderr = new EnvironmentOutputRedactor(opts.secretValues); let out = ""; let err = ""; let outputTruncated = false; const capture = (text: string, kind: "stdout" | "stderr") => { opts[kind]?.(text); const current = kind === "stdout" ? out : err; const remaining = MAX_CAPTURE_BYTES - Buffer.byteLength(current, "utf8"); const data = Buffer.from(text, "utf8"); const next = current + data.subarray(0, Math.max(0, remaining)).toString("utf8"); if (data.length > remaining) outputTruncated = true; if (kind === "stdout") out = next; else err = next; }; child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); child.stdout.on("data", (data: string) => capture(stdout.push(data), "stdout"), ); child.stderr.on("data", (data: string) => capture(stderr.push(data), "stderr"), ); let canceled = false; let escalation: ReturnType | undefined; const killTree = (signal: NodeJS.Signals) => { try { if (process.platform !== "win32" && child.pid) process.kill(-child.pid, signal); else child.kill(signal); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") child.kill(signal); } }; const terminate = () => { canceled = true; killTree("SIGTERM"); escalation ??= setTimeout(() => killTree("SIGKILL"), 5000); }; process.once("SIGINT", terminate); process.once("SIGTERM", terminate); const timeout = opts.timeoutMs ? setTimeout(terminate, opts.timeoutMs) : undefined; try { const result = await new Promise<{ exitCode: number; signal: string | null; }>((done, reject) => { child.once("error", () => reject( new Error( "environment_command_failed: Could not start the requested executable.", ), ), ); child.once("close", (code, signal) => done({ exitCode: canceled ? 130 : (code ?? 128), signal: canceled ? "SIGTERM" : signal, }), ); }); capture(stdout.push("", true), "stdout"); capture(stderr.push("", true), "stderr"); return { ...result, stdout: out, stderr: err, outputTruncated }; } finally { if (timeout) clearTimeout(timeout); if (escalation) clearTimeout(escalation); process.removeListener("SIGINT", terminate); process.removeListener("SIGTERM", terminate); } } /** Create a private binding destination without ever walking an existing symlink. */ export async function environmentPrivatePath( workspace: string, relative: string, ): Promise { validateEnvironmentPath(relative); let at = workspace; const parts = relative.split("/"); for (let i = 0; i < parts.length; i++) { at = join(at, parts[i]); try { const stat = await lstat(at); if ( stat.isSymbolicLink() || (i < parts.length - 1 && !stat.isDirectory()) ) throw new Error( "environment_binding_unsafe: A credential/state path traverses a link or file.", ); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; if (i < parts.length - 1) await mkdir(at, { mode: 0o700 }); } } return at; } export function activatedEnvironment( prepared: PreparedEnvironment, environment: ResolvedEnvironment, context?: { sessionToken?: string; convexUrl?: string; apiUrl?: string }, ): NodeJS.ProcessEnv { const values = Object.fromEntries( Object.entries(environment.credentialEnv).map(([name, entry]) => [ name, entry.value, ]), ); const inherited: NodeJS.ProcessEnv = { NODE_ENV: process.env.NODE_ENV ?? "production", }; for (const name of [ "PATH", "LANG", "LC_ALL", "TERM", "TMPDIR", "SystemRoot", "DENCH_HARNESS_A_JOB_ID", "DENCH_HARNESS_JOB_ID", "DENCH_RUN_ID", "DENCH_CHAT_ID", "DENCH_CHAT_ARTIFACTS", "DENCH_ORG_ID", "DENCH_ORG_SLUG", "DENCH_API_URL", "DENCH_HOST", "DENCH_GATEWAY_URL", "CONVEX_URL", "NEXT_PUBLIC_CONVEX_URL", "DENCH_ENVIRONMENT_WORKSPACE", ]) if (process.env[name]) inherited[name] = process.env[name]; // Only the currently authenticated Dench session is forwarded, never another // provider's ambient credentials or the native user's home/config directory. const session = context?.sessionToken ?? process.env.DENCH_AGENT_SESSION_TOKEN ?? process.env.DENCH_API_KEY; return { ...inherited, ...environment.manifest.env, ...values, HOME: prepared.workspace, XDG_CONFIG_HOME: join(prepared.workspace, ".config"), DENCH_CONFIG_DIR: join(prepared.workspace, ".dench"), ...(session ? { DENCH_AGENT_SESSION_TOKEN: session } : {}), ...(context?.convexUrl ? { CONVEX_URL: context.convexUrl } : {}), ...(context?.apiUrl ? { DENCH_API_URL: context.apiUrl, DENCH_HOST: context.apiUrl } : {}), DENCH_ENVIRONMENT_PREFIX: prepared.prefix, DENCH_ENVIRONMENT_ROOT: prepared.workspace, DENCH_ENVIRONMENT_REVISION: environment.revision, PATH: [ join(prepared.workspace, "node_modules", ".bin"), join(prepared.prefix, ".venv", "bin"), join(prepared.prefix, "venv", "bin"), process.env.PATH ?? "", ].join(delimiter), }; }