/* Maintains packages/os/site-kernel-handoff/src/leitstand/leitstand-commands.ts as an authored site-kernel-handoff authored module so agents can evolve it without rediscovering local boundaries. Do not introduce app-specific runtime composition or deployment behavior into this reusable package source file. RFC-1190: restore mission context + evidence archival in release-mode deploys RFC-1190: precedence-ordered resolveMissionContext (--mission → release build-identity → currentMission), MISSION_CONTEXT_UNRESOLVED/MISMATCH blocking, fail-closed EVIDENCE_SYNC_FAILED with --skip-evidence-sync as sole escape hatch, journaled evidence-sync-* ship steps after each certify-*, evidence.sync --site/--system flag. 2026-10-04 handoff fixes: shared 25min mission.check timeout (inner 5min race + 300s orchestrator raced cold ~7-11min captures), sweep excludeOpIds so mission.close no longer self-abandons the live ship journal (broke --resume), name-keyed journal resume state so closed-mission plan shifts can't skip unexecuted steps, wrangler dev documented as the required --base-url preview server. RFC-1249: wire werkstatt.commands.validate into packages.check + retire CMD-OUTPUT debt RFC-1284: fix review REVIEW-CODE-2026-10-15-02: RFC-1284 — single-store marker, flag=false proceed, surfaced reconcile F1: marker store unified on the git-tracked cache clone — access commands, certify entry check, and deploy reconcile all use non-smart readSystemState/writeSystemState (decisions Q-6); the mission-window blindness and the DNA-106 false-resume are gone. F2: --auto-manage-pin=false + suspension at entry skips entry-resume and proceeds on the open channel (decisions Q-7) — RFC/AC-8 text amended. F3: accessReconcile surfaced in data + summary of dev-deploy/propagate. F4: CHANGE_SUMMARY items on all touched files. F5: sternsystem.validate declares ACCESS-01/02 + system-state.yaml reads. F6: isProtectedLiveTarget exported and covered (AC-12 probe repointed). F7: fresh-read ordering proven — suspension opened mid-deploy suppresses the re-assert. RFC-1297: fetchChannelBuildIdentity moved to sternsystem/live-drift.ts (shared live-probe); rollback effect record uses gateForChannel; succeeded site rollbacks call recordSiteRollback (lastPropagated + deployment-rolled-back Bordbuch). RFC-1325: review r1 — pin-carrying probes, AC-3 write ordering, dns-aid validate parity, hardened host lint REVIEW-CODE-2026, RFC-0358, RFC-0379, RFC-0587, RFC-0608, RFC-0624, RFC-0627, RFC-0628, RFC-0629, RFC-0634, RFC-0649, RFC-0652, RFC-0653, RFC-0656, RFC-0657, RFC-0665, RFC-0668, RFC-0689, RFC-0697, RFC-0698, RFC-0700, RFC-0747, RFC-0829, RFC-0842, RFC-0866, RFC-0927, RFC-0930, RFC-0948, RFC-1097, RFC-1126, RFC-1136, RFC-1138, RFC-1149, RFC-1188, RFC-1190, RFC-1203 */ import path, { join } from "node:path"; import type { KernelCommandInput, KernelCommandResult, KernelRuntimeContext, } from "@warpgogol/werkstatt-engine/kernel"; import type { HealthCheck, DeploymentStaticConfig, DeploymentChannel, LastPropagatedChannel, PurgeResult, } from "@warpgogol/werkstatt-engine/schemas"; import { generateOperationId } from "../werkstatt/index.ts"; import { readSystemConfigSmart, readSystemStateSmart, resolveCacheClonePath, } from "../sternsystem/registry-io.ts"; import type { DeploymentAdapter, DeploymentLimits } from "./adapter.ts"; import { resolveAdapter, readBehaviorSnapshot, sourceDotenv, filterEnv } from "./adapters/index.ts"; import { collectPurgeUrls, purgeCacheByUrls, skippedPurgeResult, verifyCloudflareToken, BUILD_IDENTITY_PATH, } from "./cache-purge.ts"; import { recordSiteRollback } from "./rollback-state.ts"; import { artifactStorePreflight } from "../artifact-store/index.ts"; import { resolveOutboundAgentClient } from "./wba-signer.ts"; import { verifyBuildIdentitySignature } from "../integrity/signing.ts"; import { executeKernelCommand, KERNEL_UNIVERSAL_FLAGS } from "@warpgogol/werkstatt-engine/kernel"; import { getOrBuildWorkspaceRegistry } from "../kernel/runtime/registry-cache.ts"; import type { SmokeRunResult } from "@warpgogol/werkstatt-engine/testing/smoke"; import type { SiteE2eRunResult } from "@warpgogol/werkstatt-engine/testing/e2e"; import { authorizeAndDeploy, verifyDurableSync, authorizeMainPromotion, buildEffectRecord, writeDeploymentEffectRecord, makeR2ConfigFromEnv, resolveArtifactHash, resolveGateDecisionPath, flagSite, gateForChannel, } from "./deploy-helpers.ts"; import { buildAuthHeader } from "./deploy-execution.ts"; import { fetchChannelBuildIdentity } from "../sternsystem/live-drift.ts"; import type { Sha256Digest } from "@warpgogol/werkstatt-shared/fingerprint"; import { createDefaultIO, type WorkspaceIO } from "@warpgogol/werkstatt-shared/kernel/workspace-io"; import { failResult } from "@warpgogol/werkstatt-shared/checks/result-helpers"; function flagString(input: KernelCommandInput, key: string): string | undefined { const v = input.flags[key]; return typeof v === "string" ? v : undefined; } type Channel = "dev" | "alt" | "main"; function parseChannel(value: string | undefined, defaultValue: Channel): Channel { if (value === "dev" || value === "alt" || value === "main") return value; return defaultValue; } function _getChannelConfig(dep: DeploymentStaticConfig, channel: Channel): DeploymentChannel { const channelConfig = channel === "dev" ? dep.channels.dev : channel === "alt" ? dep.channels.alt : dep.channels.main; if (!channelConfig) { throw new Error(`[leitstand] channel '${channel}' is not defined for system`); } return channelConfig; } async function resolveConventionSecretsPath( basePath: string, io: WorkspaceIO = createDefaultIO().io, ): Promise { const filePath = path.join(basePath, ".env"); return (await io.exists(filePath)) ? filePath : undefined; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } // RFC-0668: Timeout and retry resilience for mission.check in leitstand.dev-deploy. const MISSION_CHECK_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes, per-attempt const MISSION_CHECK_MAX_RETRIES = 1; class MissionCheckTimeoutError extends Error { constructor(ms: number) { super(`mission.check timed out after ${ms}ms`); this.name = "MissionCheckTimeoutError"; } } async function withMissionCheckTimeout(promise: Promise, ms: number): Promise { return Promise.race([ promise, new Promise((_, reject) => setTimeout(() => reject(new MissionCheckTimeoutError(ms)), ms), ), ]); } // RFC-0668: Wrap mission.check with per-attempt timeout and one-time retry on // infrastructure errors (exit code 2 or any non-0/non-1). Content violations // (exit 1) are not retried — they indicate real site issues. export async function runMissionCheckWithResilience( workspaceRoot: string, missionId: string, channelUrl: string, commitSha: string, logger: { info: (m: string) => void; warn: (m: string) => void }, noReport = true, ): Promise<{ exitCode: number; data?: Record }> { const { executeKernelCommand } = await import("@warpgogol/werkstatt-engine/kernel"); for (let attempt = 0; attempt <= MISSION_CHECK_MAX_RETRIES; attempt++) { try { const result = (await withMissionCheckTimeout( executeKernelCommand({ workspaceRoot, commandName: "mission.check", argv: [ `--mission=${missionId}`, "--external-preview", `--base-url=${channelUrl}`, `--commit-sha=${commitSha}`, `--max-duration=${MISSION_CHECK_TIMEOUT_MS}`, "--channel=dev", ...(noReport ? ["--no-report"] : []), ], }), MISSION_CHECK_TIMEOUT_MS, )) as { exitCode?: number; data?: Record }; const exitCode = result.exitCode ?? 0; // Exit 0 = pass, exit 1 = content violations — return to caller, no retry if (exitCode === 0 || exitCode === 1) { return { exitCode, data: result.data }; } // Exit 2 = infrastructure error, 3+ = unexpected, 137 = signal kill — retry once if (attempt < MISSION_CHECK_MAX_RETRIES) { logger.info( `[leitstand.dev-deploy] mission.check infrastructure error (exit ${exitCode}, attempt ${attempt + 1}/${MISSION_CHECK_MAX_RETRIES + 1}), retrying...`, ); continue; } // Retry exhausted return { exitCode, data: result.data }; } catch (err) { if (err instanceof MissionCheckTimeoutError) { // Timeout is not retryable — a hung process indicates a deeper issue throw err; } // Unexpected throw from executeKernelCommand — treat as infrastructure error, retry once if (attempt < MISSION_CHECK_MAX_RETRIES) { logger.info( `[leitstand.dev-deploy] mission.check threw (attempt ${attempt + 1}/${MISSION_CHECK_MAX_RETRIES + 1}), retrying...`, ); continue; } throw err; } } throw new Error("mission.check failed after retry"); } // RFC-0649 / RFC-0657: Result of CDN freshness verification. export interface FreshnessResult { verified: boolean; cdnDistTreeHash: string | null; localDistTreeHash: string; attempts: number; error?: string; } // RFC-0657: Retry constants for verifyFreshness (hardcoded — no caller customization needed). const FRESHNESS_MAX_ATTEMPTS = 5; const FRESHNESS_BACKOFF_DELAYS_MS = [3_000, 6_000, 12_000, 24_000]; // Shared health check retry constants (RFC-0747, RFC-0846). // Used by leitstand.dev-deploy (dev health) and leitstand.promote (alt health). const _HEALTH_CHECK_MAX_ATTEMPTS = 3; const _HEALTH_CHECK_BACKOFF_DELAYS_MS = [3_000, 6_000]; // RFC-0649 / RFC-0657: Verify CDN freshness by fetching build-identity.json from the CDN URL // and comparing distTreeHash against the local build-identity. Retries up to 5 times with // exponential backoff (3s, 6s, 12s, 24s). First attempt is immediate; subsequent attempts // are separated by the backoff delays. export async function verifyFreshness( deploymentUrl: string, localDistTreeHash: string, logger: { info: (m: string) => void }, authHeaders: Record = {}, fetchImpl: typeof fetch = fetch, ): Promise { const base = deploymentUrl.replace(/\/$/, ""); const url = `${base}${BUILD_IDENTITY_PATH}`; let lastCdnDistTreeHash: string | null = null; let lastError = ""; for (let attempt = 1; attempt <= FRESHNESS_MAX_ATTEMPTS; attempt++) { if (attempt > 1) { const delayMs = FRESHNESS_BACKOFF_DELAYS_MS[attempt - 2]; logger.info( `[leitstand.dev-deploy] freshness retry ${attempt}/${FRESHNESS_MAX_ATTEMPTS} after ${delayMs / 1000}s...`, ); await sleep(delayMs); } try { const response = await fetchImpl(url, { headers: authHeaders }); if (!response.ok) { lastCdnDistTreeHash = null; lastError = `CDN freshness fetch returned HTTP ${response.status} for ${url}`; continue; } const cdnBuildIdentity = (await response.json()) as { distTreeHash?: string }; const cdnDistTreeHash = cdnBuildIdentity.distTreeHash ?? null; lastCdnDistTreeHash = cdnDistTreeHash; if (cdnDistTreeHash !== localDistTreeHash) { lastError = `CDN serving stale content: distTreeHash mismatch (cdn: ${cdnDistTreeHash}, local: ${localDistTreeHash})`; continue; } return { verified: true, cdnDistTreeHash, localDistTreeHash, attempts: attempt, }; } catch (err) { lastCdnDistTreeHash = null; lastError = `CDN freshness fetch failed: ${err instanceof Error ? err.message : String(err)}`; } } return { verified: false, cdnDistTreeHash: lastCdnDistTreeHash, localDistTreeHash, attempts: FRESHNESS_MAX_ATTEMPTS, error: lastError, }; } // RFC-0624: Post-deploy CDN cache purge step. Non-blocking on failure. async function _runPurgeStep( workspaceRoot: string, releaseId: string, deploymentUrl: string, secretsFilePath: string | undefined, logger: { info: (m: string) => void; success: (m: string) => void; warn: (m: string) => void }, ): Promise { const secretsEnv = secretsFilePath ? await sourceDotenv(secretsFilePath) : {}; const env = { ...filterEnv(process.env), ...secretsEnv }; const zoneId = env["CLOUDFLARE_ZONE_ID"]; const apiToken = env["CLOUDFLARE_API_TOKEN"]; if (!zoneId) { logger.warn("[leitstand] CLOUDFLARE_ZONE_ID not set — skipping CDN cache purge"); return skippedPurgeResult("CLOUDFLARE_ZONE_ID not set"); } if (!apiToken) { logger.warn("[leitstand] CLOUDFLARE_API_TOKEN not set — skipping CDN cache purge"); return skippedPurgeResult("CLOUDFLARE_API_TOKEN not set"); } const snapshot = await readBehaviorSnapshot(workspaceRoot, releaseId); const routes = snapshot?.routes ?? []; const urls = collectPurgeUrls(deploymentUrl, routes); logger.info(`[leitstand] Purging CDN cache for ${urls.length} URLs...`); const result = await purgeCacheByUrls(zoneId, apiToken, urls); if (result.success) { logger.success(`[leitstand] CDN cache purged (${result.purgedUrls} URLs)`); } else { logger.warn(`[leitstand] CDN cache purge failed: ${result.error}`); } return result; } async function _earlyCloudflareTokenCheck( secretsFilePath: string | undefined, logger: { info: (m: string) => void; warn: (m: string) => void }, ): Promise { const secretsEnv = secretsFilePath ? await sourceDotenv(secretsFilePath) : {}; const env = { ...filterEnv(process.env), ...secretsEnv }; const zoneId = env["CLOUDFLARE_ZONE_ID"]; const apiToken = env["CLOUDFLARE_API_TOKEN"]; if (!zoneId || !apiToken) { logger.warn( `[leitstand] ${!zoneId ? "CLOUDFLARE_ZONE_ID" : "CLOUDFLARE_API_TOKEN"} not set — CDN cache purge will be skipped and health checks may report unhealthy due to stale cache`, ); return; } const tokenCheck = await verifyCloudflareToken(zoneId, apiToken); if (!tokenCheck.valid) { logger.warn( `[leitstand] Cloudflare API token invalid: ${tokenCheck.error}. CDN cache purge will fail. Health checks will report unhealthy due to stale cache. Update CLOUDFLARE_API_TOKEN in .env before deploying.`, ); } else { logger.info("[leitstand] Cloudflare API token verified"); } } interface PreflightCheck { name: string; passed: boolean; detail: string; } async function _runPreflight( workspaceRoot: string, releaseId: string, dep: DeploymentStaticConfig, channel: Channel, channelConfig: DeploymentChannel, adapter: DeploymentAdapter, missionId: string | undefined, io: WorkspaceIO, basePath?: string, ): Promise { const checks: PreflightCheck[] = []; // 1. Release state is ready (already verified by caller, but record it) const manifestPath = path.join(workspaceRoot, "releases", releaseId, "release.yaml"); checks.push({ name: "release-ready", passed: await io.exists(manifestPath), detail: (await io.exists(manifestPath)) ? "Release manifest found" : "Release manifest missing", }); // 2. Channel exists in deployment.channels checks.push({ name: "channel-present", passed: !!channelConfig, detail: channelConfig ? `Channel '${channel}' configured` : `Channel '${channel}' not defined`, }); // 3. Convention env file existence (RFC-0761: single .env file) const envPath = basePath ? path.join(basePath, ".env") : ""; checks.push({ name: "convention-env-exists", passed: true, // info-level — always passed detail: basePath && (await io.exists(envPath)) ? `.env found at ${envPath}` : `.env not found — using process.env fallback`, }); // 4. Dist directory exists and artifact hash verifies const distPath = path.join(workspaceRoot, "releases", releaseId, "dist"); const artifactResult = await artifactStorePreflight(workspaceRoot, releaseId, distPath, io); checks.push({ name: "artifact-hash", passed: artifactResult.manifestFound && (artifactResult.hashVerified || !(await io.exists(distPath))), detail: !artifactResult.manifestFound ? "Artifact manifest not found in store" : !(await io.exists(distPath)) ? "Dist missing locally (will rehydrate from store)" : artifactResult.hashVerified ? "Dist tree hash matches artifact manifest" : "Dist tree hash mismatch — will rehydrate from store", }); // 5. Wrangler binary resolves let wranglerBinPath: string | undefined; if (missionId) { const workpieceBin = path.join( workspaceRoot, "missions", missionId, "workpiece", "node_modules", ".bin", ); if (await io.exists(workpieceBin)) { wranglerBinPath = workpieceBin; } } const wranglerCheck = await checkWranglerAvailable(workspaceRoot, wranglerBinPath, io); checks.push({ name: "wrangler-available", passed: wranglerCheck.available, detail: wranglerCheck.detail, }); // 6. Dist size within adapter-declared limits if (await io.exists(distPath)) { const sizeCheck = await checkDistSize(distPath, adapter.getLimits()); checks.push({ name: "dist-size-limit", passed: sizeCheck.withinLimit, detail: sizeCheck.detail, }); } // 7. Cloudflare API token valid (pre-flight check to avoid stale CDN after deploy) const secretsFilePath = basePath ? await resolveConventionSecretsPath(basePath) : undefined; const secretsEnv = secretsFilePath ? await sourceDotenv(secretsFilePath) : {}; const env = { ...filterEnv(process.env), ...secretsEnv }; const zoneId = env["CLOUDFLARE_ZONE_ID"]; const apiToken = env["CLOUDFLARE_API_TOKEN"]; // 8. RFC-0961: Boot-smoke evidence file exists const bootSmokeEvidencePath = path.join(workspaceRoot, "releases", releaseId, "boot-smoke.json"); checks.push({ name: "boot-smoke-evidence", passed: await io.exists(bootSmokeEvidencePath), detail: (await io.exists(bootSmokeEvidencePath)) ? "boot-smoke.json evidence found" : "boot-smoke.json evidence missing — run release.prepare to generate", }); if (!zoneId || !apiToken) { checks.push({ name: "cloudflare-token", passed: true, // info-level — deploy can proceed without purge detail: !zoneId ? "CLOUDFLARE_ZONE_ID not set — CDN purge will be skipped" : "CLOUDFLARE_API_TOKEN not set — CDN purge will be skipped", }); } else { const tokenCheck = await verifyCloudflareToken(zoneId, apiToken); checks.push({ name: "cloudflare-token", passed: tokenCheck.valid, detail: tokenCheck.valid ? "Cloudflare API token valid" : `Cloudflare API token invalid: ${tokenCheck.error}`, }); } return checks; } async function checkWranglerAvailable( workspaceRoot: string, nodeModulesBinPath?: string, io: WorkspaceIO = createDefaultIO().io, ): Promise<{ available: boolean; detail: string }> { try { const env: Record = { ...process.env } as Record; if (nodeModulesBinPath) { env.PATH = `${nodeModulesBinPath}:${process.env.PATH ?? ""}`; } const result = await io.exec("npx", ["--yes", "wrangler", "--version"], { cwd: workspaceRoot, env, }); if (result.exitCode === 0) { return { available: true, detail: `wrangler resolved: ${result.stdout.trim()}` }; } return { available: false, detail: `wrangler --version exited non-zero: ${result.stderr.trim().slice(0, 100)}`, }; } catch { return { available: false, detail: "wrangler binary not found" }; } } async function checkDistSize( distPath: string, limits: DeploymentLimits, io: WorkspaceIO = createDefaultIO().io, ): Promise<{ withinLimit: boolean; detail: string }> { let totalSize = 0; let largestFile = 0; let largestFilePath = ""; const { collectFiles } = await import("@warpgogol/werkstatt-shared/node/fs"); for (const file of await collectFiles(distPath)) { const stat = await io.stat(file); totalSize += stat.size; if (stat.size > largestFile) { largestFile = stat.size; largestFilePath = file; } } const sizeWithinLimit = totalSize <= limits.maxTotalSize; const perFileWithinLimit = largestFile <= limits.maxFileSize; const withinLimit = sizeWithinLimit && perFileWithinLimit; const totalLimitMiB = limits.maxTotalSize / 1024 / 1024; const fileLimitMiB = limits.maxFileSize / 1024 / 1024; return { withinLimit, detail: withinLimit ? `Dist size ${(totalSize / 1024 / 1024).toFixed(2)} MiB within ${totalLimitMiB.toFixed(0)} MiB total limit` : !sizeWithinLimit ? `Dist size ${(totalSize / 1024 / 1024).toFixed(2)} MiB exceeds ${totalLimitMiB.toFixed(0)} MiB total limit` : `Largest file ${largestFilePath} (${(largestFile / 1024 / 1024).toFixed(2)} MiB) exceeds ${fileLimitMiB.toFixed(0)} MiB per-file limit`, }; } function _buildLastPropagatedEntry( releaseId: string, state: LastPropagatedChannel["state"], healthy: boolean, operationId: string, purgeResult?: PurgeResult, ): LastPropagatedChannel { return { releaseId, at: new Date().toISOString(), healthy, state, operationId, leaseExpiresAt: null, purgeResult, }; } // §5.0: leitstand.dev-deploy (RFC-0628: workpiece-based dev deploy with Axiom verification gate) interface PreparedDeployContext { adapter: DeploymentAdapter; systemConfig: DeploymentStaticConfig; secretsFilePath: string | undefined; } async function prepareDeployContext( workspaceRoot: string, systemId: string, commandName: string, ): Promise { const systemConfig = await readSystemConfigSmart(workspaceRoot, systemId); if (!systemConfig.deployment) { throw new Error( `[${commandName}] system '${systemId}' has no deployment config in system-config.yaml`, ); } const adapter = resolveAdapter(systemConfig.deployment.adapter); return { adapter, systemConfig: systemConfig.deployment, secretsFilePath: undefined, }; } function resolveSecretsFilePath( workspaceRoot: string, channelConfig: DeploymentChannel | undefined, ): string | undefined { return channelConfig?.secretsFile ? path.join(workspaceRoot, channelConfig.secretsFile) : undefined; } export interface MissionContext { missionId: string; commitSha: string; source: "flag" | "build-identity" | "current-mission" | "none"; } interface MissionContextOptions { missionFlag?: string; releaseId?: string; } async function readBuildIdentityContext( workspaceRoot: string, releaseId: string, io: WorkspaceIO, ): Promise<{ missionId: string; commitSha: string } | null> { try { const identityPath = path.join( workspaceRoot, "releases", releaseId, "dist", "client", ".well-known", "build-identity.json", ); const parsed = JSON.parse(await io.readFile(identityPath)) as { missionId?: unknown; commitSha?: unknown; }; const missionId = typeof parsed.missionId === "string" ? parsed.missionId : ""; if (!missionId) return null; return { missionId, commitSha: typeof parsed.commitSha === "string" ? parsed.commitSha : "", }; } catch { return null; } } async function workpieceHeadSha( workspaceRoot: string, missionId: string, io: WorkspaceIO, ): Promise { try { const { resolveMissionDir } = await import("../kernel/mission-resolver.ts"); const missionDir = await resolveMissionDir(workspaceRoot, missionId, io); const workpieceDir = path.join(missionDir, "workpiece"); if (!(await io.exists(workpieceDir))) return ""; const headResult = await io.exec("git", ["rev-parse", "HEAD"], { cwd: workpieceDir }); return headResult.exitCode === 0 ? headResult.stdout.trim() : ""; } catch { return ""; } } // RFC-1190: mission identity is an explicit, precedence-ordered input — // --mission flag → release build-identity.json → system-state.currentMission. // The previous single-source currentMission lookup resolved to "" on every // release-mode deploy because mission.close nulls the pointer before deploys // run, silently disabling the dev mission.check, the RFC-0628 alt evidence // gate, and evidence.sync on every ship since the TLD-removal migration. export async function resolveMissionContext( workspaceRoot: string, systemId: string, io: WorkspaceIO = createDefaultIO().io, opts: MissionContextOptions = {}, ): Promise { const buildIdentity = opts.releaseId ? await readBuildIdentityContext(workspaceRoot, opts.releaseId, io) : null; if (opts.missionFlag) { if (buildIdentity && buildIdentity.missionId !== opts.missionFlag) { throw new Error( `MISSION_CONTEXT_MISMATCH: --mission '${opts.missionFlag}' contradicts build-identity missionId '${buildIdentity.missionId}' of release ${opts.releaseId}`, ); } const commitSha = buildIdentity?.commitSha || (await workpieceHeadSha(workspaceRoot, opts.missionFlag, io)); return { missionId: opts.missionFlag, commitSha, source: "flag" }; } if (buildIdentity) { return { ...buildIdentity, source: "build-identity" }; } try { const state = await readSystemStateSmart(workspaceRoot, systemId); const missionId = state.currentMission ?? ""; if (!missionId) return { missionId: "", commitSha: "", source: "none" }; const commitSha = await workpieceHeadSha(workspaceRoot, missionId, io); return { missionId, commitSha, source: "current-mission" }; } catch { return { missionId: "", commitSha: "", source: "none" }; } } function flagBool(input: KernelCommandInput, key: string): boolean { const v = input.flags[key]; return v === true || v === "true"; } export interface DevDeployResult { command: "leitstand.dev-deploy"; systemId: string; missionId: string; commitSha: string; /** RFC-1190: resolved mission identity + which source provided it. */ missionContext: MissionContext; buildState: "succeeded" | "failed"; buildSkipped: boolean; deployState: "succeeded" | "failed" | "failed-stale" | "in-progress"; deploymentUrl: string; buildIdentity: { releaseId: string; written: boolean; path: string; }; axiom: { status: "pass" | "fail" | "not-run"; errors: number; warnings: number; exitCode: number; freshness: FreshnessResult; }; smokeResult?: SmokeRunResult; e2eResult?: SiteE2eRunResult; evidenceSynced: boolean; evidenceSyncError: string | null; releaseDeployed?: string; failingPhase?: string; channelDriftWarning?: string | null; } async function emitChannelDriftWarning( workspaceRoot: string, systemId: string, devReleaseId: string, ): Promise { try { const state = await readSystemStateSmart(workspaceRoot, systemId); const altRelease = state.lastPropagated?.alt?.releaseId; const mainRelease = state.lastPropagated?.main?.releaseId; const driftChannels: string[] = []; if (altRelease && altRelease !== devReleaseId) { driftChannels.push(`alt=${altRelease}`); } if (mainRelease && mainRelease !== devReleaseId) { driftChannels.push(`main=${mainRelease}`); } if (driftChannels.length === 0) return null; return `Channel drift: dev=${devReleaseId}, ${driftChannels.join(", ")} — run leitstand.propagate + leitstand.promote to align`; } catch { return null; } } export async function runLeitstandDevDeploy( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.dev-deploy] --site is required"); const releaseId = flagString(input, "release"); const gateDecisionFlag = flagString(input, "gate-decision"); const candidateId = flagString(input, "candidate-id") ?? systemId; const artifactHashFlag = flagString(input, "artifact-hash"); if (!releaseId) { throw new Error("[leitstand.dev-deploy] --release is required"); } // RFC-1190: mission context is an explicit input — --mission flag → // release build-identity.json → system-state.currentMission. A deploy // that cannot resolve it refuses rather than silently degrading the // mission-scoped gates (dev mission.check, alt evidence gate, evidence.sync); // --skip-evidence-sync is the sole explicit escape hatch. const skipEvidenceSync = flagBool(input, "skip-evidence-sync"); const missionCtx = await resolveMissionContext(context.workspaceRoot, systemId, context.io, { missionFlag: flagString(input, "mission"), releaseId, }); if (!missionCtx.missionId && !skipEvidenceSync) { return { data: { command: "leitstand.dev-deploy", systemId, missionId: "", commitSha: "", missionContext: missionCtx, buildState: "failed", buildSkipped: false, deployState: "failed", deploymentUrl: "", buildIdentity: { releaseId, written: false, path: "" }, axiom: { status: "not-run", errors: 0, warnings: 0, exitCode: 0, freshness: { verified: false, cdnDistTreeHash: null, localDistTreeHash: "", attempts: 0, }, }, evidenceSynced: false, evidenceSyncError: null, releaseDeployed: releaseId, }, summary: `[leitstand.dev-deploy] MISSION_CONTEXT_UNRESOLVED: no mission resolvable via --mission, releases/${releaseId} build-identity, or system-state.currentMission — pass --mission or --skip-evidence-sync`, exitCode: 1, diagnostics: [ { ruleId: "MISSION_CONTEXT_UNRESOLVED", severity: "error", message: `mission context unresolved for release ${releaseId} — mission-scoped gates cannot be evaluated; pass --mission or --skip-evidence-sync`, evidence: [], }, ], } as unknown as KernelCommandResult; } const cacheCloneDir = resolveCacheClonePath(context.workspaceRoot, systemId); const gateDecisionPath = resolveGateDecisionPath( cacheCloneDir, releaseId, "dev", gateDecisionFlag, ); const releaseDir = releaseId ? path.join(context.workspaceRoot, "releases", releaseId) : undefined; const { hash: artifactHash, source: artifactHashSource, path: artifactHashPath, } = await resolveArtifactHash(artifactHashFlag, releaseDir); context.logger.info( `[deploy] artifact hash resolved from ${artifactHashSource}${artifactHashPath ? ` (${artifactHashPath})` : ""}: ${artifactHash}`, ); const authResult = await authorizeAndDeploy({ gateDecisionPath, artifactHash, candidateId, gate: "dev-deploy", durableSyncVerified: false, artifactReadinessVerified: true, forceRequested: false, skipRequested: false, waiverRequested: false, graceRequested: false, }); if (!authResult.ok) { return { ...failResult("leitstand.commands", [ `[leitstand.dev-deploy] denied: ${authResult.outcome.ruleId} — ${authResult.outcome.message}`, ]), data: { systemId, channel: "dev", releaseId: releaseId ?? "", deploymentUrl: "", state: "failed", buildIdentityVerified: false, testEvidenceVerified: false, }, diagnostics: [ { ruleId: authResult.outcome.ruleId, severity: "error", message: authResult.outcome.message, evidence: [], }, ], } as unknown as KernelCommandResult; } const operationId = generateOperationId(); const now = new Date().toISOString(); const effectRecord = buildEffectRecord( operationId, candidateId, "dev-deploy", "dev", artifactHash, authResult.outcome.decisionId, false, null, "authorized", now, ); await writeDeploymentEffectRecord(cacheCloneDir, effectRecord); const { adapter, systemConfig: depConfig } = await prepareDeployContext( context.workspaceRoot, systemId, "leitstand.dev-deploy", ); const channelConfig = depConfig.channels.dev; const secretsFilePath = resolveSecretsFilePath(context.workspaceRoot, channelConfig); const { missionId, commitSha } = missionCtx; const { executeDeployPhases } = await import("./deploy-execution.ts"); const deployResult = await executeDeployPhases( { systemId, releaseId, candidateId, artifactHash, authResult, workspaceRoot: context.workspaceRoot, cacheCloneDir, systemConfig: depConfig, adapter, operationId, gateDecisionPath, secretsFilePath, skipEvidenceSync, forceBuild: Boolean(input.flags["force-build"]), missionId, commitSha, missionContext: missionCtx, }, "dev", ); const driftWarning = deployResult.failingPhase ? null : await emitChannelDriftWarning(context.workspaceRoot, systemId, releaseId); return { data: { command: "leitstand.dev-deploy", systemId, missionId, commitSha, missionContext: missionCtx, buildState: deployResult.failingPhase === "build" ? "failed" : "succeeded", buildSkipped: deployResult.buildSkipped, deployState: deployResult.failingPhase ? "failed" : "succeeded", deploymentUrl: deployResult.deploymentUrl, buildIdentity: deployResult.buildIdentity, axiom: { status: deployResult.axiomStatus, errors: 0, warnings: 0, exitCode: deployResult.axiomExitCode, freshness: deployResult.freshness, }, evidenceSynced: deployResult.evidenceSynced, evidenceSyncError: deployResult.evidenceSyncError, releaseDeployed: releaseId, failingPhase: deployResult.failingPhase, channelDriftWarning: driftWarning, accessReconcile: deployResult.accessReconcile, }, summary: deployResult.failingPhase ? `[leitstand.dev-deploy] failed at phase: ${deployResult.failingPhase}` : driftWarning ? `[leitstand.dev-deploy] deployed to dev: ${deployResult.deploymentUrl} — WARN: ${driftWarning}` : deployResult.accessReconcile && !deployResult.accessReconcile.ok ? `[leitstand.dev-deploy] deployed to dev: ${deployResult.deploymentUrl} — WARN: ACCESS_PIN reconcile failed: ${deployResult.accessReconcile.error}` : deployResult.accessReconcile?.pinIssued ? `[leitstand.dev-deploy] deployed to dev: ${deployResult.deploymentUrl} — ACCESS_PIN issued (shown once, record it): ${deployResult.accessReconcile.pinIssued}` : `[leitstand.dev-deploy] deployed to dev: ${deployResult.deploymentUrl}`, exitCode: deployResult.failingPhase ? 1 : 0, nextSteps: deployResult.failingPhase ? [ { action: `Fix the failing phase (${deployResult.failingPhase}), then re-run: pnpm exec werkstatt run leitstand.dev-deploy --site ${systemId} --release ${releaseId}`, kind: "required", }, ] : driftWarning ? [ { action: `Propagate to alt: pnpm exec werkstatt run leitstand.propagate --site ${systemId} --release ${releaseId}`, kind: "required", }, { action: `Promote to main: pnpm exec werkstatt run leitstand.promote --site ${systemId} --release ${releaseId}`, kind: "required", }, ] : [ { action: `Propagate to alt: pnpm exec werkstatt run leitstand.propagate --site ${systemId} --release ${releaseId}`, kind: "optional", }, ], } as unknown as KernelCommandResult; } // §5.1: leitstand.propagate (RFC-0628: requires published + Axiom evidence gate with commitSha+missionId match) export interface LeitstandPropagateData { systemId: string; releaseId: string; channel: "alt"; state: "succeeded" | "failed" | "failed-stale" | "in-progress"; deploymentUrl: string; startedAt: string; completedAt: string | null; preflight: { passed: boolean; checks: PreflightCheck[] }; purgeResult?: PurgeResult; health: { state: "healthy" | "unhealthy" | "unknown"; checks: HealthCheck[] }; smokeResult?: SmokeRunResult; releaseState: "alt-deployed"; devBuildIdentityVerified: boolean; axiomEvidenceVerified: boolean; testEvidenceVerified: boolean; failingPhase?: string; /** RFC-1190: resolved mission identity + which source provided it. */ missionContext?: MissionContext; } export async function runLeitstandPropagate( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const releaseId = flagString(input, "release"); if (!releaseId) throw new Error("[leitstand.propagate] --release is required"); const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.propagate] --site is required"); // RFC-1190: mission context resolution + blocking UNRESOLVED check — see // leitstand.dev-deploy for the precedence chain and escape hatch. const skipEvidenceSync = flagBool(input, "skip-evidence-sync"); const missionCtx = await resolveMissionContext(context.workspaceRoot, systemId, context.io, { missionFlag: flagString(input, "mission"), releaseId, }); if (!missionCtx.missionId && !skipEvidenceSync) { return { ...failResult("leitstand.commands", [ `[leitstand.propagate] MISSION_CONTEXT_UNRESOLVED: no mission resolvable via --mission, releases/${releaseId} build-identity, or system-state.currentMission — pass --mission or --skip-evidence-sync`, ]), data: { releaseId, systemId, channel: "alt", deploymentUrl: "", state: "failed", releaseState: "alt-deployed", devBuildIdentityVerified: false, axiomEvidenceVerified: false, testEvidenceVerified: false, missionContext: missionCtx, }, diagnostics: [ { ruleId: "MISSION_CONTEXT_UNRESOLVED", severity: "error", message: `mission context unresolved for release ${releaseId} — mission-scoped gates cannot be evaluated; pass --mission or --skip-evidence-sync`, evidence: [], }, ], } as unknown as KernelCommandResult; } const gateDecisionFlag = flagString(input, "gate-decision"); const candidateId = flagString(input, "candidate-id") ?? systemId; const artifactHashFlag = flagString(input, "artifact-hash"); const cacheCloneDir = resolveCacheClonePath(context.workspaceRoot, systemId); const gateDecisionPath = resolveGateDecisionPath( cacheCloneDir, releaseId, "alt", gateDecisionFlag, ); const releaseDir = path.join(context.workspaceRoot, "releases", releaseId); const { hash: artifactHash, source: artifactHashSource, path: artifactHashPath, } = await resolveArtifactHash(artifactHashFlag, releaseDir); context.logger.info( `[deploy] artifact hash resolved from ${artifactHashSource}${artifactHashPath ? ` (${artifactHashPath})` : ""}: ${artifactHash}`, ); const r2Config = makeR2ConfigFromEnv(process.env as Record); let durableSyncVerified = !r2Config; if (r2Config) { durableSyncVerified = await verifyDurableSync(artifactHash, r2Config); } const authResult = await authorizeAndDeploy({ gateDecisionPath, artifactHash, candidateId, gate: "propagate-alt", durableSyncVerified, artifactReadinessVerified: true, forceRequested: false, skipRequested: false, waiverRequested: false, graceRequested: false, }); if (!authResult.ok) { return { ...failResult("leitstand.commands", [ `[leitstand.propagate] denied: ${authResult.outcome.ruleId} — ${authResult.outcome.message}`, ]), data: { releaseId, systemId, channel: "alt", deploymentUrl: "", state: "failed", releaseState: "", devBuildIdentityVerified: false, }, diagnostics: [ { ruleId: authResult.outcome.ruleId, severity: "error", message: authResult.outcome.message, evidence: [], }, ], } as unknown as KernelCommandResult; } const operationId = generateOperationId(); const now = new Date().toISOString(); const effectRecord = buildEffectRecord( operationId, candidateId, "propagate-alt", "alt", artifactHash, authResult.outcome.decisionId, durableSyncVerified, null, "authorized", now, ); await writeDeploymentEffectRecord(cacheCloneDir, effectRecord); const { adapter, systemConfig: depConfig } = await prepareDeployContext( context.workspaceRoot, systemId, "leitstand.propagate", ); const channelConfig = depConfig.channels.alt; const secretsFilePath = resolveSecretsFilePath(context.workspaceRoot, channelConfig); const { missionId, commitSha } = missionCtx; const { executeDeployPhases } = await import("./deploy-execution.ts"); const deployResult = await executeDeployPhases( { systemId, releaseId, candidateId, artifactHash, authResult, workspaceRoot: context.workspaceRoot, cacheCloneDir, systemConfig: depConfig, adapter, operationId, gateDecisionPath, secretsFilePath, skipEvidenceSync, missionId, commitSha, missionContext: missionCtx, }, "alt", ); return { data: { releaseId, systemId, channel: "alt", deploymentUrl: deployResult.deploymentUrl, state: deployResult.failingPhase ? "failed" : "succeeded", startedAt: now, completedAt: new Date().toISOString(), preflight: { passed: !deployResult.failingPhase, checks: [] }, health: { state: deployResult.healthState, checks: deployResult.healthChecks, }, purgeResult: deployResult.purgeResult, releaseState: "alt-deployed", devBuildIdentityVerified: deployResult.freshness.verified, axiomEvidenceVerified: !deployResult.failingPhase, testEvidenceVerified: deployResult.evidenceSynced, failingPhase: deployResult.failingPhase, missionContext: missionCtx, accessReconcile: deployResult.accessReconcile, }, summary: deployResult.failingPhase ? `[leitstand.propagate] failed at phase: ${deployResult.failingPhase}${deployResult.errorMessage ? ` — ${deployResult.errorMessage}` : ""}` : deployResult.accessReconcile && !deployResult.accessReconcile.ok ? `[leitstand.propagate] deployed to alt: ${deployResult.deploymentUrl} — WARN: ACCESS_PIN reconcile failed: ${deployResult.accessReconcile.error}` : `[leitstand.propagate] deployed to alt: ${deployResult.deploymentUrl}`, exitCode: deployResult.failingPhase ? 1 : 0, nextSteps: deployResult.failingPhase ? [ { action: `Fix the failing phase (${deployResult.failingPhase}), then re-run: pnpm exec werkstatt run leitstand.propagate --site ${systemId} --release ${releaseId}`, kind: "required", }, ] : [ { action: `Certify the alt gate: pnpm exec werkstatt run leitstand.certify --site ${systemId} --release ${releaseId} --gate alt`, kind: "optional", }, { action: `Promote to main: pnpm exec werkstatt run leitstand.promote --site ${systemId} --release ${releaseId}`, kind: "optional", }, ], } as unknown as KernelCommandResult; } // §5.1b: leitstand.promote (RFC-0608: alt→main with build-identity verification) export interface LeitstandPromoteData { systemId: string; releaseId: string; channel: "main"; state: "succeeded" | "failed"; deploymentUrl: string; buildIdentityVerified: boolean; testEvidenceVerified: boolean; purgeResult?: PurgeResult; healthState: "healthy" | "unhealthy" | "unknown"; smokeResult?: SmokeRunResult; releaseState: "promoted"; failingPhase?: string; /** RFC-1190: resolved mission identity + which source provided it. */ missionContext?: MissionContext; /** RFC-1138: post-promote sternsystem.sync outcome (mirrors[2+] only) */ mirror?: { synced: boolean; syncError: string | null }; } /** * Mirror sync pre-flight check (RFC-0995). * * Non-blocking advisory check that compares the bare repo's origin SHA * against refs/mirror/ and logs a warning if they differ. * All errors are caught and swallowed — the check must never block promotion. */ export async function checkMirrorSyncPreFlight( workspaceRoot: string, systemId: string, logger: { warn: (msg: string) => void }, io: WorkspaceIO = createDefaultIO().io, ): Promise { try { const { readSystemConfig, resolveMirrorPath } = await import("../sternsystem/registry-io.ts"); const config = await readSystemConfig(workspaceRoot, systemId); if (config && config.mirrors.length > 2) { const bareMirror = config.mirrors[1]; const bareRepoPath = resolveMirrorPath(workspaceRoot, bareMirror.path); if (await io.exists(bareRepoPath)) { let branch = "main"; const symRef = await io.exec("git", ["symbolic-ref", "HEAD"], { cwd: bareRepoPath }); if (symRef.exitCode === 0) branch = symRef.stdout.trim().replace("refs/heads/", ""); let originSha: string | null = null; let mirrorSha: string | null = null; const originResult = await io.exec("git", ["rev-parse", branch], { cwd: bareRepoPath }); if (originResult.exitCode === 0) originSha = originResult.stdout.trim(); const mirrorResult = await io.exec("git", ["rev-parse", `refs/mirror/${branch}`], { cwd: bareRepoPath, }); if (mirrorResult.exitCode === 0) mirrorSha = mirrorResult.stdout.trim(); if (originSha && mirrorSha && originSha !== mirrorSha) { logger.warn( `[leitstand.promote] external mirrors are out of sync (origin=${originSha?.slice(0, 8)}, mirror=${mirrorSha?.slice(0, 8)}) — run sternsystem.sync --id ${systemId} after promote to align`, ); } else if (originSha && !mirrorSha) { logger.warn( `[leitstand.promote] mirror ref not found — run sternsystem.sync --id ${systemId} to initialize external mirror tracking`, ); } } } } catch { // Non-fatal — mirror check is advisory } } export async function runLeitstandPromote( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const releaseId = flagString(input, "release"); if (!releaseId) throw new Error("[leitstand.promote] --release is required"); const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.promote] --site is required"); // RFC-1190: mission context resolution + blocking UNRESOLVED check — see // leitstand.dev-deploy for the precedence chain and escape hatch. const skipEvidenceSync = flagBool(input, "skip-evidence-sync"); const missionCtx = await resolveMissionContext(context.workspaceRoot, systemId, context.io, { missionFlag: flagString(input, "mission"), releaseId, }); if (!missionCtx.missionId && !skipEvidenceSync) { return { ...failResult("leitstand.commands", [ `[leitstand.promote] MISSION_CONTEXT_UNRESOLVED: no mission resolvable via --mission, releases/${releaseId} build-identity, or system-state.currentMission — pass --mission or --skip-evidence-sync`, ]), data: { releaseId, systemId, channel: "main", deploymentUrl: "", state: "failed", buildIdentityVerified: false, testEvidenceVerified: false, healthState: "unknown", releaseState: "promoted", missionContext: missionCtx, }, diagnostics: [ { ruleId: "MISSION_CONTEXT_UNRESOLVED", severity: "error", message: `mission context unresolved for release ${releaseId} — mission-scoped gates cannot be evaluated; pass --mission or --skip-evidence-sync`, evidence: [], }, ], } as unknown as KernelCommandResult; } const gateDecisionFlag = flagString(input, "gate-decision"); const candidateId = flagString(input, "candidate-id") ?? systemId; const artifactHashFlag = flagString(input, "artifact-hash"); const cacheCloneDir = resolveCacheClonePath(context.workspaceRoot, systemId); const gateDecisionsDir = path.join(cacheCloneDir, "gate-decisions"); const mainVerificationFlag = flagString(input, "main-verification-decision"); const mainVerificationPath = mainVerificationFlag ? path.isAbsolute(mainVerificationFlag) || path.parse(mainVerificationFlag).dir !== "" ? mainVerificationFlag : path.join(gateDecisionsDir, mainVerificationFlag) : path.join(gateDecisionsDir, `${releaseId}-main-verification.json`); const gateDecisionPath = resolveGateDecisionPath( cacheCloneDir, releaseId, "main", gateDecisionFlag, ); const releaseDir = path.join(context.workspaceRoot, "releases", releaseId); const { hash: artifactHash, source: artifactHashSource, path: artifactHashPath, } = await resolveArtifactHash(artifactHashFlag, releaseDir); context.logger.info( `[deploy] artifact hash resolved from ${artifactHashSource}${artifactHashPath ? ` (${artifactHashPath})` : ""}: ${artifactHash}`, ); const r2Config = makeR2ConfigFromEnv(process.env as Record); let durableSyncVerified = !r2Config; if (r2Config) { durableSyncVerified = await verifyDurableSync(artifactHash, r2Config); } const { authorization, mainVerification } = await authorizeMainPromotion( gateDecisionPath, mainVerificationPath, artifactHash, candidateId, durableSyncVerified, false, false, false, false, ); if (!authorization.ok) { return { ...failResult("leitstand.commands", [ `[leitstand.promote] denied: ${authorization.outcome.ruleId} — ${authorization.outcome.message}`, ]), data: { releaseId, systemId, channel: "main", deploymentUrl: "", state: "failed", buildIdentityVerified: false, testEvidenceVerified: false, healthState: "unknown", releaseState: "promoted", }, diagnostics: [ { ruleId: authorization.outcome.ruleId, severity: "error", message: authorization.outcome.message, evidence: [], }, ], } as unknown as KernelCommandResult; } // Mirror sync pre-flight check (non-blocking warning, RFC-0995) await checkMirrorSyncPreFlight(context.workspaceRoot, systemId, context.logger); const operationId = generateOperationId(); const now = new Date().toISOString(); const mainVerificationDecisionId = mainVerification.ok ? mainVerification.decisionId : null; const effectRecord = buildEffectRecord( operationId, candidateId, "promote-main", "main", artifactHash, authorization.outcome.decisionId, durableSyncVerified, mainVerificationDecisionId, "authorized", now, ); await writeDeploymentEffectRecord(cacheCloneDir, effectRecord); const { adapter, systemConfig: depConfig } = await prepareDeployContext( context.workspaceRoot, systemId, "leitstand.promote", ); const channelConfig = depConfig.channels.main; const secretsFilePath = resolveSecretsFilePath(context.workspaceRoot, channelConfig); // RFC-0896: Register custom domain DNS + Workers route and www → apex redirect // before deploying to the main channel. Both commands are idempotent. try { const { runCustomdomainRegister } = await import("../customdomain/customdomain-register.ts"); const { runRedirectRegister } = await import("../customdomain/redirect-register.ts"); const customdomainInput: KernelCommandInput = { flags: { site: systemId }, argv: [], }; await runCustomdomainRegister(customdomainInput, context); await runRedirectRegister(customdomainInput, context); } catch (err) { return { ...failResult("leitstand.commands", [ `[leitstand.promote] custom domain setup failed: ${err instanceof Error ? err.message : String(err)}`, ]), data: { releaseId, systemId, channel: "main", deploymentUrl: "", state: "failed", buildIdentityVerified: false, testEvidenceVerified: false, healthState: "unknown", releaseState: "promoted", failingPhase: "custom-domain-setup", }, } as unknown as KernelCommandResult; } const { executeDeployPhases } = await import("./deploy-execution.ts"); const deployResult = await executeDeployPhases( { systemId, releaseId, candidateId, artifactHash, authResult: authorization, workspaceRoot: context.workspaceRoot, cacheCloneDir, systemConfig: depConfig, adapter, operationId, gateDecisionPath, secretsFilePath, skipEvidenceSync, missionId: missionCtx.missionId, commitSha: missionCtx.commitSha, missionContext: missionCtx, }, "main", ); // RFC-1138: post-promote mirror sync — same non-fatal semantics as // mission.reconcile/mission.close (RFC-0705/0762). Only runs when the // promote succeeded and external mirrors (mirrors[2+]) are configured. let mirror: { synced: boolean; syncError: string | null } | undefined; if (!deployResult.failingPhase) { try { const { readSystemConfig } = await import("../sternsystem/registry-io.ts"); const config = await readSystemConfig(context.workspaceRoot, systemId); if (config && config.mirrors.length > 2) { const { executeKernelCommand } = await import("@warpgogol/werkstatt-engine/kernel"); const syncResult = (await executeKernelCommand({ workspaceRoot: context.workspaceRoot, commandName: "sternsystem.sync", argv: [`--id=${systemId}`], })) as { exitCode?: number; summary?: string }; if (syncResult.exitCode === 0) { mirror = { synced: true, syncError: null }; context.logger.info(`[leitstand.promote] external mirrors synced`); } else { mirror = { synced: false, syncError: syncResult.summary ?? "unknown error" }; context.logger.warn( `[leitstand.promote] sternsystem.sync failed (non-fatal): ${syncResult.summary ?? ""}`, ); } } } catch (err) { mirror = { synced: false, syncError: err instanceof Error ? err.message : String(err) }; context.logger.warn( `[leitstand.promote] sternsystem.sync failed (non-fatal): ${mirror.syncError}`, ); } } return { data: { releaseId, systemId, channel: "main", deploymentUrl: deployResult.deploymentUrl, state: deployResult.failingPhase ? "failed" : "succeeded", buildIdentityVerified: deployResult.freshness.verified, testEvidenceVerified: deployResult.evidenceSynced, purgeResult: deployResult.purgeResult, healthState: deployResult.healthState, releaseState: "promoted", failingPhase: deployResult.failingPhase, missionContext: missionCtx, mirror, }, summary: deployResult.failingPhase ? `[leitstand.promote] failed at phase: ${deployResult.failingPhase}` : `[leitstand.promote] promoted to main: ${deployResult.deploymentUrl}`, exitCode: deployResult.failingPhase ? 1 : 0, nextSteps: deployResult.failingPhase ? [ { action: `Fix the failing phase (${deployResult.failingPhase}), then re-run: pnpm exec werkstatt run leitstand.promote --site ${systemId} --release ${releaseId}`, kind: "required", }, ] : [ { action: `Verify health: pnpm exec werkstatt run leitstand.health --site ${systemId} --channel main`, kind: "optional", }, { action: `Close the mission: pnpm exec werkstatt run mission.close --mission `, kind: "optional", }, ], } as unknown as KernelCommandResult; } // §5.2: leitstand.status export interface LeitstandStatusData { systemId: string; channels: { dev?: { releaseId: string; state: string; healthy: boolean; at: string; purgeResult?: PurgeResult; } | null; alt?: { releaseId: string; state: string; healthy: boolean; at: string; purgeResult?: PurgeResult; } | null; main?: { releaseId: string; state: string; healthy: boolean; at: string; purgeResult?: PurgeResult; } | null; }; /** RFC-1203: set by the leitstand.channels alias wrapper; absent on canonical leitstand.status results. */ aliasedTo?: "leitstand.status"; } export async function runLeitstandStatus( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.status] --site is required"); const cacheCloneDir = resolveCacheClonePath(context.workspaceRoot, systemId); const opsDir = path.join(cacheCloneDir, "deployment-operations"); const channels: LeitstandStatusData["channels"] = {}; for (const ch of ["dev", "alt", "main"] as const) { try { const entries = await context.io.readdir(opsDir).catch(() => []); const channelRecords = entries .filter((f) => f.name.endsWith(".json")) .map((f) => path.join(opsDir, f.name)); let latest: { releaseId: string; state: string; at: string; purgeResult?: PurgeResult; } | null = null; for (const recordPath of channelRecords) { try { const content = await context.io.readFile(recordPath); const record = JSON.parse(content); if (record.channel === ch) { const at = record.timestamp ?? ""; if (!latest || at > latest.at) { latest = { releaseId: record.candidateId ?? record.operationId ?? "", state: record.state ?? "", at, purgeResult: record.purgeResult, }; } } } catch { // Skip malformed records } } if (latest) { channels[ch] = { releaseId: latest.releaseId, state: latest.state, healthy: latest.state === "succeeded", at: latest.at, purgeResult: latest.purgeResult, }; } else { channels[ch] = null; } } catch { channels[ch] = null; } } return { data: { systemId, channels }, summary: `[leitstand.status] ${systemId}: dev=${channels.dev?.state ?? "none"} alt=${channels.alt?.state ?? "none"} main=${channels.main?.state ?? "none"}`, exitCode: 0, nextSteps: [ { action: `Check pipeline state: pnpm exec werkstatt run leitstand.pipeline.check --site ${systemId}`, kind: "optional", }, ], }; } // §5.2b: leitstand.channels (RFC-1203) — relabeling alias of leitstand.status. // The wrapper owns the alias surface (summary prefix + data.aliasedTo); the // canonical handler stays untouched and thrown errors propagate verbatim. export async function runLeitstandChannels( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const result = await runLeitstandStatus(input, context); const summaryBody = result.summary?.replace(/^\[leitstand\.status\]\s*/, "") ?? ""; return { ...result, summary: `[leitstand.channels] alias of leitstand.status — ${summaryBody}`, data: result.data ? { ...result.data, aliasedTo: "leitstand.status" } : result.data, exitCode: result.exitCode ?? 0, }; } // §5.3: leitstand.rollback (RFC-0895: native wrangler rollback, unified site+service) // RFC-0926: release-aware rollback with --to-release and Worker Version ID tracking interface ResolvedRollbackTarget { versionId?: string; releaseId?: string; } async function resolveRollbackVersionId( cacheCloneDir: string, channel: Channel, targetRelease?: string, io: WorkspaceIO = createDefaultIO().io, ): Promise { const opsDir = join(cacheCloneDir, "deployment-operations"); if (!(await io.exists(opsDir))) return {}; const files = (await io.readdir(opsDir)).map((f) => f.name).filter((f) => f.endsWith(".json")); const records: Array<{ workerVersionId?: string; releaseId?: string; state: string; channel?: string; timestamp: string; }> = []; for (const file of files) { try { const content = await io.readFile(join(opsDir, file)); const parsed = JSON.parse(content); if (parsed.state === "succeeded" && parsed.workerVersionId) { records.push({ workerVersionId: parsed.workerVersionId, releaseId: parsed.releaseId, state: parsed.state, channel: parsed.channel, timestamp: parsed.timestamp, }); } } catch { // Skip malformed files } } // Sort by timestamp descending (most recent first) records.sort((a, b) => b.timestamp.localeCompare(a.timestamp)); if (targetRelease) { // Find the most recent succeeded record for the target release const match = records.find((r) => r.releaseId === targetRelease); if (match?.workerVersionId) { return { versionId: match.workerVersionId, releaseId: match.releaseId }; } return {}; } // Default: find the previous release (different from the current release in system-state) const state = await readSystemStateSmart(cacheCloneDir, path.basename(cacheCloneDir)); const currentReleaseId = state?.lastPropagated?.[channel]?.releaseId; if (!currentReleaseId) return {}; // Find the most recent succeeded record with a different releaseId const match = records.find((r) => r.releaseId && r.releaseId !== currentReleaseId); if (match?.workerVersionId) { return { versionId: match.workerVersionId, releaseId: match.releaseId }; } return {}; } export interface LeitstandRollbackData { command: "leitstand.rollback"; target: "site" | "service"; systemId?: string; serviceId?: string; channel?: Channel; workerName: string; rollbackState: "succeeded" | "failed"; rolledBackToVersionId?: string; rolledBackToReleaseId?: string; startedAt: string; completedAt: string; operationId: string; purgeResult?: PurgeResult; } export async function runLeitstandRollback( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const { workspaceRoot, logger } = context; const siteId = flagSite(input); const serviceId = flagString(input, "service"); const channel = parseChannel(flagString(input, "channel"), "main"); const toRelease = flagString(input, "to-release"); // Reject removed flags if (flagString(input, "gate-decision")) { throw new Error( "[leitstand.rollback] --gate-decision is no longer supported for rollback (RFC-0895)", ); } // Exactly one of --site or --service required if (!siteId && !serviceId) { throw new Error("[leitstand.rollback] --site or --service is required"); } if (siteId && serviceId) { throw new Error("[leitstand.rollback] --site and --service are mutually exclusive"); } const operationId = generateOperationId(); const startedAt = new Date().toISOString(); if (siteId) { return runSiteRollback( workspaceRoot, siteId, channel, operationId, startedAt, logger, context.io, toRelease, ); } else { return runServiceRollback( workspaceRoot, serviceId!, operationId, startedAt, logger, context.io, toRelease, ); } } async function runSiteRollback( workspaceRoot: string, systemId: string, channel: Channel, operationId: string, startedAt: string, logger: { info: (msg: string) => void }, io: WorkspaceIO = createDefaultIO().io, toRelease?: string, ): Promise> { const systemConfig = await readSystemConfigSmart(workspaceRoot, systemId); if (!systemConfig.deployment) { throw new Error( `[leitstand.rollback] system '${systemId}' has no deployment config in system-config.yaml`, ); } const channelConfig = channel === "dev" ? systemConfig.deployment.channels.dev : channel === "alt" ? systemConfig.deployment.channels.alt : systemConfig.deployment.channels.main; const workerName = channelConfig?.workerName ?? systemId; const channelUrl = channelConfig?.url ?? ""; // Resolve Worker Version ID for targeted rollback (RFC-0926) const cacheCloneDir = resolveCacheClonePath(workspaceRoot, systemId); const rollbackTarget = await resolveRollbackVersionId(cacheCloneDir, channel, toRelease, io); if (rollbackTarget.versionId) { logger.info( `[leitstand.rollback] resolved Worker Version ID ${rollbackTarget.versionId} for release ${rollbackTarget.releaseId ?? toRelease ?? "previous"}`, ); } else if (toRelease) { logger.info( `[leitstand.rollback] no stored Worker Version ID for release '${toRelease}', falling back to wrangler rollback without version ID`, ); } else { logger.info( `[leitstand.rollback] no previous release version ID found, falling back to wrangler rollback without version ID`, ); } // Create temp directory with minimal wrangler.json for wrangler rollback const os = await import("node:os"); const tmpDir = await io.mkdtemp(join(os.tmpdir(), "wrangler-rollback-")); const wranglerConfigPath = join(tmpDir, "wrangler.json"); await io.writeFile(wranglerConfigPath, JSON.stringify({ name: workerName })); try { logger.info( `[leitstand.rollback] rolling back ${workerName} (channel=${channel}) via wrangler rollback…`, ); const adapter = resolveAdapter(systemConfig.deployment.adapter); const rollbackResult = await adapter.rollback({ systemId, channel, wranglerConfigDir: tmpDir, workerName, versionId: rollbackTarget.versionId, }); // Purge CDN cache for site rollbacks let purgeResult: PurgeResult | undefined; if (rollbackResult.state === "succeeded" && channelUrl) { const secretsEnv = channelConfig?.secretsFile ? await sourceDotenv(join(workspaceRoot, channelConfig.secretsFile)) : {}; const env = { ...filterEnv(process.env), ...secretsEnv }; const zoneId = env["CLOUDFLARE_ZONE_ID"]; const apiToken = env["CLOUDFLARE_API_TOKEN"]; if (zoneId && apiToken) { const urls = collectPurgeUrls(channelUrl, []); purgeResult = await purgeCacheByUrls(zoneId, apiToken, urls); } } // Write deployment effect record — RFC-1297: the record carries the // channel's real gate, not a hardcoded dev-deploy. const effectRecord = buildEffectRecord( operationId, systemId, gateForChannel(channel), channel, "" as Sha256Digest, operationId, false, null, rollbackResult.state === "succeeded" ? "rolled-back" : "failed", new Date().toISOString(), rollbackTarget.versionId, rollbackTarget.releaseId, ); await writeDeploymentEffectRecord(cacheCloneDir, effectRecord); // RFC-1297: record the rollback outcome — lastPropagated[channel] names the // release now live (state "rolled-back") and Bordbuch carries a // deployment-rolled-back event. Without this, pipeline rollbacks diverged // recorded vs live state silently. let recordedReleaseId: string | null = null; if (rollbackResult.state === "succeeded") { const recorded = await recordSiteRollback({ workspaceRoot, systemId, channel, operationId, rollbackTarget, io, }); recordedReleaseId = recorded.releaseId; } const completedAt = new Date().toISOString(); const data: LeitstandRollbackData = { command: "leitstand.rollback", target: "site", systemId, channel, workerName, rollbackState: rollbackResult.state, rolledBackToVersionId: rollbackTarget.versionId, rolledBackToReleaseId: rollbackTarget.releaseId, startedAt, completedAt, operationId, purgeResult, }; if (rollbackResult.state === "failed") { return { data, summary: `[leitstand.rollback] ${systemId}: wrangler rollback failed — ${rollbackResult.stderr.slice(-200)}`, exitCode: 1, nextSteps: [ { action: `Check wrangler logs and re-run: pnpm exec werkstatt run leitstand.rollback --site ${systemId}`, kind: "required", }, ], } as unknown as KernelCommandResult; } const nextSteps: Array<{ action: string; kind: "optional" | "required" }> = [ { action: `Verify health: pnpm exec werkstatt run leitstand.health --site ${systemId} --channel ${channel}`, kind: "optional", }, ]; if (recordedReleaseId === null) { // RFC-1297: the rollback succeeded but the live release could not be // named — reality must be recorded deliberately by the operator. nextSteps.push({ action: `Live release unknown after rollback — run leitstand.verify --site ${systemId}, then record reality: pnpm exec werkstatt run leitstand.state.reconcile --site ${systemId} --channel ${channel} --from-live --reason "…"`, kind: "required", }); } return { data, exitCode: 0, summary: `[leitstand.rollback] ${systemId}: rolled back to ${rollbackTarget.releaseId ?? "previous deployment"}${rollbackTarget.versionId ? ` (version ${rollbackTarget.versionId})` : ""}`, nextSteps, } as unknown as KernelCommandResult; } finally { // Clean up temp directory await io.rm(tmpDir, { recursive: true }).catch(() => {}); } } async function runServiceRollback( workspaceRoot: string, serviceId: string, operationId: string, startedAt: string, logger: { info: (msg: string) => void }, io: WorkspaceIO = createDefaultIO().io, _toRelease?: string, ): Promise> { const { readServicesRegistry, findServiceEntry } = await import("../sternsystem/registry-io.ts"); const { runWranglerRollback, parseEnvFile, acquireServiceLock, releaseServiceLock, recordProdDeployState, } = await import("./service-deploy-helpers.ts"); const registry = await readServicesRegistry(workspaceRoot); const serviceEntry = findServiceEntry(registry, serviceId); if (!serviceEntry) { throw new Error( `[leitstand.rollback] service '${serviceId}' not found in services/registry.yaml`, ); } const serviceDir = join(workspaceRoot, "services", serviceId); if (!(await io.exists(serviceDir))) { throw new Error(`[leitstand.rollback] services/${serviceId}/ does not exist`); } const wranglerPath = join(serviceDir, "wrangler.jsonc"); if (!(await io.exists(wranglerPath))) { throw new Error( `[leitstand.rollback] services/${serviceId}/wrangler.jsonc not found — not a Cloudflare Worker service`, ); } await acquireServiceLock(workspaceRoot, serviceId, operationId, "leitstand.rollback"); try { const envPath = join(serviceDir, ".env"); const deployEnv = await parseEnvFile(envPath); logger.info( `[leitstand.rollback] rolling back ${serviceEntry.workerName} via wrangler rollback…`, ); const wranglerResult = await runWranglerRollback(serviceDir, deployEnv); if (wranglerResult.exitCode !== 0) { const completedAt = new Date().toISOString(); return { data: { command: "leitstand.rollback", target: "service", serviceId, workerName: serviceEntry.workerName, rollbackState: "failed", startedAt, completedAt, operationId, }, summary: `[leitstand.rollback] ${serviceId}: wrangler rollback failed — ${wranglerResult.stderr.slice(-200)}`, exitCode: 1, nextSteps: [ { action: `Check wrangler logs and re-run: pnpm exec werkstatt run leitstand.rollback --service ${serviceId}`, kind: "required", }, ], } as unknown as KernelCommandResult; } await recordProdDeployState(workspaceRoot, serviceId, { at: new Date().toISOString(), state: "rolled-back", operationId, }); const completedAt = new Date().toISOString(); return { data: { command: "leitstand.rollback", target: "service", serviceId, workerName: serviceEntry.workerName, rollbackState: "succeeded", startedAt, completedAt, operationId, }, exitCode: 0, summary: `[leitstand.rollback] ${serviceId}: rolled back to previous deployment`, nextSteps: [ { action: `Verify service health: pnpm exec werkstatt run leitstand.health --service ${serviceId}`, kind: "optional", }, ], } as unknown as KernelCommandResult; } finally { await releaseServiceLock(workspaceRoot, serviceId); } } // §5.4: leitstand.health export interface LeitstandHealthData { systemId: string; channel: Channel; state: "healthy" | "unhealthy" | "unknown"; checks: HealthCheck[]; } export async function runLeitstandHealth( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.health] --site is required"); const channel = parseChannel(flagString(input, "channel"), "alt"); const systemConfig = await readSystemConfigSmart(context.workspaceRoot, systemId); if (!systemConfig.deployment) { return { data: { systemId, channel, state: "unknown", checks: [] }, summary: `[leitstand.health] ${systemId}: no deployment config`, exitCode: 0, }; } const adapter = resolveAdapter(systemConfig.deployment.adapter); const channelConfig = channel === "dev" ? systemConfig.deployment.channels.dev : channel === "alt" ? systemConfig.deployment.channels.alt : systemConfig.deployment.channels.main; if (!channelConfig) { return { data: { systemId, channel, state: "unknown", checks: [] }, summary: `[leitstand.health] ${systemId}: channel ${channel} not configured`, exitCode: 0, }; } // RFC-1136: resolve the channel's deployed releaseId before probing — // the adapter loads the behavior snapshot by releaseId, so an empty id // made every health check report "unknown" (ADR-0089: explicit context, // not ambient state). const systemState = await readSystemStateSmart(context.workspaceRoot, systemId); const releaseId = flagString(input, "release") ?? systemState?.lastPropagated?.[channel]?.releaseId ?? null; if (!releaseId) { return { data: { systemId, channel, state: "unknown", checks: [] }, summary: `[leitstand.health] ${systemId} channel=${channel}: no release recorded — deploy first`, exitCode: 0, }; } try { // RFC-1188: standalone health probes are agent-directed traffic — sign // through the outbound client when the identity key resolves. const outbound = await resolveOutboundAgentClient(context.workspaceRoot, systemId, undefined, { onSigningFailure: (e) => context.logger.warn( `[leitstand.health] outbound signing failed, request sent unsigned: ${e instanceof Error ? e.message : String(e)}`, ), }); const result = await adapter.health({ systemId, deploymentUrl: channelConfig.url, channel, releaseId, expectedBehaviorSnapshotHash: "", workspaceRoot: context.workspaceRoot, agentFetch: outbound.client.fetch, }); return { data: { systemId, channel, state: result.state ?? "unknown", checks: result.checks ?? [], }, summary: `[leitstand.health] ${systemId} channel=${channel}: state=${result.state ?? "unknown"}`, exitCode: 0, }; } catch (err) { return { data: { systemId, channel, state: "unhealthy", checks: [] }, summary: `[leitstand.health] ${systemId} channel=${channel}: error=${err instanceof Error ? err.message : String(err)}`, exitCode: 0, }; } } export interface PipelineCheckResult { command: "leitstand.pipeline.check"; releaseId: string; systemId: string; releaseState: string; steps: Array<{ step: string; status: "done" | "pending" | "blocked"; detail?: string; }>; nextStep: string | null; } function determineNextStep(releaseState: string): string { switch (releaseState) { case "prepared": return "release.ready"; case "ready": return "leitstand.dev-deploy"; case "dev-deployed": return "leitstand.propagate"; case "alt-deployed": return "leitstand.promote"; case "main-deployed": return "done"; case "failed": return "leitstand.rollback"; default: return "release.prepare"; } } export async function runLeitstandPipelineCheck( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const releaseId = flagString(input, "release"); if (!releaseId) { throw new Error("[leitstand.pipeline.check] --release is required"); } const systemId = flagSite(input) ?? ""; const cacheCloneDir = resolveCacheClonePath(context.workspaceRoot, systemId); const opsDir = path.join(cacheCloneDir, "deployment-operations"); const channelStates: Record = {}; try { const entries = await context.io.readdir(opsDir); for (const f of entries) { if (!f.name.endsWith(".json")) continue; try { const record = JSON.parse(await context.io.readFile(path.join(opsDir, f.name))); const ch = record.channel as string | undefined; if (!ch) continue; const at = record.timestamp ?? ""; if (!channelStates[ch] || at > channelStates[ch].at) { channelStates[ch] = { state: record.state ?? "", at }; } } catch { // skip } } } catch { // ops dir doesn't exist yet } const devState = channelStates["dev"]?.state; const altState = channelStates["alt"]?.state; const mainState = channelStates["main"]?.state; const steps: PipelineCheckResult["steps"] = [ { step: "release.prepare", status: "done", detail: "release prepared" }, { step: "release.ready", status: "done", detail: "release ready" }, { step: "leitstand.dev-deploy", status: devState === "succeeded" ? "done" : devState === "failed" ? "blocked" : "pending", detail: devState ? `state=${devState}` : "awaiting gate decision", }, { step: "leitstand.propagate", status: altState === "succeeded" ? "done" : altState === "failed" ? "blocked" : "pending", detail: altState ? `state=${altState}` : "awaiting dev-deploy + R2 durable sync", }, { step: "leitstand.promote", status: mainState === "succeeded" ? "done" : mainState === "failed" ? "blocked" : "pending", detail: mainState ? `state=${mainState}` : "awaiting propagate-alt + main verification", }, ]; let releaseState = "ready"; if (mainState === "succeeded") releaseState = "main-deployed"; else if (altState === "succeeded") releaseState = "alt-deployed"; else if (devState === "succeeded") releaseState = "dev-deployed"; else if (devState === "failed" || altState === "failed" || mainState === "failed") releaseState = "failed"; return { data: { command: "leitstand.pipeline.check", releaseId, systemId, releaseState, steps, nextStep: determineNextStep(releaseState), }, summary: `[leitstand.pipeline.check] release=${releaseId}: state=${releaseState} next=${determineNextStep(releaseState)}`, exitCode: 0, nextSteps: [ { action: determineNextStep(releaseState), kind: "optional", }, ], }; } export interface HotfixDevDeployResult { command: "leitstand.hotfix.dev-deploy"; systemId: string; missionId: string; releaseId: string; phases: { commit: { skipped: boolean; commitSha?: string }; validate: { mode: "postbuild" | "full"; passed: boolean }; reconcile: { ok: boolean }; close: { ok: boolean }; release: { releaseId: string; prepared: boolean; ready: boolean }; certify: { gate: "dev"; decision: string; passed: boolean }; deploy: { url: string; deployed: boolean }; }; deploymentUrl: string; failingPhase?: string; } async function runPhase( workspaceRoot: string, commandName: string, argv: string[], logger: { info: (m: string) => void }, ): Promise<{ exitCode: number; data?: Record; summary?: string }> { // Pre-flight: validate flags against command schema before calling executeKernelCommand. // This catches composite-command bugs (e.g. passing --release to release.prepare) // with a clear error message, before any pipeline work begins. const registry = await getOrBuildWorkspaceRegistry(workspaceRoot); const command = registry?.commands.get(commandName); if (command) { const validFlags = new Set([ ...Object.keys(KERNEL_UNIVERSAL_FLAGS), ...Object.keys(command.flags ?? {}), ]); for (const entry of argv) { if (!entry.startsWith("--")) continue; const flagName = entry.slice(2).split("=")[0]; if (flagName && !validFlags.has(flagName)) { const msg = `Unknown flag "--${flagName}" for command "${commandName}" (called from leitstand.hotfix.dev-deploy). Valid flags: ${[...validFlags].sort().join(", ")}`; logger.info(` [hotfix] ${msg}`); const failed: { exitCode: number; data?: Record; summary?: string } = { exitCode: 1, summary: msg, }; return failed; } } } logger.info(` [hotfix] ${commandName} ${argv.join(" ")}`); const result = (await executeKernelCommand({ workspaceRoot, commandName, argv, outputFormat: "pretty", })) as { exitCode?: number; data?: Record; summary?: string }; return { exitCode: result.exitCode ?? 0, data: result.data, summary: result.summary, }; } export async function runLeitstandHotfixDevDeploy( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const { workspaceRoot, logger } = context; const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.hotfix.dev-deploy] --site is required"); const missionId = flagString(input, "mission"); if (!missionId) throw new Error("[leitstand.hotfix.dev-deploy] --mission is required"); const message = flagString(input, "message"); const forceBuild = input.flags["force-build"] === true; const workpieceDir = path.join(workspaceRoot, "missions", missionId, "workpiece"); const phases: HotfixDevDeployResult["phases"] = { commit: { skipped: false }, validate: { mode: "postbuild", passed: false }, reconcile: { ok: false }, close: { ok: false }, release: { releaseId: "", prepared: false, ready: false }, certify: { gate: "dev", decision: "", passed: false }, deploy: { url: "", deployed: false }, }; const failResult = ( failingPhase: string, _summary: string, ): KernelCommandResult => ({ data: { command: "leitstand.hotfix.dev-deploy", systemId, missionId, releaseId: phases.release.releaseId, phases, deploymentUrl: phases.deploy.url, failingPhase, }, summary: `[leitstand.hotfix.dev-deploy] failed at phase: ${failingPhase}`, exitCode: 1, nextSteps: [ { action: `Fix the failing phase (${failingPhase}), then re-run: pnpm exec werkstatt run leitstand.hotfix.dev-deploy --site ${systemId} --mission ${missionId}`, kind: "required", }, ], }); // Phase 1: Commit (skip if workpiece is clean) try { const statusResult = await context.io.exec("git", ["status", "--porcelain"], { cwd: workpieceDir, }); const gitStatus = statusResult.exitCode === 0 ? statusResult.stdout.trim() : ""; if (gitStatus === "") { phases.commit.skipped = true; logger.info(" [hotfix] Phase 1 (commit): workpiece clean, skipping"); } else { const commitArgv = [`--mission=${missionId}`]; if (message) commitArgv.push(`--message=${message}`); const commitResult = await runPhase(workspaceRoot, "mission.git.commit", commitArgv, logger); if (commitResult.exitCode !== 0) { return failResult("commit", commitResult.summary ?? "mission.git.commit failed"); } const headResult = await context.io.exec("git", ["rev-parse", "HEAD"], { cwd: workpieceDir, }); const commitSha = headResult.exitCode === 0 ? headResult.stdout.trim() : ""; phases.commit.commitSha = commitSha; logger.info(` [hotfix] Phase 1 (commit): committed as ${commitSha}`); } } catch (err) { return failResult("commit", err instanceof Error ? err.message : String(err)); } // Phase 2: Validate (postbuild first, fallback to full mission.validate) const postbuildArgv = [`--mission=${missionId}`]; if (forceBuild) postbuildArgv.push("--force-build"); const postbuildResult = await runPhase( workspaceRoot, "validate.postbuild", postbuildArgv, logger, ); if (postbuildResult.exitCode === 0) { phases.validate.mode = "postbuild"; phases.validate.passed = true; logger.info(" [hotfix] Phase 2 (validate): postbuild passed"); } else { logger.info( " [hotfix] Phase 2 (validate): postbuild failed, falling back to full mission.validate", ); const fullValidateArgv = [`--mission=${missionId}`]; if (forceBuild) fullValidateArgv.push("--force-build"); const fullResult = await runPhase(workspaceRoot, "mission.validate", fullValidateArgv, logger); if (fullResult.exitCode !== 0) { phases.validate.mode = "full"; phases.validate.passed = false; return failResult("validate", fullResult.summary ?? "mission.validate failed"); } phases.validate.mode = "full"; phases.validate.passed = true; logger.info(" [hotfix] Phase 2 (validate): full mission.validate passed"); } // Phase 3: Reconcile const reconcileResult = await runPhase( workspaceRoot, "mission.reconcile", [`--mission=${missionId}`], logger, ); if (reconcileResult.exitCode !== 0) { return failResult("reconcile", reconcileResult.summary ?? "mission.reconcile failed"); } phases.reconcile.ok = true; logger.info(" [hotfix] Phase 3 (reconcile): ok"); // Phase 4: Close const closeResult = await runPhase( workspaceRoot, "mission.close", [`--mission=${missionId}`], logger, ); if (closeResult.exitCode !== 0) { return failResult("close", closeResult.summary ?? "mission.close failed"); } phases.close.ok = true; logger.info(" [hotfix] Phase 4 (close): ok"); // Phase 5: Release (prepare + ready) // release.prepare derives the releaseId internally (sequential numbering); // we read it from the result data instead of generating it ourselves. const prepareArgv = [`--mission=${missionId}`]; if (forceBuild) prepareArgv.push("--force"); const prepareResult = await runPhase(workspaceRoot, "release.prepare", prepareArgv, logger); if (prepareResult.exitCode !== 0) { return failResult("release", prepareResult.summary ?? "release.prepare failed"); } const prepareData = prepareResult.data as { releaseId?: string } | undefined; const releaseId = prepareData?.releaseId ?? ""; if (!releaseId) { return failResult("release", "release.prepare returned no releaseId"); } phases.release.releaseId = releaseId; phases.release.prepared = true; const readyResult = await runPhase( workspaceRoot, "release.ready", [`--release=${releaseId}`], logger, ); if (readyResult.exitCode !== 0) { return failResult("release", readyResult.summary ?? "release.ready failed"); } phases.release.ready = true; logger.info(` [hotfix] Phase 5 (release): ${releaseId} prepared and ready`); // Phase 6: Certify (dev gate) const certifyArgv = [`--site=${systemId}`, `--gate=dev`, `--release=${releaseId}`]; const certifyResult = await runPhase(workspaceRoot, "leitstand.certify", certifyArgv, logger); if (certifyResult.exitCode !== 0) { return failResult("certify", certifyResult.summary ?? "leitstand.certify failed"); } const certifyData = certifyResult.data as { decisionId?: string } | undefined; phases.certify.decision = certifyData?.decisionId ?? ""; phases.certify.passed = true; logger.info(" [hotfix] Phase 6 (certify): dev gate passed"); // Phase 7: Deploy const deployArgv = [`--site=${systemId}`, `--release=${releaseId}`, `--mission=${missionId}`]; const deployResult = await runPhase(workspaceRoot, "leitstand.dev-deploy", deployArgv, logger); if (deployResult.exitCode !== 0) { return failResult("deploy", deployResult.summary ?? "leitstand.dev-deploy failed"); } const deployData = deployResult.data as { deploymentUrl?: string } | undefined; phases.deploy.url = deployData?.deploymentUrl ?? ""; phases.deploy.deployed = true; logger.info(` [hotfix] Phase 7 (deploy): deployed to ${phases.deploy.url}`); return { data: { command: "leitstand.hotfix.dev-deploy", systemId, missionId, releaseId, phases, deploymentUrl: phases.deploy.url, }, summary: `[leitstand.hotfix.dev-deploy] deployed to dev: ${phases.deploy.url}`, exitCode: 0, nextSteps: [ { action: `Propagate to alt: pnpm exec werkstatt run leitstand.propagate --site ${systemId} --release ${releaseId}`, kind: "optional", }, ], }; } // §5.5: leitstand.verify (RFC-0930) export interface LeitstandVerifyChannel { channel: Channel; url: string; /** True when the server responded with an HTTP status (even 4xx). False on network error, timeout, or 5xx. */ reachable: boolean; httpStatus: number | null; buildIdentity: { distTreeHash: string | null; releaseId: string | null; missionId: string | null; buildTimestamp: string | null; } | null; /** RFC-0931: signature verification result (null = not checked or no signature). */ signatureValid: boolean | null; /** RFC-0931: public key URL used for verification (null when not checked or no signature). */ publicKeyUrl: string | null; localState: { releaseId: string | null; workerVersionId: string | null; state: string | null; at: string | null; } | null; matchesLocal: boolean | null; error: string | null; } export interface LeitstandVerifyData { command: "leitstand.verify"; systemId: string; channels: LeitstandVerifyChannel[]; crossChannel: { allReachable: boolean; allSameHash: boolean; commonHash: string | null; hashesByChannel: Record; }; summary: string; exitCode: 0 | 1; } function flagBooleanDefault( input: KernelCommandInput, key: string, defaultValue: boolean, ): boolean { const v = input.flags[key]; if (typeof v === "boolean") return v; if (typeof v === "string") return v === "true"; return defaultValue; } function flagNumberDefault(input: KernelCommandInput, key: string, defaultValue: number): number { const v = input.flags[key]; if (typeof v === "number") return v; if (typeof v === "string") { const n = Number(v); if (!Number.isNaN(n)) return n; } return defaultValue; } function emptyVerifyResult( systemId: string, summary: string, exitCode: 0 | 1, ): KernelCommandResult { return { data: { command: "leitstand.verify", systemId, channels: [], crossChannel: { allReachable: true, allSameHash: true, commonHash: null, hashesByChannel: {}, }, summary, exitCode, }, summary, exitCode, }; } export async function runLeitstandVerify( input: KernelCommandInput, context: KernelRuntimeContext, ): Promise> { const systemId = flagSite(input); if (!systemId) throw new Error("[leitstand.verify] --site is required"); const channelFilter = flagString(input, "channel") as Channel | undefined; if ( channelFilter && channelFilter !== "dev" && channelFilter !== "alt" && channelFilter !== "main" ) { return emptyVerifyResult( systemId, `[leitstand.verify] ${systemId}: invalid channel '${channelFilter}'`, 1, ); } const compareLocal = flagBooleanDefault(input, "compare-local", true); const timeoutMs = flagNumberDefault(input, "timeout-ms", 10000); const verifySignature = input.flags["verify-signature"] === true; const systemConfig = await readSystemConfigSmart(context.workspaceRoot, systemId); if (!systemConfig.deployment) { return emptyVerifyResult(systemId, `[leitstand.verify] ${systemId}: no deployment config`, 0); } const systemState = await readSystemStateSmart(context.workspaceRoot, systemId); const authHeaders = buildAuthHeader(systemState.accessPin); const allChannels: Channel[] = ["dev", "alt", "main"]; const targetChannels = channelFilter ? allChannels.filter((ch) => ch === channelFilter) : allChannels; const channelConfigs: Array<{ channel: Channel; url: string }> = []; for (const ch of targetChannels) { const cfg = ch === "dev" ? systemConfig.deployment.channels.dev : ch === "alt" ? systemConfig.deployment.channels.alt : systemConfig.deployment.channels.main; if (cfg) { channelConfigs.push({ channel: ch, url: cfg.url }); } } if (channelFilter && channelConfigs.length === 0) { return emptyVerifyResult( systemId, `[leitstand.verify] ${systemId}: channel '${channelFilter}' not configured`, 1, ); } // RFC-1188: channel probes are platform-originated agent traffic — // sign through OutboundAgentClient when the identity key resolves. const outbound = await resolveOutboundAgentClient(context.workspaceRoot, systemId, undefined, { onSigningFailure: (e) => context.logger.warn( `[leitstand.verify] outbound signing failed, request sent unsigned: ${e instanceof Error ? e.message : String(e)}`, ), }); const fetchImpl = outbound.client.fetch; const fetchResults = await Promise.all( channelConfigs.map(async ({ channel, url }) => { const result = await fetchChannelBuildIdentity( channel, url, authHeaders, timeoutMs, fetchImpl, ); const localEntry = systemState.lastPropagated?.[channel]; const localState = localEntry ? { releaseId: localEntry.releaseId ?? null, workerVersionId: localEntry.workerVersionId ?? null, state: localEntry.state ?? null, at: localEntry.at ?? null, } : null; let matchesLocal: boolean | null = null; if (compareLocal && localState && result.buildIdentity?.releaseId) { matchesLocal = result.buildIdentity.releaseId === localState.releaseId; } else if (compareLocal && !localState) { matchesLocal = null; } let signatureValid: boolean | null = null; let pubKeyUrl: string | null = null; if (verifySignature && result.buildIdentity) { try { const base = url.replace(/\/$/, ""); const pubKeyFetchUrl = `${base}/.well-known/release-pubkey.json`; const pubKeyResponse = await fetchImpl(pubKeyFetchUrl, { headers: authHeaders, redirect: "follow", signal: AbortSignal.timeout(timeoutMs), }); if (pubKeyResponse.ok) { const pubKeyData = (await pubKeyResponse.json()) as { publicKeyHex?: string }; if (pubKeyData.publicKeyHex) { const fullIdentity = await fetchImpl(`${base}${BUILD_IDENTITY_PATH}`, { headers: authHeaders, redirect: "follow", signal: AbortSignal.timeout(timeoutMs), }); if (fullIdentity.ok) { const identityJson = (await fullIdentity.json()) as Record; signatureValid = await verifyBuildIdentitySignature({ buildIdentity: identityJson, publicKeyHex: pubKeyData.publicKeyHex, }); pubKeyUrl = pubKeyFetchUrl; } } } } catch { // Non-fatal: signature verification is best-effort } } return { channel, url, reachable: result.reachable, httpStatus: result.httpStatus, buildIdentity: result.buildIdentity, signatureValid, publicKeyUrl: pubKeyUrl, localState, matchesLocal, error: result.error, } satisfies LeitstandVerifyChannel; }), ); const hashesByChannel: Record = {}; for (const ch of fetchResults) { hashesByChannel[ch.channel] = ch.buildIdentity?.distTreeHash ?? null; } const reachableChannels = fetchResults.filter((ch) => ch.reachable && ch.buildIdentity); const allReachable = fetchResults.length > 0 && fetchResults.every((ch) => ch.reachable); const hashSet = new Set( reachableChannels .map((ch) => ch.buildIdentity!.distTreeHash) .filter((h): h is string => h !== null), ); const allSameHash = reachableChannels.length > 0 && hashSet.size === 1; const commonHash = allSameHash ? [...hashSet][0] : null; const localMismatch = compareLocal && fetchResults.some((ch) => ch.matchesLocal === false); const hasIssues = !allReachable || !allSameHash || localMismatch; const reachableCount = fetchResults.filter((ch) => ch.reachable).length; const total = fetchResults.length; let summary: string; if (hasIssues) { const issues: string[] = []; if (!allReachable) issues.push(`${reachableCount}/${total} reachable`); if (reachableChannels.length > 0 && !allSameHash) { const hashDetails = Object.entries(hashesByChannel) .map(([ch, h]) => `${ch}=${h ?? "unreachable"}`) .join(", "); issues.push(`hash drift (${hashDetails})`); } if (localMismatch) issues.push("local mismatch"); summary = `[leitstand.verify] ${systemId}: ${issues.join(", ")}`; } else { summary = `[leitstand.verify] ${systemId}: ${reachableCount}/${total} reachable, all same hash${compareLocal ? ", all match local" : ""}`; } return { data: { command: "leitstand.verify", systemId, channels: fetchResults, crossChannel: { allReachable, allSameHash, commonHash, hashesByChannel }, summary, exitCode: hasIssues ? 1 : 0, }, summary, exitCode: hasIssues ? 1 : 0, }; }