/*
RFC-0964: fleet.apply — cross-site wave orchestration command. Runs a kernel
command or pipeline across fleet sites with canary-first waves, bounded
concurrency, stop-threshold, and journal-backed crash-safe resume.
Uses journal I/O primitives (appendRecord, readJournal, findIncompleteOperation)
with a custom execution loop — runOperation is not used (it's linear/sequential,
no concurrency support).
- Do not re-implement per-site pipeline logic — delegate via executeKernelCommand/executeKernelPipeline.
- Do not block concurrent fleet.apply runs — each has its own journal; operators avoid overlapping destructive ops.
- RFC-0964: initial fleet.apply implementation — wave orchestration, concurrency, stop-threshold, journal, report.
- RFC-1149: migrate ambient filesystem and process IO to the WorkspaceIO port across engine and shared
- RFC-1249: wire werkstatt.commands.validate into packages.check + retire CMD-OUTPUT debt
*/
import { dirname, join } from "node:path";
import { randomUUID } from "node:crypto";
import type {
KernelCommandInput,
KernelCommandResult,
KernelRuntimeContext,
} from "@warpgogol/werkstatt-shared/kernel";
import { executeKernelCommand } from "../kernel/runtime/execute-command.ts";
import { executeKernelPipeline } from "../kernel/runtime/execute-pipeline.ts";
import { getOrBuildWorkspaceRegistry } from "../kernel/runtime/registry-cache.ts";
import {
discoverSystems,
resolveCacheClonePath,
readSystemState,
} from "../sternsystem/registry-io.ts";
import { appendRecord, readJournal } from "../journal/jsonl.ts";
import type { FleetSiteRecord } from "./fleet-sites-generate.ts";
import { createDefaultIO, type WorkspaceIO } from "@warpgogol/werkstatt-shared/kernel/workspace-io";
import { failResult } from "@warpgogol/werkstatt-shared/checks/result-helpers";
const FLEET_SITES_FILE = "fleet/fleet.sites.yaml";
const FLEET_OPERATIONS_DIR = "fleet/operations";
const FLEET_REPORTS_DIR = "fleet/reports";
const STALENESS_THRESHOLD_MS = 24 * 60 * 60 * 1000; // 24h
export type WaveName = "canary" | "rest";
export type SiteStatus = "ok" | "failed" | "skipped(precondition)" | "skipped(stop-threshold)";
export interface FleetApplyReportEntry {
site: string;
wave: WaveName;
status: SiteStatus;
exitCode: number | null;
durationMs: number;
error?: string;
}
export interface FleetApplyResult {
opId: string;
command: string;
wave: string;
totalSites: number;
entries: FleetApplyReportEntry[];
ok: number;
failed: number;
skipped: number;
exitCode: number;
}
interface FleetSitesFile {
sites: FleetSiteRecord[];
}
function flagString(input: KernelCommandInput, key: string): string | undefined {
const v = input.flags[key];
return typeof v === "string" ? v : undefined;
}
function flagBool(input: KernelCommandInput, key: string): boolean {
const v = input.flags[key];
return v === true || v === "true";
}
function flagInt(input: KernelCommandInput, key: string, def: number): number {
const v = input.flags[key];
if (typeof v === "string") {
const n = parseInt(v, 10);
return isNaN(n) ? def : n;
}
return def;
}
async function loadFleetSites(workspaceRoot: string, io: WorkspaceIO): Promise {
const filePath = join(workspaceRoot, FLEET_SITES_FILE);
if (!(await io.exists(filePath))) return [];
const raw = await io.readFile(filePath);
// Strip GENERATED header
const markerIdx = raw.indexOf("\n---\n");
const body = markerIdx >= 0 ? raw.slice(markerIdx + 5) : raw;
const { parse: parseYaml } = await import("yaml");
const yamlParsed = parseYaml(body) as FleetSitesFile;
return yamlParsed.sites ?? [];
}
async function isRegistryStale(
workspaceRoot: string,
systems: { id: string }[],
io: WorkspaceIO,
): Promise {
const fleetPath = join(workspaceRoot, FLEET_SITES_FILE);
if (!(await io.exists(fleetPath))) return true;
const fleetMtime = (await io.stat(fleetPath)).mtimeMs;
const now = Date.now();
if (now - fleetMtime > STALENESS_THRESHOLD_MS) return true;
for (const sys of systems) {
const configPath = join(resolveCacheClonePath(workspaceRoot, sys.id), "system-config.yaml");
if (await io.exists(configPath)) {
const configMtime = (await io.stat(configPath)).mtimeMs;
if (configMtime > fleetMtime) return true;
}
}
return false;
}
async function regenerateRegistry(
workspaceRoot: string,
logger: KernelRuntimeContext["logger"],
): Promise {
logger.info("[fleet.apply] Registry stale or missing — regenerating via fleet.sites.generate");
const { runFleetSitesGenerate } = await import("./fleet-sites-generate.ts");
const result = await runFleetSitesGenerate({ flags: {}, argv: [] }, {
io: createDefaultIO().io,
workspaceRoot,
logger,
} as KernelRuntimeContext);
const data = result.data as { sites: FleetSiteRecord[] };
return data.sites;
}
async function resolveMissionForSite(
workspaceRoot: string,
siteId: string,
): Promise {
try {
const state = await readSystemState(workspaceRoot, siteId);
return state.currentMission;
} catch {
return null;
}
}
async function commandRequiresMission(
workspaceRoot: string,
commandName: string,
): Promise {
const registry = await getOrBuildWorkspaceRegistry(workspaceRoot);
if (!registry) return false;
const def = registry.commands.get(commandName);
if (!def) return false;
return !!(def.flags && def.flags.mission);
}
async function runOneSite(
workspaceRoot: string,
logger: KernelRuntimeContext["logger"],
runSpec: string,
siteId: string,
): Promise<{ exitCode: number; error?: string }> {
const isPipeline = runSpec.startsWith("pipeline ");
const target = isPipeline ? runSpec.slice("pipeline ".length) : runSpec;
const baseFlags: Record = {};
if (!isPipeline) {
const requiresMission = await commandRequiresMission(workspaceRoot, target);
if (requiresMission) {
const missionId = await resolveMissionForSite(workspaceRoot, siteId);
if (!missionId) {
const skipped: { exitCode: number; error?: string } = {
exitCode: -1,
error: "skipped(precondition): no active mission",
};
return skipped;
}
baseFlags.mission = missionId;
}
}
try {
if (isPipeline) {
const result = await executeKernelPipeline({
workspaceRoot,
pipelineName: target,
siteName: siteId,
flags: baseFlags,
});
const exitCode = Array.isArray(result) ? (result[0]?.exitCode ?? 0) : result.exitCode;
return { exitCode };
} else {
const result = await executeKernelCommand({
workspaceRoot,
commandName: target,
siteName: siteId,
argv: [],
});
const exitCode = Array.isArray(result) ? (result[0]?.exitCode ?? 0) : result.exitCode;
return { exitCode };
}
} catch (err) {
const failed: { exitCode: number; error?: string } = {
exitCode: 1,
error: err instanceof Error ? err.message : String(err),
};
return failed;
}
}
async function runWave(
workspaceRoot: string,
logger: KernelRuntimeContext["logger"],
runSpec: string,
waveName: WaveName,
sites: FleetSiteRecord[],
concurrency: number,
stopAfterFailures: number,
completedSites: Set,
journalPath: string,
opId: string,
failureCount: number,
): Promise<{ entries: FleetApplyReportEntry[]; failureCount: number; stop: boolean }> {
const entries: FleetApplyReportEntry[] = [];
let failures = failureCount;
let stop = false;
// Simple bounded concurrency using chunking
const remaining = sites.filter((s) => !completedSites.has(s.id));
for (let i = 0; i < remaining.length; i += concurrency) {
if (stop) break;
const chunk = remaining.slice(i, i + concurrency);
const results = await Promise.allSettled(
chunk.map(async (site) => {
const start = Date.now();
const stepName = `${waveName}:${site.id}`;
// Journal: step-started
await appendRecord(journalPath, {
kind: "step-started",
opId,
step: stepName,
seq: 0,
at: new Date().toISOString(),
});
const result = await runOneSite(workspaceRoot, logger, runSpec, site.id);
const durationMs = Date.now() - start;
let status: SiteStatus;
if (result.error?.startsWith("skipped(precondition)")) {
status = "skipped(precondition)";
} else if (result.exitCode !== 0) {
status = "failed";
failures++;
} else {
status = "ok";
}
// Journal: step-done (ok) or step-failed
if (status === "ok") {
await appendRecord(journalPath, {
kind: "step-done",
opId,
step: stepName,
seq: 0,
at: new Date().toISOString(),
meta: { durationMs },
});
} else if (status === "failed") {
await appendRecord(journalPath, {
kind: "step-failed",
opId,
step: stepName,
seq: 0,
at: new Date().toISOString(),
error: result.error ?? "failed",
});
} else {
// skipped(precondition)
await appendRecord(journalPath, {
kind: "step-skipped",
opId,
step: stepName,
seq: 0,
at: new Date().toISOString(),
reason: "already-satisfied",
});
}
return {
site: site.id,
wave: waveName,
status,
exitCode: result.exitCode,
durationMs,
error: result.error,
summary: "[apply] Done",
} as FleetApplyReportEntry;
}),
);
for (const r of results) {
if (r.status === "fulfilled") {
entries.push(r.value);
if (r.value.status === "ok") {
completedSites.add(r.value.site);
}
if (r.value.status === "skipped(precondition)") {
completedSites.add(r.value.site);
}
} else {
// Promise rejected — shouldn't happen but handle it
entries.push({
site: "unknown",
wave: waveName,
status: "failed",
exitCode: 1,
durationMs: 0,
error: String(r.reason),
});
failures++;
}
}
if (failures >= stopAfterFailures) {
stop = true;
// Mark remaining sites as skipped(stop-threshold)
for (let j = i + concurrency; j < remaining.length; j++) {
entries.push({
site: remaining[j].id,
wave: waveName,
status: "skipped(stop-threshold)",
exitCode: null,
durationMs: 0,
});
}
}
}
return { entries, failureCount: failures, stop };
}
export async function runFleetApply(
input: KernelCommandInput,
context: KernelRuntimeContext,
): Promise {
const { workspaceRoot, logger } = context;
const runSpec = flagString(input, "run");
if (!runSpec) {
return {
...failResult("fleet.apply", ["--run flag is required"]),
data: { error: "missing --run flag" },
};
}
const allSites = flagBool(input, "all-sites");
const sitesFlag = flagString(input, "sites");
const waveFilter = flagString(input, "wave") ?? "all";
const concurrency = flagInt(input, "concurrency", 1);
const stopAfterFailures = flagInt(input, "stop-after-failures", 1);
const resumeOpId = flagString(input, "resume");
// Resolve site set
let sites: FleetSiteRecord[];
const { systems } = await discoverSystems(workspaceRoot);
if (await isRegistryStale(workspaceRoot, systems, context.io)) {
sites = await regenerateRegistry(workspaceRoot, logger);
} else {
sites = await loadFleetSites(workspaceRoot, context.io);
if (sites.length === 0 && systems.length > 0) {
sites = await regenerateRegistry(workspaceRoot, logger);
}
}
// Filter by --sites
if (sitesFlag) {
const requested = new Set(sitesFlag.split(",").map((s) => s.trim()));
sites = sites.filter((s) => requested.has(s.id));
} else if (!allSites) {
return {
...failResult("fleet.apply", ["must specify --all-sites or --sites "]),
data: { error: "no site selection" },
};
}
// Empty fleet
if (sites.length === 0) {
const opId = resumeOpId ?? randomUUID();
const result: FleetApplyResult = {
opId,
command: runSpec,
wave: waveFilter,
totalSites: 0,
entries: [],
ok: 0,
failed: 0,
skipped: 0,
exitCode: 0,
};
return {
exitCode: 0,
summary: "[fleet.apply] 0 sites",
data: result,
};
}
// Partition into waves
const canarySites = sites.filter((s) => s.canary);
const restSites = sites.filter((s) => !s.canary);
// Wave filter
let canaryWave: FleetSiteRecord[] = [];
let restWave: FleetSiteRecord[] = [];
if (waveFilter === "canary") {
canaryWave = canarySites;
} else if (waveFilter === "rest") {
restWave = restSites;
} else {
canaryWave = canarySites;
restWave = restSites;
}
// Empty canary wave
if (waveFilter === "canary" && canaryWave.length === 0) {
const opId = resumeOpId ?? randomUUID();
const result: FleetApplyResult = {
opId,
command: runSpec,
wave: waveFilter,
totalSites: 0,
entries: [],
ok: 0,
failed: 0,
skipped: 0,
exitCode: 0,
};
return {
exitCode: 0,
summary: "[fleet.apply] 0 canary sites",
data: result,
};
}
// Journal setup
const opId = resumeOpId ?? randomUUID();
const journalPath = join(workspaceRoot, FLEET_OPERATIONS_DIR, `${opId}.jsonl`);
await context.io.mkdir(dirname(journalPath));
// Resume: load completed sites from journal
const completedSites = new Set();
if (resumeOpId && (await context.io.exists(journalPath))) {
const records = await readJournal(journalPath);
for (const rec of records) {
if (rec.kind === "step-done") {
const siteId = rec.step.split(":")[1];
if (siteId) completedSites.add(siteId);
}
}
logger.info(
`[fleet.apply] Resuming op ${opId}, ${completedSites.size} site(s) already completed`,
);
}
// Journal: op-started
await appendRecord(journalPath, {
kind: "op-started",
opId,
op: "fleet.apply",
missionId: "",
at: new Date().toISOString(),
platformVersion: "",
});
// Preflight: Cloudflare worker count
try {
const servicesPath = join(workspaceRoot, "services", "registry.yaml");
if (await context.io.exists(servicesPath)) {
const { parse: parseYaml } = await import("yaml");
const raw = await context.io.readFile(servicesPath);
const parsed = parseYaml(raw) as { services?: unknown[] };
const workerCount = parsed.services?.length ?? 0;
const totalWorkers = workerCount + sites.length;
if (totalWorkers > 450) {
logger.warn(
`[fleet.apply] Preflight: ${totalWorkers} Cloudflare workers projected (services: ${workerCount} + sites: ${sites.length}). Account ceiling is 500.`,
);
}
}
} catch {
// Non-fatal preflight
}
// Execute waves
const allEntries: FleetApplyReportEntry[] = [];
let failureCount = 0;
let stop = false;
// Canary wave
if (canaryWave.length > 0) {
logger.info(`[fleet.apply] Starting canary wave: ${canaryWave.length} site(s)`);
const canaryResult = await runWave(
workspaceRoot,
logger,
runSpec,
"canary",
canaryWave,
concurrency,
stopAfterFailures,
completedSites,
journalPath,
opId,
failureCount,
);
allEntries.push(...canaryResult.entries);
failureCount = canaryResult.failureCount;
stop = canaryResult.stop;
// Canary wave failure → rest wave never starts (hard rule)
if (canaryResult.failureCount > 0) {
stop = true;
logger.warn(
`[fleet.apply] Canary wave had ${canaryResult.failureCount} failure(s) — rest wave skipped`,
);
}
}
// Rest wave
if (restWave.length > 0 && !stop) {
logger.info(`[fleet.apply] Starting rest wave: ${restWave.length} site(s)`);
const restResult = await runWave(
workspaceRoot,
logger,
runSpec,
"rest",
restWave,
concurrency,
stopAfterFailures,
completedSites,
journalPath,
opId,
failureCount,
);
allEntries.push(...restResult.entries);
failureCount = restResult.failureCount;
}
// Journal: op-done
await appendRecord(journalPath, {
kind: "op-done",
opId,
at: new Date().toISOString(),
});
// Build result
const ok = allEntries.filter((e) => e.status === "ok").length;
const failed = allEntries.filter((e) => e.status === "failed").length;
const skipped = allEntries.filter((e) => e.status.startsWith("skipped")).length;
const result: FleetApplyResult = {
opId,
command: runSpec,
wave: waveFilter,
totalSites: sites.length,
entries: allEntries,
ok,
failed,
skipped,
exitCode: failed > 0 ? 1 : 0,
};
// Persist report
const reportPath = join(workspaceRoot, FLEET_REPORTS_DIR, `${opId}.json`);
await context.io.mkdir(dirname(reportPath));
await context.io.writeFile(reportPath, JSON.stringify(result, null, 2));
return {
exitCode: result.exitCode,
summary: `[fleet.apply] ${ok} ok, ${failed} failed, ${skipped} skipped (opId: ${opId})`,
data: result,
};
}