/*
orchestrator — drive certification runs through their state machine to a terminal verdict.
- Do not evaluate checks — the orchestrator coordinates, evaluators decide.
- RFC-1097: sweep — werkstatt-engine clean
Sweep batch 4: 73 Compass headers on headerless engine files (certification, component-runtime, isolation, evolution, testing), real KEY_DECISIONS on 75 files (kernel, cache, dht, swim, gitmesh, runtime), ~80 purpose expansions (CONTRACT-02/PURPOSE-02), non-goals on 13 CONTRACT-03 files, CS-07 history literal fix repo-wide (253 files). Policy: .template.ts/.template.astro excludedPaths. werkstatt-engine now 0 diagnostics.
*/
import type { GateChannel } from "../contracts/identifiers.ts";
import type { ReleaseCandidateV1 } from "../contracts/candidate.ts";
import type { CertificationPolicyBundleV1 } from "../contracts/policy-bundle.ts";
import type { CertificationProfileV1 } from "../profile/schemas.ts";
import type { EvidenceEnvelopeV1 } from "../contracts/evidence.ts";
import type { GateDecisionV1 } from "../contracts/decisions.ts";
import type { Sha256Digest } from "@warpgogol/werkstatt-shared/fingerprint";
export type OrchestratorState =
| "idle"
| "locked"
| "planning"
| "producing"
| "ingesting"
| "evaluating"
| "syncing"
| "completed"
| "failed"
| "cancelled";
export interface ProducerDependencyNodeV1 {
producerId: string;
dependsOn: string[];
}
export interface ProducerPlanV1 {
ok: true;
readonly batches: readonly (readonly string[])[];
readonly producerIds: readonly string[];
}
export interface ProducerPlanFailureV1 {
ok: false;
ruleId: "CERT-ORCHESTRATOR-01" | "CERT-ORCHESTRATOR-02";
message: string;
}
export type ProducerPlanOutcomeV1 = ProducerPlanV1 | ProducerPlanFailureV1;
export function planProducers(nodes: readonly ProducerDependencyNodeV1[]): ProducerPlanOutcomeV1 {
const nodeMap = new Map();
for (const node of nodes) {
if (nodeMap.has(node.producerId)) {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-01",
message: `duplicate producerId "${node.producerId}" in dependency graph`,
};
}
nodeMap.set(node.producerId, node);
}
for (const node of nodes) {
for (const dep of node.dependsOn) {
if (!nodeMap.has(dep)) {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-02",
message: `producer "${node.producerId}" depends on unknown producer "${dep}"`,
};
}
}
}
const allIds = [...nodeMap.keys()];
const sorted: string[] = [];
const done = new Set();
const inProgress = new Set();
function topoSort(id: string): boolean {
if (done.has(id)) return true;
if (inProgress.has(id)) return false;
inProgress.add(id);
const node = nodeMap.get(id)!;
for (const dep of node.dependsOn) {
if (!topoSort(dep)) return false;
}
inProgress.delete(id);
done.add(id);
sorted.push(id);
return true;
}
for (const id of allIds) {
if (!topoSort(id)) {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-02",
message: `cycle detected in producer dependency graph involving "${id}"`,
};
}
}
const levelMap = new Map();
for (const id of sorted) {
const node = nodeMap.get(id)!;
let maxDepLevel = -1;
for (const dep of node.dependsOn) {
const depLevel = levelMap.get(dep) ?? 0;
if (depLevel > maxDepLevel) maxDepLevel = depLevel;
}
levelMap.set(id, maxDepLevel + 1);
}
const maxLevel = sorted.length > 0 ? Math.max(...[...levelMap.values()]) : -1;
const batches: string[][] = [];
for (let i = 0; i <= maxLevel; i++) {
batches.push(sorted.filter((id) => levelMap.get(id) === i));
}
return {
ok: true as const,
batches,
producerIds: sorted,
};
}
export interface GateLockV1 {
releaseId: string;
gate: GateChannel;
operationId: string;
acquiredAt: string;
holder: string;
}
export interface GateLockAcquireInputV1 {
releaseId: string;
gate: GateChannel;
operationId: string;
acquiredAt: string;
holder: string;
}
export interface GateLockManagerV1 {
acquire(input: GateLockAcquireInputV1): GateLockAcquireOutcomeV1;
release(operationId: string): boolean;
isLocked(releaseId: string, gate: GateChannel): boolean;
getLock(releaseId: string, gate: GateChannel): GateLockV1 | null;
}
export interface GateLockAcquireResultV1 {
ok: true;
lock: GateLockV1;
}
export interface GateLockAcquireFailureV1 {
ok: false;
ruleId: "CERT-ORCHESTRATOR-03";
message: string;
}
export type GateLockAcquireOutcomeV1 = GateLockAcquireResultV1 | GateLockAcquireFailureV1;
export function createGateLockManager(): GateLockManagerV1 {
const locks = new Map();
const key = (releaseId: string, gate: GateChannel) => `${releaseId}:${gate}`;
return {
acquire(input: GateLockAcquireInputV1): GateLockAcquireOutcomeV1 {
const k = key(input.releaseId, input.gate);
const existing = locks.get(k);
if (existing && existing.operationId !== input.operationId) {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-03",
message: `gate "${input.gate}" for release "${input.releaseId}" is already locked by operation "${existing.operationId}"`,
};
}
const lock: GateLockV1 = {
releaseId: input.releaseId,
gate: input.gate,
operationId: input.operationId,
acquiredAt: input.acquiredAt,
holder: input.holder,
};
locks.set(k, lock);
return { ok: true, lock };
},
release(operationId: string): boolean {
for (const [k, lock] of locks) {
if (lock.operationId === operationId) {
locks.delete(k);
return true;
}
}
return false;
},
isLocked(releaseId: string, gate: GateChannel): boolean {
return locks.has(key(releaseId, gate));
},
getLock(releaseId: string, gate: GateChannel): GateLockV1 | null {
return locks.get(key(releaseId, gate)) ?? null;
},
};
}
export interface ProducerExecutionConfigV1 {
operationId: string;
timestamp: string;
maxParallelism: number;
timeoutMs: number;
maxRetries: number;
retryDelayMs: number;
}
export const DEFAULT_PRODUCER_CONFIG: Omit =
{
maxParallelism: 4,
timeoutMs: 30000,
maxRetries: 2,
retryDelayMs: 1000,
};
export interface ProducerExecutionInputV1 {
producerId: string;
candidate: ReleaseCandidateV1;
profile: CertificationProfileV1;
policyBundle: CertificationPolicyBundleV1;
config: ProducerExecutionConfigV1;
}
export type ProducerExecutionHandlerV1 = (
input: ProducerExecutionInputV1,
) => Promise;
export interface ProgressEventV1 {
operationId: string;
producerId: string;
status: "started" | "completed" | "failed" | "retried" | "cancelled" | "skipped";
timestamp: string;
attempt: number;
detail?: string;
}
export type ProgressCallbackV1 = (event: ProgressEventV1) => void;
export interface ProducerExecutionResultV1 {
ok: true;
evidence: EvidenceEnvelopeV1[];
events: ProgressEventV1[];
}
export interface ProducerExecutionFailureV1 {
ok: false;
ruleId: "CERT-ORCHESTRATOR-04" | "CERT-ORCHESTRATOR-05" | "CERT-ORCHESTRATOR-06";
message: string;
events: ProgressEventV1[];
partialEvidence: EvidenceEnvelopeV1[];
}
export type ProducerExecutionOutcomeV1 = ProducerExecutionResultV1 | ProducerExecutionFailureV1;
export async function executeProducers(
plan: ProducerPlanV1,
handler: ProducerExecutionHandlerV1,
baseInput: Omit,
onProgress?: ProgressCallbackV1,
): Promise {
const allEvidence: EvidenceEnvelopeV1[] = [];
const events: ProgressEventV1[] = [];
const timestamp = baseInput.config.timestamp;
for (const batch of plan.batches) {
const semaphore = new Semaphore(baseInput.config.maxParallelism);
const batchResults = await Promise.allSettled(
batch.map(async (producerId) => {
const acquired = await semaphore.acquire();
if (!acquired) {
const event: ProgressEventV1 = {
operationId: baseInput.config.operationId,
producerId,
status: "skipped",
timestamp,
attempt: 0,
};
events.push(event);
onProgress?.(event);
return null;
}
try {
let lastError: Error | null = null;
for (let attempt = 0; attempt <= baseInput.config.maxRetries; attempt++) {
const startEvent: ProgressEventV1 = {
operationId: baseInput.config.operationId,
producerId,
status: "started",
timestamp,
attempt,
};
events.push(startEvent);
onProgress?.(startEvent);
try {
const evidence = await withTimeout(
handler({ ...baseInput, producerId }),
baseInput.config.timeoutMs,
);
const completeEvent: ProgressEventV1 = {
operationId: baseInput.config.operationId,
producerId,
status: "completed",
timestamp,
attempt,
};
events.push(completeEvent);
onProgress?.(completeEvent);
return evidence;
} catch (err) {
lastError = err instanceof Error ? err : new Error(String(err));
if (attempt < baseInput.config.maxRetries) {
const retryEvent: ProgressEventV1 = {
operationId: baseInput.config.operationId,
producerId,
status: "retried",
timestamp,
attempt,
detail: lastError.message,
};
events.push(retryEvent);
onProgress?.(retryEvent);
await sleep(baseInput.config.retryDelayMs);
}
}
}
const failEvent: ProgressEventV1 = {
operationId: baseInput.config.operationId,
producerId,
status: "failed",
timestamp,
attempt: baseInput.config.maxRetries,
detail: lastError?.message ?? "unknown error",
};
events.push(failEvent);
onProgress?.(failEvent);
throw lastError ?? new Error("unknown error");
} finally {
semaphore.release();
}
}),
);
for (const result of batchResults) {
if (result.status === "fulfilled" && result.value !== null) {
allEvidence.push(result.value);
} else if (result.status === "rejected") {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-04",
message: `producer execution failed: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
events,
partialEvidence: allEvidence,
};
}
}
}
return { ok: true, evidence: allEvidence, events };
}
function sleep(ms: number): Promise {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function withTimeout(promise: Promise, ms: number): Promise {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
reject(new Error(`timeout after ${ms}ms`));
}, ms);
promise.then(
(val) => {
clearTimeout(timer);
resolve(val);
},
(err) => {
clearTimeout(timer);
reject(err);
},
);
});
}
class Semaphore {
private available: number;
private waiters: Array<() => void> = [];
constructor(private readonly max: number) {
this.available = max;
}
async acquire(): Promise {
if (this.available > 0) {
this.available--;
return true;
}
return new Promise((resolve) => {
this.waiters.push(() => {
this.available--;
resolve(true);
});
});
}
release(): void {
this.available++;
const waiter = this.waiters.shift();
if (waiter) {
waiter();
}
}
}
export interface OrchestratorOperationV1 {
operationId: string;
releaseId: string;
gate: GateChannel;
candidate: ReleaseCandidateV1;
profile: CertificationProfileV1;
policyBundle: CertificationPolicyBundleV1;
state: OrchestratorState;
startedAt: string;
resumePoint?: string;
evidence: EvidenceEnvelopeV1[];
decision?: GateDecisionV1;
actionPackLocator?: string;
dossierRootHash?: Sha256Digest;
}
export interface OrchestratorResumePointV1 {
batchIndex: number;
producerIdsCompleted: Set;
}
export function createResumePoint(): OrchestratorResumePointV1 {
return {
batchIndex: 0,
producerIdsCompleted: new Set(),
};
}
export interface OrchestratorResumeResultV1 {
ok: true;
resumePoint: OrchestratorResumePointV1;
}
export interface OrchestratorResumeFailureV1 {
ok: false;
ruleId: "CERT-ORCHESTRATOR-07";
message: string;
}
export type OrchestratorResumeOutcomeV1 = OrchestratorResumeResultV1 | OrchestratorResumeFailureV1;
export function computeResumePoint(
operation: OrchestratorOperationV1,
plan: ProducerPlanV1,
): OrchestratorResumeOutcomeV1 {
const completed = new Set();
for (const ev of operation.evidence) {
const producerId = ev.producerId;
if (producerId) {
completed.add(producerId);
}
}
let batchIndex = 0;
for (let i = 0; i < plan.batches.length; i++) {
const batch = plan.batches[i];
const allComplete = batch.every((id) => completed.has(id));
if (!allComplete) {
batchIndex = i;
break;
}
batchIndex = i + 1;
}
if (batchIndex >= plan.batches.length && plan.batches.length > 0) {
return {
ok: false,
ruleId: "CERT-ORCHESTRATOR-07",
message: "all producer batches are already complete — nothing to resume",
};
}
return {
ok: true,
resumePoint: { batchIndex, producerIdsCompleted: completed },
};
}