import { createHash } from "node:crypto"; import * as fs from "node:fs"; import * as path from "node:path"; import { ExternalJobProviderError, type ExternalJobHandle, type ExternalJobResult, type ExternalJobState } from "../../api/external-job-provider.ts"; import type { ExternalJobStatus } from "../../shared/types.ts"; import { requestExternalJobOperation } from "./external-job-bridge.ts"; const STATUS_POLL_INTERVAL_MS = 1_000; export interface ExternalJobRunResult { output: string; exitCode: number; error?: string; timedOut?: boolean; stopped?: boolean; externalJob: ExternalJobStatus; } export function externalJobPromptDigest(prompt: string): string { return createHash("sha256").update(prompt).digest("hex"); } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function terminal(state: ExternalJobState): boolean { return state === "completed" || state === "failed" || state === "stopped" || state === "blocked"; } function formatError(error: ExternalJobProviderError, provider: string): string { const blocking = error.blockingJobId ? ` Blocking provider job: ${error.blockingJobId}.` : ""; return `External-job provider '${provider}' failed closed (${error.code}): ${error.message}.${blocking}`; } function isNotFound(error: unknown): boolean { return typeof error === "object" && error !== null && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT"; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function malformedStatus(statusPath: string): ExternalJobProviderError { return new ExternalJobProviderError(`Malformed external-job status '${statusPath}'. Refusing to redispatch the prompt.`, { code: "status-unreadable" }); } function parseExternalJobStatus(value: unknown, statusPath: string): ExternalJobStatus { if (!isRecord(value)) throw malformedStatus(statusPath); const provider = value.provider; const promptDigest = value.promptDigest; const state = value.state; if (typeof provider !== "string" || provider.length === 0) throw malformedStatus(statusPath); if (typeof promptDigest !== "string" || promptDigest.length === 0) throw malformedStatus(statusPath); if (state !== "queued" && state !== "running" && state !== "completed" && state !== "failed" && state !== "stopped" && state !== "blocked") throw malformedStatus(statusPath); if (value.options !== undefined && !isRecord(value.options)) throw malformedStatus(statusPath); const providerJobId = value.providerJobId; const handleUrl = value.handleUrl; const conversationUrl = value.conversationUrl; const resultArtifactPath = value.resultArtifactPath; const failureCode = value.failureCode; const failureMessage = value.failureMessage; const blockingJobId = value.blockingJobId; const startedAt = value.startedAt; const updatedAt = value.updatedAt; if (providerJobId !== undefined && (typeof providerJobId !== "string" || providerJobId.length === 0)) throw malformedStatus(statusPath); if (handleUrl !== undefined && typeof handleUrl !== "string") throw malformedStatus(statusPath); if (conversationUrl !== undefined && typeof conversationUrl !== "string") throw malformedStatus(statusPath); if (resultArtifactPath !== undefined && typeof resultArtifactPath !== "string") throw malformedStatus(statusPath); if (failureCode !== undefined && typeof failureCode !== "string") throw malformedStatus(statusPath); if (failureMessage !== undefined && typeof failureMessage !== "string") throw malformedStatus(statusPath); if (blockingJobId !== undefined && typeof blockingJobId !== "string") throw malformedStatus(statusPath); if (startedAt !== undefined && typeof startedAt !== "number") throw malformedStatus(statusPath); if (updatedAt !== undefined && typeof updatedAt !== "number") throw malformedStatus(statusPath); return { provider, promptDigest, options: value.options ?? {}, state, ...(providerJobId ? { providerJobId } : {}), ...(handleUrl ? { handleUrl } : {}), ...(conversationUrl ? { conversationUrl } : {}), ...(resultArtifactPath ? { resultArtifactPath } : {}), ...(failureCode ? { failureCode } : {}), ...(failureMessage ? { failureMessage } : {}), ...(blockingJobId ? { blockingJobId } : {}), ...(startedAt !== undefined ? { startedAt } : {}), ...(updatedAt !== undefined ? { updatedAt } : {}), }; } function readExistingExternalJob(asyncDir: string, stepIndex: number): ExternalJobStatus | undefined { const statusPath = path.join(asyncDir, "status.json"); let raw: string; try { raw = fs.readFileSync(statusPath, "utf-8"); } catch (error) { if (isNotFound(error)) return undefined; throw new ExternalJobProviderError(`Unreadable external-job status '${statusPath}'. Refusing to redispatch the prompt.`, { code: "status-unreadable", cause: error }); } let parsed: unknown; try { parsed = JSON.parse(raw); } catch (error) { throw new ExternalJobProviderError(`Malformed external-job status '${statusPath}'. Refusing to redispatch the prompt.`, { code: "status-unreadable", cause: error }); } if (!isRecord(parsed) || !Array.isArray(parsed.steps)) throw malformedStatus(statusPath); const step = parsed.steps[stepIndex]; if (step === undefined || !isRecord(step)) throw malformedStatus(statusPath); if (step.externalJob === undefined) return undefined; return parseExternalJobStatus(step.externalJob, statusPath); } function statusFromHandle(input: { provider: string; promptDigest: string; options: Record; startedAt?: number; previous?: ExternalJobStatus; handle: ExternalJobHandle; resultArtifactPath?: string; }): ExternalJobStatus { return { provider: input.provider, providerJobId: input.handle.providerJobId, promptDigest: input.promptDigest, options: input.options, handleUrl: input.handle.handleUrl ?? input.previous?.handleUrl, conversationUrl: input.handle.conversationUrl ?? input.previous?.conversationUrl, resultArtifactPath: input.resultArtifactPath ?? input.previous?.resultArtifactPath, state: input.handle.state, failureCode: input.handle.failureCode, failureMessage: input.handle.failureMessage, blockingJobId: input.handle.blockingJobId, startedAt: input.previous?.startedAt ?? input.startedAt, updatedAt: Date.now(), }; } function failureStatus(input: { provider: string; promptDigest: string; options: Record; previous?: ExternalJobStatus; code: string; message: string; blockingJobId?: string; }): ExternalJobStatus { return { provider: input.provider, providerJobId: input.previous?.providerJobId, promptDigest: input.promptDigest, options: input.options, handleUrl: input.previous?.handleUrl, conversationUrl: input.previous?.conversationUrl, resultArtifactPath: input.previous?.resultArtifactPath, state: input.blockingJobId ? "blocked" : "failed", failureCode: input.code, failureMessage: input.message, blockingJobId: input.blockingJobId, startedAt: input.previous?.startedAt, updatedAt: Date.now(), }; } function resultOutput(result: ExternalJobResult, artifactPath: string | undefined): string { if (result.output?.trim()) return result.output.trim(); if (artifactPath) return `External job finished. Result artifact: ${artifactPath}`; return "External job finished without text output."; } function blocksStartRedispatch(status: ExternalJobStatus, provider: string, promptDigest: string): boolean { return status.provider === provider && status.promptDigest === promptDigest && !status.providerJobId && status.failureCode !== "provider-unavailable"; } export async function runExternalJob(input: { provider: string; options?: Record; cwd: string; prompt: string; asyncDir: string; stepIndex: number; runId: string; agent: string; sessionId?: string; registerTimeout?: (stop: (() => void) | undefined) => void; registerStop?: (stop: (() => void) | undefined) => void; timeoutMessage?: string; stopMessage?: string; onExternalJob?: (status: ExternalJobStatus) => void; }): Promise { const provider = input.provider; const options = input.options ?? {}; const promptDigest = externalJobPromptDigest(input.prompt); const startedAt = Date.now(); let timedOut = false; let stopped = false; let current: ExternalJobStatus | undefined; const timeout = () => { timedOut = true; }; const stop = () => { stopped = true; }; input.registerTimeout?.(timeout); input.registerStop?.(stop); const publish = (status: ExternalJobStatus) => { current = status; input.onExternalJob?.(status); }; const localCancellation = () => { if (!timedOut && !stopped) return undefined; const message = stopped ? input.stopMessage ?? `Subagent stopped locally before external provider '${provider}' returned a job id. The provider start may still be running.` : input.timeoutMessage ?? `Subagent timed out locally before external provider '${provider}' returned a job id. The provider start may still be running.`; return new ExternalJobProviderError(message, { code: stopped ? "local-stop" : "local-timeout" }); }; try { current = readExistingExternalJob(input.asyncDir, input.stepIndex); let handle: ExternalJobHandle; if (current?.providerJobId) { if (current.provider !== provider || current.promptDigest !== promptDigest) { const message = `Existing external job '${current.providerJobId}' does not match provider or prompt digest. Refusing to redispatch prompt.`; const status = failureStatus({ provider, promptDigest, options, previous: current, code: "recovery-mismatch", message }); publish(status); return { output: message, exitCode: 1, error: message, externalJob: status }; } handle = await requestExternalJobOperation(input.asyncDir, { operation: "reattach", provider, providerJobId: current.providerJobId }); } else { if (current && blocksStartRedispatch(current, provider, promptDigest)) { const message = `External-job start for provider '${provider}' previously ended without a durable provider job id. Refusing to redispatch the prompt automatically.`; const status = failureStatus({ provider, promptDigest, options, previous: current, code: "start-redispatch-blocked", message }); publish(status); return { output: message, exitCode: 1, error: message, externalJob: status }; } publish({ provider, promptDigest, options, state: "queued", startedAt, updatedAt: Date.now() }); handle = await requestExternalJobOperation(input.asyncDir, { operation: "start", provider, start: { prompt: input.prompt, promptDigest, cwd: input.cwd, runId: input.runId, stepIndex: input.stepIndex, agent: input.agent, options, ...(input.sessionId ? { sessionId: input.sessionId } : {}), }, }, undefined, localCancellation); } publish(statusFromHandle({ provider, promptDigest, options, startedAt, previous: current, handle })); while (!terminal(handle.state)) { if (timedOut || stopped) { const message = stopped ? input.stopMessage ?? `Subagent stopped locally; external provider job '${handle.providerJobId}' may still be running.` : input.timeoutMessage ?? `Subagent timed out locally; external provider job '${handle.providerJobId}' may still be running.`; return { output: message, exitCode: 1, error: message, ...(timedOut ? { timedOut: true } : {}), ...(stopped ? { stopped: true } : {}), externalJob: current! }; } await sleep(STATUS_POLL_INTERVAL_MS); handle = await requestExternalJobOperation(input.asyncDir, { operation: "status", provider, providerJobId: handle.providerJobId }); publish(statusFromHandle({ provider, promptDigest, options, previous: current, handle })); } const result = await requestExternalJobOperation(input.asyncDir, { operation: "result", provider, providerJobId: handle.providerJobId }); let artifactPath = result.artifactPath; if (!artifactPath && result.output !== undefined) { artifactPath = path.join(input.asyncDir, `external-job-${input.stepIndex}.result.md`); fs.writeFileSync(artifactPath, result.output, "utf-8"); } const finalStatus = statusFromHandle({ provider, promptDigest, options, previous: current, handle: result, resultArtifactPath: artifactPath }); publish(finalStatus); const output = resultOutput(result, artifactPath); const error = result.state === "completed" ? undefined : result.failureMessage ?? `External job ${result.state}.`; return { output, exitCode: result.state === "completed" ? 0 : 1, ...(error ? { error } : {}), externalJob: finalStatus }; } catch (error) { const providerError = error instanceof ExternalJobProviderError ? error : new ExternalJobProviderError(error instanceof Error ? error.message : String(error), { code: "provider-error", cause: error }); const message = formatError(providerError, provider); const status = failureStatus({ provider, promptDigest, options, previous: current, code: providerError.code, message, ...(providerError.blockingJobId ? { blockingJobId: providerError.blockingJobId } : {}) }); publish(status); return { output: message, exitCode: 1, error: message, ...(timedOut ? { timedOut: true } : {}), ...(stopped ? { stopped: true } : {}), externalJob: status }; } finally { input.registerTimeout?.(undefined); input.registerStop?.(undefined); } }