import { findActionEditCommit } from "@sixb/core/internal/actions" import { reportRunFailure } from "@sixb/core/internal/error-reporting" import { createSixbError } from "@sixb/core/internal/errors" import type { ActionRunFailure, ActionRunRecord } from "@sixb/core/storage" import { isTerminalActionRun } from "@sixb/core/storage" import { toActionRunFailure } from "../normalize" import type { ActionRunResult, RunActionJobInput } from "../types" export function requireFinishedAt(input: { readonly actionId: string readonly runId: string readonly finishedAt: Date | undefined }): Date { if (input.finishedAt) { return input.finishedAt } throw createSixbError( "internal.unexpected", `[SixbActionWorker] Action run '${input.runId}' finished without a finishedAt timestamp.`, { details: { actionId: input.actionId, runId: input.runId } } ) } /** * Decides whether a redelivered `running` run reached a resumable boundary. * * A succeeded writeback or an authoritative ontology commit for this run means the previous attempt * got far enough that resuming is safe; anything earlier is treated as a lost lease. */ export async function resolveRedeliveredRunningRun( input: RunActionJobInput, run: ActionRunRecord ): Promise< | { readonly kind: "resume"; readonly run: ActionRunRecord } | { readonly kind: "finished"; readonly result: ActionRunResult } > { let resolution: Awaited> try { resolution = await resolveRunningRunUnderFence(input, run) } catch (error) { const latest = await input.runtime.actionRunsStorage.getById({ projectId: input.runtime.id, id: input.job.id, }) if (latest && isTerminalActionRun(latest)) { return { kind: "finished", result: skippedResult(input, latest) } } throw error } if (resolution.kind === "resume") return resolution reportRedeliveryFailure(input, resolution.run, resolution.failure) return { kind: "finished", result: failedResult(input.job.id, input.job.actionId, resolution.run, resolution.failure), } } async function resolveRunningRunUnderFence(input: RunActionJobInput, run: ActionRunRecord) { return input.runtime.storage.transaction( async (storage) => { if (!storage.actionRuns) { throw createSixbError( "internal.unexpected", "[SixbActionWorker] Action workers require transactional Action materialization fencing.", { details: { actionId: input.job.actionId, runId: input.job.id } } ) } const locked = await storage.actionRuns.lockForMaterialization({ projectId: input.runtime.id, actionId: input.job.actionId, runId: run.id, }) const commit = await findActionEditCommit({ storage, projectId: input.runtime.id, runId: run.id, }) if (locked.writeback?.status === "succeeded" || commit) { return { kind: "resume" as const, run: locked } } const failedAt = new Date() const failure = redeliveryFailure(input.job.id, locked, failedAt) const finished = await storage.actionRuns.finish({ projectId: input.runtime.id, id: input.job.id, status: "failed", finishedAt: failedAt, error: failure, }) return { kind: "failed" as const, run: finished, failure } }, { isolation: "serializable" } ) } export function failedResult( runId: string, actionId: string, run: ActionRunRecord, failure: ActionRunFailure ): ActionRunResult { return { id: runId, actionId, subject: run.subject, status: "failed", startedAt: run.startedAt ?? run.queuedAt, finishedAt: requireFinishedAt({ actionId, runId, finishedAt: run.finishedAt }), error: failure, record: run, } } function redeliveryFailure(runId: string, run: ActionRunRecord, failedAt: Date): ActionRunFailure { return toActionRunFailure( { name: "ActionRunLeaseLostError", message: `Action run '${runId}' was redelivered while already running. The previous worker may have lost its queue lease or crashed before reaching a resumable phase boundary.`, }, run.phase ?? "validation", { actionId: run.actionId, runId, at: failedAt, } ) } function reportRedeliveryFailure( input: RunActionJobInput, run: ActionRunRecord, failure: ActionRunFailure ): void { const error = createSixbError( "internal.unexpected", `[SixbActionWorker] ${run.error?.message ?? `Action run '${run.id}' lost its lease.`}`, { details: { actionId: input.job.actionId, runId: input.job.id } } ) reportRunFailure(input.runtime.errorReporterHost, error, { projectId: input.runtime.id, attempt: input.attempt, runKind: "action", run: { runId: input.job.id, actionId: input.job.actionId, }, failure, }) } function skippedResult(input: RunActionJobInput, run: ActionRunRecord): ActionRunResult { return { id: input.job.id, actionId: input.job.actionId, subject: run.subject, status: run.status, skipped: true, record: run, } }