import type { Job, Worker } from "bullmq"; import { inject, injectable } from "@codemation/core"; import type { HumanTaskStore } from "@codemation/core"; import { CodemationTelemetryAttributeNames, HumanTaskStoreToken } from "@codemation/core"; import { Engine } from "@codemation/core/bootstrap"; import { ApplicationTokens } from "../applicationTokens"; import type { AppConfig } from "../presentation/config/AppConfig"; import { RedisConnectionOptionsFactory } from "../infrastructure/scheduler/bullmq/RedisConnectionOptionsFactory"; import type { HitlTimeoutJobPayload } from "./HitlTimeoutJobScheduler"; import { HitlTimeoutJobScheduler } from "./HitlTimeoutJobScheduler"; import { ResumeTelemetryContextForRun } from "../application/telemetry/ResumeTelemetryContextForRun"; @injectable() export class HitlTimeoutWorker { private readonly taskStore: HumanTaskStore; private worker: Worker | null = null; private readonly connectionOptions: Readonly> | null; constructor( @inject(HumanTaskStoreToken) taskStore: HumanTaskStore | undefined, @inject(Engine) private readonly engine: Engine, @inject(HitlTimeoutJobScheduler) private readonly scheduler: HitlTimeoutJobScheduler, @inject(ApplicationTokens.AppConfig) appConfig: AppConfig, @inject(ResumeTelemetryContextForRun) private readonly resumeTelemetry: ResumeTelemetryContextForRun, ) { if (!taskStore) { throw new Error("HitlTimeoutWorker: HumanTaskStore is not registered."); } this.taskStore = taskStore; const redisUrl = appConfig.scheduler.kind === "bullmq" ? (appConfig.scheduler.redisUrl ?? null) : null; this.connectionOptions = redisUrl === null ? null : RedisConnectionOptionsFactory.fromConfig({ url: redisUrl }); } async start(): Promise { if (!this.connectionOptions) return; const { Worker } = await import("bullmq"); this.worker = new Worker( this.scheduler.getQueueName(), async (job: Job) => { await this.processJob(job); }, { connection: this.connectionOptions as never }, ); } async stop(): Promise { if (this.worker) { await this.worker.close(); this.worker = null; } } async processTimeoutForTask(taskId: string): Promise { const task = await this.taskStore.findById(taskId); if (!task) return; if (task.status !== "pending") return; const now = new Date(); if (task.onTimeout === "auto-accept") { await this.taskStore.markAutoAccepted(taskId); const telemetry = await this.resumeTelemetry.forTask(taskId); await telemetry?.addSpanEvent({ name: "hitl.task.timed_out", attributes: { [CodemationTelemetryAttributeNames.hitlTaskId]: taskId, policy: "auto-accept", }, }); await this.engine.resumeRun({ runId: task.runId, taskId: task.id, resumeContext: { decision: { kind: "auto_accepted", at: now }, delivery: task.deliveryRef ?? null, task: { taskId: task.id, runId: task.runId, nodeId: task.nodeId, expiresAt: task.expiresAt, resumeUrl: "", }, }, }); } else { await this.taskStore.markTimedOut(taskId); const telemetry = await this.resumeTelemetry.forTask(taskId); await telemetry?.addSpanEvent({ name: "hitl.task.timed_out", attributes: { [CodemationTelemetryAttributeNames.hitlTaskId]: taskId, policy: "halt", }, }); await this.engine.resumeRun({ runId: task.runId, taskId: task.id, resumeContext: { decision: { kind: "timed_out", at: now }, delivery: task.deliveryRef ?? null, task: { taskId: task.id, runId: task.runId, nodeId: task.nodeId, expiresAt: task.expiresAt, resumeUrl: "", }, }, }); } } private async processJob(job: Job): Promise { const data = job.data as HitlTimeoutJobPayload; if (!data || data.kind !== "hitl.timeout") { throw new Error(`Unexpected job payload for hitl.timeout queue: ${JSON.stringify(data)}`); } await this.processTimeoutForTask(data.taskId); } }