import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs'; const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' }); export async function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise { const queueUrl = process.env.SQS_CRAWL_QUEUE_URL; if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured'); await sqs.send(new SendMessageCommand({ QueueUrl: queueUrl, MessageBody: JSON.stringify({ jobId, tenantId, config }), })); } export async function startSqsWorker(): Promise { const queueUrl = process.env.SQS_CRAWL_QUEUE_URL; if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured'); const { processJob } = await import('./job-queue.js'); console.log('[sqs-worker] starting, queue:', queueUrl); while (true) { try { const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({ QueueUrl: queueUrl, WaitTimeSeconds: 20, MaxNumberOfMessages: 2, })); await Promise.all(Messages.map(async (msg) => { let jobId = '?'; try { const data = JSON.parse(msg.Body!); jobId = data.jobId; await processJob(data.jobId, data.config); await sqs.send(new DeleteMessageCommand({ QueueUrl: queueUrl, ReceiptHandle: msg.ReceiptHandle!, })); } catch (err: any) { console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message); // Do NOT delete — SQS redrive policy + DLQ handles retry } })); } catch (err: any) { console.error('[sqs-worker] receive error:', err.message); await new Promise((r) => setTimeout(r, 2000)); } } }