/** * Standalone worker process entrypoint for a dedicated crawler/worker container, * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json). * * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT * start any background loop itself, or importing a single helper from @detiq/crawler * would silently spin up a second worker inside the API process too. This file is * the actual entrypoint that keeps a dedicated worker container's process alive and * consuming jobs: `node dist/worker.js`. */ import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js'; import { logger } from './logger.js'; async function main() { logger.info('[worker] starting standalone crawler worker process'); // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once. startJobWorkerLoop(); // Daily data-retention purge: runs immediately on startup then every 24 h. // Only runs in the dedicated worker container (this file), not in the API // process, so there is exactly one purge sweep regardless of API replica count. const runPurge = () => { processDataRetentionPurge().catch((err) => { logger.error({ err: String(err) }, '[worker] data-retention purge failed'); }); }; runPurge(); setInterval(runPurge, 24 * 60 * 60 * 1000); // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets. if (process.env.JOB_QUEUE_BACKEND === 'sqs') { const { startSqsWorker } = await import('./sqs-queue.js'); startSqsWorker().catch((err) => { logger.error({ err: String(err) }, '[worker] SQS worker crashed'); process.exit(1); }); } } process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); }); process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); }); main().catch((err) => { logger.error({ err: String(err) }, '[worker] fatal startup error'); process.exit(1); });