import { defineBatchStrategyMap, type BatchOperationStrategy, } from './batching-types'; const FULLENRICH_BATCH_ITEM_KEY = 'deepline_batch_item_key'; const FULLENRICH_BATCH_SIZE = 100; type FullEnrichContactRow = Record & { custom?: Record; }; type FullEnrichBulkPayload = { name: string; webhook_url?: string; wait_for_completion?: boolean; poll_interval_ms?: number; max_wait_ms?: number; data: FullEnrichContactRow[]; }; type FullEnrichBulkResult = Record & { data?: unknown[]; }; function sortRecord(value: Record): Record { const entries = Object.entries(value) .filter(([, entry]) => entry !== undefined) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, entry]) => [key, stableValue(entry)]); return Object.fromEntries(entries); } function stableValue(value: unknown): unknown { if (Array.isArray(value)) { return value.map((entry) => stableValue(entry)); } if (value && typeof value === 'object') { return sortRecord(value as Record); } return value; } function stableStringify(value: unknown): string { return JSON.stringify(stableValue(value)); } function readPath(root: unknown, path: string): unknown { const parts = path.split('.'); let current: unknown = root; for (const part of parts) { if (!current || typeof current !== 'object') { return undefined; } current = (current as Record)[part]; } return current; } function readArrayAtPaths(root: unknown, paths: string[]): unknown[] | null { for (const path of paths) { const value = readPath(root, path); if (Array.isArray(value)) { return value; } } return null; } function readRows(payload: FullEnrichBulkPayload): FullEnrichContactRow[] { return Array.isArray(payload.data) ? payload.data : []; } function isOneRowPayload(payload: FullEnrichBulkPayload): boolean { return readRows(payload).length === 1; } function controlKey(payload: FullEnrichBulkPayload): string { return stableStringify({ webhook_url: payload.webhook_url ?? null, wait_for_completion: payload.wait_for_completion ?? null, poll_interval_ms: payload.poll_interval_ms ?? null, max_wait_ms: payload.max_wait_ms ?? null, }); } function rowIdentity(row: FullEnrichContactRow): string { return stableStringify({ first_name: row.first_name ?? null, last_name: row.last_name ?? null, domain: row.domain ?? null, company_name: row.company_name ?? null, linkedin_url: row.linkedin_url ?? null, enrich_fields: row.enrich_fields ?? null, custom: row.custom ?? null, }); } function shortStableHash(value: string): string { let hash = 0x811c9dc5; for (let index = 0; index < value.length; index += 1) { hash ^= value.charCodeAt(index); hash = Math.imul(hash, 0x01000193) >>> 0; } return hash.toString(36); } function itemKey(payload: FullEnrichBulkPayload, index = 0): string { return `dl_${index}_${shortStableHash(rowIdentity(readRows(payload)[0] ?? {}))}`; } function withBatchItemKey( row: FullEnrichContactRow, key: string, ): FullEnrichContactRow { return { ...row, custom: { ...(row.custom ?? {}), [FULLENRICH_BATCH_ITEM_KEY]: key, }, }; } function extractResultRows(fullResult: FullEnrichBulkResult | unknown) { return readArrayAtPaths(fullResult, ['data']) ?? []; } function readResultItemKey(row: unknown): string | null { if (!row || typeof row !== 'object' || Array.isArray(row)) { return null; } const custom = (row as Record).custom; if (!custom || typeof custom !== 'object' || Array.isArray(custom)) { return null; } const key = (custom as Record)[FULLENRICH_BATCH_ITEM_KEY]; return typeof key === 'string' && key.trim() ? key.trim() : null; } const fullenrichBulkSelfBatchStrategy: BatchOperationStrategy< FullEnrichBulkPayload, FullEnrichBulkPayload, FullEnrichBulkResult, { data: FullEnrichBulkResult }, unknown > = { sourceOperation: 'fullenrich_bulk_enrich', batchOperation: 'fullenrich_bulk_enrich', kind: 'identifier_batch', maxBatchSize: FULLENRICH_BATCH_SIZE, canBatchWith(left, right) { return ( isOneRowPayload(left) && isOneRowPayload(right) && controlKey(left) === controlKey(right) ); }, toBucketKey(payload) { if (!isOneRowPayload(payload)) { return `fullenrich_bulk_enrich:passthrough:${stableStringify(payload)}`; } return `fullenrich_bulk_enrich:${controlKey(payload)}`; }, toItemKey(payload) { return itemKey(payload); }, compile(payloads) { const first = payloads[0]; const rows = payloads.flatMap((payload, index) => readRows(payload).map((row) => withBatchItemKey(row, itemKey(payload, index)), ), ); return { batchOperation: 'fullenrich_bulk_enrich', batchPayload: { name: payloads.length === 1 ? first?.name || 'deepline-fullenrich-batch' : `deepline-fullenrich-batch-${payloads.length}`, ...(first?.webhook_url ? { webhook_url: first.webhook_url } : {}), ...(first?.wait_for_completion !== undefined ? { wait_for_completion: first.wait_for_completion } : {}), ...(first?.poll_interval_ms !== undefined ? { poll_interval_ms: first.poll_interval_ms } : {}), ...(first?.max_wait_ms !== undefined ? { max_wait_ms: first.max_wait_ms } : {}), data: rows, }, items: payloads.map((payload, index) => ({ itemKey: itemKey(payload, index), payload, })), }; }, splitResult(fullResult, compiled) { const resultRows = extractResultRows(fullResult); const rowsByItemKey = new Map(); for (const row of resultRows) { const key = readResultItemKey(row); if (key) { rowsByItemKey.set(key, row); } } return compiled.items.map((item, index) => { const matchedRow = rowsByItemKey.get(item.itemKey) ?? (index < resultRows.length ? resultRows[index] : null); const singleResult = { ...fullResult, data: matchedRow ? [matchedRow] : [], }; return { itemKey: item.itemKey, result: { data: singleResult }, rawResult: matchedRow, }; }); }, }; export const fullenrichBatchStrategies = defineBatchStrategyMap({ fullenrich_bulk_enrich: fullenrichBulkSelfBatchStrategy, });