{"version":3,"file":"load-pdf.mjs","names":[],"sources":["../../../../../../../../ai/src/rag/loaders/load-pdf.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport { PDF_PARSE_INSTALL_INSTRUCTIONS } from \"./errors\";\nimport type { LoadPdfOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * The slice of `pdf-parse`'s result we consume. The peer returns more\n * (`info`, `metadata`, `version`); we only need the extracted `text` and\n * page count, so we type just those to keep the dependency at arm's length.\n */\ntype PdfParseResult = {\n /** Concatenated text of every page. */\n text: string;\n /** Number of pages in the document. */\n numpages: number;\n /** Document info dictionary — `Title` lifted into metadata when present. */\n info?: { Title?: string } & Record;\n};\n\n/** The `pdf-parse` module's callable default export. */\ntype PdfParseFn = (\n data: Buffer | Uint8Array,\n options?: {\n /**\n * Per-page renderer `pdf-parse` calls once per page in document order and\n * `await`s — may return the page text synchronously or as a promise.\n */\n pagerender?: (page: unknown) => string | Promise;\n },\n) => Promise;\n\n// ============================================================\n// Lazily-loaded pdf-parse (OPTIONAL peer)\n// ============================================================\n\nlet pdfParse: PdfParseFn | undefined;\nlet isModuleExists: boolean | undefined;\nlet loadingPromise: Promise | undefined;\n\n/**\n * Settle the lazy import of `pdf-parse` once, concurrency-safe. A bare\n * `catch` flips the flag to `false`; the curated\n * {@link PDF_PARSE_INSTALL_INSTRUCTIONS} surfaces at first\n * {@link loadPdf} call, never a raw module-resolution stack trace. Mirrors\n * the guard moderation detector's `loadOpenAi`.\n */\nfunction loadPdfParse(): Promise {\n if (isModuleExists !== undefined) {\n return Promise.resolve();\n }\n\n if (loadingPromise) {\n return loadingPromise;\n }\n\n loadingPromise = (async () => {\n try {\n // Literal specifier so `vi.mock(\"pdf-parse\")` can intercept it in tests.\n // Typed via the ambient `pdf-parse` shim in this directory, so the bare\n // import resolves even though the OPTIONAL peer is not a dependency.\n const mod = (await import(\"pdf-parse\")) as {\n default?: PdfParseFn;\n } & Partial;\n // pdf-parse ships CommonJS — the callable is `module.exports`, surfaced\n // as `default` under ESM interop. Fall back to the namespace itself for\n // bundlers that hoist the callable to the top level.\n pdfParse = mod.default ?? (mod as unknown as PdfParseFn);\n isModuleExists = typeof pdfParse === \"function\";\n } catch {\n isModuleExists = false;\n }\n })();\n\n return loadingPromise;\n}\n\n/**\n * Coerce a {@link RagDocument}-compatible binary input into a `Buffer` for\n * `pdf-parse`. Accepts a Node `Buffer`, an `ArrayBuffer`, or a typed array\n * (`Uint8Array`) — the shapes a file read / fetch body hands back.\n */\nfunction toBuffer(input: Buffer | ArrayBuffer | Uint8Array): Buffer {\n if (Buffer.isBuffer(input)) {\n return input;\n }\n\n if (input instanceof ArrayBuffer) {\n return Buffer.from(input);\n }\n\n return Buffer.from(input.buffer, input.byteOffset, input.byteLength);\n}\n\n/**\n * Load a PDF's bytes into {@link RagDocument}(s) via the OPTIONAL `pdf-parse`\n * peer. The peer is resolved lazily on the FIRST call (not at import) so\n * importing `@warlock.js/ai` never forces it to be installed; when it is\n * absent the curated {@link PDF_PARSE_INSTALL_INSTRUCTIONS} is thrown as a\n * plain `Error` (a missing optional peer is an infrastructure fault, not a\n * content problem).\n *\n * By default the whole PDF becomes a single document carrying\n * `metadata.pageCount`. With `perPage: true`, each page becomes its own\n * document (`id` suffixed `#p`, `metadata.page` set) so citations stay\n * page-precise. Document `metadata.title` comes from the PDF info\n * dictionary's `Title` (unless overridden), and `metadata.loader` is\n * `\"pdf\"`. The output is the exact shape `index()` consumes.\n *\n * @example\n * import { readFile } from \"node:fs/promises\";\n * const kb = ai.rag({ embedder, store });\n * await kb.index(await loadPdf(await readFile(\"guide.pdf\"), { id: \"guide\" }));\n *\n * @example\n * // One document per page for page-precise citations:\n * await kb.index(await loadPdf(bytes, { id: \"manual\", perPage: true }));\n *\n * @param input - The PDF bytes (`Buffer`, `ArrayBuffer`, or `Uint8Array`).\n * @param options - `perPage` plus shared `id` / `metadata` / `tags`\n * ({@link LoadPdfOptions}).\n * @returns A {@link RagLoaderResult} ready for `rag.index()`.\n * @throws {Error} carrying {@link PDF_PARSE_INSTALL_INSTRUCTIONS} when the\n * `pdf-parse` peer is not installed.\n */\nexport async function loadPdf(\n input: Buffer | ArrayBuffer | Uint8Array,\n options: LoadPdfOptions = {},\n): Promise {\n await loadPdfParse();\n\n if (!isModuleExists || !pdfParse) {\n throw new Error(PDF_PARSE_INSTALL_INSTRUCTIONS);\n }\n\n const id = options.id ?? DEFAULT_ID;\n const perPage = options.perPage ?? false;\n\n if (perPage) {\n return loadPerPage(input, id, options);\n }\n\n const parsed = await pdfParse(toBuffer(input));\n const text = parsed.text.trim();\n const title = parsed.info?.Title?.trim();\n\n // An image-only / empty PDF extracts no text — emit nothing so index()\n // never receives a no-op record.\n if (text.length === 0) {\n return [];\n }\n\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: id,\n loader: \"pdf\",\n pageCount: parsed.numpages,\n ...(title ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n\n/** One page of a parsed PDF — the text-layer item list `pagerender` sees. */\ntype PdfPage = {\n getTextContent: (\n options?: unknown,\n ) => Promise<{ items: { str: string }[] }>;\n};\n\n/**\n * Per-page variant: render each page separately via `pdf-parse`'s\n * `pagerender` hook, accumulating one document per non-empty page. Each\n * carries `metadata.page` (1-based) and `metadata.pageCount`, and its id is\n * the base id suffixed `#p` so every page-document is distinctly\n * identified for citation.\n *\n * `pdf-parse` calls `pagerender` once per page in document order and\n * `await`s the returned string, so capturing each page's joined text content\n * here gives reliable page boundaries the concatenated `text` lacks.\n */\nasync function loadPerPage(\n input: Buffer | ArrayBuffer | Uint8Array,\n id: string,\n options: LoadPdfOptions,\n): Promise {\n const pages: string[] = [];\n\n const parsed = await pdfParse!(toBuffer(input), {\n pagerender: async (page: unknown): Promise => {\n const rendered = await renderPage(page as PdfPage);\n pages.push(rendered);\n return rendered;\n },\n });\n\n const title = parsed.info?.Title?.trim();\n const docs: RagDocument[] = [];\n\n pages.forEach((pageText, index) => {\n const text = pageText.trim();\n\n if (text.length === 0) {\n return;\n }\n\n const pageNumber = index + 1;\n\n docs.push({\n id: `${id}#p${pageNumber}`,\n text,\n metadata: {\n source: id,\n loader: \"pdf\",\n page: pageNumber,\n pageCount: parsed.numpages,\n ...(title ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n });\n });\n\n return docs;\n}\n\n/**\n * Join a single page's text-layer items in reading order, inserting a space\n * between items so adjacent words do not run together. Mirrors the essence\n * of `pdf-parse`'s default renderer without depending on its internals, so\n * the per-page hook stays stable across `pdf-parse` versions. A page with no\n * text layer (scanned image) renders to an empty string and is dropped.\n */\nasync function renderPage(page: PdfPage): Promise {\n if (typeof page?.getTextContent !== \"function\") {\n return \"\";\n }\n\n const content = await page.getTextContent({\n normalizeWhitespace: true,\n disableCombineTextItems: false,\n });\n\n return content.items\n .map((item) => item.str)\n .join(\" \")\n .replace(/\\s+/g, \" \")\n .trim();\n}\n"],"mappings":";;;;AAKA,MAAM,aAAa;AAgCnB,IAAI;AACJ,IAAI;AACJ,IAAI;;;;;;;;AASJ,SAAS,eAA8B;CACrC,IAAI,mBAAmB,QACrB,OAAO,QAAQ,QAAQ;CAGzB,IAAI,gBACF,OAAO;CAGT,kBAAkB,YAAY;EAC5B,IAAI;GAIF,MAAM,MAAO,MAAM,OAAO;GAM1B,WAAW,IAAI,WAAY;GAC3B,iBAAiB,OAAO,aAAa;EACvC,QAAQ;GACN,iBAAiB;EACnB;CACF,EAAC,CAAE;CAEH,OAAO;AACT;;;;;;AAOA,SAAS,SAAS,OAAkD;CAClE,IAAI,OAAO,SAAS,KAAK,GACvB,OAAO;CAGT,IAAI,iBAAiB,aACnB,OAAO,OAAO,KAAK,KAAK;CAG1B,OAAO,OAAO,KAAK,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AACrE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,eAAsB,QACpB,OACA,UAA0B,CAAC,GACD;CAC1B,MAAM,aAAa;CAEnB,IAAI,CAAC,kBAAkB,CAAC,UACtB,MAAM,IAAI,MAAM,8BAA8B;CAGhD,MAAM,KAAK,QAAQ,MAAM;CAGzB,IAFgB,QAAQ,WAAW,OAGjC,OAAO,YAAY,OAAO,IAAI,OAAO;CAGvC,MAAM,SAAS,MAAM,SAAS,SAAS,KAAK,CAAC;CAC7C,MAAM,OAAO,OAAO,KAAK,KAAK;CAC9B,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CAIvC,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAgBV,OAAO,CAAC;EAZN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,WAAW,OAAO;GAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;GACzB,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb;;;;;;;;;;;;AAoBA,eAAe,YACb,OACA,IACA,SACwB;CACxB,MAAM,QAAkB,CAAC;CAEzB,MAAM,SAAS,MAAM,SAAU,SAAS,KAAK,GAAG,EAC9C,YAAY,OAAO,SAAmC;EACpD,MAAM,WAAW,MAAM,WAAW,IAAe;EACjD,MAAM,KAAK,QAAQ;EACnB,OAAO;CACT,EACF,CAAC;CAED,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CACvC,MAAM,OAAsB,CAAC;CAE7B,MAAM,SAAS,UAAU,UAAU;EACjC,MAAM,OAAO,SAAS,KAAK;EAE3B,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,aAAa,QAAQ;EAE3B,KAAK,KAAK;GACR,IAAI,GAAG,GAAG,IAAI;GACd;GACA,UAAU;IACR,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,WAAW,OAAO;IAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;IACzB,GAAG,QAAQ;GACb;GACA,MAAM,QAAQ;EAChB,CAAC;CACH,CAAC;CAED,OAAO;AACT;;;;;;;;AASA,eAAe,WAAW,MAAgC;CACxD,IAAI,OAAO,MAAM,mBAAmB,YAClC,OAAO;CAQT,QAAO,MALe,KAAK,eAAe;EACxC,qBAAqB;EACrB,yBAAyB;CAC3B,CAAC,EAEa,CAAC,MACZ,KAAK,SAAS,KAAK,GAAG,CAAC,CACvB,KAAK,GAAG,CAAC,CACT,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;AACV"}