{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport type { KnowledgeDocumentId, KnowledgeScopeId } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n  readonly scopeId: KnowledgeScopeId\n  readonly documentId: KnowledgeDocumentId\n  readonly source: LoadedKnowledgeSource\n  readonly contentHash?: string\n  readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n  Effect.gen(function* () {\n    const store = yield* SearchIndexStore\n    yield* store.markDocumentError({\n      scopeId: input.scopeId,\n      documentId: input.documentId,\n      message: error.message\n    })\n  }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n  Effect.gen(function* () {\n    yield* Effect.annotateCurrentSpan({\n      'knowledge_search.scope_id': input.scopeId,\n      'knowledge_search.document_id': input.documentId,\n      'knowledge_search.source_type': input.source.source._tag\n    })\n    const store = yield* SearchIndexStore\n    const extractor = yield* KnowledgeExtractor\n    const chunker = yield* KnowledgeChunker\n    const embedder = yield* KnowledgeEmbedder\n    const summarizer = yield* KnowledgeSummarizer\n\n    yield* store\n      .upsertDocument({\n        document: {\n          id: input.documentId,\n          scopeId: input.scopeId,\n          source: input.source.source,\n          status: 'processing',\n          metadata: input.source.metadata\n        }\n      })\n      .pipe(\n        Effect.mapError(\n          error =>\n            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n        )\n      )\n\n    const extracted = yield* extractor\n      .extract(input.source)\n      .pipe(\n        Effect.mapError(\n          error =>\n            new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n        )\n      )\n\n    const chunks = yield* chunker\n      .chunk({\n        scopeId: input.scopeId,\n        documentId: input.documentId,\n        content: extracted.content,\n        maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n        metadata: extracted.metadata\n      })\n      .pipe(\n        Effect.mapError(\n          error =>\n            new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n        )\n      )\n\n    const indexed = yield* Effect.all(\n      {\n        embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(\n          Effect.mapError(\n            error =>\n              new KnowledgeIngestionError({\n                message: error.message,\n                stage: 'embed',\n                cause: error\n              })\n          )\n        ),\n        summary: summarizer\n          .summarize({\n            content: extracted.content,\n            sourceTitle: extracted.title,\n            metadata: extracted.metadata\n          })\n          .pipe(\n            Effect.mapError(\n              error =>\n                new KnowledgeIngestionError({\n                  message: error.message,\n                  stage: 'summarize',\n                  cause: error\n                })\n            )\n          )\n      },\n      { concurrency: 'unbounded' }\n    )\n\n    if (indexed.embeddings.length !== chunks.length) {\n      return yield* Effect.fail(\n        new KnowledgeIngestionError({\n          message: 'Embedding count did not match chunk count',\n          stage: 'embed'\n        })\n      )\n    }\n\n    const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({\n      chunk,\n      embedding\n    }))\n\n    yield* store\n      .replaceDocumentChunks({\n        scopeId: input.scopeId,\n        documentId: input.documentId,\n        chunks: indexedChunks\n      })\n      .pipe(\n        Effect.mapError(\n          error =>\n            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n        )\n      )\n\n    const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)\n\n    return yield* store\n      .markDocumentReady({\n        scopeId: input.scopeId,\n        documentId: input.documentId,\n        title: indexed.summary.title ?? extracted.title,\n        summary: indexed.summary.summary ?? extracted.summary,\n        contentHash: input.contentHash,\n        tokenCount,\n        chunkCount: chunks.length\n      })\n      .pipe(\n        Effect.mapError(\n          error =>\n            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n        )\n      )\n  }).pipe(\n    Effect.withSpan('knowledge_search.ingestDocument'),\n    Effect.catch(error =>\n      markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))\n    )\n  )\n\nexport type KnowledgeIngestionPipeline = {\n  readonly ingest: (\n    input: IngestKnowledgeDocumentInput\n  ) => ReturnType<typeof ingestKnowledgeDocument>\n}\n\nexport const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({\n  ingest: ingestKnowledgeDocument\n})\n"],"mappings":";;;;;;;;;AAmBA,MAAM,uBAAuB,OAAqC,UAChE,OAAO,IAAI,aAAa;CAEtB,QAAO,OADc,kBACR,kBAAkB;EAC7B,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,MAAM;CACjB,CAAC;AACH,CAAC,EAAE,KAAK,OAAO,YAAY,OAAO,IAAI,CAAC;AAEzC,MAAa,2BAA2B,UACtC,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,oBAAoB;EAChC,6BAA6B,MAAM;EACnC,gCAAgC,MAAM;EACtC,gCAAgC,MAAM,OAAO,OAAO;CACtD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,YAAY,OAAO;CACzB,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,OAAO;CACxB,MAAM,aAAa,OAAO;CAE1B,OAAO,MACJ,eAAe,EACd,UAAU;EACR,IAAI,MAAM;EACV,SAAS,MAAM;EACf,QAAQ,MAAM,OAAO;EACrB,QAAQ;EACR,UAAU,MAAM,OAAO;CACzB,EACF,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,YAAY,OAAO,UACtB,QAAQ,MAAM,MAAM,EACpB,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAW,OAAO;CAAM,CAAC,CAC1F,CACF;CAEF,MAAM,SAAS,OAAO,QACnB,MAAM;EACL,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,UAAU;EACnB,WAAW,MAAM,aAAA;EACjB,UAAU,UAAU;CACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,UAAU,OAAO,OAAO,IAC5B;EACE,YAAY,SAAS,WAAW,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,EAAE,KAClE,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;EACA,SAAS,WACN,UAAU;GACT,SAAS,UAAU;GACnB,aAAa,UAAU;GACvB,UAAU,UAAU;EACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;CACJ,GACA,EAAE,aAAa,YAAY,CAC7B;CAEA,IAAI,QAAQ,WAAW,WAAW,OAAO,QACvC,OAAO,OAAO,OAAO,KACnB,IAAI,wBAAwB;EAC1B,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,MAAM,gBAAgBA,MAAI,IAAI,QAAQ,QAAQ,UAAU,EAAE,KAAK,CAAC,OAAO,gBAAgB;EACrF;EACA;CACF,EAAE;CAEF,OAAO,MACJ,sBAAsB;EACrB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,QAAQ;CACV,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,aAAa,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,YAAY,CAAC;CAE9E,OAAO,OAAO,MACX,kBAAkB;EACjB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,OAAO,QAAQ,QAAQ,SAAS,UAAU;EAC1C,SAAS,QAAQ,QAAQ,WAAW,UAAU;EAC9C,aAAa,MAAM;EACnB;EACA,YAAY,OAAO;CACrB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;AACJ,CAAC,EAAE,KACD,OAAO,SAAS,iCAAiC,GACjD,OAAO,OAAM,UACX,oBAAoB,OAAO,KAAK,EAAE,KAAK,OAAO,cAAc,OAAO,KAAK,KAAK,CAAC,CAAC,CACjF,CACF;AAQF,MAAa,+BAA2D,EACtE,QAAQ,wBACV"}