{"version":3,"sources":["jsdelivr-header.js","/npm/@polyfact/vectorizer@1.0.5/dist/index.js"],"names":[],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;AACA,ACNA,aACA,IAAI,gBAAmB,MAAQ,KAAK,kBAAqB,OAAO,OAAS,SAAU,EAAG,EAAG,EAAG,QAC7E,IAAP,IAAkB,EAAK,GAC3B,IAAI,EAAO,OAAO,yBAAyB,EAAG,GACzC,KAAS,QAAS,GAAQ,EAAE,WAAa,EAAK,UAAY,EAAK,gBAClE,EAAO,CAAE,YAAY,EAAM,IAAK,WAAa,OAAO,EAAE,EAAI,IAE5D,OAAO,eAAe,EAAG,EAAI,EAChC,EAAI,SAAU,EAAG,EAAG,EAAG,QACT,IAAP,IAAkB,EAAK,GAC3B,EAAE,GAAM,EAAE,EACb,GACG,mBAAsB,MAAQ,KAAK,qBAAwB,OAAO,OAAS,SAAU,EAAG,GACxF,OAAO,eAAe,EAAG,UAAW,CAAE,YAAY,EAAM,MAAO,GAClE,EAAI,SAAS,EAAG,GACb,EAAW,QAAI,CACnB,GACI,aAAgB,MAAQ,KAAK,cAAiB,SAAU,GACxD,GAAI,GAAO,EAAI,WAAY,OAAO,EAClC,IAAI,EAAS,CAAC,EACd,GAAW,MAAP,EAAa,IAAK,IAAI,KAAK,EAAe,YAAN,GAAmB,OAAO,UAAU,eAAe,KAAK,EAAK,IAAI,gBAAgB,EAAQ,EAAK,GAEtI,OADA,mBAAmB,EAAQ,GACpB,CACX,EACI,gBAAmB,MAAQ,KAAK,iBAAoB,SAAU,GAC9D,OAAQ,GAAO,EAAI,WAAc,EAAM,CAAE,QAAW,EACxD,EACA,OAAO,eAAe,QAAS,aAAc,CAAE,OAAO,IACtD,QAAQ,gBAAa,EACrB,MAAM,WAAa,QAAQ,YACrB,cAAgB,QAAQ,iBACxB,SAAW,gBAAgB,QAAQ,WACnC,MAAQ,QAAQ,qCAChB,KAAO,aAAa,QAAQ,SAC5B,KAAO,gBAAgB,QAAQ,OACrC,IAAI,WAMJ,SAAS,iBAAiB,GACtB,MAAM,EAAS,GAaf,OAZa,KAAK,QAAQ,YAAY,GACjC,SAAS,IACV,MAAM,EAAW,KAAK,QAAQ,EAAK,GAC7B,EAAO,KAAK,QAAQ,SAAS,GACnC,GAAI,GAAQ,EAAK,cACb,EAAO,QAAQ,iBAAiB,QAE/B,CACD,MAAM,EAAU,KAAK,QAAQ,aAAa,EAAU,CAAE,SAAU,UAChE,EAAO,KAAK,CAAE,KAAM,EAAU,WAClC,KAEG,CACX,CACA,SAAS,SAAS,EAAO,GACrB,MAAM,EAAU,GAChB,IAAI,EAAS,IAAI,GACjB,MAAM,EAAa,KAAK,KAAK,EAAO,OAAS,GAC7C,IAAK,IAAI,EAAI,EAAG,EAAI,EAAY,IAC5B,EAAQ,KAAK,EAAO,OAAO,EAAG,IAElC,OAAO,CACX,CACA,SAAS,WAAW,EAAO,EAAU,GACjC,GAAqB,IAAjB,EAAM,OACN,MAAM,IAAI,MAAM,oCAEpB,MAAM,EAAS,GAaf,OAZA,EAAM,SAAS,IACX,MAAM,GAAiB,EAAI,cAAc,aAAa,EAAK,QAAS,GAChE,EAAe,OAAS,EACxB,EAAe,SAAQ,CAAC,EAAG,KACvB,EAAO,KAAK,CAAE,KAAM,GAAG,EAAK,QAAQ,IAAK,QAAS,GAAI,IAI1D,EAAO,KAAK,GAEhB,EAAS,EAAK,IAEX,CACX,EAhDA,SAAW,GACP,EAAW,EAAsB,UAAI,GAAK,YAC1C,EAAW,EAAgB,IAAI,GAAK,MACpC,EAAW,EAAkB,MAAI,GAAK,OACzC,CAJD,CAIG,WAAa,QAAQ,aAAe,QAAQ,WAAa,CAAC,IA6C7D,MAAM,WACF,WAAA,CAAY,EAAe,EAAU,GACjC,GAAI,GAAY,EACZ,MAAM,IAAI,MAAM,sCAEpB,KAAK,cAAgB,EACrB,KAAK,SAAW,GAAY,KAC5B,KAAK,SAAW,CACpB,CACA,6BAAM,GAEF,GADA,QAAQ,IAAI,0BAA2B,KAAK,UAAW,KAAK,WACvD,KAAK,SAAU,CAChB,QAAQ,KAAK,uBACb,MAAM,QAAa,EAAI,WAAW,cAAc,KAAK,eACrD,QAAQ,IAAI,GACZ,KAAK,SAAW,EAAK,GACrB,QAAQ,KAAK,2BAA2B,KAAK,WACjD,CACJ,CACA,gBAAM,CAAW,GACb,IAAI,EAAI,EACR,KAAoC,QAA7B,EAAK,KAAK,qBAAkC,IAAP,OAAgB,EAAS,EAAG,OACpE,MAAM,IAAI,MAAM,sBAEpB,MAAM,EAAQ,GACd,IAAK,MAAM,KAAY,EACnB,GAAI,GAAY,KAAK,QAAQ,WAAW,GACpC,IACI,MAAM,EAAQ,KAAK,QAAQ,aAAa,GACxC,QAAQ,KAAK,oBACb,MAAM,QAAsB,EAAI,WAAW,YAAY,EAAO,KAAK,eACnE,QAAQ,KAAK,uBACb,EAAM,KAAK,CACP,KAAM,EACN,QAAS,EACT,SAA+C,QAApC,EAAK,EAAS,MAAM,KAAK,aAA0B,IAAP,EAAgB,EAAK,KAEhF,QAAQ,KAAK,eAAe,IAChC,CACA,MAAO,GACH,QAAQ,MAAM,kBAAkB,OAAc,EAAM,UACxD,MAGA,QAAQ,MAAM,6CAGtB,OAAO,CACX,CACA,cAAM,CAAS,GACX,MAAM,EAAS,GA8Bf,MAAM,EAAW,EAAM,KAAK,GA7B5B,eAAiC,EAAU,GAAU,GACjD,MAAM,EAAgB,EAAU,CAAC,EAAI,CAAE,YAAY,GAC7C,EAAS,IAAI,MAAM,UAAU,EAAU,GAC7C,IACI,MAAM,QAAa,EAAO,OACpB,EAAW,KAAK,SAAS,GAC/B,GAAI,EACA,IAAK,IAAI,EAAI,EAAG,EAAI,EAAK,OAAQ,IAC7B,EAAO,KAAK,CACR,QAAS,EAAK,GAAG,YACjB,KAAM,EAAI,EACV,SAAU,EACV,KAAM,SAKd,EAAO,KAAK,CACR,QAAS,EAAK,GAAG,YACjB,KAAM,EACN,SAAU,EACV,KAAM,GAGlB,CACA,MAAO,GACH,MAAM,IAAI,MAAM,kBAAkB,OAAc,EAAM,UAC1D,CACJ,CACqC,CAAkB,KAAK,QAAQ,MAEpE,aADM,QAAQ,IAAI,GACX,CACX,CACA,iBAAA,CAAkB,GACd,IAAK,MAAM,QAAQ,GACf,MAAM,IAAI,MAAM,0CAA0C,GAE9D,OAAO,SAAS,QAAQ,QAAQ,EAAM,KAAK,IACvC,MAAM,EAAW,KAAK,QAAQ,GAC9B,GAAI,KAAK,QAAQ,WAAW,GAAW,CACnC,GAAI,KAAK,QAAQ,SAAS,GAAU,cAChC,OAAO,iBAAiB,GAIxB,MAAO,CAAC,CAAE,KAAM,EAAU,QADV,KAAK,QAAQ,aAAa,EAAU,CAAE,SAAU,WAGxE,CAEI,MAAM,IAAI,MAAM,aAAa,qBACjC,IAER,CACA,eAAM,CAAU,EAAO,EAAa,WAAW,WAC3C,GAAI,IAAe,WAAW,UAC1B,OAAO,KAAK,kBAAkB,GAE7B,GAAI,IAAe,WAAW,IAC/B,OAAO,KAAK,SAAS,GAEpB,GAAI,GAAc,WAAW,MAC9B,OAAO,KAAK,WAAW,GAGvB,MAAM,IAAI,MAAM,sBAExB,CACA,eAAM,CAAU,EAAO,GACnB,GAAqB,IAAjB,EAAM,OACN,MAAM,IAAI,MAAM,wCAMpB,SAAS,EAAkB,GACvB,EAAS,EAAM,EAAM,OAAQ,YACjC,OANM,KAAK,0BAQX,MAAM,EAAS,SADf,EAAQ,WAAW,EAAO,KAAK,UAN/B,SAAsB,GAClB,EAAS,EAAM,EAAM,OAAQ,QACjC,IAK+B,IACzB,EAAS,GACf,IAAK,IAAI,EAAI,EAAG,EAAI,EAAO,OAAQ,IAAK,CACpC,MAAM,EAAQ,EAAO,SACf,QAAQ,IAAI,EAAM,KAAI,MAAO,IAC/B,GAAsB,OAAlB,KAAK,SACL,MAAM,IAAI,MAAM,yBAEpB,MAAM,QAAgB,EAAI,WAAW,cAAc,KAAK,SAAU,KAAK,UAAU,GAAO,KAAK,SAAU,KAAK,eAC5G,EAAO,OAAO,OAAO,OAAO,OAAO,CAAC,EAAG,GAAO,CAAE,MAAO,QAAQ,KAC/D,EAAO,KAAK,GACZ,EAAkB,EAAK,IAE/B,CACA,OAAO,CACX,CACA,WAAA,GAEI,GADA,QAAQ,IAAI,cAAe,KAAK,UACV,OAAlB,KAAK,SACL,MAAM,IAAI,MAAM,yBAEpB,OAAO,KAAK,QAChB,EAEJ,QAAQ,QAAU","file":"/npm/@polyfact/vectorizer@1.0.5/dist/index.js","sourceRoot":"","sourcesContent":["/**\n * Minified by jsDelivr using Terser v5.39.0.\n * Original file: /npm/@polyfact/vectorizer@1.0.5/dist/index.js\n *\n * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files\n */\n","\"use strict\";\nvar __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {\n    if (k2 === undefined) k2 = k;\n    var desc = Object.getOwnPropertyDescriptor(m, k);\n    if (!desc || (\"get\" in desc ? !m.__esModule : desc.writable || desc.configurable)) {\n      desc = { enumerable: true, get: function() { return m[k]; } };\n    }\n    Object.defineProperty(o, k2, desc);\n}) : (function(o, m, k, k2) {\n    if (k2 === undefined) k2 = k;\n    o[k2] = m[k];\n}));\nvar __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {\n    Object.defineProperty(o, \"default\", { enumerable: true, value: v });\n}) : function(o, v) {\n    o[\"default\"] = v;\n});\nvar __importStar = (this && this.__importStar) || function (mod) {\n    if (mod && mod.__esModule) return mod;\n    var result = {};\n    if (mod != null) for (var k in mod) if (k !== \"default\" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);\n    __setModuleDefault(result, mod);\n    return result;\n};\nvar __importDefault = (this && this.__importDefault) || function (mod) {\n    return (mod && mod.__esModule) ? mod : { \"default\": mod };\n};\nObject.defineProperty(exports, \"__esModule\", { value: true });\nexports.SourceType = void 0;\nconst polyfact_1 = require(\"polyfact\");\nconst splitString_1 = require(\"./splitString\");\nconst lodash_1 = __importDefault(require(\"lodash\"));\nconst pdf_1 = require(\"langchain/document_loaders/fs/pdf\");\nconst path = __importStar(require(\"path\"));\nconst fs_1 = __importDefault(require(\"fs\"));\nvar SourceType;\n(function (SourceType) {\n    SourceType[SourceType[\"DIRECTORY\"] = 0] = \"DIRECTORY\";\n    SourceType[SourceType[\"PDF\"] = 1] = \"PDF\";\n    SourceType[SourceType[\"AUDIO\"] = 2] = \"AUDIO\";\n})(SourceType = exports.SourceType || (exports.SourceType = {}));\nfunction readDirRecursive(dir) {\n    const result = [];\n    const list = fs_1.default.readdirSync(dir);\n    list.forEach((file) => {\n        const filePath = path.resolve(dir, file);\n        const stat = fs_1.default.statSync(filePath);\n        if (stat && stat.isDirectory()) {\n            result.push(...readDirRecursive(filePath));\n        }\n        else {\n            const content = fs_1.default.readFileSync(filePath, { encoding: \"utf-8\" });\n            result.push({ path: filePath, content });\n        }\n    });\n    return result;\n}\nfunction batchify(array, size) {\n    const batched = [];\n    let copied = [...array];\n    const numOfChild = Math.ceil(copied.length / size);\n    for (let i = 0; i < numOfChild; i++) {\n        batched.push(copied.splice(0, size));\n    }\n    return batched;\n}\nfunction splitFiles(files, maxToken, progress) {\n    if (files.length === 0) {\n        throw new Error(\"You need to add files to split !\");\n    }\n    const result = [];\n    files.forEach((file) => {\n        const splittedString = (0, splitString_1.splitString)(file.content, maxToken);\n        if (splittedString.length > 1) {\n            splittedString.forEach((s, i) => {\n                result.push({ path: `${file.path}_${i}`, content: s });\n            });\n        }\n        else {\n            result.push(file);\n        }\n        progress(file);\n    });\n    return result;\n}\nclass Vectorizer {\n    constructor(clientOptions, maxToken, memoryId) {\n        if (maxToken <= 0) {\n            throw new Error(\"maxToken must be a positive number\");\n        }\n        this.clientOptions = clientOptions;\n        this.memoryId = memoryId || null;\n        this.maxToken = maxToken;\n    }\n    async ensureMemoryInitialized() {\n        console.log(\"ensureMemoryInitialized\", this.memoryId, !this.memoryId);\n        if (!this.memoryId) {\n            console.info(\"Creating memory ...\");\n            const test = await (0, polyfact_1.createMemory)(this.clientOptions);\n            console.log(test);\n            this.memoryId = test.id;\n            console.info(`Memory created with id: ${this.memoryId}`);\n        }\n    }\n    async readAudios(filePaths) {\n        var _a, _b;\n        if (!((_a = this.clientOptions) === null || _a === void 0 ? void 0 : _a.token)) {\n            throw new Error(\"No token provided.\");\n        }\n        const files = [];\n        for (const filePath of filePaths) {\n            if (filePath && fs_1.default.existsSync(filePath)) {\n                try {\n                    const audio = fs_1.default.readFileSync(filePath);\n                    console.info(`Transcribing ...`);\n                    const transcription = await (0, polyfact_1.transcribe)(audio, this.clientOptions);\n                    console.info(`Transcription done.`);\n                    files.push({\n                        path: filePath,\n                        content: transcription,\n                        filename: (_b = filePath.split(\"/\").pop()) !== null && _b !== void 0 ? _b : \"\",\n                    });\n                    console.info(`Added file: ${filePath}`);\n                }\n                catch (error) {\n                    console.error(`Error reading \"${filePath}\": ${error.message}`);\n                }\n            }\n            else {\n                console.error(\"Invalid filePath : no file path provided.\");\n            }\n        }\n        return files;\n    }\n    async readPDFs(files) {\n        const output = [];\n        async function extractPDFContent(filePath, perPage = true) {\n            const loaderOptions = perPage ? {} : { splitPages: false };\n            const loader = new pdf_1.PDFLoader(filePath, loaderOptions);\n            try {\n                const docs = await loader.load();\n                const filename = path.basename(filePath);\n                if (perPage) {\n                    for (let i = 0; i < docs.length; i++) {\n                        output.push({\n                            content: docs[i].pageContent,\n                            page: i + 1,\n                            filename: filename,\n                            path: filePath,\n                        });\n                    }\n                }\n                else {\n                    output.push({\n                        content: docs[0].pageContent,\n                        page: 1,\n                        filename: filename,\n                        path: filePath,\n                    });\n                }\n            }\n            catch (error) {\n                throw new Error(`Error reading \"${filePath}\": ${error.message}`);\n            }\n        }\n        const promises = files.map((file) => extractPDFContent(path.resolve(file)));\n        await Promise.all(promises);\n        return output;\n    }\n    readFilesFromRepo(files) {\n        if (!Array.isArray(files)) {\n            throw new Error(`Expected an array, but received ${typeof files}`);\n        }\n        return lodash_1.default.flatten(files.map((item) => {\n            const filePath = path.resolve(item);\n            if (fs_1.default.existsSync(filePath)) {\n                if (fs_1.default.statSync(filePath).isDirectory()) {\n                    return readDirRecursive(filePath);\n                }\n                else {\n                    const content = fs_1.default.readFileSync(filePath, { encoding: \"utf-8\" });\n                    return [{ path: filePath, content }];\n                }\n            }\n            else {\n                throw new Error(`The path \"${filePath}\" does not exist.`);\n            }\n        }));\n    }\n    async readFiles(files, sourceType = SourceType.DIRECTORY) {\n        if (sourceType === SourceType.DIRECTORY) {\n            return this.readFilesFromRepo(files);\n        }\n        else if (sourceType === SourceType.PDF) {\n            return this.readPDFs(files);\n        }\n        else if (sourceType == SourceType.AUDIO) {\n            return this.readAudios(files);\n        }\n        else {\n            throw new Error(\"Unknown source type\");\n        }\n    }\n    async vectorize(files, progress) {\n        if (files.length === 0) {\n            throw new Error(\"No files provided for vectorization!\");\n        }\n        await this.ensureMemoryInitialized();\n        function splitProgess(file) {\n            progress(file, files.length, \"split\");\n        }\n        function vectorizeProgress(file) {\n            progress(file, files.length, \"vectorize\");\n        }\n        files = splitFiles(files, this.maxToken, splitProgess);\n        const batchs = batchify(files, 10);\n        const result = [];\n        for (let i = 0; i < batchs.length; i++) {\n            const batch = batchs[i];\n            await Promise.all(batch.map(async (file) => {\n                if (this.memoryId === null) {\n                    throw new Error(\"Memory ID is not set.\");\n                }\n                const updated = await (0, polyfact_1.updateMemory)(this.memoryId, JSON.stringify(file), this.maxToken, this.clientOptions);\n                file = Object.assign(Object.assign({}, file), { saved: Boolean(updated) });\n                result.push(file);\n                vectorizeProgress(file);\n            }));\n        }\n        return result;\n    }\n    getMemoryId() {\n        console.log(\"getMemoryId\", this.memoryId);\n        if (this.memoryId === null) {\n            throw new Error(\"Memory ID is not set.\");\n        }\n        return this.memoryId;\n    }\n}\nexports.default = Vectorizer;\n"]}