#!/usr/bin/env ts-node import fs from 'fs'; import path from 'path'; import { config as setupEnv } from 'dotenv-flow'; import connectToDatabase from '@/config/db'; import { AlgoliaSearchIndex, AlgoliaIndexKind } from '@/services'; import Page from '../src/entities/Page'; setupEnv(); const algoliaIndex = AlgoliaSearchIndex.fromIndex(AlgoliaIndexKind.PAGES); type AlgoliaPage = { objectID: string; name?: string; price?: number; retailer?: string; url?: string; urlQuery?: string; availability?: boolean; product_id?: string; featured?: boolean; img?: string; discoverable: boolean; ['lowest_price.price']?: number; }; const main = async () => { await connectToDatabase(); let unindexedAlgoliaPages = new Array(); let indexedPageCount = 0; const ids = JSON.parse( await fs.promises.readFile( path.join(__dirname, 'data', 'featured-pages-firebase-ids-3-16-22.json'), 'utf8' ) ); console.log('ids:', ids); await new Promise((resolve, reject) => { Page.createQueryBuilder('Page') .where('Page.firebaseId IN (:...ids)', { ids }) .andWhere('Page.discoverable = true') .andWhere('Page.featured = true') .leftJoinAndSelect('Page.lowestPricePage', 'lowestPricePage') .stream() .then((stream) => { stream.on('data', (pgPage: Record) => { const page = Page.create( Object.entries(pgPage).reduce((prev, [key, value]) => { return { ...prev, [key.startsWith('Page_') ? key.slice(5) : key]: value, }; }, {}) ); unindexedAlgoliaPages.push({ objectID: page.firebaseId, name: page.name, price: page.price, retailer: page.retailer, url: page.url, urlQuery: page.urlQuery, availability: page.availability, product_id: page.productFirebaseId, featured: page.featured, img: page.img, ['lowest_price.price']: (pgPage['lowestPricePage_price'] as number) ?? undefined, discoverable: page.discoverable ?? false, }); if (unindexedAlgoliaPages.length >= 100) { stream.pause(); algoliaIndex .createOrUpdateRecords(unindexedAlgoliaPages) .then(() => { indexedPageCount = indexedPageCount + unindexedAlgoliaPages.length; console.log(`indexed ${indexedPageCount} pages`); unindexedAlgoliaPages = []; stream.resume(); }) .catch((error) => { stream.destroy(); reject(error); }); } }); stream.on('error', reject); stream.on('end', () => { if (unindexedAlgoliaPages.length >= 0) { algoliaIndex .createOrUpdateRecords(unindexedAlgoliaPages) .then(() => { resolve(undefined); indexedPageCount = indexedPageCount + unindexedAlgoliaPages.length; console.log(`indexed ${indexedPageCount} pages`); unindexedAlgoliaPages = []; }) .catch((error) => { stream.destroy(); reject(error); }); } }); stream.on('close', resolve); }); }); }; main().catch((error) => { console.error(error); process.exit(1); });