import { config as setupEnv } from 'dotenv-flow'; import connect from '@/config/db'; import { Page, Product } from '@/entities'; import { TypeormDataLoader } from '@/entities/utils'; import logger from '@/logger'; import { ContextType } from '@/types'; setupEnv(); const main = async () => { await connect(); let unupdatedProducts = new Array(); let indexedProductCount = 0; const dbDataLoader = new TypeormDataLoader(); const updateProducts = async () => { // uncomment to ensure too many pages don't get indexed // if (indexedPageCount > 10000) { // stream.pause(); // stream.destroy(); // throw new Error('too many pages indexed'); // } await Promise.all( unupdatedProducts.map(async (product) => { const context = { dbDataLoader } as ContextType; const lowestPricePage = await product.lowestPricePage(context); if (lowestPricePage) { return Page.update( { productFirebaseId: product.firebaseId, }, { lowestPricePageId: lowestPricePage.id, } ); } }) ); indexedProductCount = indexedProductCount + unupdatedProducts.length; logger.info(`updated ${indexedProductCount} products`); unupdatedProducts = []; }; const stream = await Product.createQueryBuilder().stream(); await new Promise((resolve, reject) => { stream.on('data', (pgProduct: Record) => { const product = Product.create( Object.entries(pgProduct).reduce((prev, [key, value]) => { return { ...prev, [key.startsWith('Product_') ? key.slice(8) : key]: value, }; }, {}) ); unupdatedProducts.push(product); if (unupdatedProducts.length >= 100) { stream.pause(); updateProducts() .then(() => { stream.resume(); }) .catch((error) => { stream.destroy(error); reject(error); }); } }); stream.on('end', async () => { resolve(undefined); if (unupdatedProducts.length >= 0) { try { updateProducts().then(resolve); } catch (error) { stream.destroy(); reject(error); throw error; } } }); }); }; console.time('main'); main() .then(() => { console.timeEnd('main'); }) .catch((error) => { console.error(error); process.exit(1); });