#!/usr/bin/env ts-node import { config as setupEnv } from 'dotenv-flow'; setupEnv(); import { DocumentData, Firestore, QueryDocumentSnapshot, } from '@google-cloud/firestore'; import { In } from 'typeorm'; import { Click, Hunt, Lead, Page, Product } from '../src/entities'; import connect from '../src/config/db'; type SequenceableFunction = () => Promise; /** * Run an array of functions that return promises in sequence */ export const sequence = async (fns: SequenceableFunction[]): Promise => { await fns.reduce(async (prev, next) => { await prev.then(next); }, Promise.resolve()); /** * This additional await ensures this function won't return until after the * last `SequenceableFunction` also returns. * * @see[the javascript event loop](https://nodejs.org/en/docs/guides/event-loop-timers-and-nexttick/) * for details */ await new Promise((resolve) => setTimeout(resolve, 0)); }; const migrateClicks = async ({ firestore, pageFirebaseIds, productFirebaseIds, }: { firestore: Firestore; pageFirebaseIds: Set; productFirebaseIds: Set; }) => { const collection = firestore.collection('clicks'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); await sequence( documents.map((doc) => async () => { const data = doc.data(); const click = Click.create({ firebaseId: doc.id, ...(data['page_id'] && pageFirebaseIds.has(data['page_id']) ? { pageFirebaseId: data['page_id'], } : {}), ...(data['product_id'] && productFirebaseIds.has(data['product_id']) ? { productFirebaseId: data['product_id'], } : {}), ...(data['ref_page_id'] && pageFirebaseIds.has(data['ref_page_id']) ? { refPageFirebaseId: data['ref_page_id'], } : {}), retailer: data['retailer'], url: data['url'], ...(data['created_at'] ? { createdAt: new Date(data['created_at']), } : {}), }); await click.save(); }) ); }; const migrateHunts = async ({ firestore }: { firestore: Firestore }) => { const collection = firestore.collection('hunts'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); await sequence( documents.map((doc) => async () => { const data = doc.data(); const hunt = Hunt.create({ firebaseId: doc.id, email: data['email'], url: data['url'], ...(data['created_at'] ? { createdAt: new Date(data['created_at']), } : {}), }); await hunt.save(); }) ); }; const migrateLeads = async ({ firestore }: { firestore: Firestore }) => { const collection = firestore.collection('leads'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); await sequence( documents.map((doc) => async () => { const data = doc.data(); // don't save leads that don't have an email if (!data['email']) { return; } const lead = Lead.create({ firebaseId: doc.id, email: data['email'], ...(data['created_at'] ? { createdAt: new Date(data['created_at']), } : {}), }); await lead.save(); }) ); }; /** * @returns {Set} a set of product ids that were created */ const migrateProducts = async ({ firestore, }: { firestore: Firestore; }): Promise> => { const collection = firestore.collection('products'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); const productFirebaseIds = new Set(); await sequence( documents.map((doc) => async () => { const data = doc.data(); productFirebaseIds.add(doc.id); const product = Product.create({ firebaseId: doc.id, ...(data['created_at'] ? { createdAt: new Date(data['created_at']), } : {}), }); await product.save(); }) ); return productFirebaseIds; }; /** * @returns {Set} a set of page ids that were created */ const migratePages = async ({ firestore, }: { firestore: Firestore; }): Promise> => { const collection = firestore.collection('pages'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); const pageFirebaseIds = new Set(); await sequence( documents.map((doc) => async () => { const data = doc.data(); const page = Page.create({ firebaseId: doc.id, availability: data['availability'], distributor: data['distributor'], distributorSKU: data['distributor_sku'], img: data['img'], name: data['name'], productFirebaseId: data['product_id'], productSKU: data['product_sku'], price: data['price'], retailer: data['retailer'], url: data['url'], ...(data['urlOriginal'] ? { urlOriginal: data['urlOriginal'], } : {}), urlQuery: data['urlQuery'], ...(data['created_at'] ? { createdAt: new Date(data['created_at']), } : {}), asin: data['asin'], dimensions: data['dimensions'], colorCode: data['color_code'], }); await page.save({ listeners: false }); pageFirebaseIds.add(doc.id); }) ); return pageFirebaseIds; }; const addLowestPricePages = async ({ firestore }: { firestore: Firestore }) => { // @TODO: should i just cache these ids instead of refetching. There might // be more data here that breaks this script const collection = firestore.collection('pages'); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); await sequence( documents.map((doc) => async () => { const data = doc.data(); if (!data.lowest_price?.id) { // console.log( // `could not find lowest price page for ${ // doc.id // } with data: ${JSON.stringify(data, null, 2)} ` // ); return; } const lowestPricePage = await Page.findOne({ firebaseId: data.lowest_price.id, }); if (!lowestPricePage) { console.log( `could not find lowest price page for ${doc.id} with firebaseId: ${data.lowest_price?.id} ` ); return; } await Page.update( { firebaseId: doc.id }, { lowestPricePage: lowestPricePage }, { listeners: false } ); }) ); }; const runPageAfterUpdateListeners = async () => { const pages = await Page.find({}); await sequence( pages.map((page) => async () => { return page.afterUpdate(); }) ); }; const addFeatured = async ({ firestore }: { firestore: Firestore }) => { const collection = firestore .collection('pages') .where('featured', '==', true); const snapshot = await collection.get(); const documents = new Array>(); snapshot.forEach((doc) => { documents.push(doc); }); await Page.update( { firebaseId: In(documents.map(({ id }) => id)), }, { featured: true, } ); await Promise.all( ( await Page.find({ firebaseId: In(documents.map(({ id }) => id)), featured: true, }) ).map(async (page) => { return page.afterUpdate(); }) ); }; const main = async () => { await connect(); const firestore = new Firestore(); // add all raw data await migrateHunts({ firestore }); await migrateLeads({ firestore }); const productFirebaseIds = await migrateProducts({ firestore }); const pageFirebaseIds = await migratePages({ firestore }); // const pageFirebaseIds = new Set( // (await Page.find({ select: ['firebaseId'] })).map( // ({ firebaseId }) => firebaseId // ) // ); // const productFirebaseIds = new Set( // (await Product.find({ select: ['firebaseId'] })).map( // ({ firebaseId }) => firebaseId // ) // ); await migrateClicks({ firestore, pageFirebaseIds, productFirebaseIds }); // link data together (foreign keys) await addLowestPricePages({ firestore }); // run after update listener for pages await runPageAfterUpdateListeners(); await addFeatured({ firestore }); }; main().catch((error) => { console.error(error); process.exit(1); });