/* tslint:disable:no-console */ const envPath = process.env.NODE_ENV ? `.env.${process.env.NODE_ENV}` : '.env.development'; console.log('envPath', envPath); require('dotenv').config({ path: envPath }); import { Message } from '@floom/xbot'; import ApolloClient from 'apollo-client'; import gql from 'graphql-tag'; import plugins from '../../plugins/datocms-access/'; import { SiteClient } from 'datocms-client'; import { TYPE } from '../../plugins/datocms-access/constants'; import slugify from 'slugify'; import { getClient } from '../access/floomx-api'; import axios from 'axios'; const {Builder} = require('@floom/xbot'); async function executeSequentially(params: T[], action: (t: T) => Promise): Promise { // console.log(`executeSequentially ${params.length} items params[0]=${params[0]}`); return params.reduce(async (previousPromise: Promise, param: T, index: number) => { await previousPromise; // console.log(`${index + 1}/${params.length} param=${JSON.stringify(param, null, 4)}`); return action(param); }, Promise.resolve()); } function buildClient(environment): ApolloClient { if (environment === 'production') { return getClient(process.env.FLOOMX_API_URL_PRODUCTION, process.env.WEBSITE_GEN_API_KEY_PRODUCTION); } else if (environment === 'staging') { return getClient(process.env.FLOOMX_API_URL_STAGING, process.env.WEBSITE_GEN_API_KEY_STAGING); } else { console.error('Cant buildClient for environment=', environment); process.exit(1); } } const fxQuery = gql` { products(where: { active: true channels_some:{ channel:Website } }) { id active title description slug sku promotable merchant { title customSiteConfig{ customSiteProjectId } } types: type { title fxId: id slug } variations { title price type media { src id } } } } `; const datoMerchantsQuery = gql` { allMerchants{ id projectId } } `; const PAGE_SIZE = 100; const tokens = {}; const QUERY = gql` query AllProducts($pageSize: IntType, $beginningNode: IntType) { allProducts(first:$pageSize, skip: $beginningNode) { id fxId:floomxId merchant{ projectId } } } `; function validateVariables(): void { const errors = []; const envVars = [ 'FLOOMX_API_URL_STAGING', 'FLOOMX_API_URL_PRODUCTION', 'WEBSITE_GEN_API_KEY_STAGING', 'WEBSITE_GEN_API_KEY_PRODUCTION' ]; envVars.forEach(envVar => { const value = process.env[envVar]; if (value) { console.log(`Got value for ${envVar}=${value}`); } else { errors.push(`No value for ${envVar}`); } }); if (errors.length > 0) { errors.forEach(error => { console.error(error); }); process.exit(1); } } const loadProductsFromDato = async (beginningNode): Promise => { return plugins.api.query( { query: QUERY, variables: {pageSize: PAGE_SIZE, beginningNode}, fetchPolicy: 'no-cache' } ); }; const getAllProducts = async (): Promise => { let datoItems = await loadProductsFromDato(0) .then(response => response.data.allProducts); if (datoItems.length === PAGE_SIZE) { let beginningNode = PAGE_SIZE; let done = false; while (!done) { const extraItems = await loadProductsFromDato(beginningNode) .then(response => response.data.allProducts); datoItems = datoItems.concat(extraItems); beginningNode += PAGE_SIZE; if (extraItems.length !== PAGE_SIZE) { done = true; } } } return datoItems; }; const removeDatoProducts = async (datoIds: string[]): Promise => { return executeSequentially(datoIds, datoId => { return plugins.client.items.destroy(datoId); }); }; const extractDuplicates = (datoProducts: any[]): any[] => { const uniqueProducts = []; return datoProducts.filter(datoProduct => { const exists = uniqueProducts.some(uniqueProduct => { return uniqueProduct.fxId === datoProduct.fxId && uniqueProduct.merchant.projectId === datoProduct.merchant.projectId; }); if (!exists) { uniqueProducts.push(datoProduct); } return exists; }); }; const extractOrphans = (datoMerchants: any[], datoProducts: any[]): any[] => { const projectIds = datoMerchants.map(datoMerchant => { return datoMerchant.projectId; }); return datoProducts.filter(datoProduct => { return !projectIds.some(projectId => { return datoProduct.merchant.projectId === projectId; }); }); }; const getClientForProject = async (projectId: any): Promise => { if (!projectId) { throw Error('getClientForProject no projectId'); } if (tokens[projectId]) { console.log('Got token for projectId=', projectId); return SiteClient(tokens[projectId]); } else { console.log('Fetching token for projectId=', projectId); } return plugins.tokens.get(projectId) .then(token => { console.log('Got token for projectId=', projectId, 'Token=', token); tokens[projectId] = token.token; return SiteClient(token.token); }); }; async function fetchProducts(): Promise { const {data: {products: productionProducts}} = await buildClient('production').query({ query: fxQuery }); if (productionProducts) { console.log(`Got ${productionProducts.length} productionProducts`); } else { console.error(`Couldnt get productionProducts`); } const {data: {products: stagingProducts}} = await buildClient('staging').query({ query: fxQuery }); if (stagingProducts) { console.log(`Got ${stagingProducts.length} stagingProducts`); } else { console.error(`Couldnt get stagingProducts`); } return productionProducts.concat(stagingProducts); } async function execute(): Promise { console.log('execute()'); validateVariables(); const fxProducts = await fetchProducts(); console.log(`Got ${fxProducts.length} fxProducts`); const {data: {allMerchants: datoMerchants}} = await plugins.api.query({ query: datoMerchantsQuery }); const datoProducts = await getAllProducts(); const updatedProjectIds = new Set(); console.log('datoProducts.length', datoProducts.length); const datoProductIdsToRemove = datoProducts .filter((datoProduct: any) => { return !fxProducts.some((fxProduct: any) => { return fxProduct.id.toString() === datoProduct.fxId.toString(); }); }).map((datoProduct: any) => { updatedProjectIds.add(datoProduct.merchant.projectId); return datoProduct.id; }); const datoProductIdsDuplicates = extractDuplicates(datoProducts).map(datoProduct => { updatedProjectIds.add(datoProduct.merchant.projectId); return datoProduct.id; }); const datoProductIdsOrphans = extractOrphans(datoMerchants, datoProducts).map(datoProduct => { updatedProjectIds.add(datoProduct.merchant.projectId); return datoProduct.id; }); const fxProductsToInsert = fxProducts .filter((fxProduct: any) => { if (!fxProduct.merchant.customSiteConfig || !fxProduct.merchant.customSiteConfig.customSiteProjectId) { return false; } return datoMerchants.some(datoMerchant => { // console.log('datoMerchant.projectId', datoMerchant.projectId); // console.log('fxProduct.merchant.projectId', fxProduct.merchant.customSiteConfig.customSiteProjectId); return datoMerchant.projectId === fxProduct.merchant.customSiteConfig.customSiteProjectId; }); }) .filter((fxProduct: any) => { return !datoProducts.some((datoProduct: any) => { return fxProduct.id.toString() === datoProduct.fxId.toString(); }); }); console.log('datoProductIdsDuplicates.length', datoProductIdsDuplicates.length); console.log('datoProductIdsOrphans.length', datoProductIdsOrphans.length); console.log('datoProductIdsToRemove.length', datoProductIdsToRemove.length); console.log('fxProductsToInsert.length', fxProductsToInsert.length); if (datoProductIdsDuplicates.length > 0 || datoProductIdsOrphans.length > 0 || datoProductIdsToRemove.length) { let builder = new Builder() .addHeader('Removing products') .beginSection() .addTitledTextField('Removing', datoProductIdsToRemove.length); if (datoProductIdsOrphans.length > 0) { builder = builder .addTitledTextField('Orphans', datoProductIdsOrphans.length); } if (datoProductIdsDuplicates.length > 0) { builder = builder .addTitledTextField('Duplicates', datoProductIdsDuplicates.length); } const message = builder.endSection().build(); post(message); } console.log('Removing duplicates'); await removeDatoProducts(datoProductIdsDuplicates); console.log('Removing orphans'); await removeDatoProducts(datoProductIdsOrphans); console.log('Removing disabled products'); await removeDatoProducts(datoProductIdsToRemove); await executeSequentially(fxProductsToInsert, fxProduct => { // tslint:disable-next-line:max-line-length const imageUrl = encodeURI(`https://floomx.imgix.net/${fxProduct.variations[0].media[0].src.trim()}?auto=format&fit=crop&mode=crop&jpegQuality=80&interlace=true&fm=pjpg&ixlib=react-8.5.1&h=600&w=600`); const merchants = datoMerchants.filter((datoMerchant: any) => { return datoMerchant.projectId === fxProduct.merchant.customSiteConfig.customSiteProjectId; }); if (merchants.length === 0) { console.error('No merchants for ', fxProduct.merchant.title); console.error('Datomerchants=', JSON.stringify(datoMerchants, null, 4)); process.exit(1); } return executeSequentially(merchants, merchant => { updatedProjectIds.add(merchant.projectId); return getClientForProject(merchant.projectId) .then(projectClient => { console.log('Attempting to upload image: ', imageUrl); return projectClient.uploadImage(imageUrl) .then((image: any) => { console.log('Uploaded image imageUrl', imageUrl); return projectClient.items.create({ itemType: TYPE.product, merchant: merchant.id, title: fxProduct.title, floomxId: fxProduct.id.toString(), images: [image.toString()], slug: slugify(fxProduct.slug, {lower: true, remove: /[?*+~.()'"!:@]/g}) }); }); }) .then(() => { const message = new Builder() .addHeader('Uploaded product') .beginSection() .addTitledTextField('Project ID', merchant.projectId) .addTitledTextField('Product', fxProduct.title) .withImage(imageUrl) .endSection() .build(); return post(message); }) .catch(error => { console.error('Error uploading image', error); }); }); }); updatedProjectIds.forEach(async projectId => { const message = new Builder() .addHeader('Updating products') .beginSection() .addTitledTextField('Project ID', projectId) .endSection() .build(); await post(message); await triggerBuild(projectId); }); } async function triggerBuild(projectId: string): Promise { try { // tslint:disable-next-line:max-line-length await axios.get(`https://${projectId}.netlify.com/.netlify/functions/build-site?build-hook-key=${process.env.BUILD_HOOK_KEY}&source=Product-Sync`); } catch (e) { console.log(`cannot build site: https://${projectId}.netlify.com`); console.error(e.message); } } async function post(message: Message): Promise { return message.post(process.env.SLACK_BOT_KEY, 'x-sync'); } execute() .then(result => { console.log(result || 'Finished'); }) .catch(error => { console.error(error); });