#!/usr/bin/env ts-node import fs from 'fs'; import path from 'path'; import { URL } from 'url'; import { config as setupEnv } from 'dotenv-flow'; import kebabCase from 'lodash.kebabcase'; import { SitemapAndIndexStream, SitemapStream } from 'sitemap'; import { IsNull, Not } from 'typeorm'; import connectToDatabase from '@/config/db'; import { Page } from '@/entities'; import logger from '@/logger'; setupEnv(); const getBaseUrl = (): string => { switch (process.env.NODE_ENV) { case 'test': return 'test/'; case 'staging': return 'https://www.spoken-staging.com'; default: return 'https://www.spoken.io'; } }; const baseUrl = getBaseUrl(); const createURLElementFromURL = (url: string): string => { return `${url}${new Date().toISOString()}monthly1.0`; }; const retailerPaths = [ 'urbanoutfitters', 'luluandgeorgia', 'birchlane', 'allmodern', 'perigold', 'jossandmain', 'dotandbo', 'onekingslane', 'pier1', 'burkedecor', 'houzz', 'worldmarket', 'amara', 'kathykuohome', 'kohls', 'ashleyfurniture', 'wayfair', '1stopbedrooms', 'bellacor', 'bloomingdales', 'colemanfurniture', 'emmamason', 'englishelm', 'graysonluxury', 'highfashionhome', 'homeshoppingmalls', 'interiorhomescapes', 'jcpenney', 'laylagrayce', 'macys', 'makerandmoss', 'neimanmarcus', 'paynesgray', 'raymourflanigan', 'rcwilley', 'saksfifthavenue', 'scoutandnimble', 'sears', 'target', 'topmodern', ].sort(); const getSitemapPageURLs = (): string[] => { const baseUrl = getBaseUrl(); const pages = [ ...['signup.js', 'disclaimer.js', 'request.js'], ...retailerPaths.map((retailer) => `shop/${retailer}`), ]; return pages.map((page) => { page = page.replace(/\.js$/, ''); return `${baseUrl}/${page}`; }); }; const main = async () => { await connectToDatabase(); let pagesAdded = 0; const sms = new SitemapAndIndexStream({ limit: 50000, // SitemapAndIndexStream will call this user provided function every time // it needs to create a new sitemap file. You merely need to return a stream // for it to write the sitemap urls to and the expected url where that // sitemap will be hosted getSitemapStream: (i) => { const sitemapStream = new SitemapStream({ hostname: getBaseUrl() }); const sitemapPath = `./sitemap-${i}.xml`; const directory = path.join(__dirname, '../../web/public/sitemap'); if (!fs.existsSync(directory)) { fs.mkdirSync(directory, { recursive: true }); } const ws = sitemapStream.pipe( fs.createWriteStream(path.join(directory, sitemapPath)) ); return [ new URL(sitemapPath, `${getBaseUrl()}/sitemap/`).toString(), sitemapStream, ws, ]; }, }); const staticPages = getSitemapPageURLs(); staticPages.forEach((url) => { sms.write({ changefreq: 'monthly', priority: 1.0, url, }); }); const stream = await Page.createQueryBuilder('Page') .where({ img: Not(IsNull()), discoverable: true, }) .stream(); stream .pipe(sms) .pipe( fs.createWriteStream(path.join(__dirname, '../../web/public/sitemap.xml')) ); await new Promise((resolve, reject) => { stream.on('data', (pgPage) => { const page = Page.create( Object.entries(pgPage).reduce((prev, [key, value]) => { return { ...prev, [key.startsWith('Page_') ? key.slice(5) : key]: value, }; }, {}) ); const slug = page.name && page.retailer ? `/${kebabCase(page.retailer)}-${kebabCase(page.name)}` : page.name ? `/${kebabCase(page.name)}` : ''; sms.write({ url: `${baseUrl}/page${slug}/${page.firebaseId}`, priority: 1.0, changefreq: 'monthly', }); pagesAdded += 1; if (pagesAdded % 1000 === 0) { logger.info(`Processed ${pagesAdded} pages`); } }); stream.on('end', () => { logger.info(`Processed ${pagesAdded} pages`); sms.end(() => { resolve(); }); }); stream.on('error', (error) => { reject(error); }); }); }; main().catch((error) => { logger.error(error); process.exit(1); });