#!/usr/bin/env ts-node
import fs from 'fs';
import path from 'path';
import { URL } from 'url';
import { config as setupEnv } from 'dotenv-flow';
import kebabCase from 'lodash.kebabcase';
import { SitemapAndIndexStream, SitemapStream } from 'sitemap';
import { IsNull, Not } from 'typeorm';
import connectToDatabase from '@/config/db';
import { Page } from '@/entities';
import logger from '@/logger';
setupEnv();
const getBaseUrl = (): string => {
switch (process.env.NODE_ENV) {
case 'test':
return 'test/';
case 'staging':
return 'https://www.spoken-staging.com';
default:
return 'https://www.spoken.io';
}
};
const baseUrl = getBaseUrl();
const createURLElementFromURL = (url: string): string => {
return `${url}${new Date().toISOString()}monthly1.0`;
};
const retailerPaths = [
'urbanoutfitters',
'luluandgeorgia',
'birchlane',
'allmodern',
'perigold',
'jossandmain',
'dotandbo',
'onekingslane',
'pier1',
'burkedecor',
'houzz',
'worldmarket',
'amara',
'kathykuohome',
'kohls',
'ashleyfurniture',
'wayfair',
'1stopbedrooms',
'bellacor',
'bloomingdales',
'colemanfurniture',
'emmamason',
'englishelm',
'graysonluxury',
'highfashionhome',
'homeshoppingmalls',
'interiorhomescapes',
'jcpenney',
'laylagrayce',
'macys',
'makerandmoss',
'neimanmarcus',
'paynesgray',
'raymourflanigan',
'rcwilley',
'saksfifthavenue',
'scoutandnimble',
'sears',
'target',
'topmodern',
].sort();
const getSitemapPageURLs = (): string[] => {
const baseUrl = getBaseUrl();
const pages = [
...['signup.js', 'disclaimer.js', 'request.js'],
...retailerPaths.map((retailer) => `shop/${retailer}`),
];
return pages.map((page) => {
page = page.replace(/\.js$/, '');
return `${baseUrl}/${page}`;
});
};
const main = async () => {
await connectToDatabase();
let pagesAdded = 0;
const sms = new SitemapAndIndexStream({
limit: 50000,
// SitemapAndIndexStream will call this user provided function every time
// it needs to create a new sitemap file. You merely need to return a stream
// for it to write the sitemap urls to and the expected url where that
// sitemap will be hosted
getSitemapStream: (i) => {
const sitemapStream = new SitemapStream({ hostname: getBaseUrl() });
const sitemapPath = `./sitemap-${i}.xml`;
const directory = path.join(__dirname, '../../web/public/sitemap');
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, { recursive: true });
}
const ws = sitemapStream.pipe(
fs.createWriteStream(path.join(directory, sitemapPath))
);
return [
new URL(sitemapPath, `${getBaseUrl()}/sitemap/`).toString(),
sitemapStream,
ws,
];
},
});
const staticPages = getSitemapPageURLs();
staticPages.forEach((url) => {
sms.write({
changefreq: 'monthly',
priority: 1.0,
url,
});
});
const stream = await Page.createQueryBuilder('Page')
.where({
img: Not(IsNull()),
discoverable: true,
})
.stream();
stream
.pipe(sms)
.pipe(
fs.createWriteStream(path.join(__dirname, '../../web/public/sitemap.xml'))
);
await new Promise((resolve, reject) => {
stream.on('data', (pgPage) => {
const page = Page.create(
Object.entries(pgPage).reduce((prev, [key, value]) => {
return {
...prev,
[key.startsWith('Page_') ? key.slice(5) : key]: value,
};
}, {})
);
const slug =
page.name && page.retailer
? `/${kebabCase(page.retailer)}-${kebabCase(page.name)}`
: page.name
? `/${kebabCase(page.name)}`
: '';
sms.write({
url: `${baseUrl}/page${slug}/${page.firebaseId}`,
priority: 1.0,
changefreq: 'monthly',
});
pagesAdded += 1;
if (pagesAdded % 1000 === 0) {
logger.info(`Processed ${pagesAdded} pages`);
}
});
stream.on('end', () => {
logger.info(`Processed ${pagesAdded} pages`);
sms.end(() => {
resolve();
});
});
stream.on('error', (error) => {
reject(error);
});
});
};
main().catch((error) => {
logger.error(error);
process.exit(1);
});