import * as async from 'async' import * as fs from 'fs' import * as path from 'path' import puppeteer from 'puppeteer' import { createOutDir } from './lib/createOutDir' import { findRoutes } from './lib/findRoutes' import { formatOutputFilename } from './lib/formatOutputFilename' import { getPageHtmlContent } from './lib/getPageHtmlContent' import { loadPagesMetadata } from './lib/loadPagesMetadata' import { parsePageHtmlContent } from './lib/parseHtmlContent' import { startProductionServer } from './lib/startProductionServer' import { htmlToMarkdown } from './lib/toMarkdown' import { TransformerConfig } from './types/config.types' export const loadConfig = (projectDir: string) => { const configPath = path.join(projectDir, '.unbody', 'transformer.config.json') return JSON.parse(fs.readFileSync(configPath, 'utf-8')) as TransformerConfig } export const transform = async (config: TransformerConfig) => { const pagesMetadata = loadPagesMetadata({ config }) const routes = await findRoutes({ config }).then((routes) => routes.filter(([route]) => { const metadata = pagesMetadata.get(route) return !!metadata && metadata.frontMatter?.searchable !== false }), ) const outDir = await createOutDir(config.outDir) await outDir.init() const server = config.server.enable && (await startProductionServer(config.projectDir)) const browser = config.browser.enable && (await puppeteer.launch({ headless: true })) const tasks: async.AsyncFunction[] = [] for (const route of routes) { const task: async.AsyncFunction = async (callback) => { try { const pageFile = path.join( config.projectDir, config.distDir, 'server', route[1], ) const content = await getPageHtmlContent({ config, browser: browser || undefined, server: server ? { hostname: server.hostname, port: server.port, // basePath: nextConfig.basePath || '/', basePath: '/', } : undefined, pagePath: route[0], pageFile, }) if (!content) { console.info('not content found for page: ', route[0]) return } const parsed = await parsePageHtmlContent( content.html, pagesMetadata.get(route[0]), config, ) if (!parsed) { console.info('no content found for page: ', route[0]) return } const filename = await formatOutputFilename({ pagePath: route[0], extension: '.md', filename: pageFile, relativeFilename: route[1], }) await outDir.write( filename, await htmlToMarkdown(parsed.html, parsed.metadata), ) callback(null, filename as any) } catch (e) { console.error('Failed to transform page: ', route[0]) console.error(e) callback(e as Error, '' as any) } return null } tasks.push(task) } if (browser && config.browser.concurrency) { await async.parallelLimit(tasks, config.browser.concurrency) } else { await Promise.allSettled(tasks.map((task) => task(() => {}))) } if (browser) { await browser.close() } }