import fs from "node:fs"; import crypto from "node:crypto"; import path from "node:path"; import chalk from "chalk"; import fm from "front-matter"; import log from "loglevel"; import { fdir } from "fdir"; import { buildURL, execGit, slugToFolder, Document, Redirect, } from "../content/index.js"; import { DEFAULT_LOCALE, HTML_FILENAME, MARKDOWN_FILENAME, VALID_LOCALES, } from "../libs/constants/index.js"; import { CONTENT_ROOT, CONTENT_TRANSLATED_ROOT } from "../libs/env/index.js"; import { DocFrontmatter } from "../libs/types/document.js"; const CONFLICTING = "conflicting"; const ORPHANED = "orphaned"; const DEFAULT_LOCALE_LC = DEFAULT_LOCALE.toLowerCase(); export async function syncAllTranslatedContent(locale: string) { if (!CONTENT_TRANSLATED_ROOT) { throw new Error( "CONTENT_TRANSLATED_ROOT must be set to sync translated content!" ); } const redirects = new Map(); const api = new fdir() .withFullPaths() .withErrors() .filter((filePath) => { return ( filePath.endsWith(HTML_FILENAME) || filePath.endsWith(MARKDOWN_FILENAME) ); }) .crawl(path.join(CONTENT_TRANSLATED_ROOT, locale)); const files = [...(api.sync() as any)]; const stats = { conflictingDocs: 0, movedDocs: 0, orphanedDocs: 0, redirectedDocs: 0, renamedDocs: 0, totalDocs: files.length, }; for (const f of files) { const { conflicting, moved, followed, orphaned, redirect, renamed } = await syncTranslatedContent(f, locale); if (conflicting) { stats.conflictingDocs += 1; } if (moved) { stats.movedDocs += 1; } if (followed) { stats.redirectedDocs += 1; } if (orphaned) { stats.orphanedDocs += 1; } if (redirect) { redirects.set(redirect[0], redirect[1]); } if (renamed) { stats.renamedDocs += 1; } } if (redirects.size > 0) { Redirect.add(locale, [...redirects.entries()]); } return stats; } function resolve(slug: string) { if (!slug) { return slug; } const url = buildURL(DEFAULT_LOCALE_LC, slug); const resolved = Redirect.resolve(url); const doc = Document.read(Document.urlToFolderPath(resolved)); return doc?.metadata.slug ?? slug; } function mdOrHtmlExists(folderPath: string) { return ( fs.existsSync(path.join(folderPath, MARKDOWN_FILENAME)) || fs.existsSync(path.join(folderPath, HTML_FILENAME)) ); } export async function syncTranslatedContent( inFilePath: string, locale: string ) { if (!CONTENT_TRANSLATED_ROOT) { throw new Error( "CONTENT_TRANSLATED_ROOT must be set to sync translated content!" ); } const status = { redirect: null, conflicting: false, followed: false, moved: false, orphaned: false, renamed: false, }; const rawDoc = fs.readFileSync(inFilePath, "utf-8"); const fileName = path.basename(inFilePath); const { attributes: oldMetadata, body: rawBody } = fm(rawDoc); const resolvedSlug = resolve(oldMetadata.slug); const metadata = { ...oldMetadata, slug: resolvedSlug, }; if ( oldMetadata.slug.startsWith(ORPHANED) || oldMetadata.slug.startsWith(CONFLICTING) ) { return status; } // Any case-sensitive change is (at least) a rename. status.renamed = oldMetadata.slug !== metadata.slug; // Any case-insensitive change is a move. status.moved = status.renamed && oldMetadata.slug.toLowerCase() !== metadata.slug.toLowerCase(); if (status.moved) { log.log( chalk.bold(`Original redirect: ${oldMetadata.slug} → ${metadata.slug}`) ); status.followed = true; } const dehash = () => { const hash = metadata.slug.indexOf("#"); if (hash < 0) { return; } status.moved = true; log.log(chalk.yellow(`${metadata.slug} contains #, stripping`)); metadata.slug = metadata.slug.substring(0, hash); }; const getFileDir = () => { return path.join( CONTENT_TRANSLATED_ROOT, locale, slugToFolder(metadata.slug) ); }; dehash(); let folderPath = getFileDir(); status.orphaned = !mdOrHtmlExists( path.join(CONTENT_ROOT, DEFAULT_LOCALE_LC, slugToFolder(metadata.slug)) ); if (!status.renamed && !status.orphaned) { return status; } if (status.orphaned) { log.log(chalk.yellow(`orphaned: ${inFilePath}`)); status.followed = false; metadata.slug = `${ORPHANED}/${metadata.slug}`; status.moved = true; folderPath = getFileDir(); if (mdOrHtmlExists(folderPath)) { const filePath = path.join(folderPath, fileName); log.log(`${inFilePath} → ${filePath}`); throw new Error(`file: ${filePath} already exists!`); } } else if (status.moved && mdOrHtmlExists(folderPath)) { console.log(`unrooting ${inFilePath} (conflicting translation)`); metadata.slug = `${CONFLICTING}/${metadata.slug}`; status.conflicting = true; folderPath = getFileDir(); if (mdOrHtmlExists(folderPath)) { metadata.slug = `${metadata.slug}_${crypto .createHash("md5") .update(oldMetadata.slug) .digest("hex")}`; folderPath = getFileDir(); } } status.redirect = [ buildURL(VALID_LOCALES.get(locale), oldMetadata.slug), buildURL(VALID_LOCALES.get(locale), metadata.slug), ]; const filePath = path.join(folderPath, fileName); log.log(`${inFilePath} → ${filePath}`); await Document.updateWikiHistory( path.join(CONTENT_TRANSLATED_ROOT, locale.toLowerCase()), oldMetadata.slug, metadata.slug ); if (status.moved) { moveContent(path.dirname(inFilePath), folderPath); metadata.original_slug = oldMetadata.slug; } Document.saveFile(filePath, Document.trimLineEndings(rawBody), metadata); return status; } // Move all regular files (excluding subdirectories) from one directory to another, // and delete the source directory if it's empty. function moveContent(inFileDir: string, outFileDir: string) { const files = fs.readdirSync(inFileDir, { encoding: "utf-8", withFileTypes: true, }); fs.mkdirSync(outFileDir, { recursive: true }); const regularFiles = files .filter((file) => file.isFile()) .map((file) => file.name); for (const filename of regularFiles) { const source = path.join(inFileDir, filename); execGit(["mv", source, outFileDir], { cwd: CONTENT_TRANSLATED_ROOT }); } // assuming that the source directory is empty if (files.length === regularFiles.length) { try { fs.rmdirSync(inFileDir); } catch (e: any) { if (e.code !== "ENOTEMPTY") { throw e; } } } } export async function syncTranslatedContentForAllLocales() { let moved = 0; for (const locale of VALID_LOCALES.keys()) { if (locale === DEFAULT_LOCALE_LC) { continue; } const { movedDocs } = await syncAllTranslatedContent(locale); moved += movedDocs; } return moved; }