// html-escaper's five-entity table is XML-safe (`'` → the numeric `'`). import { escape as escapeXml } from "html-escaper"; import { customStaticRoutes, discoverPagesSync, hasGeneratedChangelog, } from "../astro/pages.ts"; import { normalizeBasePath, withBasePath } from "../core/base-path.ts"; import type { BlumeProject } from "../core/project-graph.ts"; import { siteRoot } from "../core/site-url.ts"; /** * Astro's reserved error routes. A user-authored override (`pages/404.astro`, * `pages/500.astro`, or a `404.md` content page — see `writeNotFoundPage` in * `astro/generate.ts`) is neither dynamic nor private, so it would otherwise * be emitted — but error pages aren't crawlable destinations and must stay out * of the sitemap. */ const ERROR_ROUTES = new Set(["/404", "/500"]); /** A `` element (W3C date) when the page has a valid modified date. */ const lastmodTag = (value: string | undefined): string => { if (!value) { return ""; } const date = new Date(value); return Number.isNaN(date.getTime()) ? "" : `${date.toISOString().slice(0, 10)}`; }; /** One emitted sitemap artifact: its dist-root filename and XML body. */ export interface SitemapFile { name: string; xml: string; } /** * The sitemaps.org cap on `` entries in a single file. Beyond it, * `sitemap.xml` becomes a sitemap index pointing at numbered chunk files — * search engines reject an oversized urlset outright. */ const URLS_PER_FILE = 50_000; const renderUrlset = ( urls: string[] ): string => ` ${urls.join("\n")} `; /** * Build the sitemap files from the route manifest plus the routes the manifest * can't see: custom `.astro` pages (most importantly a custom landing `/`) and * the generated `/changelog` index. Returns null when the sitemap is disabled * or no `site` is configured (absolute URLs are required for a valid sitemap). * Drafts, hidden, and `noindex` pages are excluded. * * Sites within the per-file URL cap get the single classic `sitemap.xml`; * larger sites get `sitemap.xml` as a sitemap index over numbered * `sitemap-N.xml` chunks, all served from the same directory robots.txt * already points at. */ export const buildSitemapFiles = ( project: BlumeProject ): SitemapFile[] | null => { const { site } = project.config.deployment; if (!(site && project.config.seo.sitemap)) { return null; } const base = siteRoot(site); // Routes carry `basePath`; a `deployment.base` subdirectory is layered on top. const deployBase = normalizeBasePath(project.config.deployment.base); // Archived-version pages leave the sitemap when the version is noindexed, // or when their canonical points at a still-existing latest equivalent — // listing a URL whose canonical says "index the other page" invites the // noindexed-page-in-sitemap incoherence Docusaurus is known for. A page // that exists only in an archived version stays listed (self-canonical). const { versions } = project.config; const archivedById = new Map( (versions?.archived ?? []).map((version) => [version.id, version]) ); const currentKeys = versions ? new Set( project.graph.pages.flatMap((page) => page.version === "" ? [`${page.versionKey}\u0000${page.locale}`] : [] ) ) : null; const archivedExcluded = (page: (typeof project.graph.pages)[number]) => { if (!page.version) { return false; } // A non-empty version always names a configured archived entry — that is // the only way detection assigns one. const archived = archivedById.get(page.version); return Boolean( archived?.noindex || (archived?.canonical === "latest" && currentKeys?.has(`${page.versionKey}\u0000${page.locale}`)) ); }; const seen = new Set(); const urls: string[] = []; const pushUrl = (route: string, lastModified?: string): void => { // `` must be a well-formed, XML-escaped URL: percent-encode the path, // then escape XML metacharacters (notably `&`) so a route like // `/Tips & Tricks` doesn't produce invalid XML that gets the whole sitemap // rejected. const loc = escapeXml(encodeURI(`${base}${route}`)); if (seen.has(loc)) { return; } seen.add(loc); urls.push(` ${loc}${lastmodTag(lastModified)}`); }; for (const page of project.graph.pages) { if ( page.meta.draft || page.meta.sidebar.hidden || page.meta.seo.noindex || ERROR_ROUTES.has(page.route) || archivedExcluded(page) ) { continue; } pushUrl(withBasePath(deployBase, page.route), page.lastModified); } // Custom `.astro` pages and the generated changelog index mount outside // `basePath` (they're injected at their pattern — see `blumeIntegration`), so // only the deployment base layers onto their URLs. const userPages = project.context.pagesRoot ? discoverPagesSync(project.context.pagesRoot) : []; const extraRoutes = customStaticRoutes(userPages).filter( (route) => !ERROR_ROUTES.has(route) ); if (hasGeneratedChangelog(project, userPages)) { extraRoutes.push("/changelog"); } for (const route of extraRoutes) { pushUrl(withBasePath(deployBase, route)); } urls.sort(); if (urls.length <= URLS_PER_FILE) { return [{ name: "sitemap.xml", xml: renderUrlset(urls) }]; } const chunks: SitemapFile[] = []; const references: string[] = []; for (let start = 0; start < urls.length; start += URLS_PER_FILE) { const name = `sitemap-${chunks.length + 1}.xml`; chunks.push({ name, xml: renderUrlset(urls.slice(start, start + URLS_PER_FILE)), }); // Chunks sit next to sitemap.xml, so their URLs layer the same deployment // base robots.txt uses for the index. const loc = escapeXml( encodeURI(`${base}${withBasePath(deployBase, `/${name}`)}`) ); references.push(` ${loc}`); } const index = ` ${references.join("\n")} `; return [{ name: "sitemap.xml", xml: index }, ...chunks]; };