import * as dntShim from "./_dnt.shims.js"; import { call, type Operation, resource, spawn, useAbortSignal, } from "effection"; import { join } from "./deps/jsr.io/@std/path/1.1.6/mod.js"; import { ensureDir } from "./deps/jsr.io/@std/fs/1.0.24/ensure_dir.js"; import { stringify } from "./deps/jsr.io/@libs/xml/6.0.8/stringify.js"; import { parse } from "./deps/jsr.io/@libs/xml/6.0.8/parse.js"; import { useDownloader } from "./downloader.js"; export interface StaticalizeOptions { host: URL; base: URL; dir: string; strict?: boolean; concurrency?: number; retries?: number; } export interface Staticalizer { urls: ReadonlySet; staticalize(): Operation; } export function useStaticalizer( options: StaticalizeOptions, ): Operation { let { host, base, dir, strict, concurrency, retries } = options; return resource(function* (provide) { let signal = yield* useAbortSignal(); let urls: Set = yield* call(async () => { let url = new URL("/sitemap.xml", host); let response = await fetch(url, { signal }); if (!response.ok) { let error = new Error( `GET ${url} ${response.status} ${response.statusText}`, ); error.name = `SitemapError`; throw error; } let xml: SitemapXML; try { let text = await response.text(); xml = parse(text, { flatten: { attributes: false, empty: false, text: true }, }) as unknown as SitemapXML; } catch (cause) { // Reading or parsing the sitemap body can fail on its own (e.g. a // gzip/transport decode error, or malformed XML); attach the URL so the // failure is traceable rather than a bare stack trace. let error = new Error(`GET ${url} could not be read as a sitemap`, { cause, }); error.name = `SitemapError`; throw error; } let entries = xml.urlset.url ?? xml.urlset.urls ?? []; let list = Array.isArray(entries) ? entries : [entries]; return new Set( list.filter(Boolean).map((entry) => { let loc = typeof entry === "string" ? entry : entry.loc; return new URL(loc); }), ); }); let downloader = yield* useDownloader({ host, base, outdir: dir, strict, concurrency, retries, }); yield* provide({ urls, *staticalize() { yield* call(() => ensureDir(dir)); for (let url of urls) { yield* downloader.download(url.toString()); } let sitemap = yield* spawn(function* () { let xml = stringify({ urlset: { "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9", "urls": [...urls].map((url) => { let loc = new URL(url); loc.host = base.host; loc.port = base.port; loc.protocol = base.protocol; return { loc: { "#text": loc } }; }), }, }); yield* call(() => dntShim.Deno.writeFile( join(dir, "sitemap.xml"), new TextEncoder().encode(xml), ) ); }); yield* sitemap; yield* downloader; }, }); }); } export interface SitemapURL { loc: string; lastmod?: string; changefreq?: string; priority?: string; } interface SitemapXML { urlset: { url?: SitemapEntry | SitemapEntry[]; urls?: SitemapEntry | SitemapEntry[]; }; } type SitemapEntry = SitemapURL | string;