// robots.txt generation. Environment-aware, and the default is the safe one: // anything that is not explicitly production emits `Disallow: /`, because a // branch deploy that gets indexed outranks and cannibalises the real site. import { DEFAULT_ENVIRONMENT, type Environment, type SeoRobotsGroup, tryResolveEnvironment, } from '@ultimat3/core'; import { absoluteUrl } from './xml'; /** One `User-agent` group — the very shape `app.config.ts`'s `seo.robots.groups` declares. */ export type RobotsGroup = SeoRobotsGroup; export interface RobotsConfig { baseUrl: string; /** Omitted means "resolve from the environment"; anything but `production` disallows all. */ environment?: Environment | undefined; groups?: readonly RobotsGroup[]; /** * Paths every crawler is kept out of — `app.config.ts`'s `seo.robots.disallow`. Added to EVERY * group, and a `User-agent: *` group is emitted to carry it when none is declared. Production * only: a non-production `robots.txt` already disallows everything. */ disallow?: readonly string[]; /** Sitemap paths or absolute URLs. Only emitted in production. */ sitemaps?: readonly string[]; /** Extra lines appended verbatim, e.g. a `Host:` directive. */ extra?: readonly string[]; } /** * Fail-closed: only the exact string `production` opts a deploy into indexing. A branch deploy * (`staging`), a laptop, a typo and an unset variable are all "not production" and all disallow. */ export function isIndexable(environment: Environment): boolean { return environment === 'production'; } /** * `@ultimat3/core` owns the read of `ULTIMATE_ENV`; this module owns only what an unreadable one * means for a crawler. A typo in that key throws there, and a `robots.txt` render can be the first * thing in a web process to ask — nothing in the container's boot path resolves the environment * unconditionally — so an unnameable deploy resolves to core's own default here. The body was * already going to be `Disallow: /`; a 500 would only cost the operator the reason. */ function ambientEnvironment(): Environment { return tryResolveEnvironment() ?? DEFAULT_ENVIRONMENT; } function agents(userAgent: string | readonly string[]): readonly string[] { return typeof userAgent === 'string' ? [userAgent] : userAgent; } export function buildRobots(config: RobotsConfig): string { const environment = config.environment ?? ambientEnvironment(); const lines: string[] = [`# environment: ${environment}`]; if (!isIndexable(environment)) { // No sitemap either: advertising one invites a crawl we just refused. lines.push('User-agent: *', 'Disallow: /'); return `${lines.join('\n')}\n`; } const declared: readonly RobotsGroup[] = config.groups === undefined || config.groups.length === 0 ? [{ userAgent: '*', allow: ['/'] }] : config.groups; // A crawler obeys the ONE group that names it most specifically and never reads `*` beside it, // so "every crawler" means every group. Appending to `*` alone left the list out of exactly // the groups an app had written for a named crawler — and out of the file when no group named `*`. const extra = config.disallow ?? []; const everywhere = declared.map((group) => ({ ...group, disallow: [...new Set([...(group.disallow ?? []), ...extra])], })); const namesEveryone = declared.some((group) => agents(group.userAgent).includes('*')); const groups: readonly RobotsGroup[] = extra.length === 0 || namesEveryone ? everywhere : [...everywhere, { userAgent: '*', disallow: extra }]; for (const group of groups) { for (const agent of agents(group.userAgent)) lines.push(`User-agent: ${agent}`); for (const path of group.allow ?? []) lines.push(`Allow: ${path}`); for (const path of group.disallow ?? []) lines.push(`Disallow: ${path}`); if (group.crawlDelay !== undefined) lines.push(`Crawl-delay: ${group.crawlDelay}`); lines.push(''); } for (const sitemap of config.sitemaps ?? ['/sitemap.xml']) { lines.push(`Sitemap: ${absoluteUrl(config.baseUrl, sitemap)}`); } for (const line of config.extra ?? []) lines.push(line); return `${lines.join('\n').trimEnd()}\n`; } /** * `rule` as a crawler reads it: a PREFIX of the path, `*` standing for any run of characters and a * trailing `$` anchoring the end. An empty rule disallows nothing. */ function disallowPattern(rule: string): RegExp | undefined { if (rule === '') return undefined; const anchored = rule.endsWith('$'); const body = (anchored ? rule.slice(0, -1) : rule) .split('*') .map((literal) => literal.replace(/[.+?^${}()|[\]\\]/g, '\\$&')) .join('.*'); return new RegExp(`^${body}${anchored ? '$' : ''}`); } /** * A test for "do these `Disallow:` rules keep a crawler out of this path". What the sitemap asks * before it lists a URL: one that `robots.txt` blocks is reported as "submitted URL blocked by * robots.txt", and the two files are written from one config. */ export function disallowMatcher(rules: readonly string[]): (path: string) => boolean { const patterns = rules.flatMap((rule) => disallowPattern(rule) ?? []); return (path) => patterns.some((pattern) => pattern.test(path)); }