import type { ISeoCrawlerStatus, ISeoRobotsPolicy, } from '../../service/seo/seo.interface'; /** What the settings say about one AI crawler; "default" leaves it to the text. */ export type RobotsChoice = 'allow' | 'block' | 'default'; export type RobotsPreset = 'open' | 'answers' | 'closed' | 'text'; type Crawler = Pick; const aiIds = ( crawlers: Crawler[], kinds?: ISeoCrawlerStatus['kind'][] ): string[] => crawlers .filter(crawler => kinds ? kinds.includes(crawler.kind) : crawler.kind !== 'search' ) .map(crawler => crawler.id); export const choiceOf = ( policy: ISeoRobotsPolicy, id: string ): RobotsChoice => { if (policy.block.includes(id)) { return 'block'; } return policy.allow.includes(id) ? 'allow' : 'default'; }; export const withChoice = ( policy: ISeoRobotsPolicy, id: string, choice: RobotsChoice ): ISeoRobotsPolicy => ({ allow: [ ...policy.allow.filter(item => item !== id), ...(choice === 'allow' ? [id] : []), ], block: [ ...policy.block.filter(item => item !== id), ...(choice === 'block' ? [id] : []), ], }); /** * The lists a preset stands for: every AI crawler in, the answer engines in * and the training crawlers out (with the names that decide training and * answers at once, since they cannot be split), all of them out, or none * named. */ export const presetPolicy = ( preset: RobotsPreset, crawlers: Crawler[] ): ISeoRobotsPolicy => { switch (preset) { case 'open': return { allow: aiIds(crawlers), block: [] }; case 'answers': return { allow: aiIds(crawlers, ['answers']), block: aiIds(crawlers, ['training', 'mixed']), }; case 'closed': return { allow: [], block: aiIds(crawlers) }; default: return { allow: [], block: [] }; } }; export const samePolicy = ( a: ISeoRobotsPolicy, b: ISeoRobotsPolicy ): boolean => { const same = (x: string[], y: string[]): boolean => x.length === y.length && x.every(item => y.includes(item)); return same(a.allow, b.allow) && same(a.block, b.block); }; /** The preset the lists match, if any. */ export const matchingPreset = ( policy: ISeoRobotsPolicy, crawlers: Crawler[] ): RobotsPreset | null => (['open', 'answers', 'closed', 'text'] as RobotsPreset[]).find(preset => samePolicy(policy, presetPolicy(preset, crawlers)) ) ?? null; /** * The search engines a new file would shut out that the file served now * lets in: saving `Disallow: /` for everyone took the site out of Google * with nothing but a counter to say so. */ export const shutOutSearchEngines = ( before: Pick[], after: Pick[] ): string[] => after .filter( crawler => crawler.kind === 'search' && crawler.status === 'blocked' && before.some( earlier => earlier.id === crawler.id && earlier.status !== 'blocked' ) ) .map(crawler => crawler.product);