/* eslint-env jest */ import { choiceOf, matchingPreset, presetPolicy, shutOutSearchEngines, withChoice, } from '../robots-policy'; const crawlers = [ { id: 'googlebot', kind: 'search' as const }, { id: 'oai-searchbot', kind: 'answers' as const }, { id: 'perplexitybot', kind: 'answers' as const }, { id: 'gptbot', kind: 'training' as const }, { id: 'ccbot', kind: 'training' as const }, ]; describe('robots policy', () => { it('builds each preset from the AI crawlers only, never the search engines', () => { expect(presetPolicy('open', crawlers)).toEqual({ allow: ['oai-searchbot', 'perplexitybot', 'gptbot', 'ccbot'], block: [], }); expect(presetPolicy('answers', crawlers)).toEqual({ allow: ['oai-searchbot', 'perplexitybot'], block: ['gptbot', 'ccbot'], }); expect(presetPolicy('closed', crawlers).block).not.toContain('googlebot'); expect(presetPolicy('text', crawlers)).toEqual({ allow: [], block: [] }); }); it('keeps a name that decides answers and training out of "Answers, not training"', () => { const withGoogle = [ ...crawlers, { id: 'google-extended', kind: 'mixed' as const }, { id: 'amzn-searchbot', kind: 'answers' as const }, ]; expect(presetPolicy('answers', withGoogle)).toEqual({ allow: ['oai-searchbot', 'perplexitybot', 'amzn-searchbot'], block: ['gptbot', 'ccbot', 'google-extended'], }); expect(presetPolicy('open', withGoogle).allow).toContain('google-extended'); expect(presetPolicy('closed', withGoogle).block).toContain( 'google-extended' ); expect( matchingPreset( { allow: [ 'oai-searchbot', 'perplexitybot', 'amzn-searchbot', 'google-extended', ], block: ['gptbot', 'ccbot'], }, withGoogle ) ).toBeNull(); }); it('moves one crawler between the lists and reads its choice back', () => { const open = presetPolicy('open', crawlers); const blocked = withChoice(open, 'gptbot', 'block'); expect(choiceOf(blocked, 'gptbot')).toBe('block'); expect(blocked.allow).not.toContain('gptbot'); expect(choiceOf(withChoice(blocked, 'gptbot', 'default'), 'gptbot')).toBe( 'default' ); }); it('recognises a preset whatever the order of the lists', () => { expect( matchingPreset( { allow: ['perplexitybot', 'oai-searchbot'], block: ['ccbot', 'gptbot'], }, crawlers ) ).toBe('answers'); expect( matchingPreset({ allow: ['gptbot'], block: [] }, crawlers) ).toBeNull(); }); it('names the search engines a new file would shut out', () => { const row = ( id: string, kind: 'search' | 'answers', status: 'allowed' | 'blocked' | 'partial' ) => ({ id, kind, product: id, status }); const now = [ row('googlebot', 'search', 'allowed'), row('bingbot', 'search', 'partial'), row('duckduckbot', 'search', 'blocked'), row('gptbot', 'answers', 'allowed'), ]; const next = [ row('googlebot', 'search', 'blocked'), row('bingbot', 'search', 'blocked'), row('duckduckbot', 'search', 'blocked'), row('gptbot', 'answers', 'blocked'), ]; expect(shutOutSearchEngines(now, next)).toEqual(['googlebot', 'bingbot']); expect(shutOutSearchEngines(next, next)).toEqual([]); }); });