import { ChevronRightIcon } from 'lucide-react'; import { choiceOf, matchingPreset, presetPolicy, withChoice, type RobotsChoice, type RobotsPreset, } from '../../lib/seo/robots-policy'; import { classNames } from '../../lib/utils'; import type { ISeoCrawlerStatus, ISeoRobotsPolicy, } from '../../service/seo/seo.interface'; import { Badge } from '../ui/badge'; import { Button } from '../ui/button'; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from '../ui/select'; export const ROBOTS_PRESETS: { value: RobotsPreset; label: string; hint: string; }[] = [ { value: 'open', label: 'Open to AI', hint: 'Every AI crawler may read the site: it can be cited in AI answers and the next models learn about it.', }, { value: 'answers', label: 'Answers, not training', hint: 'The answer engines may read and cite the site; the crawlers that train models are kept out, Google-Extended included, so Gemini apps no longer answer from the pages. Google Search, AI Overviews and AI Mode are unaffected.', }, { value: 'closed', label: 'Closed to AI', hint: 'Every AI crawler is kept out. The site cannot be quoted or linked in ChatGPT, Perplexity, Claude, Alexa or Gemini app answers.', }, { value: 'text', label: 'As the text says', hint: 'No AI crawler is named by the settings; each follows the rules of the text below.', }, ]; const CHOICES: { value: RobotsChoice; label: string }[] = [ { value: 'allow', label: 'Allow' }, { value: 'block', label: 'Block' }, { value: 'default', label: 'As the text says' }, ]; const GROUPS: { kind: ISeoCrawlerStatus['kind']; title: string; description: string; }[] = [ { kind: 'answers', title: 'Answer engines', description: 'Fetch pages to answer questions and cite them now. Kept out, the site is never quoted or linked in these answers.', }, { kind: 'mixed', title: 'Answers + training', description: 'One name decides both. Google-Extended covers Gemini’s training and the pages Gemini apps answer from: kept out, Google no longer trains on the site and Gemini apps stop citing it. Google Search, AI Overviews and AI Mode read the site through Googlebot either way.', }, { kind: 'training', title: 'Model training', description: 'Collect text for the next versions of the models. Kept out, today’s answers are unaffected, but the models learn less about the brand.', }, { kind: 'search', title: 'Search engines', description: 'Set by the text below. Google’s AI Overviews and Bing’s Copilot answers are built from these crawlers, not from the AI ones above.', }, ]; const statusText = (crawler: ISeoCrawlerStatus): string => { if (crawler.status === 'blocked') { return 'Kept out'; } if (crawler.status === 'partial') { return `Kept out of ${crawler.blocked.join(', ')}`; } return 'Can read the site'; }; /** Whose rules decide: the crawler's own group, or the one for every crawler. */ const ruleSource = (crawler: ISeoCrawlerStatus): string => crawler.named ? 'named in robots.txt' : 'rules for all crawlers'; const summary = (rows: ISeoCrawlerStatus[]): string => { if (rows.length === 1) { return statusText(rows[0]); } const open = rows.filter(row => row.status === 'allowed').length; return open === rows.length ? `All ${rows.length} can read the site` : `${open} of ${rows.length} can read the site`; }; const StatusDot = ({ status, }: { status: ISeoCrawlerStatus['status']; }): JSX.Element => ( ); /** * Which AI crawlers may read the site: presets for the usual choices, then * one row per crawler with the choice and what the file lets it read. * The status comes from the file the draft would serve. */ export const RobotsCrawlers = ({ crawlers, policy, onChange, disabled, }: { crawlers: ISeoCrawlerStatus[]; policy: ISeoRobotsPolicy; onChange: (policy: ISeoRobotsPolicy) => void; disabled?: boolean; }): JSX.Element => { const preset = matchingPreset(policy, crawlers); // The list sits in the control column of a settings row, so its rows // follow its own width: three columns only where the name keeps room. return (
{ROBOTS_PRESETS.map(item => ( ))}

{preset ? ROBOTS_PRESETS.find(item => item.value === preset)?.hint : 'A choice of your own, crawler by crawler.'}

{GROUPS.map(group => { const rows = crawlers.filter(crawler => crawler.kind === group.kind); if (rows.length === 0) { return null; } const allOpen = rows.every(row => row.status === 'allowed'); return (
{group.title} {summary(rows)} {group.description}
    {rows.map(crawler => (
  • {crawler.token} {crawler.company} · {crawler.product}
    {statusText(crawler)} {' '} · {ruleSource(crawler)}
    {group.kind !== 'search' && ( )}
  • ))}
); })}
); };