import { useEffect, useState } from 'react'; import { CopyIcon, ExternalLinkIcon, HistoryIcon, RotateCcwIcon, } from 'lucide-react'; import { toast } from 'sonner'; import { OptionRow, OptionTip } from '../../components/seo/option-row'; import { SeoPageError, SeoPageLoading } from '../../components/seo/seo-page'; import { Button } from '../../components/ui/button'; import { Callout } from '../../components/ui/callout'; import { Checkbox } from '../../components/ui/checkbox'; import ConfirmDialog from '../../components/ui/confirm-dialog'; import { Textarea } from '../../components/ui/textarea'; import { RobotsCrawlers } from '../../components/seo/robots-crawlers'; import { samePolicy, shutOutSearchEngines } from '../../lib/seo/robots-policy'; import type { ISeoFiles, ISeoHtaccessCheck, ISeoRobotsPolicy, ISeoRobotsPreview, } from '../../service/seo/seo.interface'; import { checkSeoHtaccess, fetchSeoFiles, previewSeoRobots, restoreSeoHtaccess, saveSeoHtaccess, saveSeoRobots, } from '../../service/seo/seo.service'; const NO_POLICY: ISeoRobotsPolicy = { allow: [], block: [] }; export interface SeoFilesState { files: ISeoFiles | null; setFiles: (files: ISeoFiles) => void; error: string; } interface EditorProps extends SeoFilesState { /** Told when the editor starts or stops holding unsaved changes. */ onDirtyChange?: (dirty: boolean) => void; } /** * Both files, read once for the page that holds the two editors, so each * stays mounted with its draft while another section is on screen. * * @returns {SeoFilesState} The files, their setter and the load error. */ export const useSeoFiles = (): SeoFilesState => { const [files, setFiles] = useState(null); const [error, setError] = useState(''); useEffect(() => { fetchSeoFiles() .then(setFiles) .catch(reason => setError((reason as Error).message)); }, []); return { files, setFiles, error }; }; const formatDate = (iso: string): string => new Date(iso).toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short', }); /** * The robots.txt editor: which AI crawlers may read the site, the text of * the file (empty means the default one) and the file crawlers would get, * previewed from the unsaved changes before they are saved, with what a * file written by hand drops from the default one. */ export const SeoRobotsEditor = ({ files, setFiles, error, onDirtyChange, }: EditorProps): JSX.Element => { const [draft, setDraft] = useState(null); const [policyDraft, setPolicyDraft] = useState(null); const [preview, setPreview] = useState(null); const [saving, setSaving] = useState(false); /** The search engines the draft would shut out, while that is asked. */ const [shutOut, setShutOut] = useState(null); const robots = files?.robots; const value = draft ?? robots?.content ?? ''; const policy = policyDraft ?? robots?.policy ?? NO_POLICY; const dirty = !!robots && ((draft !== null && draft !== robots.content) || (policyDraft !== null && !samePolicy(policyDraft, robots.policy))); useEffect(() => { onDirtyChange?.(dirty); }, [dirty, onDirtyChange]); useEffect(() => { if (!dirty) { setPreview(null); return undefined; } const timer = window.setTimeout(() => { previewSeoRobots(value, policy) .then(setPreview) .catch(() => setPreview(null)); }, 400); return () => window.clearTimeout(timer); }, [dirty, value, policy]); if (error) { return ; } if (!robots) { return ; } const shown = preview ?? { served: robots.served, crawlers: robots.crawlers, warnings: robots.warnings, lost: robots.lost, }; const save = async (content: string, confirmed = false): Promise => { setSaving(true); try { if (!confirmed) { const next = preview ?? (await previewSeoRobots(content, policy)); const lost = shutOutSearchEngines(robots.crawlers, next.crawlers); if (lost.length > 0) { setShutOut(lost); return; } } setShutOut(null); setFiles(await saveSeoRobots(content, policy)); setDraft(null); setPolicyDraft(null); toast.success( content.trim() === '' && draft !== null ? 'robots.txt back to the default text' : 'robots.txt saved' ); } catch (reason) { toast.error((reason as Error).message); } finally { setSaving(false); } }; const discard = (): void => { setDraft(null); setPolicyDraft(null); }; return (
{(robots.rivals ?? []).length > 0 && ( {(robots.rivals ?? []).join(', ')} writes robots.txt while it is active, so what is saved here reaches crawlers only once Recomaze takes over (SEO → Dashboard → Import & switch). Until then, change robots.txt in {(robots.rivals ?? []).join(', ')}. )} {robots.physical && ( A robots.txt file exists on the server and is served instead of this content, so neither the text nor the AI crawler choices reach crawlers. Copy what it needs here, then delete or rename the file. )} {!robots.public && ( Settings → Reading asks search engines not to index this site, so the default file blocks everything and the AI crawler choices are not applied. Change that setting first. )}