import { IconRobot as Bot, IconBrain as Brain, IconFileText as FileText, IconSearch as Search, IconSparkles as Sparkles, } from '@tabler/icons-react'; import { useQuery } from '@tanstack/react-query'; import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from 'recharts'; import { PageHeader } from '@/components/PageHeader'; import { Badge } from '@/components/ui/badge'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; import { type ChartConfig, ChartContainer, ChartLegend, ChartLegendContent, ChartTooltip, ChartTooltipContent, } from '@/components/ui/chart'; import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle, } from '@/components/ui/empty'; import { Skeleton } from '@/components/ui/skeleton'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow, } from '@/components/ui/table'; import { apiJson } from '@/lib/api'; import { ContentPanel } from '../../components/ContentPanel'; /** Crawler classification, mirrored from the server's shared crawler registry. */ type CrawlerCategory = 'ai-training' | 'ai-search' | 'search'; interface CrawlerTotal { crawler: string; operator: string; category: CrawlerCategory; hits: number; } interface CrawlerDay { day: string; hits: number; byCategory: Record; } interface CrawlerTopPath { path: string; hits: number; crawlers: string[]; } /** Response of `GET /api/v1/admin/crawlers`. */ export interface CrawlerStatsResponse { windowDays: number; totals: CrawlerTotal[]; daily: CrawlerDay[]; topPaths: CrawlerTopPath[]; } /** Days of history requested. No sibling admin page ships a window selector yet. */ const WINDOW_DAYS = 30; export const crawlerStatsKey = ['admin', 'crawlers', WINDOW_DAYS]; export function fetchCrawlerStats(days: number = WINDOW_DAYS): Promise { return apiJson( `/api/v1/admin/crawlers?days=${days}`, {}, 'Failed to fetch crawler activity' ); } const CATEGORY_LABELS: Record = { 'ai-search': 'AI search', 'ai-training': 'AI training', search: 'Search engine', }; const CATEGORY_BADGE: Record = { 'ai-search': 'default', 'ai-training': 'secondary', search: 'outline', }; const dailyConfig = { 'ai-search': { label: CATEGORY_LABELS['ai-search'], color: 'var(--chart-1)' }, 'ai-training': { label: CATEGORY_LABELS['ai-training'], color: 'var(--chart-2)' }, search: { label: CATEGORY_LABELS.search, color: 'var(--chart-3)' }, } satisfies ChartConfig; const CATEGORY_ORDER: CrawlerCategory[] = ['ai-search', 'ai-training', 'search']; function CategoryBadge({ category }: { category: CrawlerCategory }) { return {CATEGORY_LABELS[category]}; } function KpiCard({ label, value, hint, icon: Icon, loading, }: { label: string; value: string; hint: string; icon: React.ComponentType<{ className?: string }>; loading: boolean; }) { return (
{label}
{loading ? ( ) : (

{value}

)}
{loading ? ( ) : ( {hint} )}
); } /** Short day label for the x-axis: `2026-08-24` โ†’ `Aug 24`. */ function formatDay(day: string): string { const [year, month, date] = day.split('-').map(Number); if (!year || !month || !date) return day; return new Date(Date.UTC(year, month - 1, date)).toLocaleDateString(undefined, { month: 'short', day: 'numeric', timeZone: 'UTC', }); } export default function AdminCrawlers() { const { data, isLoading } = useQuery({ queryKey: crawlerStatsKey, queryFn: () => fetchCrawlerStats(), }); const totals = data?.totals ?? []; const daily = data?.daily ?? []; const topPaths = data?.topPaths ?? []; const windowDays = data?.windowDays ?? WINDOW_DAYS; const byCategory = CATEGORY_ORDER.reduce( (acc, category) => { acc[category] = totals .filter((t) => t.category === category) .reduce((sum, t) => sum + t.hits, 0); return acc; }, { 'ai-search': 0, 'ai-training': 0, search: 0 } as Record ); const totalHits = totals.reduce((sum, t) => sum + t.hits, 0); const aiHits = byCategory['ai-search'] + byCategory['ai-training']; const hasData = totalHits > 0; const chartData = daily.map((d) => ({ day: formatDay(d.day), 'ai-search': d.byCategory['ai-search'], 'ai-training': d.byCategory['ai-training'], search: d.byCategory.search, })); const num = (n: number) => n.toLocaleString(); const windowLabel = `Last ${windowDays} days`; return ( <> {!isLoading && !hasData ? ( No crawler visits yet Visits from AI crawlers such as GPTBot, ClaudeBot, and PerplexityBot โ€” plus search engines like Googlebot and Bingbot โ€” will appear here once they fetch a page. If this stays empty, check that the site is deployed on a public domain and that robots.txt still allows crawlers. ) : ( <> {/* Crawler KPIs */}
{/* Daily trend */} Crawler hits over time Document fetches per day, stacked by crawler type {isLoading ? ( ) : ( } /> } /> {CATEGORY_ORDER.map((category) => ( ))} )}
{/* Per-crawler totals */} Crawlers Who visited, and how often {isLoading ? ( ) : totals.length === 0 ? (

No crawler visits in this window.

) : (
Crawler Operator Type Hits {totals.map((row) => ( {row.crawler} {row.operator} {num(row.hits)} ))}
)}
{/* Most-crawled paths */} Most-crawled pages What crawlers actually fetch from your site {isLoading ? ( ) : topPaths.length === 0 ? (

No crawled pages in this window.

) : (
Path Crawlers Hits {topPaths.map((row) => ( {row.path}
{row.crawlers.map((crawler) => ( {crawler} ))}
{num(row.hits)}
))}
)}
)}
); }