/** * Link classifier — assigns priority and category to discovered URLs. * * Categories: * - 'nav' : site navigation (homepage, top-level pages) — highest priority * - 'auth' : login/register/password flows — high priority * - 'app' : authenticated app screens — high priority * - 'content' : blog/docs/articles — medium priority * - 'api' : REST/GraphQL API endpoints — detected, not crawled as screens * - 'resource' : static assets (JS/CSS/img/fonts) — skip * - 'external' : different domain — skip unless cross-domain crawl enabled * - 'low-value' : pagination, sort, filter query strings — deprioritize */ export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value'; export interface ClassifiedLink { url: string; category: LinkCategory; priority: number; // 0-100, higher = crawl sooner reason: string; } const RESOURCE_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\?|$)/i; const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i; const AUTH_PATTERN = /\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\b/i; const LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i; const PAGINATION_PATTERN = /\/page\/\d+|\/p\/\d+|\bpage=\d+/i; export function classifyLink(url: string, baseUrl: string): ClassifiedLink { let parsed: URL; let baseHost: string; try { parsed = new URL(url); baseHost = new URL(baseUrl).hostname; } catch { return { url, category: 'low-value', priority: 0, reason: 'invalid URL' }; } const path = parsed.pathname + parsed.search; const isExternal = parsed.hostname !== baseHost; if (isExternal) return { url, category: 'external', priority: 0, reason: 'different domain' }; if (RESOURCE_EXT.test(path)) return { url, category: 'resource', priority: 0, reason: 'static asset extension' }; if (API_PATTERN.test(path)) return { url, category: 'api', priority: 10, reason: 'API path pattern' }; if (AUTH_PATTERN.test(path)) return { url, category: 'auth', priority: 90, reason: 'auth flow pattern' }; if (PAGINATION_PATTERN.test(path)) return { url, category: 'low-value', priority: 15, reason: 'pagination' }; if (LOW_VALUE_PATTERN.test(path)) return { url, category: 'low-value', priority: 20, reason: 'filter/sort query param' }; const segments = parsed.pathname.split('/').filter(Boolean); if (segments.length === 0) return { url, category: 'nav', priority: 100, reason: 'homepage' }; if (segments.length === 1) return { url, category: 'nav', priority: 80, reason: 'top-level nav' }; if (segments.length <= 3) return { url, category: 'app', priority: 60, reason: 'app screen' }; return { url, category: 'content', priority: 40, reason: 'deep content page' }; } export function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[] { return urls.map(u => classifyLink(u, baseUrl)) .sort((a, b) => b.priority - a.priority); } export function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[] { return classified.filter(l => l.category !== 'resource' && l.category !== 'external' && l.category !== 'api'); }