/**
 * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
 * Implements the subset of robots.txt spec used in practice:
 *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
 */

interface RobotsRule {
  userAgent: string;
  disallow: string[];
  allow: string[];
}

function parseRobots(text: string): RobotsRule[] {
  const rules: RobotsRule[] = [];
  let current: RobotsRule | null = null;

  for (const rawLine of text.split('\n')) {
    const line = rawLine.split('#')[0].trim();
    if (!line) { current = null; continue; }
    const colonIdx = line.indexOf(':');
    if (colonIdx === -1) continue;
    const key = line.slice(0, colonIdx).trim().toLowerCase();
    const value = line.slice(colonIdx + 1).trim();

    if (key === 'user-agent') {
      // Start new block only when current block already has rules, otherwise
      // accumulate multiple User-agent lines into one block.
      if (!current || current.disallow.length > 0 || current.allow.length > 0) {
        current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
        rules.push(current);
      } else {
        current.userAgent = value.toLowerCase();
      }
    } else if (key === 'disallow' && current && value) {
      current.disallow.push(value);
    } else if (key === 'allow' && current && value) {
      current.allow.push(value);
    }
  }

  return rules;
}

function isAllowedByRules(rules: RobotsRule[], userAgent: string, pathname: string): boolean {
  const ua = userAgent.toLowerCase();
  const exactRules = rules.filter(r => r.userAgent === ua);
  const wildcardRules = rules.filter(r => r.userAgent === '*');
  const applicable = exactRules.length > 0 ? exactRules : wildcardRules;

  if (applicable.length === 0) return true;

  // Collect all matched allow/disallow directives; longest path wins.
  const matched: Array<{ path: string; allow: boolean }> = [];
  for (const rule of applicable) {
    for (const p of rule.allow) {
      if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
    }
    for (const p of rule.disallow) {
      if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
    }
  }

  if (matched.length === 0) return true;
  matched.sort((a, b) => b.path.length - a.path.length);
  return matched[0].allow;
}

export class RobotsCache {
  private readonly cache = new Map<string, RobotsRule[]>();

  async isAllowed(url: string, userAgent = 'ZeTa-Crawler'): Promise<boolean> {
    let origin: string;
    let pathname: string;
    try {
      const u = new URL(url);
      origin = u.origin;
      pathname = u.pathname || '/';
    } catch {
      return true;
    }

    if (!this.cache.has(origin)) {
      try {
        const res = await fetch(`${origin}/robots.txt`, {
          headers: { 'User-Agent': userAgent },
          signal: AbortSignal.timeout(5_000),
        });
        this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
      } catch {
        this.cache.set(origin, []); // network error or timeout = no restrictions
      }
    }

    return isAllowedByRules(this.cache.get(origin)!, userAgent, pathname);
  }
}