/**
 * Robots.txt & Sitemap Discovery Module (D-007)
 *
 * Extracts API hints from robots.txt and sitemap.xml:
 * - robots.txt: Disallow/Allow directives that hint at API paths
 * - sitemap.xml: URLs that point to API documentation or developer pages
 *
 * This module provides low-confidence hints that can guide further discovery
 * rather than definitive API patterns.
 */
/**
 * A hint extracted from robots.txt or sitemap.xml
 */
export interface ApiHint {
    /** The path or URL that was discovered */
    path: string;
    /** Source of this hint */
    source: 'robots.txt' | 'sitemap.xml';
    /** Type of hint */
    type: 'api-path' | 'documentation' | 'developer-portal' | 'spec-file' | 'graphql';
    /** How the hint was derived */
    reason: string;
    /** Confidence in this hint (0-1) */
    confidence: number;
}
/**
 * Parsed robots.txt directives
 */
export interface ParsedRobotsTxt {
    /** Disallow directives for user-agent * or all user agents */
    disallowPaths: string[];
    /** Allow directives for user-agent * or all user agents */
    allowPaths: string[];
    /** Sitemap URLs referenced */
    sitemapUrls: string[];
    /** Raw content for debugging */
    rawContent?: string;
}
/**
 * Parsed sitemap entry
 */
export interface SitemapEntry {
    /** URL location */
    loc: string;
    /** Last modified date */
    lastmod?: string;
    /** Change frequency */
    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
    /** Priority */
    priority?: number;
}
/**
 * Parsed sitemap (can be sitemap or sitemap index)
 */
export interface ParsedSitemap {
    /** Type of sitemap */
    type: 'sitemap' | 'sitemapindex';
    /** URL entries (for sitemap) */
    entries: SitemapEntry[];
    /** Child sitemap URLs (for sitemapindex) */
    sitemapUrls: string[];
}
/**
 * Result from robots/sitemap discovery
 */
export interface RobotsSitemapDiscoveryResult {
    /** Whether any hints were found */
    found: boolean;
    /** Discovered API hints */
    hints: ApiHint[];
    /** Parsed robots.txt if found */
    robotsTxt?: ParsedRobotsTxt;
    /** Parsed sitemap if found */
    sitemap?: ParsedSitemap;
    /** Locations that were probed */
    probedLocations: string[];
    /** Time taken for discovery (ms) */
    discoveryTime: number;
    /** Error message if discovery failed */
    error?: string;
}
/**
 * Options for robots/sitemap discovery
 */
export interface RobotsSitemapDiscoveryOptions {
    /** Maximum time to spend probing (ms) */
    timeout?: number;
    /** Whether to follow sitemap index references */
    followSitemapIndex?: boolean;
    /** Maximum sitemap entries to analyze */
    maxSitemapEntries?: number;
    /** Headers to send with probe requests */
    headers?: Record<string, string>;
    /** Custom fetch function */
    fetchFn?: (url: string, init?: RequestInit) => Promise<Response>;
}
/**
 * Parse robots.txt content into structured format
 */
export declare function parseRobotsTxt(content: string): ParsedRobotsTxt;
/**
 * Extract API hints from parsed robots.txt
 */
export declare function extractHintsFromRobotsTxt(robotsTxt: ParsedRobotsTxt): ApiHint[];
/**
 * Parse sitemap XML content
 */
export declare function parseSitemap(content: string): ParsedSitemap;
/**
 * Extract API hints from parsed sitemap
 */
export declare function extractHintsFromSitemap(sitemap: ParsedSitemap): ApiHint[];
/**
 * Discover API hints from robots.txt and sitemap.xml
 */
export declare function discoverRobotsSitemap(domain: string, options?: RobotsSitemapDiscoveryOptions): Promise<RobotsSitemapDiscoveryResult>;
/** Default cache TTL: 1 hour */
export declare const DEFAULT_CACHE_TTL_MS: number;
/**
 * Get cached discovery result if valid
 * Uses unified discovery cache with tenant isolation
 */
export declare function getCachedRobotsSitemap(domain: string): Promise<RobotsSitemapDiscoveryResult | null>;
/**
 * Cache a discovery result
 * Uses unified discovery cache
 */
export declare function cacheRobotsSitemap(domain: string, result: RobotsSitemapDiscoveryResult, ttlMs?: number): Promise<void>;
/**
 * Clear cache for a specific domain or all domains
 */
export declare function clearRobotsSitemapCache(domain?: string): Promise<void>;
/**
 * Discover with caching
 * Uses unified discovery cache with failed domain tracking
 */
export declare function discoverRobotsSitemapCached(domain: string, options?: RobotsSitemapDiscoveryOptions): Promise<RobotsSitemapDiscoveryResult>;
/**
 * Filter hints by type
 */
export declare function filterHintsByType(hints: ApiHint[], types: ApiHint['type'][]): ApiHint[];
/**
 * Get hints above a confidence threshold
 */
export declare function filterHintsByConfidence(hints: ApiHint[], minConfidence: number): ApiHint[];
/**
 * Sort hints by confidence (highest first)
 */
export declare function sortHintsByConfidence(hints: ApiHint[]): ApiHint[];
/**
 * Get just API path hints (for direct API discovery)
 */
export declare function getApiPathHints(hints: ApiHint[]): ApiHint[];
/**
 * Get documentation hints (for doc discovery)
 */
export declare function getDocumentationHints(hints: ApiHint[]): ApiHint[];
//# sourceMappingURL=robots-sitemap-discovery.d.ts.map