/**
 * Crawl upstream documentation for football data providers.
 *
 * Fetches documentation from the best available source (llms.txt, ReadTheDocs,
 * GitHub, OpenAPI specs) and writes markdown files with provenance frontmatter.
 *
 * IMPORTANT: This crawler stores upstream content faithfully. It does NOT
 * paraphrase, summarise, or interpret. The content in the output files should
 * be directly traceable to the source URL.
 *
 * One exception, for pages reached through an llms.txt index (llms_indexes):
 * the crawler removes markup and bulk, and marks each removal in the text.
 * It drops the OpenAPI definition ReadMe appends and SVG diagrams, cuts long
 * example payloads with a note, replaces a data-point table repeated from an
 * earlier page with a line naming that page, and repeats a heading with
 * "(continued)" where it splits a long section. Every sentence that remains is
 * the source's own.
 *
 * Usage:
 *   npm run crawl                         # crawl all providers with sources
 *   npm run crawl -- --provider kloppy    # crawl one provider
 *   npm run crawl -- --discover           # probe sources without crawling
 */
interface CrawledDoc {
    category: string;
    content: string;
    source_url: string;
    source_type: string;
}
/**
 * Extract main content from an HTML page and convert to markdown.
 * Uses Mozilla's Readability (same algorithm as Firefox Reader View)
 * to isolate the main content, then Turndown to convert to markdown.
 */
export declare function htmlToMarkdown(html: string, url: string): string | null;
/**
 * Drop the categories a provider's registry entry excludes. Kept separate from
 * the crawl loop so the rule is testable without going near the network.
 */
export declare function applyCategoryExclusions<T extends {
    category: string;
}>(docs: T[], excluded: string[] | undefined): T[];
/**
 * Crawl an llms.txt or llms-full.txt file.
 * Already structured for LLM consumption — chunk by top-level heading.
 */
export declare function crawlLlmsTxt(content: string, sourceUrl: string): CrawledDoc[];
/**
 * Read the links out of an llms.txt index. Page links point at a page's
 * markdown copy (`.md`); index links point at a further `llms.txt`. Links to
 * another host are ignored, so an index cannot send the crawl off-site.
 */
export declare function parseLlmsIndex(content: string, indexUrl: string): {
    pages: string[];
    indexes: string[];
};
/** The category a page is stored under: the last path segment, without `.md`. */
export declare function llmsPageCategory(pageUrl: string): string;
/**
 * Turn a ReadMe-hosted page's markdown copy into plain markdown.
 *
 * ReadMe appends the endpoint's full OpenAPI definition to every reference page;
 * the spec is mirrored separately (specs/), so everything from that heading on is
 * dropped. Accordions become `###` headings, because their titles are where one
 * data-point table ends and the next begins. Other MDX components and list markup
 * are unwrapped to their text. Long example payloads are cut to their opening.
 */
export declare function cleanLlmsPage(text: string): string;
/**
 * Split any `##`/`###` section longer than `max` at paragraph breaks, repeating
 * its heading with "(continued)" so each piece is still a searchable section.
 */
export declare function splitOversizedSections(text: string, max?: number): string;
/**
 * Reference pages repeat the same data-point tables (the competition, the
 * season, the venue) on every endpoint that returns them. Keep the first copy
 * of each `###` section and replace later identical copies with one line per
 * page naming where the table is.
 */
export declare function dedupeSharedSections<T extends {
    category: string;
    content: string;
}>(docs: T[]): T[];
/**
 * Drop each `##`/`###` section whose heading is listed, with any deeper
 * sections nested under it. Headings match case-insensitively.
 */
export declare function dropSections(text: string, headings: string[] | undefined): string;
/**
 * Crawl the pages listed by one or more llms.txt indexes, following nested
 * indexes. Excluded categories are skipped before they are fetched, and
 * excluded sections are dropped from the pages that are kept.
 */
export declare function crawlLlmsIndexes(indexUrls: string[], excluded: string[] | undefined, fetcher?: (url: string) => Promise<string | null>, pauseMs?: number, excludedSections?: string[]): Promise<CrawledDoc[]>;
export declare function slugify(text: string): string;
export {};