/** * Sitemap and paginated HTML sync engines for source-sync. * * Owns: the three generic sync drivers (syncSitemapSource, * syncHtmlListSource, and their typed variants) that back the majority of * reference and package-registry sources. Registry-specific business logic * is NOT in this file; each registry owns its own options construction. */ import type { AssetCatalogEntry, SourceDefinition } from "../../../types.js"; import type { SourceSyncContext, SourceSyncSourceState } from "./types.js"; import type { AssetKind, CompatibilityMode } from "../../../types.js"; /** * Normalizes sitemap cursor IDs so hostname changes (e.g. skills.sh → * www.skills.sh) don't cause previously persisted cursor state to be lost. * * Matches skills.sh and www.skills.sh as equivalent — the canonical www * subdomain was adopted to avoid a 308 Permanent Redirect that Node.js * https.request does not follow. Old cursors stored under skills.sh still * resume correctly. */ declare function normalizeSitemapCursorId(cursorId: string): string; /** * Syncs a reference source whose items are discovered via a sitemap. Each leaf * sitemap URL set is scanned and matching item URLs are turned into indexed * reference catalog entries. */ export declare function syncSitemapReferenceSource(source: SourceDefinition, context: SourceSyncContext, options: { rootSitemapUrl: string; leafSitemapPredicate?: (url: URL) => boolean; itemUrlPredicate?: (url: URL) => boolean; itemAssetKind: AssetKind; itemCompatibilityMode: CompatibilityMode; itemInstallMethod: string; rootUrlExclusions?: Set; }): Promise; /** * Syncs a package registry source whose items are discovered via a sitemap. * Package names are extracted from each item URL using the caller-supplied * `packageNameFromUrl` function. */ export declare function syncSitemapPackageRegistrySource(source: SourceDefinition, context: SourceSyncContext, options: { rootSitemapUrl: string; leafSitemapPredicate?: (url: URL) => boolean; itemUrlPredicate?: (url: URL) => boolean; packageNameFromUrl: (url: URL) => string | undefined; }): Promise; /** * Syncs a reference source whose items are discovered by scraping paginated * HTML pages. Links matching `options.linkPattern` are turned into indexed * reference catalog entries. */ export declare function syncHtmlReferenceSource(source: SourceDefinition, context: SourceSyncContext, options: { pageUrlTemplate: string; pageUrlForNumber?: (pageNumber: number) => string; linkPattern: RegExp; itemAssetKind: AssetKind; itemCompatibilityMode: CompatibilityMode; itemInstallMethod: string; rootUrlExclusions?: Set; }): Promise; /** * Syncs a package registry source whose items are discovered by scraping * paginated HTML pages. Package names are extracted from matched link URLs * using the caller-supplied `packageNameFromPath` function. */ export declare function syncHtmlPackageRegistrySource(source: SourceDefinition, context: SourceSyncContext, options: { pageUrlTemplate: string; pageUrlForNumber?: (pageNumber: number) => string; linkPattern: RegExp; packageNameFromPath: (url: URL) => string | undefined; }): Promise; /** * Generic sitemap-driven sync engine. Fetches the root sitemap index, iterates * leaf sitemaps with per-cursor resumable pagination, and calls `buildItem` for * each discovered URL. Used by typed wrappers above. */ export declare function syncSitemapSource(source: SourceDefinition, context: SourceSyncContext, rootSitemapUrl: string, options: { leafSitemapPredicate?: (url: URL) => boolean; itemUrlPredicate?: (url: URL) => boolean; buildItem: (url: URL) => AssetCatalogEntry | null; rootUrlExclusions?: Set; }): Promise; /** * Generic paginated HTML sync engine. Iterates numbered pages of a URL * template, extracts links matching `linkPattern`, and calls `buildItem` for * each. Stops when a page returns no new links or the page budget is exhausted. */ export declare function syncHtmlListSource(source: SourceDefinition, context: SourceSyncContext, pageUrlTemplate: string, linkPattern: RegExp, options: { buildItem: (url: URL) => AssetCatalogEntry | null; pageUrlForNumber?: (pageNumber: number) => string; rootUrlExclusions?: Set; }): Promise; /** * Minimal internals exposed for focused unit tests. */ export declare const htmlSyncInternals: { normalizeSitemapCursorId: typeof normalizeSitemapCursorId; }; export {};