/**
* Sitemap and paginated HTML sync engines for source-sync.
*
* Owns: the three generic sync drivers (syncSitemapSource,
* syncHtmlListSource, and their typed variants) that back the majority of
* reference and package-registry sources. Registry-specific business logic
* is NOT in this file; each registry owns its own options construction.
*/
import type { AssetCatalogEntry, SourceDefinition } from "../../../types.js";
import type { SourceSyncContext, SourceSyncSourceState } from "./types.js";
import type { AssetKind, CompatibilityMode } from "../../../types.js";
/**
* Normalizes sitemap cursor IDs so hostname changes (e.g. skills.sh →
* www.skills.sh) don't cause previously persisted cursor state to be lost.
*
* Matches skills.sh and www.skills.sh as equivalent — the canonical www
* subdomain was adopted to avoid a 308 Permanent Redirect that Node.js
* https.request does not follow. Old cursors stored under skills.sh still
* resume correctly.
*/
declare function normalizeSitemapCursorId(cursorId: string): string;
/**
* Syncs a reference source whose items are discovered via a sitemap. Each leaf
* sitemap URL set is scanned and matching item URLs are turned into indexed
* reference catalog entries.
*/
export declare function syncSitemapReferenceSource(source: SourceDefinition, context: SourceSyncContext, options: {
rootSitemapUrl: string;
leafSitemapPredicate?: (url: URL) => boolean;
itemUrlPredicate?: (url: URL) => boolean;
itemAssetKind: AssetKind;
itemCompatibilityMode: CompatibilityMode;
itemInstallMethod: string;
rootUrlExclusions?: Set;
}): Promise;
/**
* Syncs a package registry source whose items are discovered via a sitemap.
* Package names are extracted from each item URL using the caller-supplied
* `packageNameFromUrl` function.
*/
export declare function syncSitemapPackageRegistrySource(source: SourceDefinition, context: SourceSyncContext, options: {
rootSitemapUrl: string;
leafSitemapPredicate?: (url: URL) => boolean;
itemUrlPredicate?: (url: URL) => boolean;
packageNameFromUrl: (url: URL) => string | undefined;
}): Promise;
/**
* Syncs a reference source whose items are discovered by scraping paginated
* HTML pages. Links matching `options.linkPattern` are turned into indexed
* reference catalog entries.
*/
export declare function syncHtmlReferenceSource(source: SourceDefinition, context: SourceSyncContext, options: {
pageUrlTemplate: string;
pageUrlForNumber?: (pageNumber: number) => string;
linkPattern: RegExp;
itemAssetKind: AssetKind;
itemCompatibilityMode: CompatibilityMode;
itemInstallMethod: string;
rootUrlExclusions?: Set;
}): Promise;
/**
* Syncs a package registry source whose items are discovered by scraping
* paginated HTML pages. Package names are extracted from matched link URLs
* using the caller-supplied `packageNameFromPath` function.
*/
export declare function syncHtmlPackageRegistrySource(source: SourceDefinition, context: SourceSyncContext, options: {
pageUrlTemplate: string;
pageUrlForNumber?: (pageNumber: number) => string;
linkPattern: RegExp;
packageNameFromPath: (url: URL) => string | undefined;
}): Promise;
/**
* Generic sitemap-driven sync engine. Fetches the root sitemap index, iterates
* leaf sitemaps with per-cursor resumable pagination, and calls `buildItem` for
* each discovered URL. Used by typed wrappers above.
*/
export declare function syncSitemapSource(source: SourceDefinition, context: SourceSyncContext, rootSitemapUrl: string, options: {
leafSitemapPredicate?: (url: URL) => boolean;
itemUrlPredicate?: (url: URL) => boolean;
buildItem: (url: URL) => AssetCatalogEntry | null;
rootUrlExclusions?: Set;
}): Promise;
/**
* Generic paginated HTML sync engine. Iterates numbered pages of a URL
* template, extracts links matching `linkPattern`, and calls `buildItem` for
* each. Stops when a page returns no new links or the page budget is exhausted.
*/
export declare function syncHtmlListSource(source: SourceDefinition, context: SourceSyncContext, pageUrlTemplate: string, linkPattern: RegExp, options: {
buildItem: (url: URL) => AssetCatalogEntry | null;
pageUrlForNumber?: (pageNumber: number) => string;
rootUrlExclusions?: Set;
}): Promise;
/**
* Minimal internals exposed for focused unit tests.
*/
export declare const htmlSyncInternals: {
normalizeSitemapCursorId: typeof normalizeSitemapCursorId;
};
export {};