/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Download and manifest utilities for `mailwoman corpus fetch `. */ import type { PathBuilder, PathBuilderLike } from "path-ts"; /** * The option base every `mailwoman corpus fetch ` module extends. */ /** * How long a failed transfer waits before the next attempt. */ export declare const DEFAULT_RETRY_DELAY_MS = 5000; export interface BaseFetchOptions { /** * Destination root for downloaded source data. * * Each source writes its own subdirectory. */ outRoot: PathBuilder; /** * Pause between transfer retries, in milliseconds. * * Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}. * * A test that exercises the failure path pays this delay once per retry in real time. * Measured at 20.1 s for the two failing-transfer cases in `geonames-postal.test.ts`, * which is the whole cost of that file. * The retry count is the behavior under test there. * * The pause between attempts is not, so it is a caller's to shorten. */ retryDelayMs?: number; } /** * The per-run result every fetch module returns. * The command maps `failed > 0` to exit code 1. */ export interface FetchSummary { fetched: number; skipped: number; failed: number; failedCodes: string[]; } /** * The sibling `manifest.json` shape the single-file fetch modules write: origin URL + * fetch timestamp + byte count + sha256, so downstream adapters can verify provenance. */ export interface SourceManifest { source_url: string; downloaded_at: string; filename: string; sha256: string; bytes: number; } /** * A status worth retrying: rate limiting or a server-side failure. */ /** * An HTTP failure that includes its status, so callers branch on `error.status` * rather than on message prose. * * The prose route shipped a real flake: a caller classified "not published upstream" * with `message.includes("404")`, and the message contains the URL. * An ephemeral test-server port such as `:40453` satisfies it while the actual status is 500. * * Roughly 1–2% of ephemeral ports contain the substring. * That collision can fail a CI run and vanish locally. */ export declare class HTTPStatusError extends Error { readonly status: number; constructor(status: number, message: string); } export declare function isTransientStatus(status: number): boolean; export interface DownloadOptions { url: string; dest: PathBuilderLike; /** * Per-attempt timeout. * * Default 10 minutes — these are multi-GB government dumps. */ timeoutMs?: number; /** * Extra attempts after the first, taken only on transient statuses or network errors. * * Default 0. */ retries?: number; /** * Delay between attempts. * * Default 5s. */ retryDelayMs?: number; headers?: Record; report?: (line: string) => void; } /** * Download `url` to `dest` with per-attempt timeout and transient-status retry. * * @throws On a non-transient http status or once retries are exhausted. * Returns the byte count written. */ export declare function downloadToFile(options: DownloadOptions): Promise<{ bytes: number; }>; export interface StreamDownloadOptions { headers?: Record; timeoutMs: number; retries: number; retryDelayMs: number; } /** * Stream an http download to disk, returning the final http status (0 on network error after retries). * * Follows redirects (the Census and OpenAddresses endpoints both 302 to their real hosts). * * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk * (the buffered helper reads via `arrayBuffer()`) and returns the HTTP status instead of throwing. * Per-file result collectors and two-URL fallback ladders consume that result. */ export declare function streamDownload(url: string, dest: PathBuilderLike, opts: StreamDownloadOptions): Promise; //# sourceMappingURL=network.d.ts.map