/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Download and manifest utilities for `mailwoman corpus fetch `. */ import type { PathBuilderLike } from "path-ts"; /** * The option base every `mailwoman corpus fetch ` module extends. */ /** * How long a failed transfer waits before the next attempt. */ export declare const DEFAULT_RETRY_DELAY_MS = 5000; export interface BaseFetchOptions { /** * Destination root for downloaded source data. Each source writes its own subdirectory. */ outRoot: PathBuilderLike; /** * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}. * * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten. */ retryDelayMs?: number; } /** * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1. */ export interface FetchSummary { fetched: number; skipped: number; failed: number; failedCodes: string[]; } /** * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count + * sha256, so downstream adapters can verify provenance. */ export interface SourceManifest { source_url: string; downloaded_at: string; filename: string; sha256: string; bytes: number; } /** * A status worth retrying: rate limiting or a server-side failure. */ /** * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500. * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI * run and vanishes locally. */ export declare class HTTPStatusError extends Error { readonly status: number; constructor(status: number, message: string); } export declare function isTransientStatus(status: number): boolean; export interface DownloadOptions { url: string; dest: string; /** * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps. */ timeoutMs?: number; /** * Extra attempts after the first, taken only on transient statuses or network errors. Default 0. */ retries?: number; /** * Delay between attempts. Default 5s. */ retryDelayMs?: number; headers?: Record; report?: (line: string) => void; } /** * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status * or once retries are exhausted. Returns the byte count written. */ export declare function downloadToFile(options: DownloadOptions): Promise<{ bytes: number; }>; export interface StreamDownloadOptions { headers?: Record; timeoutMs: number; retries: number; retryDelayMs: number; } /** * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts). * * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and * two-URL fallback ladders consume. */ export declare function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise; //# sourceMappingURL=network.d.ts.map