/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter. * * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the * CDF version it follows, so this module reads that answer first and downloads the file it names. * * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the * `2026-10-03 08:00:00` publish: * * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a * `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes * 0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a * publish replaced mid-transfer is refused rather than recorded. * 2. **The host honors `Range`.** An interrupted transfer resumes through * {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host * compresses small responses on the fly and a range must count the archive's own bytes. * 3. **Each publish has its own file name**, date-prefixed as * `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that * name, so a partial transfer of one publish never resumes into another, and the adapter reads the * newest name in the directory. * * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter * streams the member out of the archive rather than reading an extracted copy. * * The receipt written beside the archive records the URL, the publish date, the byte count and the * sha256 of the archive, which identify the bytes a corpus row was read from. */ import { APIClient } from "@mailwoman/core/api"; import { type PathBuilderLike } from "path-ts"; import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"; /** * The publish API's answer for the newest golden copy and its delta files. */ export declare const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"; /** * The file the publish API names for the Level 1 golden copy in CSV. */ export interface GLEIFPublishedFile { /** * The publish timestamp the API states, as `2026-10-03 08:00:00`. */ publishDate: string; url: string; /** * The archive's length in bytes, as the API states it. */ bytes: number; recordCount: number; cdfVersion: string; } /** * The receipt written beside the archive. */ export interface GLEIFGoldenCopyReceipt { source: string; source_url: string; api_url: string; publish_date: string; cdf_version: string; /** * The record count the API states for this publish. */ record_count: number; downloaded_at: string; filename: string; bytes: number; sha256: string; /** * The CSV member inside the archive, and its decompressed length as the central directory states it. */ csv_member: string; csv_member_bytes: number; license: string; license_url: string; attribution: string; } /** * Read which file the publish API names as the current Level 1 golden copy in CSV. * * @throws When the answer omits any of the five fields, naming the one it lacks, * rather than downloading a file whose length or date cannot be checked. */ export declare function readGLEIFLatestPublish(client: Pick): Promise; /** * The archive's file name, which is the last path segment of the URL the API names. */ export declare function gleifArchiveFilename(url: string): string; /** * The directory the adapter reads under a fetch root. */ export declare function gleifInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike; /** * Whether the archive on disk is the publish the API names and the length the receipt records. * * Exported so a test can exercise the decision without a transfer. */ export declare function isGLEIFArchiveCurrent(recorded: GLEIFGoldenCopyReceipt | null, published: GLEIFPublishedFile, path: PathBuilderLike, verifyDigest: boolean): Promise; export interface DownloadGLEIFOptions { /** * Where the archive and the receipt are written. * The directory itself is the adapter's `inputPath`. */ outputDir: PathBuilderLike; /** * Re-read the archive's sha256 on a re-run instead of comparing its byte count. */ verifyDigest?: boolean; /** * Download even where the receipt already records the current publish. */ force?: boolean; retryDelayMs?: number; report?: (line: string) => void; } export type FetchGLEIFOptions = BaseFetchOptions & Omit; /** * Download the current Level 1 golden copy into `options.outputDir` and write its receipt. * * Re-runnable: a receipt naming the current publish, with the archive still on disk * at the recorded length, makes no request for the body. * Archives of earlier publishes are removed once the new one is verified, * so the directory holds one publish. */ export declare function downloadGLEIF(client: Pick, options: DownloadGLEIFOptions): Promise; /** * Download the current golden copy into `/gleif/`. * * The registry entry point. * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself. */ export declare function fetchGLEIF(options: FetchGLEIFOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=gleif.d.ts.map