/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Download the CORDIS project archives for the `cordis` adapter. * * CORDIS serves one CSV archive per framework programme at * `https://cordis.europa.eu/data/cordis-projects-csv.zip`, where the programme is `fp7`, * `h2020` or `HORIZON` (Horizon Europe), the spelling the host serves. These are the distributions the * data.europa.eu records `cordisfp7projects`, `cordish2020projects` and * `cordis-eu-research-projects-under-horizon-europe-2021-2027` link. Each archive holds an * `organization.csv` beside the project, topic and link tables, and the adapter reads that member * straight out of the archive, so the archive is what this module keeps. * * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`. * * Measured on 2026-10-03, the host answers `HEAD` with `accept-ranges: bytes`, a `content-length` * and a `last-modified` for each archive: `fp7` 32,953,397 bytes last modified `Thu, 02 Jan 2025 * 10:27:58 GMT`, `h2020` 55,219,250 bytes and `HORIZON` 36,908,998 bytes, both last modified on * 22 September 2026. The re-run check compares that pair against the manifest, so a rerun makes * one small request per archive, and an interrupted transfer resumes through * {@linkcode resumableDownload}. * * A delivered archive is checked for its `organization.csv` member before the manifest records it, * so an error page served under the archive's name never reaches the adapter. */ import { APIClient } from "@mailwoman/core/api"; import { type PathBuilderLike } from "path-ts"; import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"; /** * The programmes CORDIS publishes an archive for, in the spelling the host serves. */ export declare const CORDIS_PROGRAMMES: readonly string[]; /** * The download URL for one programme's archive. */ export declare function cordisArchiveURL(programme: string): string; /** * The file name one programme's archive is kept under, which is the URL's own last segment. */ export declare function cordisArchiveFilename(programme: string): string; /** * The page whose license statement the adapter quotes. */ export declare const CORDIS_LEGAL_NOTICE_URL = "https://cordis.europa.eu/about/legal"; /** * What the manifest records per archive. * * {@linkcode SourceManifest}'s five fields describe the archive as it arrived. * `last_modified` is what the re-run check compares before it downloads the body again. */ export interface CORDISArchiveManifest extends SourceManifest { /** * The `last-modified` the host served the archive under, or `null` where it served none. */ last_modified: string | null; } /** * What the host states about one archive without sending it. */ export interface CORDISArchiveHead { lastModified: string | null; contentLength: number | null; } /** * Read what the host states about one archive. * * A header the host does not send is reported as `null` rather than raised on, * and the caller then downloads rather than reading the absence as a match. */ export declare function readCORDISArchiveHead(client: Pick, programme: string): Promise; /** * Whether an archive on disk is still the one the manifest describes and the host still serves. * * The `HEAD` decides first, then the file's own length, then, when asked, its digest. */ export declare function isCORDISArchiveCurrent(recorded: CORDISArchiveManifest | undefined, head: CORDISArchiveHead, path: PathBuilderLike, verifyDigest: boolean): Promise; export interface DownloadCORDISOptions { /** * Where the archives and the manifest are written. * The directory itself is the adapter's `inputPath`. */ outputDir: PathBuilderLike; /** * Which programmes to fetch. * Defaults to {@linkcode CORDIS_PROGRAMMES}. */ programmes?: readonly string[]; /** * Re-read each archive's sha256 on a re-run instead of comparing its byte count. */ verifyDigest?: boolean; /** * Download even where the `HEAD` agrees with the manifest. */ force?: boolean; /** * Pause between range attempts, in milliseconds. */ retryDelayMs?: number; report?: (line: string) => void; } export type FetchCORDISOptions = BaseFetchOptions & Omit; /** * The directory `#adapters/cordis/adapter` reads under a fetch root. */ export declare function cordisInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike; /** * Download each requested programme's archive into `options.outputDir` beside one manifest. * * Re-runnable per archive: one whose `HEAD` matches the manifest and whose file is * still on disk at the recorded length makes no request for the body. */ export declare function downloadCORDIS(client: Pick, options: DownloadCORDISOptions): Promise; /** * Download the CORDIS archives into `/cordis/`. * * The registry entry point. */ export declare function fetchCORDIS(options: FetchCORDISOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=cordis.d.ts.map