/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Harvest ČÚZK's INSPIRE Addresses archives into the directory `#cz/adapters/cuzk/adapter` reads. * * ČÚZK serves the Addresses (AD) theme as a two-level ATOM service rather than as a WFS. The * service document at `https://atom.cuzk.gov.cz/AD/AD.xml` holds one entry per municipality, each * pointing at a dataset feed, and each dataset feed offers that municipality's addresses as a * zipped GML in two projections. Measured 2026-10-02: the service document answered HTTP 200 with * 8,373,817 bytes holding 6,258 entries, sha256 * `9b1d51a721ab601fa74307cce5ce20f74d12dd29512f87be29cd1072f0a2d83b`. * * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so * this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does * not declare. * * Every one of the service document's `rights` elements reads `žádné podmínky neplatí`, INSPIRE's * controlled value for no conditions applying to access and use, and the address-source register * elects it on that basis. `#cz/adapters/cuzk/adapter` records the same value on every row. * * Four properties of this service decide the harvest's shape, and each was measured rather than * assumed: * * 1. **The service document carries each municipality's file modification time.** An entry's * `` equals the archive's `Last-Modified` to the second, checked on six municipalities * on 2026-10-02 — `584061` reads `2026-06-04T02:19:33+02:00` against * `Thu, 04 Jun 2026 00:19:33 GMT`, and `531529`, `551481`, `545031`, `587044` and `584282` * agree the same way. So one 8.4 MB request states the freshness of all 6,258 archives, and a * re-run requests only the municipalities whose stated time moved. The alternative — a * conditional request per municipality — costs 6,258 round trips to learn the same thing. * 2. **The download URL is composed from a base the feed itself declares.** The service document's * own `rel="next"` links are `https://services.cuzk.gov.cz/gml/inspire/ad/epsg-4258` and * `…/epsg-5514`, and a dataset feed's `alternate` link for the same municipality is that base * plus `/.zip`. Composing from the declared base rather than reading 6,258 dataset feeds * halves the harvest's requests. {@linkcode HarvestCzCuzkOptions.resolveThroughDatasetFeed} * reads the publisher's own href instead, for a run that will not compose a URL. * 3. **The `type` attribute describes the data rather than the file.** A dataset feed's link * advertises `application/gml+xml` while the host serves `application/zip`, so the GML is inside * the archive and the advertised type cannot decide what to write. The archive is stored as it * arrives: the adapter opens it with `adm-zip`, because at least one archive's central directory * records its member's size as the ZIP64 sentinel `0xFFFFFFFF`. * 4. **A dataset feed's `length` is accurate here, and is still not used.** `584061.zip` claims * 24,846 bytes and the host's `content-length` says 24,846. Denmark's feed understates its file * by 23 times, so every byte count this module records is counted off the delivered body. * * The archives are small — a systematic sample of 50 of the 6,258 municipalities on 2026-10-02 ran * from 5,236 to 1,642,586 bytes with a median of 16,591 — so the harvest is bound by round trips * rather than by bandwidth, and it is dispatched serially through one `APIClient`. ČÚZK publishes * no rate limit. {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS} holds the harvest to four requests per * second against one government host, which is under half of the ten per second the same 50 * requests measured, and a caller that has asked ČÚZK for more raises it. */ import { APIClient } from "@mailwoman/core/api"; import { type PathBuilderLike } from "path-ts"; import type { AtomFeed } from "#tools/fetch/atom"; import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"; /** * The ATOM service document listing one dataset feed per municipality. */ export declare const CZ_CUZK_SERVICE_FEED_URL = "https://atom.cuzk.gov.cz/AD/AD.xml"; /** * The attribution the model card must carry for this source. */ export declare const CZ_CUZK_ATTRIBUTION = "\u010Cesk\u00FD \u00FA\u0159ad zem\u011Bm\u011B\u0159ick\u00FD a katastr\u00E1ln\u00ED (\u010C\u00DAZK), INSPIRE t\u00E9ma Adresy"; /** * The two projections the service offers, keyed by the path segment the feed's own base URL ends in. * * The adapter reads either. * `epsg-4258` is ETRS89, in degrees, and is the projection the adapter's * component joins were verified against. */ export declare const CZ_CUZK_PROJECTIONS: { readonly "epsg-4258": "ETRS89"; readonly "epsg-5514": "S-JTSK"; }; /** * One of {@linkcode CZ_CUZK_PROJECTIONS}. */ export type CzCuzkProjection = keyof typeof CZ_CUZK_PROJECTIONS; /** * The minimum spacing between two requests to `services.cuzk.gov.cz`, in milliseconds. * * Four requests per second. * A systematic sample of 50 archives on 2026-10-02 answered a serial `HEAD` in 100 ms each, * so this leaves the host more than half the rate it demonstrated. */ export declare const CZ_CUZK_REQUEST_INTERVAL_MS = 250; /** * One municipality's dataset, as the service document states it. */ export interface CzMunicipalityDataset { /** * The municipality code, which is both the `obec` code and the archive's file name stem. */ code: string; /** * The entry's title, `INSPIRE - adresní místa - obec: Unkovice [584061]`. */ title: string; /** * The `` the service document states, which is this archive's modification time. */ updated: string; /** * The dataset feed for this municipality, which lists the same data in both projections. */ datasetFeedURL: string; /** * The archive, composed from the base the feed declares for the requested projection. */ downloadURL: string; /** * The file name the archive is written under, which is what the adapter globs. */ filename: string; } /** * The municipality code of one entry, read from its INSPIRE identifier and checked against its title. * * Both spellings are read because the code is the file name the archive is composed * and stored under, so a disagreement between the publisher's two statements of it * would silently store one municipality's addresses under another's name. * * @throws When either statement is missing or the two disagree, which reads as a change * in the publisher's layout rather than as a municipality without a code. */ export declare function municipalityCodeOf(entry: { title: string; identifierCode: string | null; }): string; /** * The base URL the feed declares for one projection. * * Matched on the href's last path segment rather than on the link's `title`, * because the segment is the EPSG code itself while the title is a Czech-language label. * * @throws When the feed declares no base for the requested projection, naming the ones it does * declare, so a publisher that moves its files reports itself instead of composing a dead URL. */ export declare function projectionBaseURL(feed: AtomFeed, projection: CzCuzkProjection): string; /** * The municipalities the service document lists, with the archive URL for one projection. * * @throws When the feed lists no entry, when two entries claim one municipality code — which * would make one archive overwrite the other — or when an entry carries no dataset feed link. */ export declare function readCzCuzkServiceFeed(feed: AtomFeed, projection: CzCuzkProjection): readonly CzMunicipalityDataset[]; /** * The archive link a dataset feed states for one projection. * * Read only when {@linkcode HarvestCzCuzkOptions.resolveThroughDatasetFeed} is set. * The feed holds one entry per projection, each with one `alternate` link, and the projection is * identified by the href's own path segment for the same reason {@linkcode projectionBaseURL} uses it. * * @throws When the dataset feed states no link for the requested projection. */ export declare function datasetFeedArchiveURL(feed: AtomFeed, projection: CzCuzkProjection, context: string): string; /** * One archive, as the harvest's manifest records it. * * {@linkcode SourceManifest}'s five fields plus what a re-run needs: the modification time * the service document stated for this municipality, and the one the host served it under. */ export interface CzCuzkArchiveEntry extends SourceManifest { municipality_code: string; /** * The `` the service document stated when this archive was fetched. * * A later run that reads the same value for this municipality leaves the file alone. * This is the field that makes the harvest resumable without a request per municipality. */ feed_updated: string; /** * The `Last-Modified` the host served the archive under, or `null` where it served none. * * Recorded rather than compared: {@linkcode CzCuzkArchiveEntry.feed_updated} * answers the same question without a request. */ last_modified: string | null; } /** * The harvest's `MANIFEST.json`. */ export interface CzCuzkHarvestManifest extends SourceCollectionManifest { /** * The projection the archives in this directory are in. * * Recorded because the two projections share a file name, so a directory holding * both would be read by the adapter as two copies of every address. */ projection: CzCuzkProjection; /** * How many municipalities the service document listed when this manifest was written. * * The denominator for `files`, read from the feed on each run rather than fixed here. */ municipalities_listed: number; files: CzCuzkArchiveEntry[]; } export interface HarvestCzCuzkOptions { /** * Where the archives and the manifest are written, which is the adapter's `inputPath`. */ outputDir: PathBuilderLike; /** * Which projection to harvest. * Defaults to `epsg-4258`. */ projection?: CzCuzkProjection; /** * Harvest only these municipality codes, for a probe or for a repair of named municipalities. * * A code the service document does not list is reported as a failure rather than ignored. */ municipalities?: readonly string[]; /** * Stop after this many municipalities, counting the ones already on disk. * * A bounded run rather than all 6,258. */ limit?: number; /** * Take each archive's URL from its own dataset feed instead of composing it * from the base the service document declares. * * Costs one extra request per municipality and asks the publisher rather than composing. */ resolveThroughDatasetFeed?: boolean; /** * Re-read the sha256 of every archive already on disk instead of comparing its byte count. * * The default compares the recorded byte count against the file's size, which is one `stat`. * This re-hashes, which reads every archive in the directory. */ verifyDigests?: boolean; signal?: AbortSignal; report?: (line: string) => void; } /** * Harvest the municipalities the service document lists into `options.outputDir`. * * The client is injected so the harvest is testable without the network and so a caller * decides the pacing. {@linkcode fetchCzCuzk} is the registry entry point and supplies both. * * @returns What was fetched, what was already current, and which municipality codes failed. */ export declare function harvestCzCuzk(client: Pick, options: HarvestCzCuzkOptions): Promise; export interface FetchCzCuzkOptions extends BaseFetchOptions, Pick { /** * The minimum spacing between two requests, in milliseconds. * * Defaults to {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS}. */ minRequestIntervalMs?: number; } /** * Harvest ČÚZK's INSPIRE Addresses archives into `/cz-cuzk/`. * * Re-runnable: a municipality whose recorded modification time still matches the service * document's and whose archive is still on disk at the recorded length costs no request. * * A full harvest is 6,259 requests — the service document plus one per municipality — * and an estimated 396 MB, from a mean of 63,337 bytes over a systematic sample * of 50 of the 6,258 archives on 2026-10-02. * The sample ran from 5,236 to 1,642,586 bytes, so the total is an estimate from * 50 measurements rather than a measurement of all 6,258. */ export declare function fetchCzCuzk(options: FetchCzCuzkOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=cuzk.d.ts.map