/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Harvests Estonia's INSPIRE Addresses (AD) theme into the newline-delimited JSON file that * `#ee/adapters/ads/adapter` reads. * * Maa- ja Ruumiamet publishes the Aadressiandmete süsteem through GeoServer at * {@linkcode EE_ADS_WFS}. The service's dataset record states Creative Commons CC0 1.0 in * `otherConstraints`, which the address-source register elected under `spdx` `CC0-1.0`. CC0 reserves * no act, so paging the service and storing what it returns is granted and the harvest needs no * attribution clause. * * `#tools/fetch/inspire-addresses` owns the capabilities read and the GeoJSON page read, and * `#tools/fetch/wfs-harvest` owns the appending, the manifest and the resumption. This module states * what is particular to Estonia, which is four measurements taken on 2026-10-02: * * 1. The payload needs no component resolution. GeoServer flattens the application schema here, so * each `component_xlink_href` has a sibling `component_xlink_title` carrying the referenced * feature's value, and the adapter reads the titles. The harvest therefore takes one feature type * rather than the five a harmonized INSPIRE service splits an address across. * 2. `resultType=hits` at `startIndex=1` reports `numberMatched="729973"` and a feature page reports * the same number, so this service's count is real. It is read through * `readCheckedWFSFeatureCount`, which proves the number against a page before a caller uses it, * and no count is written into this file. * 3. The service honors a sort. `ImplementsSorting` is `TRUE`, `sortBy=gml_id` answers HTTP 200 and * `sortBy=gml_id D` returns the type's last features first, which is what proves the parameter is * applied rather than accepted and dropped. `sortBy=gml:identifier` and `sortBy=inspireId` answer * HTTP 400 `Illegal property name`, so the sort property is one of the flattened names. * 4. A `count=10000` page of `application/json` is 31,198,909 bytes and takes about 6 seconds. At * 729,973 features that is 73 pages and about 2.3 GB, plus three requests for the capabilities and * the checked count. * * The harvest writes one feature per line, which is what lets the adapter stream the whole extract * without holding a page. The `FeatureCollection` envelope is dropped: its counts describe one page * and the manifest records them per page instead. */ import { type APIClient } from "@mailwoman/core/api"; import type { PathBuilderLike } from "path-ts"; import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"; import { type WFSHarvestManifest } from "#tools/fetch/wfs-harvest"; /** * The service this module harvests. */ export declare const EE_ADS_WFS = "https://inspire.geoportaal.ee/geoserver/AD_Address/wfs"; /** * The qualified name the service publishes its addresses under. * * Read from the capabilities document per run. * This constant is the name a refusal quotes when the document stops publishing an Addresses type. */ export declare const EE_ADS_TYPE_NAME = "AD_Address:AD.Address"; /** * The property the pages are ordered by. * * A flattened property name, because the service refuses `gml:identifier` * and `inspireId` with HTTP 400 `Illegal property name`. */ export declare const EE_ADS_SORT_BY = "gml_id"; /** * Features per page. * * The service's `CountDefault` is 1,000,000 and it honors `count=10000`, which is where the page * size is set: a 10,000-feature page is 31,198,909 bytes, and a larger page buys fewer requests * at the cost of holding a larger body in memory and losing more work when one request fails. */ export declare const EE_ADS_PAGE_SIZE = 10000; /** * The file the adapter's `inputPath` points at, inside the harvest directory. */ export declare const EE_ADS_HARVEST_FILE = "address.jsonl"; /** * One page of features as newline-delimited JSON, one feature per line. * * A trailing newline on a non-empty page, so the next page's first feature starts its own line. */ export declare function featuresAsJSONL(features: readonly unknown[]): string; export interface HarvestADSOptions { /** * Where the harvest is written. * * The adapter reads {@linkcode EE_ADS_HARVEST_FILE} inside it. */ outputDir: PathBuilderLike; /** * Features per page, defaulting to {@linkcode EE_ADS_PAGE_SIZE}. */ pageSize?: number; /** * Stop once the manifest holds this many pages, for a probe rather than a full harvest. */ maxPages?: number; signal?: AbortSignal; report?: (line: string) => void; } /** * Harvests the service into `options.outputDir` and writes the manifest beside the data file. * * The capabilities document is read first, so the type name, the JSON format * and the paging support come from the service rather than from this file. * A service that stops publishing any of the three raises here rather than * producing a harvest of the wrong thing. * * @returns The manifest as written. */ export declare function harvestADSEE(client: Pick, options: HarvestADSOptions): Promise; export type FetchADSEEOptions = BaseFetchOptions & Pick & { /** * An injected client, for a test that supplies its own transport. * * A caller that leaves it out gets a paced client of this module's own. */ client?: Pick; }; /** * Harvests Estonia's addresses under `/ads/`, for `mailwoman corpus fetch ads-ee`. * * Re-runnable: a complete harvest whose file still hashes to the manifest's digest makes * no request, and an interrupted one resumes at the page after the last that reached disk. */ export declare function fetchADSEE(options: FetchADSEEOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=ads.d.ts.map