/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Downloads the `Matrikkelen - Adresse` CSV extracts Kartverket publishes through Geonorge. * * ## Two areas, because the adapter emits two jurisdictions * * `#no/adapters/matrikkelen/adapter` reads `kommunenummer` to decide a row's country, and * municipality 2100 is Svalbard rather than the mainland. Geonorge publishes the dataset per area * from one 375-entry area list, and the two entries this repository reads are `0000` `Hele landet` * and `2100` `Svalbard`. The mainland extract carries no Svalbard row, so taking only `0000` would * leave `SJ` with no rows while the adapter still claimed to cover it. * * Each area is a separate archive at the dataset's conventional download path, so no download * order has to be placed through Geonorge's order API: * * …/MatrikkelenAdresse/CSV/Basisdata___4258_MatrikkelenAdresse_CSV.zip * * ## Why the member is extracted * * The adapter opens `opts.inputPath` with `CSVSpliterator.fromAsync`, which reads a delimited file * rather than an archive, so the member is written out beside the archive it came from. The member * is named `matrikkelenAdresse.csv` in every area's archive, inside a directory named after the * archive, so each area is written under its own area-code directory and the publisher's own file * name is kept. * * The archive is kept rather than removed, because its sha256 is the value the address-source * register records for the publication and the member's is not. * * ## Freshness * * Geonorge serves `last-modified` and `content-length` on the archive, and a regenerated extract * changes both. A skip requires a stated `last-modified`: where the service states none, both * sides read `null`, an equality test would hold, and an archive of unknown age would be kept for * as long as the service stayed silent. * * ## The header is checked on arrival * * A renamed column reaches the adapter as an empty string on every row rather than as an error, so * the extract's header is read through the same `CSVSpliterator` the adapter uses and checked * against the nine columns the adapter indexes by name. The file's first column name carries a * byte-order mark, and none of the nine is that column, so the check does not depend on the mark * being stripped. */ import { APIClient } from "@mailwoman/core/api"; import { type PathBuilderLike } from "path-ts"; import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"; /** * One area of the dataset, as Geonorge's own area list for it spells the two parts of its file name. */ export interface MatrikkelenArea { /** * The area code, which is also the directory each area is written under. */ code: string; /** * The area name as the download path spells it, which is not always the list's `name`: * area `0000` is named `Hele landet` in the list and `Norge` in the path. */ pathName: string; } /** * The areas a fetch takes when a caller names none. * * `0000` is the mainland and `2100` is Svalbard, which are the two areas the * address-source register carries rows for. * Svalbard is also published as `fylke` 21, which holds the same rows as kommune 2100. */ export declare const MATRIKKELEN_AREAS: readonly MatrikkelenArea[]; /** * The area code of the whole-country extract, which carries every mainland municipality. */ export declare const MATRIKKELEN_MAINLAND_AREA = "0000"; /** * The projection the CSV extracts are taken in. * * EPSG:4258 is the geographic projection, and it is the one the register measured. * The same extract is published in EPSG:25833 and differs only in its coordinates. */ export declare const MATRIKKELEN_PROJECTION = "4258"; /** * The directory the dataset's per-area archives sit in. */ export declare const MATRIKKELEN_DOWNLOAD_ROOT = "https://nedlasting.geonorge.no/geonorge/Basisdata/MatrikkelenAdresse/CSV"; /** * The dataset's metadata record, which is where the elected license is stated. */ export declare const MATRIKKELEN_METADATA_URL = "https://kartkatalog.geonorge.no/api/getdata/f7df7a18-b30f-4745-bd64-d0863812350c"; /** * The name of the member inside every area's archive, and of the file the adapter reads. */ export declare const MATRIKKELEN_MEMBER_FILENAME = "matrikkelenAdresse.csv"; /** * The column delimiter the publisher writes. */ export declare const MATRIKKELEN_DELIMITER = ";"; /** * The columns `#no/adapters/matrikkelen/adapter` reads by name. * * Checked against the extract's header on arrival, because a renamed column reaches * the adapter as an empty string on every row rather than as an error. */ export declare const MATRIKKELEN_REQUIRED_COLUMNS: readonly string[]; /** * The attribution CC BY 4.0 §3(a)(1) requires on a publication derived from these rows. */ export declare const MATRIKKELEN_ATTRIBUTION = "Kartverket"; /** * The license the address-source register elected for this publisher. * * Kartverket's metadata record states CC BY 4.0 across five agreeing fields, * and the adapter stamps the same identifier on every row. */ export declare const MATRIKKELEN_LICENSE = "CC-BY-4.0"; /** * The archive's name for one area, which is also the name it is written under. */ export declare function matrikkelenArchiveFilename(area: MatrikkelenArea): string; /** * The archive's URL for one area. */ export declare function matrikkelenArchiveURL(area: MatrikkelenArea): string; /** * What one area's run recorded. * * `last_modified` is the service's own header and the one freshness signal it offers, * so a changed value is the one reason to download that area again. * `bytes` and `sha256` describe the archive, which is the artifact the * address-source register records a digest for. */ export interface MatrikkelenFileManifest extends SourceManifest { area_code: string; last_modified: string | null; member_filename: string; member_bytes: number; member_sha256: string; } /** * The recorded entry for one area, or `undefined` where the manifest holds none that can decide a skip. * * `loadCollectionFiles` reads the shared collection shape, which states what every * source's manifest states and not this source's area fields. * An entry written before those fields existed, or written with no stated `last_modified`, * cannot answer whether the archive on disk is current, and this reports that as * no recorded entry rather than as an entry that disagrees. */ export declare function recordedMatrikkelenArea(entry: SourceManifest | undefined): MatrikkelenFileManifest | undefined; /** * What the service's HEAD response states about one area's archive. * * Both fields read `null` where the header is absent, rather than an empty string or zero. * An absent `last-modified` is the service declining to state a version, which is a * different fact from a version that happens to match the one on disk. */ export interface MatrikkelenPublication { lastModified: string | null; reportedBytes: number | null; } /** * Reads the service's HEAD response for one area. * * Separate from {@linkcode downloadMatrikkelen} because this one request carries the whole freshness * decision, and the download itself runs on global `fetch`, which a unit test cannot intercept. */ export declare function readMatrikkelenPublication(client: Pick, area: MatrikkelenArea, options?: { signal?: AbortSignal; }): Promise; /** * Whether one area's archive and extract on disk are the ones the service currently serves. * * A skip requires the service to state a `last-modified` value. * Where it states none, both sides read `null` and an equality test would hold, * which would keep an archive of unknown age for as long as the service stayed silent. */ export declare function matrikkelenPublicationIsRecorded(recorded: MatrikkelenFileManifest, bytesOnDisk: number, memberBytesOnDisk: number, publication: MatrikkelenPublication): boolean; /** * The areas named by their codes, or every area in {@linkcode MATRIKKELEN_AREAS}. * * @throws Naming the codes that the area list does not carry, so a typed code * reports itself rather than reading as a fetch of no areas. */ export declare function matrikkelenAreasFor(codes: readonly string[] | undefined): readonly MatrikkelenArea[]; /** * Per-invocation options. */ export interface DownloadMatrikkelenOptions { outputDir: PathBuilderLike; /** * The areas to take, defaulting to {@linkcode MATRIKKELEN_AREAS}. */ areas?: readonly MatrikkelenArea[]; /** * Downloads each area even where the manifest's `last_modified` and byte counts still match. */ force?: boolean; retries?: number; retryDelayMs?: number; signal?: AbortSignal; report?: (line: string) => void; } /** * Downloads one area's archive, extracts the member the adapter reads and checks its header. * * @returns The manifest entry for the area, which the collection manifest carries. */ export declare function downloadMatrikkelenArea(client: Pick, area: MatrikkelenArea, options: { outputDir: PathBuilderLike; recorded?: MatrikkelenFileManifest; force?: boolean; retries?: number; retryDelayMs?: number; signal?: AbortSignal; report?: (line: string) => void; }): Promise<{ entry: MatrikkelenFileManifest; downloaded: boolean; }>; /** * Downloads every requested area and writes the collection manifest beside them. * * An area that fails is counted and the rest are still taken, because the two areas are * separate publications and Svalbard's 57 KiB does not depend on the mainland's 145 MiB. */ export declare function downloadMatrikkelen(client: Pick, options: DownloadMatrikkelenOptions): Promise; /** * The path `#no/adapters/matrikkelen/adapter` reads for one area, given the root a fetch wrote under. * * The area is a parameter because the adapter covers two jurisdictions and reads one * file at a time: `0000` carries Norway's rows and `2100` carries Svalbard's. */ export declare function matrikkelenInputPath(outRoot: BaseFetchOptions["outRoot"], areaCode?: string): PathBuilderLike; /** * Per-invocation options for the registry entry. */ export interface FetchMatrikkelenOptions extends BaseFetchOptions { /** * Geonorge area codes, defaulting to every area in {@linkcode MATRIKKELEN_AREAS}. */ areas?: readonly string[]; force?: boolean; retries?: number; retryDelayMs?: number; signal?: AbortSignal; } /** * The registry entry. */ export declare function fetchMatrikkelen(options: FetchMatrikkelenOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=matrikkelen.d.ts.map