import { Row, SearchType, TenantCtx } from "./storage.mjs"; import { EngineError } from "./errors.mjs"; import { IcebergTableName, PartitionKeyEncoding } from "./iceberg/schema.mjs"; import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig } from "./iceberg/catalog.mjs"; /** * Identifies one fact slice — the atomic unit a sink emits. * `(table, site, searchType, date)`. `userId` rides along on `ctx` for * tenant-scoped sinks (local/in-memory); the prod Iceberg table is global * and keys only on `siteId`. */ interface SinkSlice { ctx: TenantCtx; table: IcebergTableName; /** GSC search-type partition. */ searchType: SearchType; /** Calendar day (PT), `YYYY-MM-DD`. The slice's `month(date)` partition. */ date: string; } /** * Outcome of a sink write. `rowCount` is the number of rows accepted; * `bytes` is best-effort — `IcebergAppendSink` does not report it (undefined there). */ interface SinkWriteResult { rowCount: number; bytes?: number; } /** * Static description of a sink. Production ingestion is append-only under the * v5 stability-cutoff model. Test and revision-path adapters may expose an * overwrite capability explicitly. */ interface SinkCapabilities { /** When true, re-emitting a slice accumulates duplicate rows. */ appendOnly: boolean; /** Whether the sink exposes partition overwrite semantics. */ canOverwrite?: boolean; } /** Partition-overwrite writer for revision paths. */ interface SliceOverwriteWriter { overwriteSlice: (slice: SinkSlice, rows: readonly Row[]) => Promise; } /** * Outcome of `Sink.close()` — which tables' buffered rows reached durable * storage and which failed. * * `IcebergAppendSink.emit` only BUFFERS; the durable Iceberg commit happens * in `close()`, one `icebergAppend()` per table. The ingest ledger * (`sinkAsIngestEngine`) records a `(site, table, searchType, date)` slice * ONLY after the table holding it appears in `flushed` — a table in `failed` * leaves its slices un-recorded so the next sync re-emits them. This is what * keeps the D1 ledger from ever running ahead of Iceberg. */ interface SinkCloseResult { /** Tables whose buffered rows committed durably. */ flushed: IcebergTableName[]; /** * Tables whose flush failed — their slices must NOT be ledger-recorded. Each * carries a typed `sink-table-flush-failed` `EngineError`; the human-readable * cause is on `error.message`. */ failed: { table: IcebergTableName; error: EngineError; }[]; } interface Sink { readonly capabilities: SinkCapabilities; /** * Emit the fact rows for one slice. Append semantics — for `IcebergAppendSink` * this commits one Iceberg snapshot. Re-emitting the same slice produces * DUPLICATE rows; exactly-once is enforced upstream by the D1 ingested-days * ledger, which only calls `emit` for a slice once. * * `rows` carry the table's data columns; the sink injects the partition * identity columns (`site_id`, `search_type`) from `slice` — callers MUST * NOT pre-populate them. */ emit: (slice: SinkSlice, rows: readonly Row[]) => Promise; /** * Flush any buffered rows and release resources, returning which tables * reached durable storage. Idempotent — a second `close()` after a flush * reports empty `flushed`/`failed` (nothing left buffered). * * `IcebergAppendSink` buffers in `emit` and commits one `icebergAppend()` * per table HERE; `flushed`/`failed` reflect those per-table commits. * In-memory / local sinks write durably in `emit`, so they report every * table they received rows for as `flushed`. */ close: () => Promise; } /** Construction options shared by all sink implementations. */ interface SinkOptions { now?: () => number; } /** * `IcebergAppendSink`-specific options — the R2 Data Catalog coordinates the * sink appends to via `icebird`. The catalog config shape (`IcebergCatalogConfig`) * is type-only here so this contract file stays implementation-free. */ interface IcebergAppendSinkOptions extends SinkOptions { /** R2 Data Catalog connection config (catalog URI, warehouse, namespace, token, S3 creds). */ catalog: IcebergCatalogConfig; /** * Options forwarded to the lazy catalog connection. Supply a shared * `CatalogCache` here to serve the REST `/v1/config` probe from durable cache * across short-lived Worker isolates. */ connect?: ConnectIcebergOptions; /** * Retry policy for the per-table `icebergAppend()` commit, applied on R2 * Data Catalog 429 ("too many commits") rate-limits. Optional — production * uses the defaults; tests inject a synchronous `sleep`. */ commitRetry?: CommitRetryOptions; /** * Partition-key encoding (default `'int'` for new catalogs). `'int'` writes * BOTH `site_id` and `search_type` as INT — the caller MUST pass the numeric * `site_id` (a numeric string is fine; it's `Number()`-coerced) in * `slice.ctx.siteId`. Pass `'string'` explicitly for legacy catalogs. A small * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See * {@link import('./iceberg/schema').PartitionKeyEncoding}. */ encoding?: PartitionKeyEncoding; } export { IcebergAppendSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter };