import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "../storage.mjs"; import { ColumnDef } from "../schema.mjs"; import { EngineError } from "../errors.mjs"; import "../contracts.mjs"; import { TenantCtx } from "@gscdump/contracts"; import { SearchType } from "gscdump/query"; interface RollupCtx extends TenantCtx { /** When the rollup was built. Stamped into payload + filename. */ builtAt: number; } /** * Tenant-scoped engine surface a rollup builder needs. Subset of * `StorageEngine.runSQL` so rollups stay testable without a full engine. */ interface RollupEngine { runSQL: (opts: { ctx: TenantCtx; fileSets: Record; table?: TableName$1; sql: string; params?: unknown[]; /** * Restrict every manifest lookup to a single GSC search-type slice. The * rollup runner forwards `RebuildRollupsOptions.searchType` so the * aggregated facts never mix web + non-web rows. Undefined preserves * the legacy cross-type union (web-only tenants). */ searchType?: SearchType; }) => Promise<{ rows: Row$1[]; }>; /** * Read the live manifest for a (tenant, table[, searchType]) cohort — * cheap, no parquet decode. Builders use this to chunk a full-history scan * into byte-bounded windows (see `WINDOW_BYTE_BUDGET`) so a single `runSQL` * call never ships an oversized Arrow IPC payload across the Workers * service-binding RPC (32MiB hard cap). */ listPartitions: (opts: { ctx: TenantCtx; table: TableName$1; searchType?: SearchType; }) => Promise>; } /** * One rollup definition. Build runs SQL over the tenant's facts and/or reads * from entity stores via `dataSource`, returning a JSON-serializable payload * that the runner timestamps + writes. */ interface RollupDef { id: string; /** * Window in days the rollup covers. `null` means full history. Used by * the runner to populate `windowDays` in the payload metadata so readers * can validate freshness. */ windowDays: number | null; /** * Storage format. `'json'` (default) wraps the build payload in a * `RollupEnvelope` and writes as a JSON blob. `'parquet'` expects `build` * to return rows matching `parquetColumns` and writes a parquet file plus * a tiny JSON sidecar envelope that points at it, so metadata * (`builtAt` / `windowDays`) stays readable without decoding parquet. */ format?: 'json' | 'parquet'; /** * Column schema for parquet output. Required when `format === 'parquet'`. * Types map the same way as the fact-table encoder: VARCHAR / DATE go * through BYTE_ARRAY/UTF8; BIGINT → INT64; INTEGER → INT32; DOUBLE → DOUBLE. */ parquetColumns?: readonly ColumnDef[]; /** Sort-key column names for parquet row-group stats. Optional. */ parquetSortKey?: readonly string[]; /** * When true, this rollup's payload is independent of GSC slice (e.g. entity * rollups sourced from sitemap / indexing snapshots, not slice-partitioned * fact tables). The runner rejects calls that pass `searchType` alongside * a slice-orthogonal def so the output never lands under a per-slice prefix * that the read path won't look at. */ sliceOrthogonal?: boolean; build: (deps: { engine: RollupEngine; ctx: TenantCtx; /** * Tenant-scoped object store. Rollups that aggregate over entity * snapshots (e.g. indexing metadata) read JSON docs through this. * Pure-SQL rollups can ignore it. */ dataSource: DataSource; /** * UTC millis the trailing window anchors to — its inclusive END. Equals * the newest synced/finalized data date when the runner is given * `dataEndDate`, otherwise wall-clock build time. Builders derive window * cutoffs from this (e.g. the trailing-28d boundary) and inline a date * literal so the SQL stays portable across DuckDB builds without the ICU * extension (Workers DuckDB — `CURRENT_DATE` lives in ICU). */ windowAnchorMs: number; /** * GSC search-type slice the runner was invoked for. Builders forward * this to every `engine.runSQL` call so the aggregated facts come * from one cohort. Undefined preserves the legacy cross-type union * (used by web-only tenants and admin paths). */ searchType?: SearchType; }) => Promise; } /** * Wire shape persisted to R2/disk. Readers can rely on the `version` + `builtAt`. * Parquet rollups write this envelope as a sidecar whose `payload` points at * the co-located `.parquet` object via `{ parquetKey, rowCount }`. */ interface RollupEnvelope { version: 1; id: string; builtAt: number; windowDays: number | null; payload: T; } interface ParquetRollupPointer { parquetKey: string; rowCount: number; /** * MULTI-FILE rollup: when set, the rollup is the UNION of these parquet keys * (disjoint by the grain's partition column, e.g. `date` for the resumable * `query_canonical_daily` build). Readers MUST union all keys; `parquetKey` * stays populated (the first part) for single-file readers. Avoids a JS * merge/re-encode of the whole rollup — the scaling bottleneck for a * cross-invocation resumable build. */ parquetKeys?: string[]; } declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string; declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string; interface RollupBucket { list: (opts: { prefix: string; cursor?: string; }) => Promise<{ objects: Array<{ key: string; }>; truncated?: boolean; cursor?: string; }>; get: (key: string) => Promise<{ text: () => Promise; } | null>; } declare function readLatestRollup(bucket: RollupBucket, ctx: TenantCtx, id: string, searchType?: SearchType): Promise | null>; interface RebuildRollupsOptions { engine: RollupEngine; dataSource: DataSource; ctx: TenantCtx; defs: readonly RollupDef[]; now?: () => number; /** * Build rollups for a single GSC search-type slice. Threads into every * builder's `engine.runSQL` call so the aggregated facts come from one * cohort, and namespaces the output object keys under a `/` * segment so per-slice rollups coexist without overwriting each other. * Undefined preserves the legacy cross-type behaviour (one rollup over * the union of all slices, written to the legacy path) — fine for web- * only tenants and explicit cross-type admin views. */ searchType?: SearchType; /** * ISO date (`YYYY-MM-DD`) of the newest synced/finalized day. Trailing- * window rollups (28d/90d) anchor their window END here instead of * wall-clock build time, so a "last 28 days" rollup covers the 28 days of * data that actually exist — not 28 days back from whenever the job ran, * which would include GSC's 2-3 day empty tail. Omit for the legacy * wall-clock behaviour. */ dataEndDate?: string; } interface RebuildRollupResult { id: string; /** JSON envelope key. For parquet rollups this is the sidecar pointer. */ objectKey: string; /** Parquet payload key. Present only when `format === 'parquet'`. */ parquetKey?: string; /** Envelope byte size; for parquet rollups does NOT include parquet bytes. */ bytes: number; /** Parquet payload byte size when `format === 'parquet'`. */ parquetBytes?: number; builtAt: number; /** * Set when this def's build/encode/write failed. The runner records the * failure and continues with the remaining defs so one bad rollup never * aborts the rest. Successful defs have no `error`. The human-readable * message (including the stack when available) lives on `error.message`. */ error?: EngineError; } declare function rebuildRollups(opts: RebuildRollupsOptions): Promise; export { ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, readLatestRollup, rebuildRollups, rollupKey, rollupParquetKey };