import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; import * as Redacted from "effect/Redacted"; import * as Provider from "../../Provider.ts"; import { Resource } from "../../Resource.ts"; import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; import type { Providers } from "../Providers.ts"; declare const TypeId: "Cloudflare.Pipelines.Sink"; type TypeId = typeof TypeId; /** * Batching policy controlling when the sink rolls (closes and uploads) * the current output file. */ export interface SinkRollingPolicy { /** * Roll the file once it reaches this size in bytes. */ fileSizeBytes?: number; /** * Roll the file after this many seconds without new events. */ inactivitySeconds?: number; /** * Roll the file at most this many seconds after it was opened. * @default 300 */ intervalSeconds?: number; } /** * Configuration of an `r2` sink writing raw files to an R2 bucket. */ export interface SinkR2Config { /** * Name of the destination R2 bucket. The bucket must already exist. */ bucket: string; /** * R2 S3-compatible credentials the sink uses to write objects. * Write-only — Cloudflare never echoes them back. */ credentials: { /** R2 access key id (the API token id). */ accessKeyId: Redacted.Redacted; /** R2 secret access key (SHA-256 hex of the API token value). */ secretAccessKey: Redacted.Redacted; }; /** * Key prefix under which output objects are written. */ path?: string; /** * Time-based partitioning of output object keys. */ partitioning?: { /** * strftime-style pattern, e.g. `year=%Y/month=%m/day=%d`. */ timePattern?: string; }; /** * Naming of output files within a partition. */ fileNaming?: { /** Prefix prepended to each file name. */ prefix?: string; /** Suffix appended to each file name. */ suffix?: string; /** * Strategy generating the unique part of each file name. * @default "uuid_v7" */ strategy?: "serial" | "uuid" | "uuid_v7" | "ulid"; }; /** * When the sink rolls output files. */ rollingPolicy?: SinkRollingPolicy; /** * Jurisdiction the bucket was created in (`eu`, `fedramp`), when not * the default. */ jurisdiction?: string; } /** * Configuration of an `r2_data_catalog` sink writing Iceberg tables via * the R2 Data Catalog. */ export interface SinkR2DataCatalogConfig { /** * Name of the R2 bucket backing the catalog. The bucket must already * exist and have the Data Catalog enabled. */ bucket: string; /** * Name of the Iceberg table to write. */ tableName: string; /** * Catalog namespace the table lives in. * @default "default" */ namespace?: string; /** * Cloudflare API token with R2 Data Catalog permissions. Write-only — * Cloudflare never echoes it back. */ token: Redacted.Redacted; /** * When the sink rolls output files. */ rollingPolicy?: SinkRollingPolicy; } /** * Output file format written by the sink. */ export type SinkFormat = { /** Newline-delimited JSON output. */ type: "json"; } | { /** Parquet output. */ type: "parquet"; /** * Compression codec. * @default "zstd" */ compression?: "uncompressed" | "snappy" | "gzip" | "zstd" | "lz4"; /** Target row-group size in bytes. */ rowGroupBytes?: number; }; interface SinkBaseProps { /** * Name of the sink. Unique per account; must be alphanumeric and * underscores only (it is referenced as a SQL table name). If omitted, * a unique name is generated from the app, stage, and logical ID. * * Sinks have no update API, so changing this (or any other) property * triggers a replacement. * @default ${app}_${id}_${stage}_${suffix} */ name?: string; /** * Output file format. * @default { type: "json" } */ format?: SinkFormat; } export type SinkProps = (SinkBaseProps & { /** * Sink type — `r2` writes raw files to an R2 bucket. */ type: "r2"; /** * R2 destination configuration. */ config: SinkR2Config; }) | (SinkBaseProps & { /** * Sink type — `r2_data_catalog` writes Iceberg tables via the R2 * Data Catalog. */ type: "r2_data_catalog"; /** * R2 Data Catalog destination configuration. */ config: SinkR2DataCatalogConfig; }); export interface SinkAttributes { /** Cloudflare-assigned sink identifier. */ sinkId: string; /** Account that owns the sink. */ accountId: string; /** Sink name (unique per account). */ name: string; /** Sink type. */ type: "r2" | "r2_data_catalog"; /** Destination R2 bucket name. */ bucket: string; /** Key prefix output objects are written under (r2 sinks). */ path: string | undefined; /** When the sink was created. */ createdAt: string; /** When the sink was last modified. */ modifiedAt: string; } export type Sink = Resource; /** * A Cloudflare Pipelines sink — the destination of the Pipelines product. * A SQL {@link Pipeline} reads events from a {@link Stream} and * writes them to a sink, which stores them in R2 either as raw files * (`r2`) or as Iceberg tables via the R2 Data Catalog * (`r2_data_catalog`). * * Sinks have no update API: every property change triggers a * replacement. With engine-generated names this is seamless (the new * sink gets a fresh name before the old one is deleted); with an * explicit `name` the create-before-delete replacement collides, so * prefer generated names. * ### Creating a Sink * **Example:** R2 sink with JSON output * The S3-compatible credentials are derived from a Cloudflare API token: * the access key id is the token id and the secret is the SHA-256 hex * digest of the token value. * ```typescript * const bucket = yield* Cloudflare.R2.Bucket("events", {}); * * const sink = yield* Cloudflare.Pipelines.Sink("events-sink", { * type: "r2", * config: { * bucket: bucket.bucketName, * credentials: { * accessKeyId: yield* Config.redacted("R2_ACCESS_KEY_ID"), * secretAccessKey: yield* Config.redacted("R2_SECRET_ACCESS_KEY"), * }, * path: "ingest", * rollingPolicy: { intervalSeconds: 30 }, * }, * }); * ``` * * **Example:** Parquet output * ```typescript * const sink = yield* Cloudflare.Pipelines.Sink("parquet-sink", { * type: "r2", * config: { bucket: bucket.bucketName, credentials }, * format: { type: "parquet", compression: "zstd" }, * }); * ``` * * ### R2 Data Catalog * **Example:** Iceberg table sink * ```typescript * const sink = yield* Cloudflare.Pipelines.Sink("iceberg-sink", { * type: "r2_data_catalog", * config: { * bucket: bucket.bucketName, * tableName: "events", * namespace: "default", * token: yield* Config.redacted("CATALOG_TOKEN"), * }, * }); * ``` * * @see https://developers.cloudflare.com/pipelines/ * * @resource * @product Pipelines * @category Storage & Databases */ export declare const Sink: import("../../Resource.ts").ResourceClass; /** * Returns true if the given value is a Sink resource. */ export declare const isSink: (value: unknown) => value is Sink; export declare const SinkProvider: () => import("effect/Layer").Layer, never, CloudflareEnvironment | import("../../Stack.ts").Stack | import("../../Stage.ts").Stage | pipelines.CloudflareOpContext>; export {}; //# sourceMappingURL=Sink.d.ts.map