import * as s3tables from "@distilled.cloud/aws/s3tables"; import * as Effect from "effect/Effect"; import * as Schedule from "effect/Schedule"; import { isResolved } from "../../Diff.ts"; import { createPhysicalName } from "../../PhysicalName.ts"; import * as Provider from "../../Provider.ts"; import { Resource } from "../../Resource.ts"; import type { Providers } from "../Providers.ts"; import type { TableBucketArn } from "./TableBucket.ts"; /** * A field in an Apache Iceberg table schema. */ export interface IcebergSchemaField { /** * Name of the field. */ name: string; /** * Iceberg field type, e.g. `int`, `long`, `string`, `boolean`, * `timestamp`, `date`, `decimal(10,2)`. */ type: string; /** * Whether the field is required (non-nullable). * @default false */ required?: boolean; } export interface TableProps { /** * ARN of the table bucket that owns the table. Changing it replaces the * table. */ tableBucket: TableBucketArn | string; /** * Name of the namespace that owns the table. Changing it replaces the * table. */ namespace: string; /** * Name of the table. Must be 1-255 characters of lowercase letters, * numbers, and underscores, beginning with a letter or number. Changing * the name replaces the table. * @default a deterministic name derived from the app, stage, and logical ID */ name?: string; /** * Open table format. Only `ICEBERG` is supported. * @default "ICEBERG" */ format?: "ICEBERG"; /** * Iceberg schema for the table, applied at create time. Schema evolution * after creation is a data-plane concern; changing this replaces the * table. */ schema?: { /** * Ordered list of schema fields. */ fields: IcebergSchemaField[]; }; } export interface Table extends Resource< "AWS.S3Tables.Table", TableProps, { tableArn: string; name: string; namespace: string; tableBucketArn: string; versionToken: string; metadataLocation: string | undefined; warehouseLocation: string; format: string; type: string; createdAt: Date; }, never, Providers > {} /** * A fully-managed Apache Iceberg table within an S3 Tables {@link Namespace}. * * S3 Tables manages the table's storage, metadata, and maintenance * (compaction, snapshot expiration). Query it through engines like Amazon * Athena, Amazon EMR, or Apache Spark via the S3 Tables Iceberg catalog. * ### Creating Tables * **Example:** Table with an Iceberg schema * ```typescript * import * as S3Tables from "alchemy/AWS/S3Tables"; * * const bucket = yield* S3Tables.TableBucket("Analytics"); * const ns = yield* S3Tables.Namespace("Events", { * tableBucket: bucket.tableBucketArn, * }); * const table = yield* S3Tables.Table("PageViews", { * tableBucket: bucket.tableBucketArn, * namespace: ns.namespace, * schema: { * fields: [ * { name: "id", type: "long", required: true }, * { name: "url", type: "string" }, * { name: "ts", type: "timestamp" }, * ], * }, * }); * ``` * * @resource */ export const Table = Resource("AWS.S3Tables.Table"); const createTableName = (id: string, props: { name?: string | undefined }) => Effect.gen(function* () { if (props.name) { return props.name; } // Table names allow lowercase letters, numbers, and underscores only. const base = yield* createPhysicalName({ id, maxLength: 60, lowercase: true, }); return base.replaceAll("-", "_"); }); const buildMetadata = ( props: TableProps, ): s3tables.TableMetadata | undefined => props.schema ? { iceberg: { schema: { fields: props.schema.fields.map((f) => ({ name: f.name, type: f.type, required: f.required, })), }, }, } : undefined; export const TableProvider = () => Provider.succeed(Table, { stables: ["tableArn", "name", "namespace", "tableBucketArn"], // Tables are scoped to a parent namespace; the engine drives lifecycle // from state rather than ambient enumeration. list: () => Effect.succeed([]), read: Effect.fn(function* ({ id, olds, output }) { const tableBucketArn = output?.tableBucketArn ?? olds?.tableBucket; const namespace = output?.namespace ?? olds?.namespace; if (typeof tableBucketArn !== "string" || typeof namespace !== "string") { return undefined; } const name = output?.name ?? (yield* createTableName(id, olds ?? {})); return yield* s3tables .getTable({ tableBucketARN: tableBucketArn, namespace, name }) .pipe( Effect.map((t): Table["Attributes"] => ({ tableArn: t.tableARN, name: t.name, namespace: t.namespace[0] ?? namespace, tableBucketArn, versionToken: t.versionToken, metadataLocation: t.metadataLocation, warehouseLocation: t.warehouseLocation, format: t.format, type: t.type, createdAt: t.createdAt, })), Effect.catchTag("NotFoundException", () => Effect.succeed(undefined)), ); }), diff: Effect.fn(function* ({ id, news, olds }) { if (!isResolved(news)) return; if ( news.tableBucket !== olds?.tableBucket || news.namespace !== olds?.namespace ) { return { action: "replace" } as const; } const oldName = yield* createTableName(id, olds ?? {}); const newName = yield* createTableName(id, news); if (oldName !== newName) { return { action: "replace" } as const; } if ((news.format ?? "ICEBERG") !== (olds?.format ?? "ICEBERG")) { return { action: "replace" } as const; } // Schema is fixed at create time; schema evolution is out of band. const oldSchema = JSON.stringify(olds?.schema ?? null); const newSchema = JSON.stringify(news.schema ?? null); if (oldSchema !== newSchema) { return { action: "replace" } as const; } }), reconcile: Effect.fn(function* ({ id, news, output, session }) { const tableBucketArn = news.tableBucket as string; const namespace = news.namespace as string; const name = output?.name ?? (yield* createTableName(id, news)); // Observe — read live state; the table may have been deleted // out-of-band even if `output` cached its ARN. let table = yield* s3tables .getTable({ tableBucketARN: tableBucketArn, namespace, name }) .pipe( Effect.map((t) => t), Effect.catchTag("NotFoundException", () => Effect.succeed(undefined)), ); // Ensure — create if missing, tolerating a concurrent create. if (table === undefined) { yield* s3tables .createTable({ tableBucketARN: tableBucketArn, namespace, name, format: news.format ?? "ICEBERG", metadata: buildMetadata(news), }) .pipe( Effect.asVoid, Effect.catchTag("ConflictException", () => Effect.void), ); // Eventual consistency: getTable can briefly 404 a table that // createTable just returned. table = yield* s3tables .getTable({ tableBucketARN: tableBucketArn, namespace, name }) .pipe( Effect.retry({ while: (e) => e._tag === "NotFoundException", schedule: Schedule.max([ Schedule.exponential(500), Schedule.recurs(8), ]), }), ); } yield* session.note(table.tableARN); return { tableArn: table.tableARN, name: table.name, namespace: table.namespace[0] ?? namespace, tableBucketArn, versionToken: table.versionToken, metadataLocation: table.metadataLocation, warehouseLocation: table.warehouseLocation, format: table.format, type: table.type, createdAt: table.createdAt, }; }), delete: Effect.fn(function* ({ output }) { // Deliberately NO versionToken: destroy is unconditional. Runtime // commits (UpdateTableMetadataLocation) rotate the token, so the // persisted one may be stale and would fail with ConflictException. yield* s3tables .deleteTable({ tableBucketARN: output.tableBucketArn, namespace: output.namespace, name: output.name, }) .pipe(Effect.catchTag("NotFoundException", () => Effect.void)); }), });