import * as Clock from "effect/Clock"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import { decodeFqn, encodeFqn } from "../FQN.ts"; import { STATE_STORE_VERSION } from "./HttpStateApi.ts"; import { StateStoreError, type PersistedState, type StateService } from "./State.ts"; import { encodeState, reviveState } from "./StateEncoding.ts"; /** * A state store over an object storage bucket (S3, Cloud Storage, …). * * The store is laid out exactly like the local state store's file tree, * with the bucket (plus an optional prefix) in place of `.alchemy/state`: * * ``` * {prefix}{stack}/{stage}/{fqn}.json * {prefix}{stack}/{stage}/__stack_output__.json * {prefix}{stack}/{stage}/__lease__.json * ``` * * Concurrent deploys of one stack and stage share the `__lease__.json` * object, written with conditional puts. The second deploy fails * immediately instead of mixing rows. The holder refreshes the lease until * the store closes; a killed process stops refreshing, the lease expires, * and the next deploy can take it. * * Each backend supplies a {@link BucketObjects}: plain object operations * plus a version token (an S3 ETag, a Cloud Storage generation) for the * conditional writes the lease needs. */ /** The bookkeeping object that stores a stack's resolved output. */ const OUTPUT_FILE = "__stack_output__.json"; /** * Per `(stack, stage)` lease. A delete of the stage leaves it in place so * the lock outlives the objects it was guarding; the store deletes it on * close. */ /** The lease token `alchemy state migrate` leaves on a stage it moved to DynamoDB. */ const MIGRATED_TOKEN = "migrated:dynamodb"; const LEASE_FILE = "__lease__.json"; /** How long a lease stays valid without a refresh. */ const LEASE_TTL_MS = 60_000; /** How long a passing lease check is trusted. A failure is never cached. */ const LEASE_CHECK_TTL_MS = 5_000; /** A write condition, in terms of the backend's version token. */ export type WriteCondition = /** Only create the object; fail if it exists. */ | { readonly ifAbsent: true } /** Only overwrite the object at exactly this version. */ | { readonly ifVersion: string }; /** * The object operations a bucket backend implements. Every operation is * self-contained (`R = never`) and fails with {@link StateStoreError}. */ export interface BucketObjects { /** Every key under `prefix`, across pagination. */ readonly listKeys: (prefix: string) => Effect.Effect, StateStoreError>; /** * The immediate "directory" names under `prefix` (delimiter `/`), without * the prefix or the trailing slash. */ readonly listChildren: (prefix: string) => Effect.Effect, StateStoreError>; /** An object's text and version, or `undefined` when it doesn't exist. */ readonly read: ( key: string, ) => Effect.Effect< { readonly text: string; readonly version: string } | undefined, StateStoreError >; /** * Write an object. Returns its new version, or `undefined` when * `condition` did not hold. */ readonly write: ( key: string, body: string, condition?: WriteCondition, ) => Effect.Effect; /** * Delete an object, only at `ifVersion` when given. Deleting a missing * object, or one at another version, succeeds without deleting. */ readonly delete: (key: string, ifVersion?: string) => Effect.Effect; /** Delete many objects. Missing objects are ignored. */ readonly deleteMany: (keys: ReadonlyArray) => Effect.Effect; } export interface BucketStateOptions { /** The store's id, e.g. `"s3"` or `"gcs"`. */ readonly id: string; /** How errors name the store, e.g. `"S3"` or `"Cloud Storage"`. */ readonly label: string; /** Key prefix within the bucket. A trailing `/` is added. */ readonly prefix?: string; /** * The bucket's objects. Runs once, on the first state operation, and is * cached: ensure the bucket exists here, never at construction. */ readonly objects: Effect.Effect; } /** * Tokens of the leases this process holds, so a conflict with one of them * can say the holder is an earlier, still-open build in the same process. */ const heldInProcess = new Set(); interface LeaseRecord { readonly token: string; readonly expiresAt: number; } interface HeldLease { readonly checkLive: Effect.Effect; } /** * A passing check is trusted for `ttlMs`, so a burst of operations does * one round-trip. A failing check is never cached. */ const amortizeCheck = ( checkLive: Effect.Effect, ttlMs: number, ): Effect.Effect => { let lastOkAt: number | undefined; return Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; if (lastOkAt !== undefined && now - lastOkAt < ttlMs) { return; } yield* checkLive; lastOkAt = yield* Clock.currentTimeMillis; }); }; const parseLease = (text: string): LeaseRecord => { let parsed: unknown; try { parsed = JSON.parse(text); } catch { parsed = undefined; } return typeof parsed === "object" && parsed !== null && "token" in parsed && "expiresAt" in parsed && typeof parsed.token === "string" && typeof parsed.expiresAt === "number" ? { token: parsed.token, expiresAt: parsed.expiresAt } : { token: "", expiresAt: 0 }; }; /** * Construct a bucket-backed {@link StateService}. Leases are bound to the * ambient scope; callers that need them to outlive the first operation * bind this effect to a longer scope with `Scope.provide`. */ export const makeBucketState = (options: BucketStateOptions) => Effect.gen(function* () { const leaseScope = yield* Effect.scope; const random = yield* SecureRandom.SecureRandom; const prefix = options.prefix ? `${options.prefix.replace(/\/+$/, "")}/` : ""; const objects = yield* Effect.cached(options.objects); const use = ( f: (objects: BucketObjects) => Effect.Effect, ): Effect.Effect => objects.pipe(Effect.flatMap(f)); const stagePrefix = ({ stack, stage }: { stack: string; stage: string }) => `${prefix}${stack}/${stage}/`; const resourceKey = (request: { stack: string; stage: string; fqn: string }) => `${stagePrefix(request)}${encodeFqn(request.fqn)}.json`; const outputKey = (request: { stack: string; stage: string }) => `${stagePrefix(request)}${OUTPUT_FILE}`; const leaseKey = (request: { stack: string; stage: string }) => `${stagePrefix(request)}${LEASE_FILE}`; const readJson = (key: string) => use((o) => o.read(key)).pipe( Effect.flatMap((object) => object === undefined ? Effect.undefined : Effect.try({ try: () => JSON.parse(object.text, reviveState) as T, catch: (cause) => new StateStoreError({ message: `Failed to parse state object '${key}'`, cause: cause instanceof Error ? cause : undefined, }), }), ), ); const writeJson = (key: string, value: unknown) => use((o) => o.write(key, JSON.stringify(encodeState(value), null, 2))); /** Delete every object under `keyPrefix`, except leases. Idempotent. */ const deleteAll = (keyPrefix: string) => use((o) => o.listKeys(keyPrefix).pipe( // Leave the lease object. Releasing it here would open the stage // before this delete finishes, and the store still holds the lock. Effect.map((keys) => keys.filter((key) => !key.endsWith(`/${LEASE_FILE}`))), Effect.flatMap((keys) => (keys.length === 0 ? Effect.void : o.deleteMany(keys))), ), ); const heldError = (stack: string, stage: string, holder: string | undefined) => // The message always starts "another deploy holds": callers match on it // (the AWS migration tells a held stage from a failed one by it). new StateStoreError({ message: holder !== undefined && heldInProcess.has(holder) ? `another deploy holds the ${options.label} state lock '${stack}/${stage}': this process holds it, from an earlier operation on this stage that is still open. Run each operation in its own Effect.scoped so its build closes first.` : `another deploy holds the ${options.label} state lock '${stack}/${stage}'`, }); const lostError = (stack: string, stage: string) => new StateStoreError({ message: `the ${options.label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`, }); const readLease = (key: string) => use((o) => o.read(key)).pipe( Effect.map((object) => object === undefined ? undefined : { version: object.version, record: parseLease(object.text) }, ), ); const putLease = (key: string, record: LeaseRecord, condition: WriteCondition) => use((o) => o.write(key, JSON.stringify(record), condition)); const acquireLease = (stack: string, stage: string) => Effect.gen(function* () { const key = leaseKey({ stack, stage }); const token = yield* random.uuidV4(); const now = yield* Clock.currentTimeMillis; const fresh: LeaseRecord = { token, expiresAt: now + LEASE_TTL_MS }; let version = yield* putLease(key, fresh, { ifAbsent: true }); if (version === undefined) { const existing = yield* readLease(key); const stillHeld = existing === undefined || existing.record.expiresAt > now; if (existing?.record.token === MIGRATED_TOKEN) { return yield* new StateStoreError({ message: `${heldError(stack, stage, undefined).message}: the stage moved to a DynamoDB state store. Deploy it with a version of alchemy that has AWS.state() on DynamoDB, or run 'alchemy state migrate --rollback' with that version to move it back.`, }); } if (stillHeld) { return yield* heldError(stack, stage, existing?.record.token); } const replaced = yield* putLease(key, fresh, { ifVersion: existing.version }); if (replaced === undefined) { return yield* heldError(stack, stage, undefined); } version = replaced; } heldInProcess.add(token); const cursor = { version, lost: false }; const gate = Semaphore.makeUnsafe(1); const renew = Semaphore.withPermits( gate, 1, )( Effect.gen(function* () { if (cursor.lost) return; const renewedAt = yield* Clock.currentTimeMillis; const next = yield* putLease( key, { token, expiresAt: renewedAt + LEASE_TTL_MS }, { ifVersion: cursor.version }, ); if (next === undefined) cursor.lost = true; else cursor.version = next; }), ).pipe( Effect.retry({ while: () => !cursor.lost, schedule: Schedule.spaced("1 second"), times: 3, }), Effect.catch(() => Effect.sync(() => { cursor.lost = true; }), ), ); const verify = Semaphore.withPermits( gate, 1, )( Effect.gen(function* () { if (cursor.lost) return yield* lostError(stack, stage); const existing = yield* readLease(key); if (existing === undefined || existing.record.token !== token) { cursor.lost = true; return yield* lostError(stack, stage); } const observedAt = yield* Clock.currentTimeMillis; if (existing.record.expiresAt <= observedAt) { // The lease lapsed without anyone taking it: a refresh queued // behind a large deploy's writes missed its window. Take it // back at the version we read, so a deploy that took it in // the meantime wins and this one stops. const reclaimed = yield* putLease( key, { token, expiresAt: observedAt + LEASE_TTL_MS }, { ifVersion: existing.version }, ); if (reclaimed === undefined) { cursor.lost = true; return yield* lostError(stack, stage); } cursor.version = reclaimed; return; } cursor.version = existing.version; }), ); // Interrupt the refresh before releasing, so a refresh cannot // land after the lease has been released. yield* Scope.addFinalizer( leaseScope, Semaphore.withPermits( gate, 1, )( Effect.gen(function* () { cursor.lost = true; heldInProcess.delete(token); // Read the version we actually hold. A refresh that landed // after the cursor was last stored would make a release // against the old version a no-op, and the lease would stay. const existing = yield* readLease(key); if (existing === undefined || existing.record.token !== token) return; // Delete rather than expire: a leftover lease object would keep // a destroyed stage listed by `listStages`. yield* use((o) => o.delete(key, existing.version)); }).pipe(Effect.ignore), ), ); yield* renew.pipe( Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))), Effect.forever, Effect.interruptible, Effect.forkIn(leaseScope), ); return { checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS) } satisfies HeldLease; }); // One lease per (stack, stage), taken on the first operation and held // for the store's lifetime. The mutex keeps two concurrent first // operations in this process from contending with each other. const leaseMutex = Semaphore.makeUnsafe(1); const leases = new Map>(); const leaseFor = (stack: string, stage: string): Effect.Effect => Semaphore.withPermits( leaseMutex, 1, )( Effect.gen(function* () { const key = `${stack}/${stage}`; const existing = leases.get(key); if (existing !== undefined) return existing; // Drop a failed acquire. Caching it would make the next operation // report the same holder after that holder has gone. const cached = yield* Effect.cached( acquireLease(stack, stage).pipe( Effect.tapError(() => Effect.sync(() => { if (leases.get(key) === cached) { leases.delete(key); } }), ), ), ); leases.set(key, cached); return cached; }), ).pipe(Effect.flatMap((lease) => lease)); const guarded = ( request: { stack: string; stage: string }, op: Effect.Effect, ): Effect.Effect => leaseFor(request.stack, request.stage).pipe( Effect.flatMap((lease) => lease.checkLive), Effect.andThen(op), ); const verifyHeldLeases: Effect.Effect = Effect.suspend(() => Effect.forEach( Array.from(leases.values()), (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)), { discard: true }, ), ); const listChildren = (keyPrefix: string) => use((o) => o.listChildren(keyPrefix)).pipe(Effect.map((names) => Array.from(names))); const state: StateService = { id: options.id, getVersion: () => Effect.succeed(STATE_STORE_VERSION), listStacks: () => verifyHeldLeases.pipe(Effect.andThen(listChildren(prefix))), listStages: (stack: string) => verifyHeldLeases.pipe(Effect.andThen(listChildren(`${prefix}${stack}/`))), get: (request) => guarded(request, readJson(resourceKey(request))), getReplacedResources: Effect.fn(function* (request) { return (yield* Effect.forEach(yield* state.list(request), (fqn) => state.get({ stack: request.stack, stage: request.stage, fqn, }), )).filter((r) => r?.status === "replaced"); }), set: (request) => guarded( request, writeJson(resourceKey(request), request.value).pipe(Effect.as(request.value)), ), delete: (request) => guarded( request, use((o) => o.delete(resourceKey(request))), ), // A stack-wide delete locks every stage it is about to remove, so a // concurrent deploy fails the lease instead of racing the delete. deleteStack: ({ stack, stage }) => stage === undefined ? verifyHeldLeases.pipe( Effect.andThen(state.listStages(stack)), Effect.flatMap((stages) => Effect.forEach( stages, (found) => guarded( { stack, stage: found }, deleteAll(stagePrefix({ stack, stage: found })), ), { discard: true }, ), ), Effect.andThen(deleteAll(`${prefix}${stack}/`)), ) : guarded({ stack, stage }, deleteAll(stagePrefix({ stack, stage }))), list: (request) => guarded( request, use((o) => o.listKeys(stagePrefix(request))).pipe( Effect.map((keys) => keys .map((key) => key.slice(stagePrefix(request).length)) // Filter bookkeeping before decoding — `decodeFqn` replaces // `__` with `/`, which would turn `__stack_output__` into // `/stack_output/` and slip past the filter. .filter( (file) => file !== OUTPUT_FILE && file !== LEASE_FILE && file.endsWith(".json"), ) .map((file) => decodeFqn(file.replace(/\.json$/, ""))), ), ), ), getOutput: (request) => guarded(request, readJson(outputKey(request))), setOutput: (request) => guarded( request, writeJson(outputKey(request), request.value).pipe(Effect.as(request.value)), ), }; return state; });