/** * Copyright (c) 2025 Elara AI Pty Ltd * Licensed under BSL 1.1. See LICENSE for details. */ /** * Storage abstraction interfaces for e3 repositories. * * These interfaces enable e3-core logic to work against different backends: * - LocalBackend: Filesystem (default, for CLI and local dev) * - EfsBackend: AWS EFS (for Lambda/Fargate cloud deployment) * - S3DynamoBackend: S3 + DynamoDB (future optimization) * * The core insight: e3-core business logic is storage-agnostic. By injecting * a StorageBackend, the same code can run locally or in the cloud. */ import type { ExecutionStatus, LockState, LockOperation, DataflowRun, DatasetRef } from '@elaraai/e3-types'; import type { LockHolderInfo } from '../errors.js'; export type { LockState, LockOperation, LockHolderInfo }; /** * Repository status for lifecycle tracking. * * - 'creating': Repository is being initialized * - 'active': Repository is ready for use * - 'gc': Garbage collection is in progress * - 'deleting': Repository is being deleted */ export type RepoStatus = 'creating' | 'active' | 'gc' | 'deleting'; /** * Repository metadata. */ export interface RepoMetadata { /** Repository name */ name: string; /** Current status */ status: RepoStatus; /** When the repository was created (ISO 8601) */ createdAt: string; /** When the status last changed (ISO 8601) */ statusChangedAt: string; } /** * Result from batch operations (resumable pattern). * * Operations return { status: 'continue', cursor } if more work remains, * or { status: 'done' } when complete. This enables Step Functions orchestration. */ export interface BatchResult { /** 'continue' if more batches remain, 'done' if complete */ status: 'continue' | 'done'; /** Opaque cursor for next batch (only present if status='continue') */ cursor?: string; /** Number of items deleted in this batch */ deleted: number; } /** * A single object entry returned by gcScanObjects. */ export interface GcObjectEntry { /** SHA256 hash of the object */ hash: string; /** Last modification time (epoch ms) */ lastModified: number; /** Size of the object in bytes */ size: number; } /** * Result from scanning root hashes for GC (paginated). */ export interface GcRootScanResult { /** Root object hashes in this batch */ roots: string[]; /** Opaque cursor for next batch; undefined means scan is complete */ cursor?: unknown; } /** * Result from scanning objects for GC. */ export interface GcObjectScanResult { /** Object entries in this batch */ objects: GcObjectEntry[]; /** Opaque cursor for next batch; undefined means scan is complete */ cursor?: unknown; } /** * Content-addressed object storage. * * Objects are immutable and identified by their SHA256 hash. * The store handles deduplication automatically. * * All methods take `repo` as first parameter to identify the repository. * For local storage, `repo` is the path to the e3 repository directory. * For cloud storage, `repo` is a repository identifier used as a key prefix. */ export interface ObjectStore { /** * Write data to the object store. * @param repo - Repository identifier * @param data - Raw bytes to store * @returns SHA256 hash of the data */ write(repo: string, data: Uint8Array): Promise; /** * Write data from a stream to the object store. * @param repo - Repository identifier * @param stream - Async iterable of chunks * @returns SHA256 hash of the data */ writeStream(repo: string, stream: AsyncIterable): Promise; /** * Read an object by hash. * @param repo - Repository identifier * @param hash - SHA256 hash of the object * @returns Raw bytes * @throws {ObjectNotFoundError} If object doesn't exist */ read(repo: string, hash: string): Promise; /** * Check if an object exists. * @param repo - Repository identifier * @param hash - SHA256 hash of the object * @returns true if object exists */ exists(repo: string, hash: string): Promise; /** * Get the size of an object without reading its contents. * @param repo - Repository identifier * @param hash - SHA256 hash of the object * @returns Object metadata including size in bytes * @throws {ObjectNotFoundError} If object doesn't exist */ stat(repo: string, hash: string): Promise<{ size: number; }>; /** * List all object hashes in the store. * Used for garbage collection. * @param repo - Repository identifier * @returns Array of hashes */ list(repo: string): Promise; /** * Count objects in the store. * More efficient than list() when only the count is needed. * @param repo - Repository identifier * @returns Number of objects */ count(repo: string): Promise; } /** * Mutable reference storage for packages, workspaces, and executions. * * Unlike objects, references can be updated and deleted. * All methods take `repo` as first parameter to identify the repository. */ export interface RefStore { /** * List all packages with their versions. * @param repo - Repository identifier * @returns Array of {name, version} pairs */ packageList(repo: string): Promise<{ name: string; version: string; }[]>; /** * Resolve a package reference to its hash. * @param repo - Repository identifier * @param name - Package name * @param version - Package version * @returns Package object hash, or null if not found */ packageResolve(repo: string, name: string, version: string): Promise; /** * Write a package reference. * @param repo - Repository identifier * @param name - Package name * @param version - Package version * @param hash - Package object hash */ packageWrite(repo: string, name: string, version: string, hash: string): Promise; /** * Remove a package reference. * @param repo - Repository identifier * @param name - Package name * @param version - Package version */ packageRemove(repo: string, name: string, version: string): Promise; /** * List all workspace names. * @param repo - Repository identifier * @returns Array of workspace names */ workspaceList(repo: string): Promise; /** * Read workspace state. * @param repo - Repository identifier * @param name - Workspace name * @returns Encoded workspace state, or null if not found */ workspaceRead(repo: string, name: string): Promise; /** * Write workspace state. * @param repo - Repository identifier * @param name - Workspace name * @param state - Encoded workspace state (empty = undeployed) */ workspaceWrite(repo: string, name: string, state: Uint8Array): Promise; /** * Remove a workspace. * @param repo - Repository identifier * @param name - Workspace name */ workspaceRemove(repo: string, name: string): Promise; /** * Get execution status for a specific execution. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @param executionId - Execution ID (UUIDv7) * @returns ExecutionStatus or null if not found */ executionGet(repo: string, taskHash: string, inputsHash: string, executionId: string): Promise; /** * Write execution status. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @param executionId - Execution ID (UUIDv7) * @param status - Execution status */ executionWrite(repo: string, taskHash: string, inputsHash: string, executionId: string, status: ExecutionStatus): Promise; /** * List all execution IDs for a (taskHash, inputsHash) pair. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @returns Array of executionId values (sorted lexicographically ascending) */ executionListIds(repo: string, taskHash: string, inputsHash: string): Promise; /** * Get the latest execution status (lexicographically greatest executionId). * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @returns ExecutionStatus or null if no executions exist */ executionGetLatest(repo: string, taskHash: string, inputsHash: string): Promise; /** * Get the latest successful output hash (for cache lookup). * Iterates from latest executionId backwards, returns first success.outputHash found. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @returns Output hash or null if no successful execution exists */ executionGetLatestOutput(repo: string, taskHash: string, inputsHash: string): Promise; /** * List all executions in the repository. * @param repo - Repository identifier * @returns Array of {taskHash, inputsHash} pairs */ executionList(repo: string): Promise<{ taskHash: string; inputsHash: string; }[]>; /** * List executions for a specific task. * @param repo - Repository identifier * @param taskHash - Task object hash * @returns Array of inputsHash values */ executionListForTask(repo: string, taskHash: string): Promise; /** * List the latest execution status for every inputsHash of a task. * * Semantically equivalent to executionListForTask + executionGetLatest per * entry, but exposed as one call so backends can serve the whole set in a * single round trip. This matters for remote stores: workspaceStatus calls * this once per task, and composing it client-side from N individual * lookups made status requests O(repo history) network round trips — the * DynamoDB backend's listing query already fetches the status bytes it * would then re-fetch one by one. * * @param repo - Repository identifier * @param taskHash - Task object hash * @returns Latest execution status per inputsHash (order unspecified) */ executionListLatest(repo: string, taskHash: string): Promise>; /** * Get a specific dataflow run. * @param repo - Repository identifier * @param workspace - Workspace name * @param runId - Run ID (UUIDv7) * @returns DataflowRun or null if not found */ dataflowRunGet(repo: string, workspace: string, runId: string): Promise; /** * Write a dataflow run. * @param repo - Repository identifier * @param workspace - Workspace name * @param run - The dataflow run record */ dataflowRunWrite(repo: string, workspace: string, run: DataflowRun): Promise; /** * List all run IDs for a workspace (sorted lexicographically ascending). * @param repo - Repository identifier * @param workspace - Workspace name * @returns Array of runId values */ dataflowRunList(repo: string, workspace: string): Promise; /** * Get the latest dataflow run for a workspace. * @param repo - Repository identifier * @param workspace - Workspace name * @returns DataflowRun or null if no runs exist */ dataflowRunGetLatest(repo: string, workspace: string): Promise; /** * Delete a specific dataflow run. * @param repo - Repository identifier * @param workspace - Workspace name * @param runId - Run ID (UUIDv7) */ dataflowRunDelete(repo: string, workspace: string, runId: string): Promise; } /** * Handle to a held lock. */ export interface LockHandle { /** The resource this lock is for */ readonly resource: string; /** Release the lock. Safe to call multiple times. */ release(): Promise; } /** * Workspace locking service mediating concurrent access. * * A second implementation (e.g. the cloud's DynamoDB-lease backend) must honour * this mutual-exclusion contract, which is the contract e3-core relies on: * * - `exclusive` excludes every other holder (shared and exclusive). Deploy and * remove take it, so they fence out all readers/writers. * - `shared` coexists with other `shared` holders but is excluded by an * `exclusive` holder. Dataflow execution and record mutation take it, so they * run concurrently with each other yet never overlap a deploy. * * The lock state is stored using the LockState type from e3-types, so cloud * implementations can extend the holder variants. All methods (except * isHolderAlive) take `repo` as the first parameter. */ export interface LockService { /** * Acquire a lock on a resource in the requested mode (default `exclusive`). * * Returns null if the mode is incompatible with a current holder per the * shared/exclusive contract above (unless `wait` is set, in which case it * blocks up to `timeout`). The implementation gathers holder information * (process ID for local, request ID for Lambda, etc.) and writes the lock * state. * * @param repo - Repository identifier * @param resource - Resource identifier (e.g., "workspaces/production") * @param operation - What operation is acquiring the lock * @param options - Lock options (`mode` defaults to `exclusive`) * @returns Lock handle, or null if the lock couldn't be acquired */ acquire(repo: string, resource: string, operation: LockOperation, options?: { wait?: boolean; timeout?: number; mode?: 'shared' | 'exclusive'; }): Promise; /** * Get the current lock state. * @param repo - Repository identifier * @param resource - Resource identifier * @returns Lock state, or null if not locked */ getState(repo: string, resource: string): Promise; /** * Check if a lock holder is still alive. * * For local process locks, checks if the PID is still running. * For cloud locks, checks expiry or queries the cloud service. * * @param holder - East text-encoded holder string from LockState * @returns true if the holder is still active */ isHolderAlive(holder: string): Promise; } /** * A chunk of log output. */ export interface LogChunk { /** Log content (UTF-8) */ data: string; /** Byte offset of this chunk */ offset: number; /** Bytes in this chunk */ size: number; /** Total log file size (for pagination) */ totalSize: number; /** True if this is the end of the file */ complete: boolean; } /** * Log storage for execution stdout/stderr. * All methods take `repo` as first parameter to identify the repository. */ export interface LogStore { /** * Append data to a log stream. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @param executionId - Execution ID (UUIDv7) * @param stream - 'stdout' or 'stderr' * @param data - Data to append */ append(repo: string, taskHash: string, inputsHash: string, executionId: string, stream: 'stdout' | 'stderr', data: string): Promise; /** * Read from a log stream. * @param repo - Repository identifier * @param taskHash - Task object hash * @param inputsHash - Combined input hashes * @param executionId - Execution ID (UUIDv7) * @param stream - 'stdout' or 'stderr' * @param options - Read options * @returns Log chunk */ read(repo: string, taskHash: string, inputsHash: string, executionId: string, stream: 'stdout' | 'stderr', options?: { offset?: number; limit?: number; }): Promise; } /** * Repository lifecycle management. * * Handles repo creation, deletion, status tracking, and GC. * Follows the sub-interface pattern (storage.repos.*) like other stores. */ export interface RepoStore { /** * List all repository names. * @returns Array of repository names */ list(): Promise; /** * Check if a repository exists. * @param repo - Repository name * @returns true if repository exists */ exists(repo: string): Promise; /** * Get repository metadata. * @param repo - Repository name * @returns Metadata or null if not found */ getMetadata(repo: string): Promise; /** * Create a new repository. * Sets status to 'active' after initialization. * @param repo - Repository name * @throws {RepoAlreadyExistsError} If repository already exists */ create(repo: string): Promise; /** * Atomically set repository status. * Used for CAS (compare-and-swap) operations. * @param repo - Repository name * @param status - New status * @param expected - Optional expected current status (single or array) for CAS * @throws {RepoNotFoundError} If repository doesn't exist * @throws {RepoStatusConflictError} If expected status doesn't match */ setStatus(repo: string, status: RepoStatus, expected?: RepoStatus | RepoStatus[]): Promise; /** * Remove repository metadata/tombstone. * Called after GC completes for 'deleting' repos. * @param repo - Repository name */ remove(repo: string): Promise; /** * Delete all refs for a repo in batches. * Loop until status='done'. * @param repo - Repository name * @param cursor - Cursor from previous call (undefined for first call) * @returns Batch result with status and optional cursor */ deleteRefsBatch(repo: string, cursor?: string): Promise; /** * Delete all objects for a repo in batches. * Loop until status='done'. * @param repo - Repository name * @param cursor - Cursor from previous call (undefined for first call) * @returns Batch result with status and optional cursor */ deleteObjectsBatch(repo: string, cursor?: string): Promise; /** * Scan package references for root hashes. * @param repo - Repository name * @param cursor - Opaque cursor from previous call (undefined for first call) * @returns Root hashes and optional cursor for next batch */ gcScanPackageRoots(repo: string, cursor?: unknown): Promise; /** * Scan workspace state for root hashes. * @param repo - Repository name * @param cursor - Opaque cursor from previous call (undefined for first call) * @returns Root hashes and optional cursor for next batch */ gcScanWorkspaceRoots(repo: string, cursor?: unknown): Promise; /** * Scan execution history for root hashes. * @param repo - Repository name * @param cursor - Opaque cursor from previous call (undefined for first call) * @returns Root hashes and optional cursor for next batch */ gcScanExecutionRoots(repo: string, cursor?: unknown): Promise; /** * Scan object catalogue entries for GC. * @param repo - Repository name * @param cursor - Opaque cursor from previous call (undefined for first call) * @returns Object entries and optional cursor for next batch */ gcScanObjects(repo: string, cursor?: unknown): Promise; /** * Delete objects by hash. Idempotent — safe to retry on failure. * @param repo - Repository name * @param hashes - Object hashes to delete */ gcDeleteObjects(repo: string, hashes: string[]): Promise; } /** * Per-dataset reference storage for reactive dataflow. * * Each dataset in a workspace has its own ref file tracking its current * value and version vector. This replaces the single rootHash approach, * enabling concurrent writes and reactive re-execution. * * Ref files are stored at: workspaces//data/.ref * where uses directory separators (e.g., inputs/sales.ref). */ export interface DatasetRefStore { /** * Read a dataset ref. * @param repo - Repository identifier * @param ws - Workspace name * @param path - Dataset path (e.g., "inputs/sales" for .inputs.sales) * @returns DatasetRef or null if ref doesn't exist */ read(repo: string, ws: string, path: string): Promise; /** * Write a dataset ref atomically. * * Unconditional, last-writer-wins. Used by callers that already serialize * writes under a workspace lock (deploy, dataflow output writes). For * concurrent writers that must not clobber each other, use {@link writeIf}. * * @param repo - Repository identifier * @param ws - Workspace name * @param path - Dataset path (e.g., "inputs/sales" for .inputs.sales) * @param ref - The dataset ref to write */ write(repo: string, ws: string, path: string, ref: DatasetRef): Promise; /** * Read a dataset ref together with an opaque revision token. * * The revision identifies the exact stored bytes; pass it back to * {@link writeIf} to make a conditional write that only succeeds if nothing * changed in between. The token is store-specific and meaningful only to the * same store (a content etag locally, a counter in memory, a DynamoDB * revision attribute in the cloud) — never compare tokens across stores. * * @param repo - Repository identifier * @param ws - Workspace name * @param path - Dataset path * @returns The ref and its revision, or null if the ref does not exist */ readVersioned(repo: string, ws: string, path: string): Promise<{ ref: DatasetRef; revision: string; } | null>; /** * Conditionally write a dataset ref iff the stored revision still matches. * * The compare-and-swap primitive behind reactive-dataflow consistency: a * caller reads a revision with {@link readVersioned}, decides on a new ref, * and commits it only if no concurrent writer slipped in. Serialized * per-path so the read-compare-write is atomic across processes. * * @param repo - Repository identifier * @param ws - Workspace name * @param path - Dataset path * @param ref - The ref to write * @param expectedRevision - Revision from {@link readVersioned}; null means * "the ref must not currently exist" * @returns The new revision on success * @throws {DatasetRefConflictError} If the stored revision no longer matches */ writeIf(repo: string, ws: string, path: string, ref: DatasetRef, expectedRevision: string | null): Promise<{ revision: string; }>; /** * List all dataset ref paths in a workspace. * @param repo - Repository identifier * @param ws - Workspace name * @returns Array of dataset paths (e.g., ["inputs/sales", "tasks/etl/output"]) */ list(repo: string, ws: string): Promise; /** * Remove a single dataset ref. * @param repo - Repository identifier * @param ws - Workspace name * @param path - Dataset path */ remove(repo: string, ws: string, path: string): Promise; /** * Remove all dataset refs for a workspace. * @param repo - Repository identifier * @param ws - Workspace name */ removeAll(repo: string, ws: string): Promise; } /** * Complete storage backend combining all storage interfaces. * * This is the main abstraction point for e3-core. Functions receive a * StorageBackend instead of a repoPath, allowing the same logic to work * against different storage implementations. */ export interface StorageBackend { /** Content-addressed object storage */ readonly objects: ObjectStore; /** Mutable reference storage */ readonly refs: RefStore; /** Distributed locking service */ readonly locks: LockService; /** Execution log storage */ readonly logs: LogStore; /** Repository lifecycle management */ readonly repos: RepoStore; /** Per-dataset reference storage (reactive dataflow) */ readonly datasets: DatasetRefStore; /** * Validate that a repository exists and is properly structured. * @param repo - Repository identifier (path to e3 repository directory for local storage) * @throws {RepoNotFoundError} If repository doesn't exist or is invalid */ validateRepository(repo: string): Promise; } //# sourceMappingURL=interfaces.d.ts.map