/** * ObjectIO — transport seam for vault object byte/metadata movement. * * s3.ts holds the *semantics* of sync (symlink-record encoding, mode/mtime * stamping, created-at preservation, directory-marker filtering). Those never * change. What CAN change is the *wire transport* underneath them: * * - `S3SdkObjectIO` — the historical path. STS-vended credentials + the AWS * S3 SDK talking directly to the per-company bucket. No policy-size * ceiling concern for the BYTES, but the STS session policy that grants * access has the 2048-char IAM limit that motivated the presigned model. * * - `PresignObjectIO` — the presigned-URL path. The vault-service decides * access as a runtime DDB check (no IAM policy ceiling) and hands back * short-lived presigned GET/PUT/DELETE URLs + (for PUT) the exact headers * to replay. The client never holds AWS credentials — it requests the * signed URLs directly. * * The seam is a per-EntityContext factory resolved INSIDE s3.ts, so every * existing call site (`uploadFile(ctx, …)`, `downloadFile(ctx, …)`, …) keeps * its signature. `runRunner` selects the transport once per session via * {@link setObjectIOFactory}; absent any selection the default is the S3 SDK, * preserving today's behavior for every non-gated caller. */ import { ObjectLockChecksumRequiredError } from "./lib/s3-content-checksum.js"; export { ObjectLockChecksumRequiredError }; import type { EntityContext } from "./types.js"; import type { PresignOp, PresignKeyInput, PresignResultRow, VaultListedObject } from "./vault-client.js"; /** * Bound for one vault/S3 HTTPS request. Matches FILE_TOMBSTONE reads * (`FETCH_TOMBSTONES_TIMEOUT_MS`) and the push-transport default so a stalled * socket fails the leg as transient instead of freezing the runner. */ export declare const OBJECT_IO_REQUEST_TIMEOUT_MS = 60000; export declare function objectIORequestTimeoutMs(): number; export declare function setObjectIORequestTimeoutMsForTesting(ms: number | undefined): void; /** * The slice of {@link VaultClient} the presigned transport needs. Narrowed to * just `presign` + `listFiles` so the factory accepts any caller that exposes * those two (the real VaultClient, or a stub in tests) without depending on * the full 20-method surface. */ export interface PresignTransportClient { presign(input: { companyUid: string; op?: PresignOp; expiresIn?: number; keys: PresignKeyInput[]; }): Promise<{ results: PresignResultRow[]; expiresAt: string; }>; listFiles(companyUid: string, prefix?: string, cursor?: string): Promise<{ objects: VaultListedObject[]; cursor: string | null; truncated: boolean; }>; } /** * Conditional-write fence for a PUT (S3 conditional writes, GA 2024-11). * * `ifMatch` — only land the PUT if the remote object's ETag still equals * this value (the journal baseline / last-observed HEAD). `ifNoneMatch: "*"` * — only land the PUT if NO object exists at the key (creation fence). * Either mismatch makes S3 reject with 412 PreconditionFailed, which the * push path surfaces as a conflict instead of a silent overwrite. * * This is the storage-level backstop for the entire stale-clobber class: * a HEAD-then-PUT race, a transport bug that misreads remote state, or an * outdated client mid-pass can no longer regress a newer remote object — * S3 itself refuses. (The 2026-06-10..12 vault regression storm was this * class: stale machine copies blind-PUT over newer objects.) */ export interface PutPrecondition { /** Land only if the current remote ETag equals this (quotes optional). */ ifMatch?: string; /** Land only if no object exists at the key. */ ifNoneMatch?: "*"; } export interface PutObjectInput extends PutPrecondition { key: string; body: Buffer; contentType: string; /** S3 user metadata (x-amz-meta-*). Lowercased keys by convention. */ metadata?: Record; } export interface GetObjectResult { body: Buffer; /** S3 user metadata (keys lowercased by S3). */ metadata?: Record; } export interface GetObjectStreamResult { body: AsyncIterable; /** S3 user metadata (keys lowercased by S3). */ metadata?: Record; } export interface ListObjectsInput { prefix?: string; continuationToken?: string; } export interface ListedRemoteObject { key: string; size: number; lastModified: Date; etag: string; /** S3 storage class; omitted by older transport implementations means STANDARD. */ storageClass?: string; } export interface ListObjectsResult { objects: ListedRemoteObject[]; /** Opaque cursor for the next page; undefined when the listing is exhausted. */ nextContinuationToken?: string; } export interface HeadObjectResult { lastModified: Date; etag: string; size: number; metadata?: Record; /** S3 storage class; omitted by older transports means STANDARD. DEEP_ARCHIVE bodies cannot be fetched without a restore. */ storageClass?: string; } /** * The minimal byte/metadata transport s3.ts needs. Deliberately narrow — no * symlink, mode, or created-at concepts leak in here; those live one layer up * in s3.ts and compose on top of these five primitives. */ export interface ObjectIO { putObject(input: PutObjectInput): Promise<{ etag: string; }>; getObject(key: string): Promise; getObjectStream?(key: string): Promise; listObjects(input: ListObjectsInput): Promise; deleteObject(key: string): Promise; /** * Null ONLY when the key definitively does not exist (404). Access denial * (403 / per-key presign denial) THROWS a `name: "Forbidden"` error — it is * unknown state, never "absent". Conflating the two disables push-side * conflict guards and clobbers newer remote objects. */ headObject(key: string): Promise; /** * Optional batch pre-mint. Warms an internal URL cache for `keys` under `op` * so subsequent per-key get/head (and, when primed, put/delete) calls reuse a * pre-signed URL instead of issuing one presign request each. This is what * turns an N-file sync from N presign calls into ceil(N/chunk) — the * difference between staying under and blowing past the 100-req/hr limit on a * bulk pull. The S3 SDK transport has no presign step and omits this (the * per-call cost there is the SDK request itself, not a separate presign). * Best-effort: a failed chunk or per-key denial simply leaves those keys * uncached, and the per-key call falls back to a single presign. */ prime?(op: PresignOp, keys: PresignKeyInput[]): Promise; /** * True if a live primed PUT URL exists for `key`. Lets uploadFile/uploadSymlink * skip recomputing metadata + the created-at HEAD when a `prime("put", …)` * pre-pass already signed the metadata into the cached URL (the upload just * sends the body, replaying the cached headers). Absent (undefined) on the S3 * SDK transport → callers take their normal compute-metadata path. */ hasPrimedPut?(key: string, contentHash?: string): boolean; } /** * Direct-to-S3 transport over STS-vended credentials. The run-scoped factory * keeps one instance per entity so concurrent operations share credential * renewal; this instance rebuilds its client after a rejected credential set. */ /** * Re-resolve one entity's context, returning credentials valid *now*. Backed by * `resolveEntityContext`, which serves from cache and re-vends only inside its * refresh window. */ export type ContextRefresh = () => Promise; /** * Credential lifecycle callbacks for a long-lived direct-S3 transport. * `resolve` may serve a still-valid cached context; `force` must evict that * cache and vend a new STS session after S3 rejects the current request. */ export interface ContextRefreshPair { resolve: ContextRefresh; force: ContextRefresh; } /** Build a {@link ContextRefresh} for a given entity uid. */ export type ContextRefresherFactory = (uid: string) => ContextRefresh; export declare class S3SdkObjectIO implements ObjectIO { private client; private readonly bucket; private readonly region; private readonly resolveRefresh?; private readonly forceRefresh?; private clientGeneration; private refreshInFlight; /** * @param refresh Re-resolve this context's credentials. A callback enables * proactive SDK-provider renewal; a resolve/force pair additionally enables * one reactive cache-evicting retry after S3 rejects a request. Omit it only * where the instance is known to be short-lived. */ constructor(ctx: EntityContext, refresh?: ContextRefresh | ContextRefreshPair); private assertCredentials; private buildClient; /** * Force one fresh STS vend for all requests rejected on the same client * generation, rebuild the SDK client, then let each caller recreate and retry * its own command exactly once. */ private refreshRejectedGeneration; private sendWithCredentialRetry; putObject(input: PutObjectInput): Promise<{ etag: string; }>; getObjectStream(key: string): Promise; getObject(key: string): Promise; listObjects(input: ListObjectsInput): Promise; deleteObject(key: string): Promise; headObject(key: string): Promise; } /** * Response shape shared by every presigned GET consumer. Unlike the runtime * `fetch` response, its body is backed by Node's core HTTP client, so it never * enters the runtime-bundled undici parser path. */ export interface PresignedGetResponse { status: number; headers: Headers; body?: AsyncIterable; destroy(): void; } /** Transport seam retained for deterministic wire-level regression tests. */ export interface PresignedGetTransport { get(url: string, headers: Record | undefined): Promise; } /** Test-only override; production always uses Node's core HTTP(S) transport. */ export declare function setPresignedGetTransportForTesting(transport: PresignedGetTransport | null): void; /** * Opt-in gate for strict fail-closed enforcement of fenced presigned PUTs. * * Defaults OFF until the hq-pro files-presign server signs and echoes * If-Match/If-None-Match. While disabled, the presigned transport preserves * today's behavior: replay whatever signed headers the server returned. */ export declare const PRESIGN_FENCE_STRICT_ENV_VAR = "HQ_PRESIGN_FENCE_STRICT"; /** * Thrown when a presign mint is skipped because the per-user 100/hr vault rate * budget is exhausted. Distinct name so callers can tell "deferred, retry next * sync" apart from a real transfer failure. The key was NOT synced. */ export declare class RateLimitedError extends Error { readonly key: string; readonly op: PresignOp; constructor(key: string, op: PresignOp, options?: { cause?: unknown; }); } /** * A fenced presigned PUT is only safe if the presign service signed and echoed * the requested conditional header for replay. Missing/mismatched condition * headers mean an older server or a stale primed URL would write * unconditionally, so fail closed and let the caller retry later. */ export declare class PresignPreconditionMissingError extends Error { readonly key: string; readonly header: "if-match" | "if-none-match"; readonly retryable = true; constructor(key: string, header: "if-match" | "if-none-match"); } /** * One-way circuit breaker shared across a run's per-company transports. The * first 429 (vault rate budget exhausted) trips it; thereafter every UNCACHED * presign fails fast with {@link RateLimitedError} instead of hitting the wire. * * Without this, an exhausted budget spirals: prime chunks 429 → keys uncached * → per-file presign → each 429s (after VaultClient's own 3 retries + * backoff) → an 86-minute storm of ~10k doomed calls (observed live). Tripping * once and short-circuiting turns that into a clean fast finish: primed URLs * still work, un-primed keys are deferred, and the run reports them so the next * sync (after the rolling hour recovers) picks them up. */ export declare class RateLimitBreaker { private tripped; isTripped(): boolean; trip(): void; } /** * Transport that moves bytes over short-lived presigned URLs minted by the * vault-service. Holds no AWS credentials. `companyUid` is the EntityContext's * `uid` — the server resolves the per-company bucket from it, so cross-company * reach is structurally impossible (same authority model as the list/presign * handlers). * * URL cache: {@link prime} batch-mints URLs into `urlCache` (keyed by op+key) * so the per-file get/head calls during a sync reuse them instead of issuing a * presign request each. A single instance is shared across all s3.ts calls for * one company within a run (see {@link presignObjectIOFactory} memoization), so * a prime before the transfer loop warms the cache the loop then drains. */ export declare class PresignObjectIO implements ObjectIO { private readonly vault; private readonly companyUid; private readonly breaker; private readonly urlCache; private readonly headReuseCache; constructor(vault: PresignTransportClient, companyUid: string, breaker?: RateLimitBreaker); private cacheKey; hasPrimedPut(key: string, contentHash?: string): boolean; /** A live (non-expiring) cached URL for op+key, or undefined. */ private peekCached; /** Return and evict a live cached URL so primed batches are bounded. */ private consumeCached; private peekHeadReuse; private consumeHeadReuse; private presignSingle; private resolveGetUrlForBody; /** Drop any cached GET URL so the next resolve mints a fresh presign. */ private invalidateGetCache; /** * Resolve a presigned URL (+ replay headers) for op+key: cache hit if primed, * else a single presign. Throws on per-key denial (matches the SDK path's * access error). `extra` carries PUT contentType/metadata on the miss path. */ private resolveUrl; private requireSignedPutPreconditions; prime(op: PresignOp, keys: PresignKeyInput[]): Promise; putObject(input: PutObjectInput): Promise<{ etag: string; }>; /** * True-streaming GET with bounded open-time outer retries. * * Company bulk pulls (Pam / Indigo 2026-08-13) failed keys when a primed GET * past X-Amz-Expires=1800 returned 403 Request has expired with no re-presign. * Status-level failures (expired URL, open-time transport cut after same-URL * retries) re-presign and reopen within {@link FETCH_MAX_RETRIES}. The body * is returned as a live stream — never buffered here. * * Mid-stream EPIPE / socket hang up after a 200 cannot be recovered inside a * streaming generator without buffering (or silently restarting under a * consumer that already received prefix bytes). That recovery lives at the * whole-download seam in s3.ts `downloadFile` (fresh stream → fresh temp). */ getObjectStream(key: string): Promise; getObject(key: string): Promise; listObjects(input: ListObjectsInput): Promise; deleteObject(key: string): Promise; headObject(key: string): Promise; /** * Mint a GET URL for headObject. Returns null only on server-confirmed * FILES_PRESIGN_NOT_FOUND (absence). Per-key denials throw Forbidden. */ private mintHeadGetUrl; } export type ObjectIOFactory = (ctx: EntityContext) => ObjectIO; /** * The S3-SDK transport, wired so every instance can renew its own STS session. * * Use this rather than the bare default wherever the caller can supply a * refresher: the bare default builds instances that expire with the 15-minute * session that created them. */ export declare function stsObjectIOFactory(refresherFor: ContextRefresherFactory, forceRefresherFor?: ContextRefresherFactory): ObjectIOFactory; /** * Install the transport factory for the current process. Passing `null` * resets to the default S3 SDK transport. Called once by `runRunner` after it * resolves the caller's identity + feature-flag gate; every subsequent s3.ts * call resolves its transport through this. */ export declare function setObjectIOFactory(factory: ObjectIOFactory | null): void; /** Resolve the transport for an EntityContext using the active factory. */ export declare function resolveObjectIO(ctx: EntityContext): ObjectIO; /** * Build a factory that routes every EntityContext through the presigned-URL * transport, reusing the one already-authenticated VaultClient and deriving * the per-company authority from `ctx.uid`. */ export declare function presignObjectIOFactory(vault: PresignTransportClient, refresherFor?: ContextRefresherFactory, forceRefresherFor?: ContextRefresherFactory): ObjectIOFactory; //# sourceMappingURL=object-io.d.ts.map