import { Logger } from "./common.js"; import { Base64Encoded, Branded } from "@milaboratories/pl-model-common"; import { Readable } from "node:stream"; import { RequestListener } from "node:http"; //#region src/pframe/internal_api/http_helpers.d.ts /** Parquet file extension */ declare const ParquetExtension: ".parquet"; /** Parquet file name */ type ParquetFileName = Branded<`${string}.parquet`, "PFrameInternal.ParquetFileName">; type FileRange = { /** Start byte position (inclusive) */start: number; /** End byte position (inclusive) */ end: number; }; /** HTTP range as of RFC 9110 */ type HttpRange = { /** * Get file content in the specified byte range * * @example * ``` * GET /file.parquet HTTP/1.1 * Range: bytes=0-1023 * ``` */ type: "bounded"; /** Start byte position (inclusive) */ start: number; /** End byte position (inclusive) */ end: number; } | { /** * Get byte range starting from the specified offset * * @example * ``` * GET /file.parquet HTTP/1.1 * Range: bytes=1024- * ``` */ type: "offset"; /** Start byte position (inclusive) */ offset: number; } | { /** * Get byte range starting from the specified suffix * * @example * ``` * GET /file.parquet HTTP/1.1 * Range: bytes=-1024 * ``` */ type: "suffix"; /** End byte position (inclusive) */ suffix: number; }; /** HTTP method passed to object store */ type HttpMethod = "GET" | "HEAD"; /** HTTP response from object store */ type ObjectStoreResponse = { /** * Will be translated to 500 Internal Server Error by the handler * or 408 Request Timeout if the request was aborted */ type: "InternalError"; } | { /** Will be translated to 404 Not Found by the handler */type: "NotFound"; } | { /** Will be translated to 416 Range Not Satisfiable by the handler */type: "RangeNotSatisfiable"; /** Total file size in bytes */ size: number; } | { /** Will be translated to 200 OK or 206 Partial Content by the handler */type: "Ok"; /** Total file size in bytes */ size: number; /** File range translated from HTTP range */ range: FileRange; /** Stream of file content, undefined for HEAD requests */ data?: Readable; }; /** Common options for object store creation */ interface ObjectStoreOptions { /** Logger instance, no logging is performed when not provided */ logger?: Logger; } /** Options for file system object store creation */ interface FsStoreOptions extends ObjectStoreOptions { /** Local directory to serve files from */ rootDir: string; } interface ObjectStore { /** * Proxy HTTP(S) request for parquet file to object store. * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler * Callback API is used so that ObjectStore can limit the number of concurrent requests. */ request(filename: ParquetFileName, params: { method: HttpMethod; range?: HttpRange; signal: AbortSignal; callback: (response: ObjectStoreResponse) => Promise; }): void; } /** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */ declare abstract class BaseObjectStore implements ObjectStore { protected readonly logger: Logger; constructor(options: ObjectStoreOptions); /** Translate HTTP range to file range, @returns null if the range is not satisfiable */ protected translate(fileSize: number, range?: HttpRange): FileRange | null; /** * Proxy HTTP(S) request for parquet file to object store. * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler * Callback API is used so that ObjectStore can limit the number of concurrent requests. */ abstract request(filename: ParquetFileName, params: { method: HttpMethod; range?: HttpRange; signal: AbortSignal; callback: (response: ObjectStoreResponse) => Promise; }): Promise; } /** Configuration for {@link HttpHelpers.createCachingObjectStore} */ type CacheConfig = { /** Filesystem path where the cache persists its data */cachePath: string; /** Hard total size budget in bytes; the cache self-stabilizes near this size */ maxSizeBytes: number; /** * Max share of the budget a single file may occupy (0..1); when a file exceeds it, that * file's oldest ranges are evicted first. Stops one large scan from flushing the working set. */ admissionFraction: number; /** Upper bound on previously-evicted (ghost) files remembered to guide readmission; resident files are already byte-bounded */ maxFilesTracked: number; }; /** * Cumulative cache event counters. Surfaced both as lifetime totals (persisted across restarts) * and as a per-process session view, @see CacheMetrics. */ type CacheCounters = { /** Requests fully served from cache */hits: number; /** Requests that fell through to upstream */ misses: number; /** Bytes served from cache */ bytesServed: number; /** * Bytes requested by clients that were not cached; compare with {@link bytesFetched} for read-ahead * amplification. */ bytesMissed: number; /** Bytes downloaded from upstream on misses, including read-ahead and read-behind */ bytesFetched: number; /** Bytes evicted to keep the cache within its total size budget, @see CacheConfig.maxSizeBytes */ bytesEvictedByBudget: number; /** Bytes evicted because a single file exceeded its allowed share of the budget, @see CacheConfig.admissionFraction */ bytesEvictedByFileCap: number; /** Files promoted on observed reuse, so they are retained longer than first-time entries */ promotions: number; /** Previously evicted files that were re-requested and readmitted */ ghostReadmissions: number; }; /** Snapshot of cache state and counters, @see CachingObjectStore.getMetrics */ type CacheMetrics = { /** Current resident bytes */sizeBytes: number; /** Number of files with resident (cached) bytes */ residentFiles: number; /** Number of previously-evicted (ghost) files remembered, <= @see CacheConfig.maxFilesTracked */ ghostFiles: number; /** Counters cumulative across all sessions (persisted) */ lifetime: CacheCounters; /** The same counters scoped to the current process */ session: CacheCounters; }; /** * Options for caching object store creation, @see HttpHelpers.createCachingObjectStore. * Standalone (not extending {@link ObjectStoreOptions}): a caching store is a decorator over * `upstream`, so source-store options do not apply to it. */ interface CachingObjectStoreOptions { /** Upstream store consulted on cache misses */ upstream: ObjectStore; /** Cache configuration */ config: CacheConfig; /** Logger instance, no logging is performed when not provided */ logger?: Logger; } /** * An {@link ObjectStore} that serves byte ranges from a persistent local cache, fetching misses * from an upstream store. This is the single handle for the cache: pass it as * {@link RequestHandlerOptions.store}, read {@link CachingObjectStore.getMetrics}, and dispose it * to release the cache, @see HttpHelpers.createCachingObjectStore. */ interface CachingObjectStore extends ObjectStore, AsyncDisposable { /** Instantaneous cache state plus lifetime/session counters */ getMetrics(): CacheMetrics; /** Drop all cached data and zero the counters (test/benchmark use) */ reset(): Promise; } /** Object store base URL in format accepted by Apache DataFusion and DuckDB */ type ObjectStoreUrl = Branded; /** HTTP(S) request handler creation options */ type RequestHandlerOptions = { /** * Object store to serve files from. Compose caching by wrapping the upstream store with * {@link HttpHelpers.createCachingObjectStore} and passing the result here, * @see HttpHelpers.createFsStore */ store: ObjectStore; }; /** Server configuration options */ type HttpServerOptions = { /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */handler: RequestListener; /** Port to bind to, @default 0 for auto-assignment */ port?: number; /** Do not apply authorization middleware to @param handler */ noAuth?: true; /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */ noHttps?: true; }; /** * Long unique opaque string for use in Bearer authorization header * * @example * ```ts * request.setHeader('Authorization', `Bearer ${authToken}`); * ``` */ type HttpAuthorizationToken = Branded; /** * TLS certificate in PEM format * * @example * ```txt * -----BEGIN CERTIFICATE----- * MIIC2zCCAcOgAwIBAgIJaVW7... * ... * ...Yf9CRK8fgnukKM7TJ * -----END CERTIFICATE----- * ``` */ type PemCertificate = Branded; /** HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */ type HttpServerInfo = { /** URL of the HTTP(S) server formatted as `http{s}://:/` */url: ObjectStoreUrl; /** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */ authToken?: HttpAuthorizationToken; /** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */ encodedCaCert?: Base64Encoded; }; /** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */ interface HttpServer { /** Server configuration information for initiating connections from clients */ get info(): HttpServerInfo; /** Promise that resolves when the server is stopped */ get stopped(): Promise; /** Request server stop, returns the same promise as @see HttpServer.stopped */ stop(): Promise; } /** List of HTTP(S) related helper functions exposed by PFrame module */ interface HttpHelpers { /** * Create an object store for serving files from a local directory. * Rejects if the provided path does not exist or is not a directory. */ createFsStore(options: FsStoreOptions): Promise; /** * Wrap an upstream object store in a persistent byte-range cache backed by SQLite. * The returned store is itself an {@link ObjectStore} (pass it to {@link createRequestHandler}) * and additionally exposes metrics and reset. Dispose it to close the cache database. */ createCachingObjectStore(options: CachingObjectStoreOptions): Promise; /** * Create an HTTP request handler for serving files from an object store. * Accepts only paths of the form `/.parquet`, returns 410 otherwise. * Assumes that files are immutable (and sets cache headers accordingly). */ createRequestHandler(options: RequestHandlerOptions): RequestListener; /** * Serve HTTP(S) requests using the provided handler on localhost port. * @returns promise that resolves when the server has stopped. * * @example * ```ts * const rootDir = '/path/to/directory/with/parquet/files'; * * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => { * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`); * }); * * const server = await HttpHelpers.createHttpServer({ * handler: HttpHelpers.createRequestHandler({ store }), * }).catch((err: unknown) => { * throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`); * }); * * const { url, authToken, encodedCaCert } = server.info; * * await server.stop(); * ``` */ createHttpServer(options: HttpServerOptions): Promise; } //#endregion export { BaseObjectStore, CacheConfig, CacheCounters, CacheMetrics, CachingObjectStore, CachingObjectStoreOptions, FileRange, FsStoreOptions, HttpAuthorizationToken, HttpHelpers, HttpMethod, HttpRange, HttpServer, HttpServerInfo, HttpServerOptions, ObjectStore, ObjectStoreOptions, ObjectStoreResponse, ObjectStoreUrl, ParquetExtension, ParquetFileName, PemCertificate, RequestHandlerOptions }; //# sourceMappingURL=http_helpers.d.ts.map