import type { Transform } from 'node:stream'; import type { ReadableStream as WebReadableStream } from 'node:stream/web'; import type { ZlibOptions, ZstdOptions } from 'node:zlib'; import type { AbortableAsyncMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, Integer, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types'; import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js'; import type { TransformLogProgressOptions } from './transform/transformLogProgress.js'; import type { TransformMapOptions } from './transform/transformMap.js'; import type { TransformMapSimpleOptions } from './transform/transformMapSimple.js'; import type { TransformMapSyncOptions } from './transform/transformMapSync.js'; import type { TransformOffsetOptions } from './transform/transformOffset.js'; import type { TransformThrottleOptions } from './transform/transformThrottle.js'; import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js'; import type { TransformWarmupOptions } from './transform/transformWarmup.js'; export declare class Pipeline { private readonly source; private transforms; private destination?; private readableLimit?; private objectMode; private abortableSignal; private constructor(); static from(source: ReadableTyped): Pipeline; /** * Useful in cases when Readable is not immediately available, * but only available after an async operation is completed. * Implemented via a proxy Transform, which should be transparent. */ static fromAsyncReadable(fn: () => Promise>): Pipeline; static fromWeb(webReadableStream: WebReadableStream): Pipeline; /** * Technically same as `fromIterable` (since Array is Iterable), * but named a bit friendlier. */ static fromArray(input: T[]): Pipeline; static fromIterable(input: Iterable | AsyncIterable): Pipeline; static fromNDJsonFile(sourceFilePath: string): Pipeline; static fromFile(sourceFilePath: string): Pipeline; /** * Limits the source Readable, but using `.take(limit)` on it. * This is THE preferred way of limiting the source. */ limitSource(limit: NonNegativeInteger | undefined): this; /** * If possible - STRONGLY PREFER applying `.take(limit)` on the source Readable, * as it's a clean graceful way of limiting the Readable. Example: * * Pipeline.from(myReadable.take(10)) * * or * * Pipeline * .from(myReadable) * .limitSource(10) * * If applying `take` on Readable is not possible - use this method at your own risk. * Why warning? * The limit works by aborting the stream, and then catching the error - certainly * less clean than `.take()` on the source. */ limit(limit: NonNegativeInteger | undefined): this; chunk(chunkSize: PositiveInteger, opt?: TransformOptions): Pipeline; flatten(this: Pipeline): Pipeline; flattenIfNeeded(): Pipeline; logProgress(opt?: TransformLogProgressOptions): this; map(mapper: AbortableAsyncMapper, opt?: TransformMapOptions): Pipeline; mapSync(mapper: IndexedMapper, opt?: TransformMapSyncOptions): Pipeline; mapSimple(mapper: IndexedMapper, opt?: TransformMapSimpleOptions): Pipeline; filter(asyncPredicate: AsyncPredicate, opt?: TransformMapOptions): this; filterSync(predicate: Predicate, opt?: TransformOptions): this; offset(opt: TransformOffsetOptions): this; tap(fn: AsyncIndexedMapper, opt?: TransformOptions): this; tapSync(fn: IndexedMapper, opt?: TransformOptions): this; throttle(opt: TransformThrottleOptions): this; throttleByRSS(opt: TransformThrottleByRSSOptions): this; /** * @experimental to be removed after transformMap2 is stable */ warmup(opt: TransformWarmupOptions): this; transform(transform: TransformTyped): Pipeline; /** * Helper method to add multiple transforms at once. * Not type safe! Prefer using singular `transform()` multiple times for type safety. */ transformMany(transforms: Transform[]): Pipeline; fork(fn: (pipeline: Pipeline) => Promise, opt?: TransformOptions): this; /** * Utility method just to conveniently type-cast the current Pipeline type. * No runtime effect. */ typeCastAs(): Pipeline; setObjectMode(objectMode: boolean): this; /** * Transform the stream of Objects into a stream of JSON lines. * Technically, it goes into objectMode=false, so it's a binary stream at the end. */ toNDJson(): Pipeline; parseNDJson(this: Pipeline): Pipeline; splitOnNewline(this: Pipeline): Pipeline; parseJson(this: Pipeline | Pipeline | Pipeline): Pipeline; gzip(this: Pipeline, opt?: ZlibOptions): Pipeline; gunzip(this: Pipeline, opt?: ZlibOptions): Pipeline; zstdCompress(this: Pipeline, level?: Integer, // defaults to 3 opt?: ZstdOptions): Pipeline; zstdDecompress(this: Pipeline, opt?: ZstdOptions): Pipeline; toArray(opt?: TransformOptions): Promise; toFile(outputFilePath: string): Promise; /** * level corresponds to zstd compression level (if filename ends with .zst), * or gzip compression level (if filename ends with .gz). * Default levels are: * gzip: 6 * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level) */ toNDJsonFile(outputFilePath: string, level?: Integer): Promise; to(destination: WritableTyped): Promise; forEach(fn: AsyncIndexedMapper, opt?: TransformMapOptions & TransformLogProgressOptions): Promise; forEachSync(fn: IndexedMapper, opt?: TransformMapSyncOptions & TransformLogProgressOptions): Promise; run(): Promise; }