import * as Opensearch from "@opensearch-project/opensearch" import * as Cassandra from "cassandra-driver"; import * as Smartlocks from "smartlocks" import * as Redis from "ioredis"; import * as Kafka from "kafkajs" import * as Postgres from "pg"; import * as Etcd from "etcd3" import * as Nats from "nats" import * as fs from "fs" /** Alyxstream Task Message.*/ export declare type TaskMessage = { payload: T metadata: TaskMessageMetadata //| TaskMessageMetadata[], // can be an array of metadata because of joinByKeyWithParallelism globalState: G, [x: string]: any } export declare type TaskMessageMetadata = { key?: string | number windowKey?: string | number | null, startTime?: any, endTime?: any, windowTimeInSeconds?: number | null, windowTimeInMinutes?: number | null, windowTimeInHours?: number | null, windowElements?: any } /** Alyxstream Task instance. * @I initial message type (needed for the *inject()* function) * @C current message type * @L local storage value types * @Ls is local storage set (default: *false*) * @Ms are metadata set (default: *false*) * @Sk task storage system kind (default: *null*) */ export declare interface T { /** Initializes task metadata. * @enables *setMetadata()*, *getMetadata()*. */ withMetadata: () => T /** @requires *task.withMetadata()*. */ setMetadata: Ms extends true ? (id: any) => T : never /** Returns task metadata. * @requires *task.withMetadata()*. */ getMetadata: Ms extends true ? () => { id: any, [x: string]: any [x: number]: any } : never /** Sets the message key to *"default"* */ withDefaultKey: () => T /** Sets the event time from the message payload. */ withEventTime: (cb: (x: C) => number) => T /** Sets the message key from the message payload. */ keyBy: (cb: (x: C) => string | number) => T filter: (cb: (x: C) => boolean) => T print: (str?: any) => T /** Splits the task execution into multiple subtasks. Waits for subtasks execution and continues to the next operator passing the array of subtasks results as message. */ branch: (subtaskFuncs: ((x: C) => Promise>)[]) => T readline: () => T /** Execute a function on the message payload. Can be an async function. */ fn: (callback: (x: C) => R) => R extends Promise ? T : T /** Execute a function on the raw task message. Can be an async fucntion. */ fnRaw: (callback: (x: TaskMessage) => R) => R extends Promise ? T : T /** @deprecated use fn() instead */ customFunction: (callback: (x: C) => R) => T /** @deprecated use fn() instead */ customAsyncFunction: (callback: (x: C) => Promise) => T /** @deprecated use fnRaw() instead */ customFunctionRaw: (callback: (x: TaskMessage) => R) => T /** @deprecated use fnRaw() instead */ customAsyncFunctionRaw: (callback: (x: TaskMessage) => Promise) => T // this creates an array of metadata (may cause type errors) joinByKeyWithParallelism: ( storage: Storage, keyFunction: (x: TaskMessage) => string | number, parallelism: number ) => T parallel: any>(numberOfProcess: number, produceFunction?: Pf) => T queueSize: (storage: Storage) => T // to check if it's really a number enqueue: (storage: Storage) => T dequeue: (storage: Storage) => T multiDequeue: (storage: Storage) => T /** Sets the task internal storage system. * @enables *toStorage()*, *fromStorage()*, *flushStorage()*, *fromStorageToGlobalState()*, *disconnectStorage()*, *collect()*, *storage()*. */ withStorage: (storage: Storage) => T /** @requires *task.withStorage()*.*/ toStorage: Sk extends StorageKind ? (keyFunc: (x: TaskMessage) => string | number, valueFunc?: (x: TaskMessage) => any, ttl?: number) => T /*To check*/ : never /** @requires *task.withStorage()*.*/ toStorageList: Sk extends StorageKind ? (keyFunc: (x: TaskMessage) => string | number, valueFunc?: (x: C) => any, ttl?: number) => T /*To check*/ : never /** @requires *task.withStorage()* * @requires lists-compatible storage system */ fromStorageList: Sk extends ListStorageKind ? (keyFunc: (x: TaskMessage) => (string | number)[], valueFunc: (x: C) => R[]) => T : never /** @requires *task.withStorage()*. */ fromStorageToGlobalState: Sk extends StorageKind ? (keysFunc: (x: TaskMessage) => (string | number)[]) => T : never /** @requires *task.withStorage()*. */ disconnectStorage: Sk extends StorageKind ? () => T : never /** @requires *task.withStorage()*. */ flushStorage: Sk extends StorageKind ? (keysFunc: (x: TaskMessage) => (string | number)[]) => T : never /** @requires *task.withStorage()*. */ storage: Sk extends StorageKind ? () => Storage : never /** Sets the task in-memory key-value store. * @enables *setLocalKV()*, *setLocalKVRaw()*, *getLocalKV*(), *mergeLocalKV()*, *flushLocalKV().* */ withLocalKVStorage: () => T // define the type of items stored in storage keys /** @requires *task.withLocalKVStorage()*. */ setLocalKV: Ls extends true ? (key: string | number, func: (x: C) => L) => T : never /** @requires *task.withLocalKVStorage()*. */ setLocalKVRaw: Ls extends true ? (key: string | number, func: (x: TaskMessage) => L) => T : never /** @requires *task.withLocalKVStorage()*. */ getLocalKV: Ls extends true ? (key?: K) => K extends Exclude // check if key is provided ? T // not provided => returns full storage : T // provided => returns single storage value : never /** @requires *task.withLocalKVStorage()*. */ flushLocalKV: Ls extends true ? (key: string | number) => T : never /** @requires windowing-compatible storage system */ tumblingWindowCount: (storage: Storage, countLength: number, inactivityMilliseconds: number) => T /** @requires windowing-compatible storage system */ tumblingWindowTime: (storage: Storage, timeLengthMilliSeconds: number, inactivityMilliseconds?: number) => T /** @requires windowing-compatible storage system */ sessionWindowTime: (storage: Storage, inactivityMilliseconds: number) => T /** @requires windowing-compatible storage system */ slidingWindowCount: (storage: Storage, countLength: number, slidingLength: number, inactivityMilliseconds: number) => T /** @requires windowing-compatible storage system */ slidingWindowTime: (storage: Storage, timeLengthMilliSeconds: number, slidingLengthMilliseconds: number, inactivityMilliseconds: number) => T /** Procudes task messages iterating over the provided array. */ fromArray: (array: R[]) => T /** Procudes a single task message from the provided object. */ fromObject: (object: R) => T /** Procudes a single task message from the provided string. */ fromString: (string: string) => T /** Procudes task messages iterating by ticking at the provided time interval. */ fromInterval: (intervalMs: number, generatorFunc?: (counter: number) => R, maxSize?: number) => T/*TBD*/ fromReadableStream: (filePath: fs.PathLike, useZlib?: boolean) => T /** Produces a message to a Kafka topic. */ toKafka: C extends (Kafka.Message | Kafka.Message[]) ? (kafkaSink: KSink, topic: string, callback?: (x: C) => Kafka.Message | Kafka.Message[], options?: KSinkOptions) => T : (kafkaSink: KSink, topic: string, callback: (x: C) => Kafka.Message | Kafka.Message[], options?: KSinkOptions) => T /** Consume messages from a Kafka source. */ fromKafka: (source: KSource) => T, G, L, Ls, Sk, Ms> kafkaCommit: C extends KCommitParams ? (kafkaSource: KSource, commitParams?: KCommitParams) => T : (kafkaSource: KSource, commitParams: KCommitParams) => T /** Watch changes over an Etcd key. */ fromEtcd: (storage: Storage, key: string | number, watch?: boolean) => T, /** Consumes messages from a NATS Jetstream stream */ fromNats: (source: NatsJsSource) => T, G, L, Ls, Sk, Ms>, // dataCb is not called in this sink /** Produces a message to a NATS Jetstream stream. */ toNats: (sink: Nats.NatsConnection, topic: string, dataCb?: (x: C) => any) => T, /** Acquires a lock on a storage key using a smartlocks library mutex. */ lock: (mutex: Smartlocks.Mutex, lockKeyFn: (x: C) => string | number, retryTimeMs?: number, ttl?: number) => T, /** Releases a lock on a storage key using a smartlocks library mutex. */ release: (mutex: Smartlocks.Mutex, lockKeyFn: (x: C) => string | number) => T, /** Progressively sums messages, returning the current counter value. * @requires *number* */ sum: C extends number ? () => T : never /** Push a new message to the task. */ inject: (data: I) => Promise> /** Starts the task execution when using a source. */ close: () => Promise> /** Return the last result of the task. */ finalize: () => TaskMessage self: (cb: (task: T) => any) => T /** @requires *task.withStorage()* * @requires collect-compatible storage system */ collect: Sk extends CollectStorageKind ? ( idFunction: (x: TaskMessage) => string, keyFunction: (x: TaskMessage) => string, valueFunction: (x: TaskMessage) => R | null, waitUntil: (arr: any[], flat: any[]) => boolean, /* TBD */ emitFunction: (arr: any[], flat: any[]) => boolean, /* TBD */ ttl?: number, ) => T : never, sumMap: () => T /** Executes a groupBy for every key of the object message. */ objectGroupBy: (keyFunction: (x: C) => string | number) => T /** Aggregates array element by key in a storage system. */ aggregate: (storage: Storage, name: string, keyFunction: (x: C) => string | number) => T /** @requires *task.withLocalKVStorage().* */ mergeLocalKV: Ls extends true ? (key: K) => T : never /** Transform array * @requires *array* */ map: C extends (infer U)[] ? (func: (x: U) => R) => T : never /** Splits the task execution for each element of the array. * @requires *array* */ each: C extends (infer U)[] ? (func?: (x: U) => any) => T : never /** Filters array elements. * @requires *array* */ filterArray: C extends (infer U)[] ? (func: (x: U) => boolean) => T : never // why does this implement a number only internal reduce function? (sum) // reduce: (func: (prev: ElemOfArr, curr: ElemOfArr, currIdx?: number) => R, initialValue?: R) => Tsk /** @requires *array* */ reduce: C extends (infer U)[] ? (func?: (x: U) => number) => T : never /** Count array element by key. * @requires *array* */ countInArray: C extends (infer U)[] ? (func: (x: U) => string | number) => T : never /** Returns the array length. * @requires *array* */ length: C extends any[] ? () => T : never /** @requires *array* */ groupBy: C extends (infer U)[] ? (func: (elem: U, index?: number, array?: U[]) => any) => T : never /** @requires *task.withStorage()* * @requires *array* */ fromStorage: Sk extends null ? never : C extends any[] ? (keysFunc: (x: TaskMessage) => (string | number)[]) => T : never /** Flattens an array. * @requires *array* */ flat: C extends any[] ? () => T[], G, L, Ls, Sk, Ms> : never /** Splits a string (default separator: '\s'). * @requires *string* */ tokenize: C extends string ? (separator?: string) => T : never // prevents type errors for task extensions // [x: string]: any } export declare type TaskExtension = (first: T, ...rest: U) => void; /** Intialize an Alyxstream task. Generic type can be used to provide the initial *inject()* message type. */ export declare function Task(id?: any): T /*TBD*/ /** Extends a task by creating a custom method. This function is **type unsafe**. Consider using **fn()** with a custom callback for type safety. */ export declare function ExtendTask(name: string, extension: TaskExtension): void /** Extends a task by creating a custom method that operates on the raw task message. This function is **type unsafe**. Consider using **fnRaw()** with a custom callback for type safety. */ export declare function ExtendTaskRaw(name: string, extension: TaskExtension, any>): void export declare enum StorageKind { Memory = "Memory", Redis = "Redis", Cassandra = "Cassandra", Etcd = "Etcd", Opensearch = "Opensearch", Postgres = "Postgres" } // from IOptions.node export declare type OpensearchNode = string | string[] | Opensearch.NodeOptions | Opensearch.NodeOptions[] /** Conditional generic type for different storage configuration objects. */ export declare type StorageConfig = K extends StorageKind.Memory ? null // memory storage has no config obj : K extends StorageKind.Redis ? Redis.RedisOptions : K extends StorageKind.Cassandra ? Cassandra.ClientOptions : K extends StorageKind.Etcd ? Etcd.IOptions : K extends StorageKind.Opensearch ? OpensearchNode : K extends StorageKind.Postgres ? Postgres.ClientConfig : never /** Storage sytem to be used in Alyxstream tasks. */ export declare interface Storage { db: () => StorageEngine set: (key: string, value: any, ttl?: number | null) => Promise; /*TBD*/ get: (key: string) => Promise; /*TBD*/ push: (key: string, value: any) => Promise; /*TBD*/ flush: (key: string) => Promise; /*TBD*/ getList: K extends ListStorageKind ? (key: string) => Promise : never; /*TBD*/ slice: K extends WindowStorageKind ? (key: string, numberOfItemsToRemove: number) => Promise : never, /*TBD*/ sliceByTime: K extends WindowStorageKind ? (key: string, startTime: number) => Promise : never, /*TBD*/ disconnect: K extends WindowStorageKind ? () => Promise : never, /*TBD*/ flushStorage: K extends WindowStorageKind ? () => Promise : never, /*TBD*/ queueSize: K extends QueueStorageKind ? (data?: any) => Promise : never, enqueue: K extends QueueStorageKind ? (data: any) => Promise : never, dequeue: K extends QueueStorageKind ? () => Promise : never } type StorageEngine = K extends StorageKind.Memory ? any // internal state : K extends StorageKind.Redis ? Redis.Redis : K extends StorageKind.Cassandra ? Cassandra.Client : K extends StorageKind.Etcd ? Etcd.Etcd3 : K extends StorageKind.Opensearch ? Opensearch.Client : K extends StorageKind.Postgres ? Postgres.Client : never /** Initialize an Alyxstream storage system to be used in a task. */ export declare function MakeStorage(kind: K, config?: StorageConfig | null, id?: string | number): Storage /** Initialize an HTTP server that exposes the state of a set of Alyxstream storage systems. Endpoint: /api/v1/state/:prefix/:keys. */ export declare function ExposeStorageState(storageMap: { [x in string | number]: Storage }, config?: { port?: number }): void export declare interface KMessage { topic: string, offset: string, partition: number, headers: any, /*TBD*/ key: string, value: T } export declare type KSinkOptions = { acks?: number timeout?: number compression?: Kafka.CompressionTypes } type KCommitParams = Pick export declare interface KSource { stream: (cb: any) => Promise /*TBD*/ consumer: () => Kafka.Consumer } export declare interface KSink extends Kafka.Producer {} export declare type RekeyFunction = (s: any) => any /*TBD*/ export declare type SinkDataFunction = (s: any) => Kafka.Message /*TBD*/ type ExchangeEmitTask = T< { key: string | number, value: string }, { key: string | number, value: string }, any, void, false, null, false > export declare interface KExchange { setKeyParser: (fn: (x: OnMessage) => string | number) => void; setValidationFunction: (fn: (x: OnMessage) => boolean | any) => void; on: (fn: (x: OnMessage) => R) => Promise>; emit: (mex: EmitMessage) => Promise } export declare type DefaultExchangeMessageKind = { kind: NonNullable metadata: NonNullable<{ key: NonNullable }> spec: NonNullable } /** Initialize Kafka client. */ export declare function KafkaClient(config: Kafka.KafkaConfig): Kafka.Kafka /** Initialize a Kafka Admin client. */ export declare function KafkaAdmin(client: Kafka.Kafka): Promise /** Initialize a Kafka source (consumer). */ export declare function KafkaSource(client: Kafka.Kafka, config: { groupId: string, topics: Array<{ topic: string, fromBeginning?: boolean autoCommit?: boolean autoHeartbeat?: number parseWith?: (x: string) => any // this should be removed for type safety }> }): Promise /** Initialize a Kafka sink (producer). */ export declare function KafkaSink(client: Kafka.Kafka, config?: Kafka.ProducerConfig): Promise export declare function KafkaCommit(source: KSource, params: KCommitParams): Promise export declare function KafkaRekey(kafkaSource: KSource, rekeyFunction: RekeyFunction, kafkaSink: KSink, sinkTopic: string, sinkDataFunction: SinkDataFunction): void // DefaultExchangeMessageKind instead of any will break existent code (maybe any is better?) // A better option would be not to use a enbedded message validator, but to provide an defaultMessageValidator // and defaultKeyValidator that one can import and use /** Initialize a Kafka Exchange. */ export declare function Exchange< OnMessage = DefaultExchangeMessageKind, EmitMessage = DefaultExchangeMessageKind >( client: Kafka.Kafka, topic: string, groupId: string, sourceOptions?: { fromBeginning?: boolean autoCommit?: boolean autoHeartbeat?: number }, sinkOptions?: KSinkOptions ): Promise> export declare interface NatsJsSource { stream: (cb: any) => Promise /*TBD*/ consumer: () => void } export declare interface NatsStreamMsg { data: T, m: { msg: Nats.JsMsg, didAck: boolean, } } /** Initialize a Nats connection */ export declare function NatsClient(server: Nats.ConnectionOptions): Promise /** Initialize a Nats Jetstream source */ export declare function NatsJetstreamSource(natsCliens: Nats.NatsConnection, sources: (Nats.ConsumerConfig & { stream: string })[]): Promise type ElemOfArr = T extends (infer U)[] ? U : never; /** returns the nested element of both 1d and 2d arrays */ type NestedElem = T extends readonly (infer U)[] ? U extends readonly (infer V)[] ? V : U : never; /** List of storage systems that are suitable for windowing. */ type WindowStorageKind = StorageKind.Memory | StorageKind.Redis | StorageKind.Cassandra /** List of storage systems that are suitable for queuing. */ type QueueStorageKind = StorageKind.Redis /** List of storage systems that are suitable for lists. */ type ListStorageKind = StorageKind.Redis | StorageKind.Cassandra | StorageKind.Memory /** List of storage systems that are suitable for collect operator. */ type CollectStorageKind = StorageKind.Cassandra