import { EachBatchHandler, EachBatchPayload, EachMessageHandler, EachMessagePayload, KafkaMessage, Producer } from "kafkajs"; /** * Different strategies for naming retry topics */ export declare enum RetryTopicNaming { /** * Retry topics are named in succession based on the number of retries (e.g. `${consumerGroup}-retry-1`, `${consumerGroup}-retry-2`, etc.) */ ATTEMPT_BASED = "attempt", /** * Retry topics are named based on the configured delay in seconds (e.g. `${consumerGroup}-retry-5s`, `${consumerGroup}-retry-30s`, etc.) */ DELAY_BASED = "delay" } /** * Events that can be subscribed to for different retry-handling operations */ export declare enum AsyncRetryEvent { /** * Fired when a message is sent to a retry topic */ RETRY = "retry", /** * Fired when a message is sent to the dead-letter topic */ DEAD_LETTER = "dead-letter" } export interface AsyncRetryConfig { /** The consumer group ID that this async retry helper is handling messages for */ groupId: string; /** A previously configured (and connected) producer that can be used to publish messages into the appropriate delay and retry topics */ producer: Pick; /** The maximum number of retries for a given message (defaults to 5) */ maxRetries?: number; /** The amount of time (in milliseconds) to block an `eachMessage` call while waiting until a retry message is ready to be processed. Waits above this amount will result in the consumer pausing on the topic/partition until the message is ready. */ maxWaitTime?: number; /** A series of delays (in seconds) that will be used for each corresponding retry attempt */ retryDelays?: [number, ...number[]]; /** Strategy for how retry topics are named */ retryTopicNaming?: RetryTopicNaming; } export interface AsyncRetryMessageDetails { /** Whether this message has been routed through an async retry or not */ isRetry: boolean; /** Whether this message is ready to be processed or not */ isReady: boolean; /** The original topic the message came from */ originalTopic: string; /** The number of attempts the message has had so far */ previousAttempts: number; /** The earliest time the message should be processed */ processTime: Date; } /** * Utility type for Arrays with a minimum length of 1 */ declare type NonEmptyArray = [T, ...T[]]; export declare type AsyncRetryAwareEachBatchPayload = EachBatchPayload & { /** * Given a {KafkaMessage} object, returns details relevant to async retry handling */ asyncRetryMessageDetails: (message: KafkaMessage) => AsyncRetryMessageDetails; /** * Handles sending a message to the appropriate retry or dead-letter topic based on how many retries have already been attempted as well as the {Error} object passed in */ messageFailureHandler: (error: Error, message: KafkaMessage) => Promise; }; export declare type AsyncRetryAwareBatchHandler = (payload: AsyncRetryAwareEachBatchPayload) => Promise; /** * The full set of data provided to the {AsyncRetryAwareEachMessageHandler} when processing a message (including details specific to async retry handling) */ export declare type AsyncRetryAwareEachMessagePayload = EachMessagePayload & { /** * Whether this message attempt is a retry or not (will be false on the first attempt and true on all subsequent attempts) */ isRetry: boolean; /** * What topic the message was originally published to before any retries */ originalTopic: string; /** * How many attempts this message has had so far (will be 0 on the first attempt and 1 on the second attempt, etc.) */ previousAttempts: number; /** * The earliest time (expressed as a {Date} object) this message should be processed. For the first attempt, this will always be "now". For subsequent attempts, this will always be something <= "now" (potentially in the past if the consumer is behind where it should be) */ processTime: Date; }; export declare type AsyncRetryAwareMessageHandler = (payload: AsyncRetryAwareEachMessagePayload) => Promise; /** * A custom exception that will result in the message being sent to the dead-letter topic */ export declare class DeadLetter extends Error { constructor(reason: string); } export declare type EventPayload = { /** The message being retried or dead-letter'd */ message: KafkaMessage; /** Information about the retry-state of the message */ details: Omit; /** The Error that is triggering the message to be retried or dead-letter'd */ error: unknown; /** The destination topic for the current retry attempt or dead-letter topic name as applicable */ topic: string; }; /** * A helper that can be used with KafkaJS to facilitate sending messages to retry and/or dead-letter topics so that processing of the primary topics can proceed without being blocked by problematic messages. */ export default class AsyncRetryHelper { /** pattern that can be used when subscribing to all relevant retry topics for the consumer group */ readonly retryTopicPattern: RegExp; /** the number of seconds to wait for each retry attempt (the first retry being the zero-th number in this array) */ readonly retryDelays: NonEmptyArray; /** The topic that messages will be delivered to after all retry attempts are exhausted */ readonly deadLetterTopic: string; private readonly producer; private readonly maxRetries; private readonly maxWaitTime; private readonly _retryTopics; private readonly eventEmitter; private pausedTopicPartitions; /** * The current set of retry topics that are in use by this {AsyncRetryHelper} instance */ get retryTopics(): string[]; /** * Create a new AsyncRetryHelper instance * @param {AsyncRetryConfig} param0 */ constructor({ maxRetries, retryDelays, maxWaitTime, // 3 seconds retryTopicNaming, ...config }: AsyncRetryConfig); /** * Registers a callback function that will be called on the specified event. * @param event The name of the event * @param listener The callback function * @returns {AsyncRetryHelper} */ on(event: AsyncRetryEvent | `${AsyncRetryEvent}`, listener: (payload: EventPayload) => void): () => void; /** * Wraps the provided handler with a handler that will send the message to the appropriate retry (or dead-letter) topic if an exception is thrown * @param {AsyncRetryAwareMessageHandler} handler - your message handler that will be provided with a few extra parameters relevant to the async retry process * @returns {EachMessageHandler} - a standard message handler that can be passed to a KafkaJS consumer instance */ eachMessage(handler: AsyncRetryAwareMessageHandler): EachMessageHandler; /** * Wraps the provided handler with a handler that will provide some extra callback functions for dealing with message processing errors and retries * @param {AsyncRetryAwareBatchHandler} handler - your batch handler that will be provided with a few extra parameters relevant to the async retry process * @returns {EachBatchHandler} - a standard batch handler that can be passed to a KafkaJS consumer instance */ eachBatch(handler: AsyncRetryAwareBatchHandler): EachBatchHandler; private extractReadyMessages; private pauseUntilMessageIsReady; private asyncRetryMessageDetails; private prepareAsyncRetryHeaders; private handleMessageFailure; } export {};