///
import * as tls from 'tls'
import * as net from 'net'
export class Kafka {
constructor(config: KafkaConfig)
producer(config?: ProducerConfig): Producer
consumer(config?: ConsumerConfig): Consumer
admin(config?: AdminConfig): Admin
logger(): Logger
}
export interface KafkaConfig {
brokers: string[]
ssl?: tls.ConnectionOptions
sasl?: SASLOptions
clientId?: string
connectionTimeout?: number
authenticationTimeout?: number
requestTimeout?: number
enforceRequestTimeout?: boolean
retry?: RetryOptions
socketFactory?: ISocketFactory
logLevel?: logLevel
logCreator?: logCreator
}
export type ISocketFactory = (
host: string,
port: number,
ssl: tls.ConnectionOptions,
onConnect: () => void
) => net.Socket
export interface SASLOptions {
mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512' | 'aws'
username: string
password: string
}
export interface ProducerConfig {
createPartitioner?: ICustomPartitioner
retry?: RetryOptions
metadataMaxAge?: number
allowAutoTopicCreation?: boolean
idempotent?: boolean
transactionalId?: string
transactionTimeout?: number
maxInFlightRequests?: number
}
export interface Message {
key?: Buffer | string | null
value: Buffer | string | null
partition?: number
headers?: IHeaders
timestamp?: string
}
export interface PartitionerArgs {
topic: string
partitionMetadata: PartitionMetadata[]
message: Message
}
export type ICustomPartitioner = () => (args: PartitionerArgs) => number
export type DefaultPartitioner = (args: PartitionerArgs) => number
export type JavaCompatiblePartitioner = (args: PartitionerArgs) => number
export const Partitioners: {
DefaultPartitioner: DefaultPartitioner
JavaCompatiblePartitioner: JavaCompatiblePartitioner
}
export type PartitionMetadata = {
partitionErrorCode: number
partitionId: number
leader: number
replicas: number[]
isr: number[]
}
export interface IHeaders {
[key: string]: string
}
export interface ConsumerConfig {
groupId: string
partitionAssigners?: PartitionAssigner[]
metadataMaxAge?: number
sessionTimeout?: number
rebalanceTimeout?: number
heartbeatInterval?: number
maxBytesPerPartition?: number
minBytes?: number
maxBytes?: number
maxWaitTimeInMs?: number
retry?: RetryOptions
allowAutoTopicCreation?: boolean
maxInFlightRequests?: number
readUncommitted?: boolean
}
export interface PartitionAssigner {
new (config: { cluster: Cluster }): Assigner
}
export interface CoordinatorMetadata {
errorCode: number
coordinator: {
nodeId: number
host: string
port: number
}
}
export type Cluster = {
isConnected(): boolean
connect(): Promise
disconnect(): Promise
refreshMetadata(): Promise
refreshMetadataIfNecessary(): Promise
addTargetTopic(topic: string): Promise
findBroker(node: { nodeId: string }): Promise
findControllerBroker(): Promise
findTopicPartitionMetadata(topic: string): PartitionMetadata[]
findLeaderForPartitions(topic: string, partitions: number[]): { [leader: string]: number[] }
findGroupCoordinator(group: { groupId: string }): Promise
findGroupCoordinatorMetadata(group: { groupId: string }): Promise
defaultOffset(config: { fromBeginning: boolean }): number
fetchTopicsOffset(
topics: Array<{
topic: string
partitions: Array<{ partition: number }>
fromBeginning: boolean
}>
): Promise<{ topic: string; partitions: Array<{ partition: number; offset: string }> }>
}
export type Assignment = { [topic: string]: number[] }
export type GroupMember = { memberId: string }
export type GroupMemberAssignment = { memberId: string; memberAssignment: Buffer }
export type GroupState = { name: string; metadata: Buffer }
export type Assigner = {
name: string
version: number
assign(group: {
members: GroupMember[]
topics: string[]
userData: Buffer
}): Promise
protocol(subscription: { topics: string[]; userData: Buffer }): GroupState
}
export interface RetryOptions {
maxRetryTime?: number
initialRetryTime?: number
factor?: number
multiplier?: number
retries?: number
}
export interface AdminConfig {
retry?: RetryOptions
}
export interface ITopicConfig {
topic: string
numPartitions?: number
replicationFactor?: number
replicaAssignment?: object[]
configEntries?: object[]
}
export interface ITopicMetadata {
name: string
partitions: PartitionMetadata[]
}
export enum ResourceType {
UNKNOWN = 0,
ANY = 1,
TOPIC = 2,
GROUP = 3,
CLUSTER = 4,
TRANSACTIONAL_ID = 5,
DELEGATION_TOKEN = 6,
}
export interface ResourceConfigQuery {
type: ResourceType
name: string
configNames: string[]
}
export interface ConfigEntries {
configName: string
configValue: string
isDefault: boolean
isSensitive: boolean
readOnly: boolean
configSynonyms: ConfigSynonyms[]
}
export interface ConfigSynonyms {
configName: string
configValue: string
configSource: number
}
export interface DescribeConfigResponse {
resources: {
configEntries: ConfigEntries[]
errorCode: number
errorMessage: string
resourceName: string
resourceType: ResourceType
}[]
throttleTime: number
}
export interface IResourceConfig {
type: ResourceType
name: string
configEntries: { name: string; value: string }[]
}
type ValueOf = T[keyof T]
export type AdminEvents = {
CONNECT: 'admin.connect'
DISCONNECT: 'admin.disconnect'
REQUEST: 'admin.network.request'
REQUEST_TIMEOUT: 'admin.network.request_timeout'
REQUEST_QUEUE_SIZE: 'admin.network.request_queue_size'
}
export interface InstrumentationEvent {
id: string
type: string
timestamp: number
payload: T
}
export type ConnectEvent = InstrumentationEvent
export type DisconnectEvent = InstrumentationEvent
export type RequestEvent = InstrumentationEvent<{
apiKey: number
apiName: string
apiVersion: number
broker: string
clientId: string
correlationId: number
createdAt: number
duration: number
pendingDuration: number
sentAt: number
size: number
}>
export type RequestTimeoutEvent = InstrumentationEvent<{
apiKey: number
apiName: string
apiVersion: number
broker: string
clientId: string
correlationId: number
createdAt: number
pendingDuration: number
sentAt: number
}>
export type RequestQueueSizeEvent = InstrumentationEvent<{
broker: string
clientId: string
queueSize: number
}>
export interface SeekEntry {
partition: number
offset: string
}
export type Admin = {
connect(): Promise
disconnect(): Promise
createTopics(options: {
validateOnly?: boolean
waitForLeaders?: boolean
timeout?: number
topics: ITopicConfig[]
}): Promise
deleteTopics(options: { topics: string[]; timeout?: number }): Promise
fetchTopicMetadata(options: { topics: string[] }): Promise<{ topics: Array }>
fetchOffsets(options: {
groupId: string
topic: string
}): Promise>
fetchTopicOffsets(
topic: string
): Promise>
setOffsets(options: { groupId: string; topic: string; partitions: SeekEntry[] }): Promise
resetOffsets(options: { groupId: string; topic: string; earliest: boolean }): Promise
describeConfigs(configs: {
resources: ResourceConfigQuery[]
includeSynonyms: boolean
}): Promise
alterConfigs(configs: { validateOnly: boolean; resources: IResourceConfig[] }): Promise
logger(): Logger
on(eventName: ValueOf, listener: (...args: any[]) => void): void
events: AdminEvents
}
export const PartitionAssigners: { roundRobin: PartitionAssigner }
export interface ISerializer {
encode(value: T): Buffer
decode(buffer: Buffer): T
}
export type MemberMetadata = {
version: number
topics: string[]
userData: Buffer
}
export type MemberAssignment = {
version: number
assignment: Assignment
userData: Buffer
}
export const AssignerProtocol: {
MemberMetadata: ISerializer
MemberAssignment: ISerializer
}
export enum logLevel {
NOTHING = 0,
ERROR = 1,
WARN = 2,
INFO = 4,
DEBUG = 5,
}
export interface LogEntry {
namespace: string
level: logLevel
label: string
log: LoggerEntryContent
}
export interface LoggerEntryContent {
readonly timestamp: Date
readonly message: string
[key: string]: any
}
export type Logger = (entry: LogEntry) => void
export type logCreator = (logLevel: string) => (entry: LogEntry) => void
export type Broker = {
isConnected(): boolean
connect(): Promise
disconnect(): Promise
apiVersions(): Promise<{ [apiKey: number]: { minVersion: number; maxVersion: number } }>
metadata(
topics: string[]
): Promise<{
brokers: Array<{ nodeId: number; host: string; port: number }>
topicMetadata: Array<{
topicErrorCode: number
topic: number
partitionMetadata: PartitionMetadata[]
}>
}>
offsetCommit(request: {
groupId: string
groupGenerationId: number
memberId: string
retentionTime?: number
topics: Array<{ topic: string; partitions: Array<{ partition: number; offset: string }> }>
}): Promise
}
export type KafkaMessage = {
key: Buffer
value: Buffer
timestamp: string
size: number
attributes: number
offset: string
headers?: IHeaders
}
export interface ProducerRecord {
topic: string
messages: Message[]
acks?: number
timeout?: number
compression?: CompressionTypes
}
export type RecordMetadata = {
topicName: string
partition: number
errorCode: number
offset: string
timestamp: string
}
export interface TopicMessages {
topic: string
messages: Message[]
}
export interface ProducerBatch {
acks: number
timeout: number
compression: CompressionTypes
topicMessages: TopicMessages[]
}
export interface PartitionOffset {
partition: number
offset: string
}
export interface TopicOffsets {
topic: string
partitions: PartitionOffset[]
}
export interface Offsets {
topics: TopicOffsets[]
}
type Sender = {
send(record: ProducerRecord): Promise
sendBatch(batch: ProducerBatch): Promise
}
export type ProducerEvents = {
CONNECT: 'producer.connect'
DISCONNECT: 'producer.disconnect'
REQUEST: 'producer.network.request'
REQUEST_TIMEOUT: 'producer.network.request_timeout'
REQUEST_QUEUE_SIZE: 'producer.network.request_queue_size'
}
export type Producer = Sender & {
connect(): Promise
disconnect(): Promise
isIdempotent(): boolean
events: ProducerEvents
on(eventName: ValueOf, listener: (...args: any[]) => void): void
transaction(): Promise
logger(): Logger
}
export type Transaction = Sender & {
sendOffsets(offsets: Offsets & { consumerGroupId: string }): Promise
commit(): Promise
abort(): Promise
isActive(): boolean
}
export type ConsumerGroup = {
groupId: string
generationId: number
memberId: string
coordinator: Broker
}
export type MemberDescription = {
clientHost: string
clientId: string
memberId: string
memberAssignment: Buffer
memberMetadata: Buffer
}
export type GroupDescription = {
groupId: string
members: MemberDescription[]
protocol: string
protocolType: string
state: string
}
export type TopicPartitions = { topic: string; partitions: number[] }
export type Batch = {
topic: string
partition: number
highWatermark: string
messages: KafkaMessage[]
isEmpty(): boolean
firstOffset(): string | null
lastOffset(): string
offsetLag(): string
offsetLagLow(): string
}
export type ConsumerEvents = {
HEARTBEAT: 'consumer.heartbeat'
COMMIT_OFFSETS: 'consumer.commit_offsets'
GROUP_JOIN: 'consumer.group_join'
FETCH: 'consumer.fetch'
START_BATCH_PROCESS: 'consumer.start_batch_process'
END_BATCH_PROCESS: 'consumer.end_batch_process'
CONNECT: 'consumer.connect'
DISCONNECT: 'consumer.disconnect'
STOP: 'consumer.stop'
CRASH: 'consumer.crash'
REQUEST: 'consumer.network.request'
REQUEST_TIMEOUT: 'consumer.network.request_timeout'
REQUEST_QUEUE_SIZE: 'consumer.network.request_queue_size'
}
export type ConsumerHeartbeatEvent = InstrumentationEvent<{
groupId: string
memberId: string
groupGenerationId: number
}>
export type ConsumerCommitOffsetsEvent = InstrumentationEvent<{
groupId: string
memberId: string
groupGenerationId: number
topics: {
topic: string
partitions: {
offset: string
partition: string
}[]
}[]
}>
export interface IMemberAssignment {
[key: string]: number[]
}
export type ConsumerGroupJoinEvent = InstrumentationEvent<{
duration: number
groupId: string
isLeader: boolean
leaderId: string
groupProtocol: string
memberId: string
memberAssignment: IMemberAssignment
}>
export type ConsumerFetchEvent = InstrumentationEvent<{
numberOfBatches: number
duration: number
}>
interface IBatchProcessEvent {
topic: string
partition: number
highWatermark: string
offsetLag: string
offsetLagLow: string
batchSize: number
firstOffset: string
lastOffset: string
}
export type ConsumerStartBatchProcessEvent = InstrumentationEvent
export type ConsumerEndBatchProcessEvent = InstrumentationEvent<
IBatchProcessEvent & { duration: number }
>
export type ConsumerCrashEvent = InstrumentationEvent<{
error: Error
groupId: string
}>
export interface OffsetsByTopicPartition {
topics: TopicOffsets[]
}
export interface EachMessagePayload {
topic: string
partition: number
message: KafkaMessage
}
export interface EachBatchPayload {
batch: Batch
resolveOffset(offset: string): void
heartbeat(): Promise
commitOffsetsIfNecessary(offsets?: Offsets): Promise
uncommittedOffsets(): Promise
isRunning(): boolean
isStale(): boolean
}
/**
* Type alias to keep compatibility with @types/kafkajs
* @see https://github.com/DefinitelyTyped/DefinitelyTyped/blob/712ad9d59ccca6a3cc92f347fea0d1c7b02f5eeb/types/kafkajs/index.d.ts#L321-L325
*/
export type ConsumerEachMessagePayload = EachMessagePayload
/**
* Type alias to keep compatibility with @types/kafkajs
* @see https://github.com/DefinitelyTyped/DefinitelyTyped/blob/712ad9d59ccca6a3cc92f347fea0d1c7b02f5eeb/types/kafkajs/index.d.ts#L327-L336
*/
export type ConsumerEachBatchPayload = EachBatchPayload
export type Consumer = {
connect(): Promise
disconnect(): Promise
subscribe(topic: { topic: string | RegExp; fromBeginning?: boolean }): Promise
stop(): Promise
run(config?: {
autoCommit?: boolean
autoCommitInterval?: number | null
autoCommitThreshold?: number | null
eachBatchAutoResolve?: boolean
partitionsConsumedConcurrently?: number
eachBatch?: (payload: EachBatchPayload) => Promise
eachMessage?: (payload: EachMessagePayload) => Promise
}): Promise
seek(topicPartition: { topic: string; partition: number; offset: string }): void
describeGroup(): Promise
pause(topics: Array<{ topic: string; partitions?: number[] }>): void
resume(topics: Array<{ topic: string; partitions?: number[] }>): void
on(eventName: ValueOf, listener: (...args: any[]) => void): void
logger(): Logger
events: ConsumerEvents
}
export enum CompressionTypes {
None = 0,
GZIP = 1,
Snappy = 2,
LZ4 = 3,
ZSTD = 4,
}
export var CompressionCodecs: {
[CompressionTypes.GZIP]: () => any
[CompressionTypes.Snappy]: () => any
[CompressionTypes.LZ4]: () => any
[CompressionTypes.ZSTD]: () => any
}