{ "$meta": { "package": "@walkeros/server-destination-kafka", "version": "4.2.1", "type": "destination", "platform": [ "server" ], "docs": "https://www.walkeros.io/docs/destinations/server/kafka", "source": "https://github.com/elbwalker/walkerOS/tree/main/packages/server/destinations/kafka/src" }, "schemas": { "mapping": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "key": { "description": "Override message key mapping path for this rule (e.g. data.id). Takes precedence over settings.kafka.key.", "type": "string" }, "topic": { "description": "Override Kafka topic for this rule. Takes precedence over settings.kafka.topic.", "type": "string" } }, "additionalProperties": false }, "settings": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "kafka": { "type": "object", "properties": { "brokers": { "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Kafka broker addresses (host:port). At least one required." }, "clientId": { "description": "Kafka client ID. Default: walkeros.", "type": "string" }, "ssl": { "description": "TLS configuration. Set true for default TLS, or provide a tls.ConnectionOptions object for mTLS.", "anyOf": [ { "type": "boolean" }, { "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": {} } ] }, "sasl": { "description": "SASL authentication config. Required for Confluent Cloud, AWS MSK with IAM, etc.", "type": "object", "properties": { "mechanism": { "type": "string", "enum": [ "plain", "scram-sha-256", "scram-sha-512", "aws", "oauthbearer" ], "description": "SASL authentication mechanism." }, "username": { "description": "Username for plain/scram mechanisms.", "type": "string" }, "password": { "description": "Password for plain/scram mechanisms.", "type": "string" }, "accessKeyId": { "description": "AWS access key ID for IAM auth (mechanism: aws).", "type": "string" }, "secretAccessKey": { "description": "AWS secret access key for IAM auth (mechanism: aws).", "type": "string" }, "sessionToken": { "description": "AWS session token for temporary credentials (mechanism: aws).", "type": "string" }, "authorizationIdentity": { "description": "AWS authorization identity (mechanism: aws).", "type": "string" } }, "required": [ "mechanism" ], "additionalProperties": false, "title": "sasl" }, "connectionTimeout": { "description": "Connection timeout in ms. Default: 1000.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "requestTimeout": { "description": "Request timeout in ms. Default: 30000.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "topic": { "type": "string", "minLength": 1, "description": "Target Kafka topic name." }, "acks": { "description": "Acknowledgement level. -1 = all replicas, 0 = fire-and-forget, 1 = leader only. Default: -1.", "type": "integer", "minimum": -1, "maximum": 1 }, "timeout": { "description": "Broker response timeout in ms. Default: 30000.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "compression": { "description": "Message compression codec. Default: gzip. Snappy/LZ4/ZSTD require additional npm packages.", "type": "string", "enum": [ "none", "gzip", "snappy", "lz4", "zstd" ] }, "idempotent": { "description": "Enable idempotent producer for exactly-once delivery. Default: false.", "type": "boolean" }, "allowAutoTopicCreation": { "description": "Allow auto-creation of topics on the broker. Default: false.", "type": "boolean" }, "key": { "description": "Mapping value path for message key derivation (e.g. user.id, data.userId). Default: entity_action.", "type": "string" }, "headers": { "description": "Static headers added to every message.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } }, "retry": { "description": "Retry configuration for transient failures.", "type": "object", "properties": { "maxRetryTime": { "description": "Max total retry wait in ms. Default: 30000.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "initialRetryTime": { "description": "First retry delay in ms. Default: 300.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "retries": { "description": "Max retry count. Default: 5.", "type": "integer", "minimum": 0, "maximum": 9007199254740991 } }, "additionalProperties": false, "title": "retry" } }, "required": [ "brokers", "topic" ], "additionalProperties": false, "description": "Kafka connection and producer settings.", "title": "kafka" } }, "required": [ "kafka" ], "additionalProperties": false }, "setup": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "topic": { "description": "Topic name. Falls back to settings.kafka.topic if omitted.", "type": "string", "minLength": 1 }, "numPartitions": { "description": "Number of partitions. Required at runtime. No safe default; choose based on expected throughput and consumer parallelism.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "replicationFactor": { "description": "Replication factor. Required at runtime. Must be <= broker count. No safe default.", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "configEntries": { "description": "Topic-level config entries, e.g. { \"retention.ms\": \"604800000\" }.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" } }, "schemaRegistry": { "description": "Optional Confluent Schema Registry binding.", "type": "object", "properties": { "url": { "type": "string", "format": "uri", "description": "Schema Registry URL." }, "subject": { "type": "string", "minLength": 1, "description": "Subject name, conventionally -value." }, "schemaType": { "type": "string", "enum": [ "AVRO", "JSON", "PROTOBUF" ], "description": "Schema type." }, "schema": { "type": "string", "minLength": 1, "description": "The schema, stringified." }, "compatibility": { "description": "Subject compatibility level set after registration.", "type": "string", "enum": [ "BACKWARD", "FORWARD", "FULL", "NONE", "BACKWARD_TRANSITIVE", "FORWARD_TRANSITIVE", "FULL_TRANSITIVE" ] }, "auth": { "description": "Optional Basic auth credentials.", "type": "object", "properties": { "username": { "type": "string" }, "password": { "type": "string" } }, "required": [ "username", "password" ], "additionalProperties": false, "title": "auth" } }, "required": [ "url", "subject", "schemaType", "schema" ], "additionalProperties": false, "title": "schemaRegistry" }, "validateOnly": { "description": "Use kafkajs validateOnly mode (broker-side dry-run). Default: false.", "type": "boolean" } }, "additionalProperties": false } }, "examples": { "env": { "push": { "Kafka": { "Kafka": { "$code": "class{constructor(e){}producer(e){return A()}admin(){return{connect:()=>Promise.resolve(),disconnect:()=>Promise.resolve(),createTopics:()=>Promise.resolve(!0),fetchTopicMetadata:({topics:e})=>Promise.resolve({topics:(e??[]).map(e=>({name:e,partitions:[{partitionId:0,leader:0,replicas:[0],isr:[0]}]}))}),describeConfigs:({resources:e})=>Promise.resolve({resources:e.map(e=>({resourceName:e.name,configEntries:[]}))})}}}" }, "CompressionTypes": { "None": 0, "GZIP": 1, "Snappy": 2, "LZ4": 3, "ZSTD": 4 } } }, "simulation": [ "call:producer.send" ] }, "step": { "defaultEvent": { "title": "Default event", "description": "An event is produced to the configured Kafka topic with the full JSON body and entity_action as the message key.", "in": { "name": "page view", "data": { "domain": "www.example.com", "title": "walkerOS documentation", "referrer": "https://www.walkeros.io/", "search": "?foo=bar", "hash": "#hash", "id": "/docs/" }, "context": { "dev": [ "test", 1 ] }, "globals": { "pagegroup": "docs" }, "custom": { "completely": "random" }, "user": { "id": "us3r", "device": "c00k13", "session": "s3ss10n" }, "nested": [ { "entity": "child", "data": { "is": "subordinated" } } ], "consent": { "functional": true }, "id": "f4a164152f12794f", "trigger": "load", "entity": "page", "action": "view", "timestamp": 1700000100, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "out": [ [ "producer.send", { "topic": "walkeros-events", "messages": [ { "key": "page_view", "value": "json:event", "headers": { "content-type": "application/json" }, "timestamp": "1700000100" } ], "acks": -1, "compression": 1 } ] ] }, "ignoredEvent": { "public": false, "in": { "name": "debug noise", "data": { "string": "foo", "number": 1, "boolean": true, "array": [ 0, "text", false ] }, "context": { "dev": [ "test", 1 ] }, "globals": { "lang": "elb" }, "custom": { "completely": "random" }, "user": { "id": "us3r", "device": "c00k13", "session": "s3ss10n" }, "nested": [ { "entity": "child", "data": { "is": "subordinated" } } ], "consent": { "functional": true }, "id": "03ee573c20027dcb", "trigger": "test", "entity": "debug", "action": "noise", "timestamp": 1700000105, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "mapping": { "ignore": true }, "out": [] }, "keyFromUser": { "title": "Key from user id", "description": "A settings.kafka.key path resolves the message key from the event, here using user.id for per-user partitioning.", "in": { "name": "user signup", "data": { "plan": "pro" }, "context": { "dev": [ "test", 1 ] }, "globals": { "lang": "elb" }, "custom": { "completely": "random" }, "user": { "id": "usr-789" }, "nested": [ { "entity": "child", "data": { "is": "subordinated" } } ], "consent": { "functional": true }, "id": "fac4720400ef6d79", "trigger": "test", "entity": "user", "action": "signup", "timestamp": 1700000103, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "settings": { "kafka": { "brokers": [ "localhost:9092" ], "topic": "walkeros-events", "key": "user.id" } }, "out": [ [ "producer.send", { "topic": "walkeros-events", "messages": [ { "key": "usr-789", "value": "json:event", "headers": { "content-type": "application/json" }, "timestamp": "1700000103" } ], "acks": -1, "compression": 1 } ] ] }, "mappedData": { "title": "Mapped payload", "description": "A data mapping transforms the event payload before producing it as the Kafka message value.", "in": { "name": "order complete", "data": { "id": "ORD-400", "total": 99.99, "currency": "EUR" }, "context": { "shopping": [ "complete", 0 ] }, "globals": { "pagegroup": "shop" }, "custom": { "completely": "random" }, "user": { "id": "us3r", "device": "c00k13", "session": "s3ss10n" }, "nested": [ { "entity": "product", "data": { "id": "ers", "name": "Everyday Ruck Snack", "color": "black", "size": "l", "price": 420 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "product", "data": { "id": "cc", "name": "Cool Cap", "size": "one size", "price": 42 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "gift", "data": { "name": "Surprise" }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] } ], "consent": { "functional": true }, "id": "9b5315244df9900e", "trigger": "load", "entity": "order", "action": "complete", "timestamp": 1700000102, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "mapping": { "name": "purchase", "data": { "map": { "order_id": "data.id", "revenue": "data.total", "currency": "data.currency" } } }, "out": [ [ "producer.send", { "topic": "walkeros-events", "messages": [ { "key": "purchase", "value": "json:data", "headers": { "content-type": "application/json" }, "timestamp": "1700000102" } ], "acks": -1, "compression": 1 } ] ] }, "mappedEventName": { "title": "Renamed event", "description": "A mapping renames the event which also changes the default Kafka message key used for partitioning.", "in": { "name": "order complete", "data": { "id": "0rd3r1d", "currency": "EUR", "shipping": 5.22, "taxes": 73.76, "total": 555 }, "context": { "shopping": [ "complete", 0 ] }, "globals": { "pagegroup": "shop" }, "custom": { "completely": "random" }, "user": { "id": "us3r", "device": "c00k13", "session": "s3ss10n" }, "nested": [ { "entity": "product", "data": { "id": "ers", "name": "Everyday Ruck Snack", "color": "black", "size": "l", "price": 420 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "product", "data": { "id": "cc", "name": "Cool Cap", "size": "one size", "price": 42 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "gift", "data": { "name": "Surprise" }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] } ], "consent": { "functional": true }, "id": "ee30df04d0be2bed", "trigger": "load", "entity": "order", "action": "complete", "timestamp": 1700000101, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "mapping": { "name": "purchase" }, "out": [ [ "producer.send", { "topic": "walkeros-events", "messages": [ { "key": "purchase", "value": "json:event", "headers": { "content-type": "application/json" }, "timestamp": "1700000101" } ], "acks": -1, "compression": 1 } ] ] }, "topicOverride": { "title": "Topic override", "description": "A mapping rule overrides the destination topic so specific events are routed to a dedicated stream.", "in": { "name": "order complete", "data": { "id": "ORD-500", "total": 42 }, "context": { "shopping": [ "complete", 0 ] }, "globals": { "pagegroup": "shop" }, "custom": { "completely": "random" }, "user": { "id": "us3r", "device": "c00k13", "session": "s3ss10n" }, "nested": [ { "entity": "product", "data": { "id": "ers", "name": "Everyday Ruck Snack", "color": "black", "size": "l", "price": 420 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "product", "data": { "id": "cc", "name": "Cool Cap", "size": "one size", "price": 42 }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] }, { "entity": "gift", "data": { "name": "Surprise" }, "context": { "shopping": [ "complete", 0 ] }, "nested": [] } ], "consent": { "functional": true }, "id": "d532ab43df6d87e6", "trigger": "load", "entity": "order", "action": "complete", "timestamp": 1700000104, "timing": 3.14, "source": { "count": 1, "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9", "type": "collector", "schema": "4" } }, "mapping": { "settings": { "topic": "orders-stream" } }, "out": [ [ "producer.send", { "topic": "orders-stream", "messages": [ { "key": "order_complete", "value": "json:event", "headers": { "content-type": "application/json" }, "timestamp": "1700000104" } ], "acks": -1, "compression": 1 } ] ] } } } }