/** * Shared pattern-extraction lib — run a batch of events through the engine. * * Used by log10x_resolve_batch and log10x_poc_from_siem. Both tools feed * raw log lines through this and get back a structured list of per-pattern * records with aggregated counts, bytes, severity, service, and sample * event bodies. * * Always runs the local 10x engine (a locally-installed `tenx` binary, or * local Docker via LOG10X_TENX_MODE=docker). Events never leave the machine. * * Under the hood this uses the existing dev-cli + cli-output-parser * machinery so the engine contract stays single-source. */ export interface ExtractedPattern { /** * Engine's internal templateHash — the templater's structural * fingerprint of the field-set. Used as an INTERNAL join key * between encoded events and their template body. NEVER use this * as a user-facing identity or in mute YAML: the receiver does not * match against templateHash (it matches `symbolMessage` or * `tenx_hash`). Mute targets must use `symbolMessage` (preferred) * or `tenxHash` (patternHash) below. */ hash: string; /** * Engine-emitted Reporter-tier pattern name (symbol-lookup result). * Bound per-event via the `pattern=` anchor on the encoded line * when the engine's `apps/mcp/stdout` config emits it. This is the * default receiver match key (`compactReceiverFieldNames: * [symbolMessage]`) and the preferred user-facing identity. */ symbolMessage?: string; /** * Engine-emitted xxHash64 (11-char base64url) of `symbolMessage`, * carried per-event via the `patternHash=` anchor. Same key the * forwarder enrichment writes to the `tenx_hash` field on events. * Acts as the secondary mute target when symbolMessage isn't * available (receiver must be configured with * `compactReceiverFieldNames: [tenx_hash]`). */ tenxHash?: string; /** Template body with `$` marking variable slots. */ template: string; /** * Dominant service label, from the envelope of the events bound to this * pattern. Only populated when `positionalBindingExact` held: when the * engine's record count does not reconcile against the submitted lines, * this is left unset rather than guessed. */ service?: string; /** Dominant severity (uppercase standard form). */ severity?: string; /** Number of events in this batch matching the pattern. */ count: number; /** Total bytes across all matching events. */ bytes: number; /** * Total bytes that would ship over the wire if these events were * encoded by the engine. Sum of the raw `encoded.log` line bytes * (`~hash,val,val,...` + newline) across this pattern's events. * Measured, not estimated. Used to compute the real compact-byte * ratio in Section 6. * * Optional: missing on older CLI builds without anchored encoded * layout. Consumers treat missing as 0. */ encodedBytes?: number; /** One representative raw event (from the first encoded match). */ sampleEvent: string; /** Per-slot captured values (slot name or positional index → distinct values observed, capped at 20 samples). */ variables: Record; /** * Per-slot TRUE distinct-value count, not capped. The `variables` * field holds at most 20 sample values to keep payloads bounded; * this field carries the real cardinality measurement from the * templater. Use this for unbounded-slot detection — `variables[k].length` * is the sample size, not the cardinality. */ slotDistinctCounts?: Record; /** Epoch ms of the earliest event seen in the pulled window. `undefined` when no envelope timestamps were available. */ firstSeenMs?: number; /** Epoch ms of the latest event seen in the pulled window. */ lastSeenMs?: number; /** * Per-hour event counts within the pulled window. Keyed by epoch-hour * (Math.floor(timestampMs / 3_600_000)). Used by `poc-enrichers` to * compute growth rate, last-24h-vs-window-average acceleration, and * a 24-bucket trajectory. */ eventsByHour?: Record; } export interface ExtractedPatterns { patterns: ExtractedPattern[]; totalEvents: number; totalBytes: number; /** Number of raw input lines the caller passed in. */ inputLineCount: number; /** Wall time spent in the engine (CLI). */ templaterWallTimeMs: number; /** Always `local_cli` — the engine runs on the caller's machine. */ executionMode: 'local_cli'; /** * Which engine build actually produced the numbers: `tenx 1.1.32 (edge)` * for the host binary, `docker log10x/pipeline-10x:latest` otherwise. * * Recorded because the docker default tag is mutable. A report generated * against one `:latest` was not reproducible against another, and the * output said nothing about which build had run. Surfaced in the report's * methodology block so the engine identity travels with the numbers. */ engineBuild?: string; /** * Fraction of patterns that resolved a severity, 0..1. Feeds the * fail-closed guard: when the engine emits no severity, every pattern * reads as reducible and the ERROR safety rail cannot fire. */ severityCoverage: number; /** * Whether encoded records reconciled against input lines, so that * envelope-derived fields (service, first-seen) could be bound to patterns. * False means they were deliberately left unset rather than guessed. */ positionalBindingExact: boolean; /** Input lines submitted, and the line count the engine's records account for. */ inputLinesSubmitted: number; inputLinesAccountedFor: number; } export interface ExtractPatternsOptions { /** * When true, route the engine run through the file-output engine app * (@apps/mcp-file) * instead of the stdout-based @apps/mcp. The CLI writes templates, * encoded events, and aggregated rows to disk; the parser reads * them after the process exits. Scales to multi-million-event pulls * because no stdout buffering is involved. * * Use this for SIEM POC pulls. Default false to preserve existing * `resolve_batch` semantics for small paste-style inputs. */ useFileOutput?: boolean; /** * When true (with `useFileOutput=true`), * split the input into chunks and run multiple tenx processes in * parallel. Each chunk gets its own LOG10X_MCP_RUNTIME_NAME so * output directories don't clash; outputs are merged by * templateHash + tenx_hash. Cuts wall time roughly linearly with * core count on multi-million-event pulls. * * Default false. Enable for SIEM POC pulls > 100MB. */ chunkParallel?: boolean; /** Target chunk size in bytes when chunkParallel=true. Default 32MB. */ chunkTargetBytes?: number; /** Parallelism cap when chunkParallel=true. Default min(cpus-1, 8). */ chunkParallelism?: number; /** * When true, coerceObjectToLine does NOT recurse into nested JSON. * For events whose structured-field variance lives in the envelope * (e.g., CloudWatch fluentd-wrapped events with kubernetes labels * embedded in the message field), the recursive descent would strip * the envelope and lose all that signal. With preserveEnvelope=true, * the function stops after the first unwrap and returns the JSON * string of the envelope so the templater sees the full structure * and extracts slots from across it. * Default: false (back-compat). */ preserveEnvelope?: boolean; /** * Hint: the upstream caller already knows the bucket's pattern_hash * (e.g., pattern_examples queries by it). Older engine builds do not * anchor `patternHash=` on encoded lines, so `rec.tenxHash` ends up * undefined and the defensive value-based filter on the slot loop * cannot fire. Pass this hint to feed the filter regardless. * Default: undefined. */ bucketHashHint?: string; } /** * Templatize a batch of events and return structured pattern records. * * `events` may contain raw strings, objects with a `text`/`message`/`log` * field, or JSON lines. Everything gets stringified into newline-separated * raw text before submission. */ export declare function extractPatterns(events: unknown[], opts?: ExtractPatternsOptions): Promise; /** * Collapse patterns that share the same `symbolMessage`. * * Why this exists: the engine emits one templateHash per distinct * field-set, but several field-sets can resolve to the same Reporter-tier * `symbolMessage` (e.g., the same log line with and without a trailing * variable slot, or audit logs that differ only by which optional fields * are populated). For a user-facing top-cost view the templateHash * distinction is noise: ten rows all named "Kind Event ApiVersion …" * with slightly different identities. From an action standpoint they * also collapse to one mute target (the receiver matches on * `symbolMessage`, not on templateHash). * * Pass each `ExtractedPattern[]` through this before rendering or * enriching. Patterns without a `symbolMessage` (older CLI) are left * as-is, keyed by their templateHash. */ export declare function collapseBySymbolMessage(patterns: ExtractedPattern[]): ExtractedPattern[];