/** * Per-forwarder action-routing recipes for the Retriever cost loop. * * Sibling to `forwarder-snippets.ts`, but a different shape. Where the * drop-rule snippet emits a single SIEM-side exclude, these recipes are a * MULTI-way fan-out keyed on the engine-stamped `routeState` marker. The * receiver stamps a PER-SERVICE action (drop | offload | tier_down | * compact | sample | pass) on each service's regulator-excess slice, so the * forwarder branches one destination per action: * * - `offload` -> the forwarder's OWN native S3 output, written as full, * newline-delimited JSON under `{bucket}/{prefix}` (the * exact layout the Retriever indexes). * - `tier_down` -> a cheaper in-platform SIEM tier (Datadog Flex / * CloudWatch Infrequent-Access / ES frozen / etc). The * cheap-tier sink is destination-specific, so each recipe * leaves a clearly-labeled placeholder for it. * - `drop` -> suppressed (no output at all; the slice is shed). * - `pass` / `compact` / `sample` -> the existing SIEM destination. The * engine already carries `compact`'s encoded bytes and * `sample`'s thinning on the wire, so the forwarder just * routes them to the SIEM unchanged. * * Nothing the customer wants kept is deleted: the `offload` slice is relocated * to the customer's own bucket before the SIEM bills it, and the Retriever * fetches it back by stamped identity. This is lossless cost reduction, not * archival. * * Engine contract: * - the receiver runs with `outputOffload true`, which resolves the output * field to `fullText("tenx_hash","routeState")` and the drop filter to * `isObject` (every marked event flows back to the forwarder, full text). * - `routeState` lands as a JSON STRING (`"routeState":"drop"` / * `"routeState":"offload"` / `"routeState":"pass"` / ...), spliced inside * the event envelope. Every forwarder match MUST therefore be string * equality against the action NAME, never a boolean/truthiness test. * - `tenx_hash` ships alongside it, so the same S3 object carries the stable * identity the Retriever correlates on. * * On EVERY branch the `routeState` marker is stripped and `tenx_hash` is kept * (mirroring the original single-route drop branch). */ export type OffloadForwarderId = 'vector' | 'fluentd' | 'fluent-bit' | 'otel-collector' | 'logstash' | 'cribl'; /** Forwarders whose recipe shape is verified against the engine contract and * the forwarder's own docs. The rest are research-derived and carry a * `smokeTest` prerequisite so the caller never claims end-to-end without it. */ export declare const OFFLOAD_FORWARDERS: OffloadForwarderId[]; export interface OffloadRecipe { language: 'toml' | 'xml' | 'ini' | 'yaml' | 'ruby' | 'json' | 'text'; /** The two-route config, ready to paste. */ body: string; /** Where it goes in the user's config, and why (anchors the Reader to the * engine mechanism so the match isn't arbitrary). */ placementNote: string; /** Hard prerequisites the recipe depends on. Always includes the engine * offload mode and the forwarder-write IAM grant; per-forwarder gotchas * (contrib distro, plugin install, JSON encoding) are appended. */ prerequisites: string[]; } export interface OffloadParams { /** The Retriever input bucket (snapshot.recommendations.retrieverS3Bucket). */ bucket: string; /** * Destination type of the offload sink. Every generator below emits an S3 * sink, so `azure_blob` and `gcs` have no recipe: the render path returns * the state of play instead of a config that would write to the wrong * store. Defaults to `s3`. */ destinationType?: 's3' | 'gcs' | 'azure_blob' | 'file'; /** Azure storage account holding the container. Read when `destinationType` is `azure_blob`. */ storageAccount?: string; /** Key prefix == the Retriever `target` (default `app`). Objects land at * `{bucket}/{prefix}/...`; the indexer's S3->SQS notification picks them up. */ prefix?: string; /** AWS region of the bucket (snapshot.aws.region). */ region: string; /** The engine's `symbolMessageHashField` value. Defaults to `tenx_hash`. */ hashField?: string; /** * Keep `routeState` on the wire for every path except the S3 offload slice. * * Set this when the DESTINATION is what reads the marker to make the routing * decision. Datadog is the case: the Flex index selects the down-tiered slice * with an index filter on `@routeState:tier_down`, so a strip on the SIEM * path removes the only field that filter can match and nothing ever tiers, * with HTTP 200 and no error anywhere. Coralogix has the same property and * gets a bespoke shipper (`fluentBitCoralogixRecipe`) for it. * * The offload slice is stripped either way: those objects land in the * customer's bucket, and the Retriever should not index internal markers. */ keepMarkerAtDestination?: boolean; } /** * The state of Azure Blob as an offload sink, as one markdown block. * * Offload delivery to Azure Blob is not available. Every generator in this * file emits an `aws_s3` sink, and no Blob writer exists behind them, so a * recipe here would hand the operator a config that writes somewhere other * than the container they named. S3 and S3-compatible buckets (MinIO, Ceph) * carry the write path today. The Retriever reads Blob either way: it * indexes and queries blobs that are already in the container, so an Azure * Monitor diagnostic export into Blob is queryable now. * * Writing this as a block rather than a caveat under a recipe is the point. * A generated `aws_s3` sink with a warning above it is still a generated * `aws_s3` sink, and the operator applies it. */ export declare function azureBlobOffloadUnavailable(params: OffloadParams): string; /** Return the two-route offload recipe for the given forwarder. */ export declare function offloadRecipe(forwarder: OffloadForwarderId, params: OffloadParams): OffloadRecipe; export interface ForwarderWriteIam { /** The least-privilege IAM policy document (PutObject to the offload prefix). */ policyJson: string; /** How to attach it: EKS IRSA vs static creds. */ attachmentNote: string; } export declare function forwarderWriteIamPolicy(params: OffloadParams): ForwarderWriteIam; /** * Ready-to-apply Terraform module for the forwarder-write IAM: a role + the * scoped PutObject policy + the EKS IRSA OIDC trust (assume-role bound to one * ServiceAccount). Non-EKS attachment is noted at the bottom. The grant is * additive — the Retriever's own role only reads the bucket. */ export declare function forwarderWriteTerraform(): string; export interface SiemTierRecipe { /** 'datadog-flex' | 'cloudwatch-ia' | 'azure-basic' | 'azure-auxiliary' */ target: string; language: 'hcl' | 'text'; body: string; note: string; } /** Datadog: route `@routeState:tier_down` to a Flex-only index (cheaper queryable * tier) instead of the premium Standard index. In-platform Terraform. */ export declare function datadogFlexRecipe(opts?: { flexRetentionDays?: number; }): SiemTierRecipe; /** CloudWatch: route `routeState == "tier_down"` to an Infrequent-Access log group * (~50% cheaper ingest, still Logs-Insights queryable). The split is * forwarder-side (events go to a different log group); this is the TF for the * IA group. */ export declare function cloudwatchIaRecipe(opts?: { logGroupName?: string; }): SiemTierRecipe; /** Azure Monitor: route the tier_down slice to a Log Analytics table on the * Basic (default) or Auxiliary plan. Like CloudWatch IA, the table PLAN is * fixed at creation (via a Data Collection Rule), so the split is * forwarder-side: marked events go to a different DCR stream / table. This is * the provisioning for the cheaper-plan table + DCR. */ export declare function azureLogsTierRecipe(opts?: { plan?: 'Basic' | 'Auxiliary'; tableName?: string; }): SiemTierRecipe; export interface CoralogixTierParams { /** Coralogix ingest domain for the tenant's region, e.g. `cx498.coralogix.com` (US2). */ domain: string; /** applicationName stamped on every shipped event. */ applicationName?: string; /** subsystemName carrying the untouched premium slice. */ passSubsystem?: string; /** subsystemName the tier_down slice is moved to (what a `subsystems` policy matches). */ tierDownSubsystem?: string; } export declare function fluentBitCoralogixRecipe(p: OffloadParams & CoralogixTierParams): OffloadRecipe; /** * Elasticsearch frozen tier. VERIFIED END TO END on a live Elastic Cloud Hosted * deployment (v9.4.4, enterprise licence) on 2026-07-31, and this emits the * artifacts that run actually used, not an idealised version of them. * * What was observed: * partial-tenx-tierdown-000001 400 docs store=0b node roles=f (frozen) * tenx-app-000001 (control) 200 docs store=19.2kb node roles=himrst (hot) * and, the reason the feature is worth shipping, IDENTITY SURVIVED the move: * a term query on `tenx_hash` returned 400/400 against the partially-mounted * index and `routeState` was still aggregatable. * * Why this shape and not a row-level rule: ILM is INDEX-level. That is a good * fit for a per-event marker, because the forwarder can put the marked slice in * its own index and the policy handles the rest. Contrast ClickHouse, where * `TTL ... TO VOLUME` is evaluated per PART and takes no WHERE clause, so the * same idea needs the routing key baked into the partition key. */ export declare function elasticFrozenTierRecipe(opts?: { repository?: string; tierDownAlias?: string; keepAlias?: string; frozenMinAge?: string; }): SiemTierRecipe; /** Coralogix: move the `tier_down` slice from High (Frequent Search) to Medium * (Monitoring). Two policy forms, because which one is available depends on the * provider version. */ export declare function coralogixMonitoringRecipe(opts?: { tierDownSubsystem?: string; }): SiemTierRecipe; /** * The TCO policy HTTP contract as the product actually implements it, recovered * by running the Terraform provider under TF_LOG=DEBUG and replaying its * requests with curl until they succeeded standalone. * * This exists because the published REST documentation is wrong in five * independently reproducible ways, and a reader following it cannot succeed. */ export declare function coralogixTcoApiContract(): SiemTierRecipe; /** * Which collector writes the offloaded rows. The OpenTelemetry Collector * variant is a copy of the config that ran end to end on ClickStack 2.38.0 with * engine 1.1.79, the released image * `ghcr.io/log-10x/edge-10x@sha256:14357d8d570cb36ba6ca254802a1b8eedb11d8acf6916a936893f8e3babb41f4`. * The Vector variants are copied from the runs that exercised them: the JSON * arm in gap 2 and the Parquet arm in gap 2b on `timberio/vector:0.58.0-debian`, * results at * benchmarks/clickstack-e2e-gaps/results/clickstack-e2e-close-2026-09-15.md. */ export type ClickhouseCollector = 'otel-collector' | 'vector' | 'vector-parquet'; export interface ClickhouseOffloadParams { /** Bucket the collector writes the offloaded rows into. */ bucket: string; /** Region of that bucket. */ region: string; /** Database holding the ClickStack tables. Default `default`. */ database?: string; /** The hot table ClickStack ships. Default `otel_logs`. */ hotTable?: string; /** * S3 endpoint ClickHouse itself reads the objects through. Default is the * regional AWS endpoint. The harness ran against MinIO at * `http://cse-minio:9000`, which is why the path-style switches appear in * the collector block as comments. */ s3Endpoint?: string; /** OTLP endpoint of the 10x receiver. Default `tenx-receiver:4317`. */ engineOtlpEndpoint?: string; /** ClickStack's own OTLP endpoint, where everything not marked offload returns. */ clickstackOtlpEndpoint?: string; /** Host and port of the HyperDX API. Default `clickstack:8000`. */ hyperdxApi?: string; /** The engine's `symbolMessageHashField` value. Default `tenx_hash`. */ hashField?: string; } export interface ClickhouseRecipePart { language: 'yaml' | 'toml' | 'sql' | 'bash' | 'text'; body: string; note: string; } export interface ClickhouseOffloadRecipeParts { collector: ClickhouseRecipePart & { variant: ClickhouseCollector; /** True only for the variant the harness actually ran. */ exercised: boolean; }; /** The tables, the view, the Merge table and the counts table, as SQL. */ ddl: ClickhouseRecipePart; /** Adding the Merge table to HyperDX as a second source, over its API. */ hyperdx: ClickhouseRecipePart; /** Mandatory. Rendered with every variant, never trimmed. */ honesty: string[]; } /** * The honesty block. It states what the saving is, what the cold path costs, * and the engine version this recipe requires. Every number quoted is from the * single run in * benchmarks/clickstack-e2e/results/clickstack-e2e-2026-09-15.md, measured on * 50,000 lines of the released capture with a dropped cache before each query. * * Exported so a caller can assert it is present rather than re-derive it. */ export declare function clickhouseOffloadHonesty(): string[]; /** * The ClickHouse offload recipe: the collector that writes the objects, the SQL * that reads them back beside the hot table, the HyperDX source, and the * honesty block. * * Pass a collector to get one variant; the render function below shows both so * the customer picks. */ export declare function clickhouseOffloadRecipe(params: ClickhouseOffloadParams, collector?: ClickhouseCollector): ClickhouseOffloadRecipeParts; /** * The full ClickHouse offload section. Substitutes for the generic forwarder * section, the way the Coralogix shipper does: the generic recipes write * newline JSON into the Retriever's `{bucket}/app/` layout and strip * `routeState`, and neither is what a ClickHouse cold table reads. */ export declare function renderClickhouseOffloadSection(params: ClickhouseOffloadParams, collector?: ClickhouseCollector): string; /** Forwarders besides the detected one, stable order, for the "also supports" * hint. */ export declare function otherOffloadForwarders(detected: OffloadForwarderId): OffloadForwarderId[]; /** Forwarders whose recipe shape is verified against the engine contract + * the forwarder's own docs (no runtime smoke-test caveat). */ export declare const VERIFIED_OFFLOAD_FORWARDERS: OffloadForwarderId[]; /** * Build the "Forwarder offload" markdown section for the retriever plan. * Pass the detected forwarder (or null to show the two verified leads). * Always renders the loop framing, the forwarder-write IAM grant, the SIEM * down-tier alternatives, and the fetch-back pointer. */ export declare function renderOffloadSection(params: OffloadParams, forwarder: OffloadForwarderId | null, rawDestination?: string): string; /** * Parameters for the serverless (Lambda + OTel collector extension) recipe. * The destination side stays Coralogix-shaped because that is the proven * estate; the collector parts are destination-agnostic. */ export interface LambdaExtensionParams { /** AWS region of the estate. */ region: string; /** Offload bucket (customer-owned S3) for the `offload` slice. Optional — * without it the recipe emits the SIEM + drop routing only. */ bucket?: string; /** Key prefix for offloaded objects. Default `app`. */ prefix?: string; /** Coralogix ingest domain, e.g. `cx498.coralogix.com`. */ domain?: string; /** applicationName stamped on shipped events. Default `tenx`. */ applicationName?: string; /** ARN of the engine extension layer once published. Placeholder until then. */ engineLayerArn?: string; } /** Multi-part recipe: each part is pasteable on its own. */ export interface ServerlessExtensionRecipe { /** Additions to the customer's existing collector-extension config. */ collector: OffloadRecipe; /** The engine's environment + invocation, extension-side. */ engine: OffloadRecipe; /** What declares the engine into the execution environment, and the * lifecycle contract the extension bootstrap must honor. */ executionEnvironment: OffloadRecipe; } /** * The OTel-extension pairing for a 100%-Lambda estate: the customer's * collector extension keeps its receivers and its Coralogix exporter; two * loopback hops to the engine extension are spliced in between. * * Grounded in measurements (2026-08-08, local execution-environment lab — * see SERVERLESS_TASK1_LIFECYCLE_REPORT.md in the workspace root): * - engine 1.1.57 native + otelcol paired over loopback inside one * sandbox; 5,400/5,400 records round-tripped, zero dupes, with * `tenx_hash` + `routeState` arriving as LOG-RECORD ATTRIBUTES * - a 118 s cgroup freeze mid-burst lost nothing and duplicated nothing * - the engine does NOT drain on bare SIGTERM (0/30,000 delivered when * killed mid-burst) — the extension bootstrap owns the SHUTDOWN drain */ export declare function lambdaOtelExtensionRecipe(p: LambdaExtensionParams): ServerlessExtensionRecipe; export interface AzureStreamsRecipe { /** Azure-side: hub creation + diagnostic settings routing logs into it. */ hub: OffloadRecipe; /** The collector that consumes the hub and pairs with the engine. */ collector: OffloadRecipe; /** The engine's environment + invocation beside that collector. */ engine: OffloadRecipe; /** * The apply half of auto-tuning: the engine's gitops pull lane delivers a * policy repo's mute file to a stable path (GH_DEST) that * rateReceiverLookupFile points into. Cloud-agnostic — the recompute half * is setup_recurring (github_actions kind writes the same repo). */ autotune: OffloadRecipe; } /** * The stream topology for Azure surfaces that cannot host a second process: * the platform streams its logs to an Event Hub, and one central engine * consumes the hub behind a collector, using the same loopback pairing as * every other topology. * * The collector settings are CERTIFIED, not inferred: verified end to end * against a live Event Hub (Basic tier) with otelcol-contrib 0.158 and * engine 1.1.68 — events returned carrying tenx_hash + routeState with * distinct pattern identities per message type. The three prerequisites * marked "measured" below are failures observed in that run. */ export declare function azureStreamsRecipe(p?: { eventHubNamespace?: string; }): AzureStreamsRecipe;