/** * The proxy that stands in the path, and does as little as possible there. * * The decision lives in `@trazum/core`'s `gatewayDecision`, which never sees a * prompt and cannot return a modified request. This file moves bytes: read a * body, ask, and either forward it **unchanged** or answer with the refusal. * The split is the safety property — everything that could go wrong in a * judgement is tested without a socket, and everything that could go wrong on * a socket has no judgement in it. * * **Loopback only, and the address is not a flag.** Same posture as `serve` * since 1.44, and more load-bearing here: this thing has somebody's provider * credential passing through it. `127.0.0.1` is compiled in. * * **The credential is not even borrowed.** The caller's own `authorization` * and `x-api-key` headers are forwarded untouched and never read, never * stored, never logged, and never put in a URL. Trazum holds no key for the * gateway and has no way to make a call of its own through it — which is a * stronger promise than the connector's *borrowed, never held*, and the right * one for a component sitting between somebody and their provider. * * **The upstream is compiled in.** A flag naming the host would turn this into * a credential-forwarding open proxy: anything that could rewrite a config on * disk could point a company's API key at a machine it chose. `checkedEndpoint` * has guarded Trazum's outbound calls on that principle since 1.14, and here * there is no caller-supplied endpoint at all. * * **Nothing about the payload is written down.** The body is read to count * tokens and to find the model, then forwarded and dropped. It is never * logged, never stored, and never included in a refusal — the store has held * aggregates since 1.42 and standing in the path changes nothing about that. */ import { once } from 'node:events'; import { createServer } from 'node:http'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { estimateTokens, gatewayDecision, streamingUsageReader, usageFromResponse } from '@trazum/core'; import type { GatewayDecision, GatewayPolicy, GatewayStanding, LimitsConfig, MeasuredPosition, PricingCatalogue, WaiveEntry, } from '@trazum/core'; /** Compiled in. See the module note. */ export const BIND_HOST = '127.0.0.1'; export const DEFAULT_GATEWAY_PORT = 7318; /** * Bodies larger than this are refused unread. * * Larger than `serve`'s limit because a real request carries a real prompt, * and smaller than unbounded because a proxy that buffers whatever it is * handed is a memory exhaustion away from taking down the application it was * installed to protect. */ export const MAX_GATEWAY_BODY_BYTES = 8 * 1024 * 1024; /** * Where each provider actually is, and the one path this speaks for it. * * Deliberately narrow. A gateway that forwarded any path would be a general * proxy for somebody's API key, and the budget decision only has meaning for * the endpoint that spends tokens. */ export interface Upstream { origin: string; /** * The one path, or the one *shape* of path when the model is part of it. * * A pattern is not a widening. It exists because Google puts the model in * the URL rather than the body, so "one path" cannot be written down as a * literal for that provider — and every pattern here is anchored at both * ends with the model segment restricted to characters a model id is made * of, which is a narrower grammar than a literal comparison against a string * somebody could have put a `?` or a `..` in. */ path: string | RegExp; /** * Where the model name is, when the request body does not carry one. * * Gemini's body has `contents` and no `model` field. Reading it out of the * validated path is the only honest source: the alternative is a gateway * that forwards a call it could not price, which is the one thing standing * in the path was for. */ modelIn?: 'path'; /** * Paths that spend no tokens, forwarded without a budget decision. * * **Refusing these would be the wrong answer, not a stricter one.** * `count_tokens` is the call you make to find out whether you can afford the * other one; answering it with a 402 blinds a caller at exactly the moment * they are trying to behave. And a budget refusal only means something when * there is money on the line: a call that spends nothing has nothing to * judge, so judging it would be theatre with a real cost. * * **What this does and does not widen.** The origin is still compiled in, so * the credential can still only ever reach one host. What grows is the set * of *operations* somebody who can reach the loopback port may perform with * it, and that is why the list is literal strings only, enumerated here, and * why each entry has to be written into `docs/gateway.md` before a guard * will let it exist. A pattern here would be a widening with no budget check * behind it, which is the general-proxy shape this gateway is built to * refuse. * * Nothing that spends belongs here. `/v1/messages/batches` is the near miss: * it looks administrative and it bills. */ free?: readonly FreePath[]; } /** One path that costs nothing, and the method it answers on. */ export interface FreePath { method: 'GET' | 'POST'; path: string; } export const UPSTREAMS: Readonly> = { anthropic: { origin: 'https://api.anthropic.com', path: '/v1/messages', /** * The two an agent asks for beside the call itself. * * `POST /v1/messages/count_tokens` returns a token count and bills nothing: * `packages/core/src/tokenizer.ts` has called it for `--exact-tokens` since * the band harness needed a ground truth, and this repository documents it * as free in the same breath every time it suggests using it. * * `GET /v1/models` lists what the account may call. It carries no body, * spends nothing, and is the call a client makes on startup to find out * what exists. * * Both are here because a coding agent pointed at this gateway hits them * within its first second and got a 404 from a proxy that was otherwise * working, which reads as the gateway being broken rather than narrow. */ free: [ { method: 'POST', path: '/v1/messages/count_tokens' }, { method: 'GET', path: '/v1/models' }, ], }, openai: { origin: 'https://api.openai.com', path: '/v1/chat/completions' }, /** * DeepSeek's host is not a new fact: `scripts/measure-token-band.mjs` has * sent a real API key to `https://api.deepseek.com/chat/completions` since * the band harness learned a second provider. Reusing the endpoint this * repository already trusts with a credential is the difference between * adding an upstream and inventing one — and the path has no `/v1`, which is * the kind of detail recall gets wrong. */ deepseek: { origin: 'https://api.deepseek.com', path: '/chat/completions' }, /** * Mistral, on the same rule as DeepSeek and from the same evidence. * * `scripts/measure-token-band.mjs` sends a real key to * `https://api.mistral.ai/v1/chat/completions`, and it was run: the corpus is * measured against Mistral's own tokenizer and the fixture is committed. The * guard in `trusted-hosts.test.js` is what forced this entry to exist rather * than letting the harness quietly become a second place credentials leave * from — *"a measuring script is not a side door"*. Reusing the endpoint the * repository already trusts with a credential is the difference between * adding an upstream and inventing one, and this path does carry `/v1` where * DeepSeek's does not. */ mistral: { origin: 'https://api.mistral.ai', path: '/v1/chat/completions' }, /** * Google, on the same rule and from a fuller record than DeepSeek's. * * `packages/core/src/llm.ts` has sent a real key to * `https://generativelanguage.googleapis.com` at * `/v1beta/models/{model}:generateContent`, with the key in an * `x-goog-api-key` header rather than the query string, since the Gemini * provider landed. `packages/core/src/usage.ts` has read the counts that * come back. Host, path, credential header and response shape are all * facts this repository already holds — nothing here was recalled. * * Only `:generateContent`. `:streamGenerateContent` and `:countTokens` are * different operations whose shapes nobody here has established, and a * gateway that forwards an operation it cannot read is a general proxy for * somebody's key with extra steps. */ google: { origin: 'https://generativelanguage.googleapis.com', path: /^\/v1beta\/models\/([A-Za-z0-9._-]+):generateContent$/, modelIn: 'path', }, }; /** * The path this gateway forwards for a provider, as a person reads it. * * One phrasing, used by the refusal, the documentation guard and the security * allowlist — because three renderings of the same fact is how the page, the * refusal and the test come to disagree about what is actually forwarded. */ export function forwards(upstream: Upstream): string { return typeof upstream.path === 'string' ? upstream.path : '/v1beta/models/{model}:generateContent'; } /** * The paths this gateway forwards without judging, as a person reads them. * * Same reason `forwards` exists: the refusal body, the documentation guard and * the page itself have to render this from one place, or they drift into three * different accounts of what is actually reachable. */ export function alsoForwards(upstream: Upstream): readonly string[] { return (upstream.free ?? []).map((entry) => `${entry.method} ${entry.path}`); } /** * Whether this request is the one call this gateway speaks for, and the model * the path named if it named one. * * The returned path is **built here**, never the caller's string echoed back. * A pattern that matched is evidence the request was well formed; it is not a * licence to forward whatever matched it. Nothing reaches the upstream URL * except the compiled-in origin and a path assembled from a model id that has * already been restricted to `[A-Za-z0-9._-]`. */ export function route( upstream: Upstream, method: string | undefined, url: string | undefined, ): { path: string; model: string | null; spends: boolean } | null { if (url === undefined) return null; /** * The free paths are checked first and compared as whole strings, method * included. A `startsWith` here would forward `/v1/models/../messages` and * anything else a caller could suffix onto a prefix that looked harmless. */ for (const entry of upstream.free ?? []) { if (method === entry.method && url === entry.path) { return { path: entry.path, model: null, spends: false }; } } // Everything below spends, and only POST spends. if (method !== 'POST') return null; if (typeof upstream.path === 'string') { return url === upstream.path ? { path: upstream.path, model: null, spends: true } : null; } const matched = upstream.path.exec(url); const model = matched?.[1]; // A pattern that matched but captured nothing would otherwise build a path // containing the word `undefined` and forward it. Refusing is the only // answer: there is no model, so there is nothing to price the call against. if (model === undefined) return null; return { path: `/v1beta/models/${model}:generateContent`, model, spends: true }; } /** * Headers Trazum adds or removes. Everything else the caller sent is forwarded * verbatim, including their credential, which this never reads. */ /** Why a forwarded call's cost could not be measured. */ export type UnmeasuredCause = 'stream-broke' | 'no-usage-event' | 'no-usage-in-body'; const HOP_BY_HOP = new Set([ 'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailer', 'transfer-encoding', 'upgrade', 'host', 'content-length', ]); export interface GatewayContext { provider: string; catalogue: PricingCatalogue; policy: GatewayPolicy; /** Where the budget stands, refreshed by the caller — never read per request. */ standing: () => GatewayStanding | null; /** The `limits` block, when the config carries one. */ limits?: LimitsConfig; /** * The measured position for one call's scopes — from an index built once * at start, like `standing`. Never a file read in the request path. */ position?: (call: { label?: string; session?: string }) => MeasuredPosition; /** The config's `waive` list — a silenced limit forwards, on the record. */ waivers?: readonly WaiveEntry[]; /** * Called after a forwarded call returns, with the provider's own counts. * * Counts only. There is no parameter here that could carry a prompt, which * is what makes "nothing about the payload is written down" a fact about the * interface rather than a discipline. */ record: (measured: { model: string; label: string | null; inputTokens: number; outputTokens: number; cacheReadTokens: number; cacheWriteTokens: number; substituted: boolean; }) => void; /** * A call that was forwarded and whose cost could not be measured. * * The money is spent either way — the provider generated what it generated — * and the period's total will be lower than the bill by however much these * came to. Naming them is the only honest option: a zero would be a * measurement, and inventing an estimate would merge the two halves this * product spent an arc separating. * * Two causes, and the second is not a failure at all: * * - `stream-broke` — the connection died before the event carrying the * counts. Rare, and a real error. * - `no-usage-event` — the stream simply carried no counts. On OpenAI that is * **every streaming call** unless the caller passed `stream_options: * {include_usage: true}`, so this is the common case rather than the * exception, and an operator who is not told will read a total that is * quietly missing most of their traffic. */ unmeasured?: (cause: UnmeasuredCause) => void; /** A line for the operator's terminal. Never given a body, ever. */ note: (line: string) => void; /** Injected so the proxy is testable against a stub upstream. */ fetchImpl?: typeof fetch; } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let size = 0; for await (const chunk of request) { const buffer = Buffer.from(chunk as Buffer); size += buffer.length; if (size > MAX_GATEWAY_BODY_BYTES) return null; chunks.push(buffer); } return Buffer.concat(chunks).toString('utf8'); } /** * What the request is asking for, without keeping any of it. * * The token count is the heuristic estimator's — the same one every other * estimate in this product uses, with the same documented error band. Counting * exactly would mean an API call to count before the API call, which is a * round trip in a hot path to make a budget decision marginally sharper. * * The returned object holds no text. That is the point: everything downstream * of here, including the decision and the record, is structurally incapable of * carrying a prompt. */ function describe(body: string, provider: string, modelFromPath: string | null): { model: string; inputTokens: number | null; maxOutputTokens: number | null; label: string | null; session: string | null; } | null { let parsed: unknown; try { parsed = JSON.parse(body); } catch { return null; } if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return null; const request = parsed as Record; /** * The body's model, or the one the validated path named. * * Never both, and the path never overrides a body that carried one: a * gateway that preferred the URL could price a call as one model and forward * it as another, which is the substitution this product exists to refuse. */ const model = typeof request.model === 'string' ? request.model : modelFromPath; if (model === null) return null; // Every text field the wire format puts in front of the model, counted and // then dropped. `JSON.stringify` of the messages over-counts by the // structural characters, which is the safe direction for a budget: an // estimate that runs high refuses slightly early rather than allowing // slightly late. const parts: string[] = []; if (typeof request.system === 'string') parts.push(request.system); if (Array.isArray(request.messages)) parts.push(JSON.stringify(request.messages)); if (Array.isArray(request.input)) parts.push(JSON.stringify(request.input)); // Gemini names the same two things differently: the system prompt is a // `systemInstruction` document rather than a string, and the turns are // `contents` rather than `messages`. Counted the same way and dropped the // same way. if (typeof request.systemInstruction === 'object' && request.systemInstruction !== null) { parts.push(JSON.stringify(request.systemInstruction)); } if (Array.isArray(request.contents)) parts.push(JSON.stringify(request.contents)); const text = parts.join('\n'); const generation = typeof request.generationConfig === 'object' && request.generationConfig !== null ? (request.generationConfig as Record).maxOutputTokens : undefined; const max = provider === 'anthropic' ? request.max_tokens : request.max_completion_tokens ?? request.max_tokens ?? generation; return { model, inputTokens: text === '' ? null : estimateTokens(text), maxOutputTokens: typeof max === 'number' && Number.isFinite(max) ? max : null, /** * `metadata.trazum_label`, and nothing inferred. * * A label is what makes a per-workload bill possible, and guessing one * from a path or a user agent would attribute somebody's spend to a * workload they never named. */ label: labelOf(request), /** * `metadata.trazum_session` — the same seam as the label, for the same * reason: a per-session ceiling can only bind conversations that name * themselves, and inferring one would attribute spend to a conversation * nobody declared. The value is used to judge and never forwarded, * recorded, or printed. */ session: metadataField(request, 'trazum_session'), }; } function labelOf(request: Record): string | null { return metadataField(request, 'trazum_label'); } function metadataField(request: Record, key: string): string | null { const metadata = request.metadata; if (typeof metadata !== 'object' || metadata === null || Array.isArray(metadata)) return null; const value = (metadata as Record)[key]; return typeof value === 'string' && value.trim() !== '' ? value : null; } /** The refusal, as the caller's SDK will receive it. */ function refusalBody(decision: Extract): string { return `${JSON.stringify( { schemaVersion: 1, error: { type: 'trazum_budget_refusal', message: decision.because }, reason: decision.reason, cause: decision.cause, restsOn: decision.restsOn, standing: decision.standing, estimatedUsd: decision.estimatedUsd, alternatives: decision.alternatives, // The limits policy, judged by the same function every door calls — // the 402 body carries the judgement, not a paraphrase of it. policy: decision.policy, }, null, 2, )}\n`; } export function buildGateway(context: GatewayContext): Server { const upstream = UPSTREAMS[context.provider]; const doFetch = context.fetchImpl ?? fetch; return createServer((request: IncomingMessage, response: ServerResponse) => { void (async () => { if (upstream === undefined) { response.writeHead(500, { 'content-type': 'application/json' }); response.end(`${JSON.stringify({ error: 'no upstream configured for this provider' })}\n`); return; } const routed = route(upstream, request.method, request.url); if (routed === null) { // The one path that spends tokens, plus the short list that spends // nothing. A gateway forwarding anything else is a general proxy for // somebody's API key. response.writeHead(404, { 'content-type': 'application/json' }); response.end( // A refusal never arrives bare: what it does forward, said from the // upstream table rather than written out again here. `${JSON.stringify({ error: 'not a path this gateway forwards', forwards: `POST ${forwards(upstream)}`, ...(alsoForwards(upstream).length === 0 ? {} : { alsoForwards: alsoForwards(upstream) }), })}\n`, ); return; } /** * A path that spends nothing is forwarded and nothing else happens to it. * * **Deliberately the dumbest branch in this file.** It reaches no * decision, records no usage and never substitutes a model, because * there is no money to judge, no counts to keep and nothing to swap. Any * of those would be a figure invented about a call that cost nothing, * which is worse than the 404 this replaces. * * It is placed before `readBody`'s model check on purpose: `GET * /v1/models` has no body and no model, and running it through machinery * that exists to price a call would refuse it with "could not read a * model out of this request", which is true and useless. */ if (!routed.spends) { const passthrough = new Headers(); for (const [name, value] of Object.entries(request.headers)) { if (HOP_BY_HOP.has(name.toLowerCase()) || value === undefined) continue; passthrough.set(name, Array.isArray(value) ? value.join(', ') : value); } const sent = request.method === 'GET' ? '' : await readBody(request); if (sent === null && request.method === 'POST') { response.writeHead(413, { 'content-type': 'application/json' }); response.end(`${JSON.stringify({ error: 'request body too large' })}\n`); return; } let free: Response; try { free = await doFetch(`${upstream.origin}${routed.path}`, { method: request.method === 'GET' ? 'GET' : 'POST', headers: passthrough, ...(request.method === 'GET' ? {} : { body: sent ?? '' }), }); } catch (error) { response.writeHead(502, { 'content-type': 'application/json' }); response.end( `${JSON.stringify({ error: { type: 'trazum_upstream_unreachable', message: error instanceof Error ? error.message : String(error), }, })}\n`, ); return; } const headers: Record = {}; free.headers.forEach((value, name) => { if (!HOP_BY_HOP.has(name.toLowerCase())) headers[name] = value; }); response.writeHead(free.status, headers); response.end(Buffer.from(await free.arrayBuffer())); return; } const body = await readBody(request); if (body === null) { response.writeHead(413, { 'content-type': 'application/json' }); response.end(`${JSON.stringify({ error: 'request body too large' })}\n`); return; } const described = describe(body, context.provider, routed.model); if (described === null) { response.writeHead(400, { 'content-type': 'application/json' }); response.end(`${JSON.stringify({ error: 'could not read a model out of this request' })}\n`); return; } const decision = gatewayDecision( { provider: context.provider, ...described }, context.standing(), { catalogue: context.catalogue, policy: context.policy, ...(context.limits === undefined ? {} : { limits: context.limits }), ...(context.waivers === undefined ? {} : { waivers: context.waivers }), ...(context.position === undefined ? {} : { position: context.position({ ...(described.label === null ? {} : { label: described.label }), ...(described.session === null ? {} : { session: described.session }), }), }), }, ); if (decision.kind === 'refuse') { /** * **402, deliberately, and never 429.** * * Every provider SDK retries a 429 automatically — that is what the * code means to them — so answering a budget refusal with one turns a * single refusal into a retry storm against a gateway that will refuse * every time. 402 Payment Required is both literally correct and in * nobody's default retry list. */ response.writeHead(402, { 'content-type': 'application/json' }); response.end(refusalBody(decision)); context.note(`refused ${described.model}: ${decision.reason}`); return; } const outgoing = new Headers(); for (const [name, value] of Object.entries(request.headers)) { if (HOP_BY_HOP.has(name.toLowerCase()) || value === undefined) continue; outgoing.set(name, Array.isArray(value) ? value.join(', ') : value); } /** * The body forwarded is the body received, **byte for byte**, except on * a configured substitution — which replaces exactly one field and says * so in the record. */ let forwarded = body; if (decision.kind === 'substitute') { const parsed = JSON.parse(body) as Record; parsed.model = decision.to.id; forwarded = JSON.stringify(parsed); context.note(`substituted ${described.model} → ${decision.to.id}: ${decision.configuredReason}`); } else if (decision.unjudged !== null) { context.note(`forwarded unjudged (${decision.unjudged}): fail-open`); } let upstreamResponse: Response; try { upstreamResponse = await doFetch(`${upstream.origin}${routed.path}`, { method: 'POST', headers: outgoing, body: forwarded, }); } catch (error) { /** * The upstream is unreachable. This is **not** a budget refusal and * must not look like one: the caller needs to tell "your provider is * down" from "you are out of money", and a proxy that blurs them sends * somebody to fix the wrong thing. */ response.writeHead(502, { 'content-type': 'application/json' }); response.end( `${JSON.stringify({ error: { type: 'trazum_upstream_unreachable', message: error instanceof Error ? error.message : String(error) }, })}\n`, ); return; } const back: Record = {}; upstreamResponse.headers.forEach((value, name) => { if (!HOP_BY_HOP.has(name.toLowerCase())) back[name] = value; }); /** Counts only ever reach `record`; the body is never kept, either way. */ const recordUsage = (usage: ReturnType): void => { if (usage === null) return; context.record({ model: decision.kind === 'substitute' ? decision.to.id : described.model, label: described.label, substituted: decision.kind === 'substitute', ...usage, }); }; /** * A streamed answer is relayed as it arrives. * * Until 1.52 this method read `await upstreamResponse.text()` for every * response, which for `"stream": true` — nearly all production traffic — * held the entire answer and then delivered it at once. Time to first * token became the total generation time. This page argues that reading a * budget file per request would put Trazum's latency between the caller * and their provider; buffering a stream was a far larger version of that * in the same file. * * The provider decides, not the request: a body asking to stream can * still come back whole, and `content-type` is what actually arrived. */ const streaming = (upstreamResponse.headers.get('content-type') ?? '').includes( 'text/event-stream', ); if (!streaming || upstreamResponse.body === null) { const text = await upstreamResponse.text(); let measured: unknown; try { measured = JSON.parse(text); } catch { measured = null; } const buffered = usageFromResponse(context.provider, measured); /** * The buffered path was the silent one. * * 1.52 taught the streaming path to say *this call is unmeasured* * when no usage event arrived, and left this branch recording nothing * and saying nothing — the same silence, on the other side of one * `if`. A guard that covers one branch of a fork reads as coverage of * the fork. * * Only on a response the provider called **ok**. An upstream error * carries no counts because it produced none, and its own status * already says so; announcing those as unmeasured calls would bury the * ones that actually spent money. */ if (buffered === null && upstreamResponse.ok) { // The word "body" is deliberately not in this sentence. A security // test refuses any `note(...)` mentioning it, prose or variable, // because the cost of that rule being blunt is one word of wording // and the cost of it being clever is somebody's prompt in a log. context.note(`${described.model} answered with no usage counts — this call is unmeasured`); context.unmeasured?.('no-usage-in-body'); } recordUsage(buffered); response.writeHead(upstreamResponse.status, back); response.end(text); return; } response.writeHead(upstreamResponse.status, back); const reader = streamingUsageReader(context.provider); const decoder = new TextDecoder(); try { for await (const chunk of upstreamResponse.body as AsyncIterable) { // Counted on the way past, then forwarded unchanged. The bytes the // caller receives are the bytes the provider sent. reader.push(decoder.decode(chunk, { stream: true })); if (!response.write(chunk)) { await once(response, 'drain'); } } reader.push(decoder.decode()); } catch (error) { /** * The stream broke partway. The head is already sent, so there is no * status left to change and no refusal to render — destroying the * socket is the only way to tell the caller the answer is incomplete * rather than short. * * The money is spent and unmeasured: the provider generated whatever it * generated, and the counts ride the event this stream never reached. * Recorded as a broken stream rather than as the partial counts, which * would be a measurement of the part that arrived and read as the cost. */ context.note( `stream broke before its usage event: ${error instanceof Error ? error.message : String(error)} — this call is unmeasured`, ); context.unmeasured?.('stream-broke'); response.destroy(); return; } const streamed = reader.done(); if (streamed === null) { /** * The stream ended cleanly and carried no counts. Not an error, and on * OpenAI not even unusual — but the call is still unmeasured, and the * operator has to hear it from here rather than infer it from a total * that looks too small. */ context.unmeasured?.('no-usage-event'); } recordUsage(streamed); response.end(); })(); }); } export function listenGateway( server: Server, where: { port: number } | { socket: string }, ): Promise { return new Promise((resolve, reject) => { server.once('error', reject); if ('socket' in where) { server.listen(where.socket, () => resolve(where.socket)); return; } server.listen(where.port, BIND_HOST, () => { const address = server.address(); const port = typeof address === 'object' && address !== null ? address.port : where.port; resolve(`http://${BIND_HOST}:${port}`); }); }); }