/** * TokenIn's decisions classifiers: Clef is the Sai default, alongside the gateway's other decisions * models (Clef Flash, Perplexity Decider, GPT-6 Luna). They all take the same System One request and * answer in the same format, so one implementation serves every model; the model id in the request body * picks the deployment. * * Transport: the gateway serves them as chat deployments, so a request is `POST {baseUrl}/chat/completions` * with one user message whose content is the JSON System One request `{state, questions}`, and the * answer is the System One body (`{answers, usage}`) as the reply's message content. (Its * `/v1/systemone` route is not deployed.) Protocol parsing and retries stay upstream-owned: this runs * upstream's System One classifier and only changes the envelope through its `onPayload` and `fetch` hooks. * * Images are validated by the shared gate in `classifier-context.ts` (at most four PNG/JPEG/WebP images, * 2 MiB of aggregate base64) and then travel inside the request's `state` array as * `{ type: "image_url", image_url: { url: "data:..." } }` parts; the gateway rejects a top-level `images`. * Only models that declare image input accept them; a caller that supplies images to any other model is * refused before anything is sent. * * Billing: the gateway bills more than the provider's raw charge and puts what the account was charged * in the `x-litellm-response-cost` header (0 for a response-cache hit). That is the only cost reported. * The `usage.cost` in the body is the provider's raw charge (and 0 for some models), and the catalog * price below is a placeholder, so neither is ever presented as cost. */ import type { ClassifierApi, ClassifierContext, ClassifierModel, ClassifierOptions, ClassifierResult, ImageContent, Usage, } from "@earendil-works/pi-ai"; import { classify as classifySystemOne } from "@earendil-works/pi-ai/api/typesafe-system-one"; import { fitsClassifierRequest, saiImageRejectionMessage, tokenInWirePayload, validateSaiImages } from "./classifier-context.ts"; /** * `cost` is a required catalog field, NOT a price. Context windows are the ones pi-ai's OpenRouter * catalog lists for the same models. Earlier live checks with solid-colour images were text-only * observations: Perplexity Decider and GPT-6 Luna answered every colour correctly, while Clef's and * Clef Flash's token counts followed the base64 text. Clef is now configured with image input pending * T19's image-dependent live verification; that is configuration, not verified vision, and Clef Flash * remains text-only. */ function tokenInDecisionsModel( id: string, name: string, contextWindow: number, vision = false, ): ClassifierModel<"typesafe-system-one"> { return { type: "classifier", provider: "tokenin", id, name: `${name} (service-priced)`, api: "typesafe-system-one", baseUrl: "https://lite.andlet.me/v1", input: vision ? ["text", "image"] : ["text"], contextWindow, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, }; } /** Cloudflare's Clef: the Sai default, 65,536 tokens of context, configured with image input pending T19's image-dependent live verification. */ export const TOKENIN_SAI_CLASSIFIER = tokenInDecisionsModel("cloudflare/clef", "Clef", 65536, true); /** Cloudflare's Clef Flash: 65,536 tokens of context. */ export const TOKENIN_CLEF_FLASH_CLASSIFIER = tokenInDecisionsModel("cloudflare/clef-flash", "Clef Flash", 65536); /** Every decisions model Token-In serves, the Sai default Clef first. */ export const TOKENIN_CLASSIFIERS: readonly ClassifierModel<"typesafe-system-one">[] = [ TOKENIN_SAI_CLASSIFIER, TOKENIN_CLEF_FLASH_CLASSIFIER, tokenInDecisionsModel("perplexity/pplx-decider-v1.1-27b", "Perplexity Decider 1.1 27B", 262144, true), tokenInDecisionsModel("openai/gpt-6-luna-decisions", "GPT-6 Luna Decisions", 1050000, true), ]; const COST_HEADER = "x-litellm-response-cost"; const CACHE_HIT_HEADER = "x-litellm-cache-key"; /** The request without its images: state and questions, as before. */ const MAX_TEXT_BYTES = 64 * 1024; function record(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } export interface ServicePricedClassifierResult extends ClassifierResult { /** Token counts without a monetary claim: the catalog has no price for these calls. */ unpricedUsage?: Omit; /** What the gateway billed for the call; absent when it did not say. */ reportedCost?: number; } function imagePart(image: ImageContent): unknown { return { type: "image_url", image_url: { url: `data:${image.mimeType};base64,${image.data}` } }; } /** The System One request upstream built, as the gateway's chat body, with validated images added to its `state`. */ function chatBody(payload: unknown, images: readonly ImageContent[]): unknown { const wire = tokenInWirePayload(payload); if (!record(wire) || typeof wire.model !== "string") throw new Error("TokenIn classifier requires a model"); if (!fitsClassifierRequest(wire, MAX_TEXT_BYTES)) throw new Error("TokenIn classifier payload exceeds 64 KiB"); const { model, ...request } = wire; const state = images.length > 0 ? [wire.state, ...images.map(imagePart)] : wire.state; return { model, messages: [{ role: "user", content: JSON.stringify({ ...request, state }) }] }; } /** The System One body a chat completion carries as its message content, or undefined. */ function systemOneBody(completion: unknown): Record | undefined { const choices = record(completion) ? completion.choices : undefined; const message = Array.isArray(choices) && record(choices[0]) ? choices[0].message : undefined; const content = record(message) ? message.content : undefined; if (typeof content !== "string") return undefined; try { const body: unknown = JSON.parse(content); return record(body) ? body : undefined; } catch { return undefined; } } /** The amount the account was charged for this response, or undefined when the gateway did not say. */ function billedCost(headers: Headers): number | undefined { if ((headers.get(CACHE_HIT_HEADER) ?? "").trim() !== "") return 0; const reported = headers.get(COST_HEADER); if (reported === null || reported.trim() === "") return undefined; const cost = Number(reported); return Number.isFinite(cost) && cost >= 0 ? cost : undefined; } /** A transport abstention with no request: the same error result upstream would produce, without sending. */ function imageRejectedResult(model: ClassifierModel, errorMessage: string): ServicePricedClassifierResult { return { api: model.api, provider: model.provider, model: model.id, answers: {}, stopReason: "error", errorMessage, timestamp: Date.now(), }; } export const classifyTokenIn = async ( model: ClassifierModel, context: ClassifierContext, options?: ClassifierOptions, ): Promise => { let billed: number | undefined; let booleanConfidence: Record = {}; let scoreLegend: Record | string[]> = {}; const requestFetch = options?.fetch ?? globalThis.fetch; // Direct registry calls must not bypass the shared gate: refuse an invalid or unsupported set // atomically, before anything is sent or clipped. const supplied = context.images ?? []; let images: ImageContent[] = []; if (supplied.length > 0) { if (!model.input.includes("image")) return imageRejectedResult(model, "TokenIn classifier model does not accept images"); const validation = validateSaiImages(supplied); if (!validation.ok) return imageRejectedResult(model, saiImageRejectionMessage(validation.reason)); images = validation.images; } // Upstream's System One client rejects images, so a model that reads them gets them through `state` // instead; the validated images are the only ones that can reach the wire. const { images: _inState, ...textContext } = context; const result: ServicePricedClassifierResult = await classifySystemOne(model, images.length > 0 ? textContext : context, { ...options, onPayload: async (payload, requestModel) => { const transformed = await options?.onPayload?.(payload, requestModel); return chatBody(transformed === undefined ? payload : transformed, images); }, fetch: async (input, init) => { // Reset per attempt: a retry must not inherit pricing from another response. billed = undefined; booleanConfidence = {}; scoreLegend = {}; const url = new URL(input instanceof Request ? input.url : String(input)); url.pathname = url.pathname.replace(/\/systemone$/u, "/chat/completions"); const response = await requestFetch(url, init); if (!response.ok) return response; billed = billedCost(response.headers); // An unreadable envelope becomes an empty body: upstream reports the missing answers once, // instead of this layer failing the attempt and having it retried (and billed) again. const body = systemOneBody(await response.json().catch(() => undefined)) ?? {}; if (record(body.answers)) for (const [id, answer] of Object.entries(body.answers)) { if (!record(answer)) continue; if (typeof answer.confidence === "number" && Number.isFinite(answer.confidence) && answer.confidence >= 0 && answer.confidence <= 1) booleanConfidence[id] = answer.confidence; const legend = answer.legend; if (Array.isArray(legend) && legend.every((value) => typeof value === "string")) scoreLegend[id] = legend; else if (record(legend)) { const labels: Record = {}; for (const [key, label] of Object.entries(legend)) if (/^\d+$/.test(key) && typeof label === "string") labels[key] = label; if (Object.keys(labels).length === Object.keys(legend).length) scoreLegend[id] = labels; } } return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } }); }, }); for (const [id, confidence] of Object.entries(booleanConfidence)) { const answer = result.answers[id]; if (answer?.type === "bool") { const reported = { ...answer, confidence }; // not a fresh literal: the public type has no `confidence` result.answers[id] = reported; } } for (const [id, legend] of Object.entries(scoreLegend)) { const answer = result.answers[id]; if (answer?.type === "score") { const reported = { ...answer, legend }; result.answers[id] = reported; } } if (result.usage) { const { cost: _catalogEstimate, ...counts } = result.usage; result.unpricedUsage = counts; delete result.usage; } if (billed !== undefined) result.reportedCost = billed; return result; };