/** * TokenIn's decisions classifiers: Clef is the Sai default, alongside the gateway's other decisions * models (Clef Flash, Perplexity Decider, GPT-6 Luna). They all take the same System One request and * answer in the same format, so one implementation serves every model; the model id in the request body * picks the deployment. * * Transport: the gateway serves them as chat deployments, so a request is `POST {baseUrl}/chat/completions` * with one user message whose content is the JSON System One request `{state, questions}`, and the * answer is the System One body (`{answers, usage}`) as the reply's message content. (Its * `/v1/systemone` route is not deployed.) The upstream client owns transport; the Sai facade disables * its retries so the bounded request owner can retry. This adapter validates wire answers individually * because upstream's parser otherwise rejects an entire partial batch. * * Images retain the shared count/format/signature/encoded-size checks, but all valid images are * currently refused because their gateway token accounting is unknown. The completed text request is * also checked for known image representations and capped in UTF-8 bytes before each HTTP attempt. * This local byte limit is not a token-fit guarantee. * * Billing: the gateway bills more than the provider's raw charge and puts what the account was charged * in the `x-litellm-response-cost` header (0 for a response-cache hit). That is the only cost reported. * The `usage.cost` in the body is the provider's raw charge (and 0 for some models), and the catalog * price below is a placeholder, so neither is ever presented as cost. */ import type { ClassifierApi, ClassifierContext, ClassifierModel, ClassifierOptions, ClassifierResult, Usage, } from "@earendil-works/pi-ai"; import { classify as classifySystemOne } from "@earendil-works/pi-ai/api/typesafe-system-one"; import { fitsClassifierRequest, inspectClassifierMaterial, MAX_CLASSIFIER_BODY_BYTES, normalizeClassifierAnswers, saiImageRejectionMessage, tokenInWirePayload, validateSaiImages, type SaiAnswerRejection } from "./classifier-context.ts"; /** * `cost` is a required catalog field, NOT a price. Context windows are the ones pi-ai's OpenRouter * catalog lists for the same models. Earlier live checks with solid-colour images were text-only * observations: Perplexity Decider and GPT-6 Luna answered every colour correctly, while Clef's and * Clef Flash's token counts followed the base64 text. Clef is now configured with image input pending * T19's image-dependent live verification; that is configuration, not verified vision, and Clef Flash * remains text-only. */ function tokenInDecisionsModel( id: string, name: string, contextWindow: number, vision = false, ): ClassifierModel<"typesafe-system-one"> { return { type: "classifier", provider: "tokenin", id, name: `${name} (service-priced)`, api: "typesafe-system-one", baseUrl: "https://lite.andlet.me/v1", input: vision ? ["text", "image"] : ["text"], contextWindow, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, }; } /** Jev 1.14: the Sai default, 64,000 tokens of context, configured with image input (not yet live-verified). */ export const TOKENIN_SAI_CLASSIFIER = tokenInDecisionsModel("jev-1.14", "Jev 1.14", 64000, true); /** Cloudflare's Clef: 65,536 tokens of context, configured with image input pending T19's image-dependent live verification. */ export const TOKENIN_CLEF_CLASSIFIER = tokenInDecisionsModel("cloudflare/clef", "Clef", 65536, true); /** Cloudflare's Clef Flash: 65,536 tokens of context. */ export const TOKENIN_CLEF_FLASH_CLASSIFIER = tokenInDecisionsModel("cloudflare/clef-flash", "Clef Flash", 65536); /** Every decisions model Token-In serves, the Sai default Jev 1.14 first. */ export const TOKENIN_CLASSIFIERS: readonly ClassifierModel<"typesafe-system-one">[] = [ TOKENIN_SAI_CLASSIFIER, tokenInDecisionsModel("jev-1.13", "Jev 1.13", 64000), TOKENIN_CLEF_CLASSIFIER, TOKENIN_CLEF_FLASH_CLASSIFIER, tokenInDecisionsModel("perplexity/pplx-decider-v1.1-27b", "Perplexity Decider 1.1 27B", 262144, true), tokenInDecisionsModel("openai/gpt-6-luna-decisions", "GPT-6 Luna Decisions", 1050000, true), ]; const COST_HEADER = "x-litellm-response-cost"; const CACHE_HIT_HEADER = "x-litellm-cache-key"; /** The request without its images: state and questions, as before. */ const MAX_TEXT_BYTES = 64 * 1024; function record(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } /** Content-free failure identity retained across upstream's string-only error normalization. */ export type SaiClassifierFailure = | { kind: "http"; httpStatus: number; retryAfterMs?: number; contextOverflow?: true } | { kind: "network" } | { kind: "local"; reason: "overflow" | "image-budget-unavailable" }; /** A local, non-retryable refusal from an adapter payload or fetch guard. */ export class SaiClassifierRefusal extends Error { readonly reason: "overflow" | "image-budget-unavailable"; constructor(reason: "overflow" | "image-budget-unavailable") { super("TokenIn classifier request refused locally"); this.reason = reason; } } export interface ServicePricedClassifierResult extends ClassifierResult { /** Token counts without a monetary claim: the catalog has no price for these calls. */ unpricedUsage?: Omit; /** What the gateway billed for the call; absent when it did not say. */ reportedCost?: number; /** Content-free per-question rejections, including entries omitted from `answers`. */ answerRejections?: Record; /** A successful HTTP reply did not contain a System One answers object. */ malformedResponse?: true; /** Safe identity for retry policy and diagnostics; never provider text or request content. */ saiFailure?: SaiClassifierFailure; } /** The text System One request as the gateway's single chat message. */ function chatBody(payload: unknown): unknown { try { if (inspectClassifierMaterial(payload) !== "ok") throw new SaiClassifierRefusal("image-budget-unavailable"); const wire = tokenInWirePayload(payload); if (!record(wire) || typeof wire.model !== "string") throw new SaiClassifierRefusal("image-budget-unavailable"); if (!fitsClassifierRequest(wire, MAX_TEXT_BYTES)) throw new SaiClassifierRefusal("overflow"); const { model, ...request } = wire; return { model, messages: [{ role: "user", content: JSON.stringify(request) }] }; } catch (error) { if (error instanceof SaiClassifierRefusal) throw error; throw new SaiClassifierRefusal("image-budget-unavailable"); } } /** Inspect the exact completed body passed to fetch; never report its contents. */ export function assertClassifierBody(body: unknown): void { if (typeof body !== "string" || Buffer.byteLength(body, "utf8") > MAX_CLASSIFIER_BODY_BYTES) throw new SaiClassifierRefusal("overflow"); try { const outer: unknown = JSON.parse(body); if (!record(outer) || typeof outer.model !== "string" || !Array.isArray(outer.messages) || outer.messages.length !== 1 || !record(outer.messages[0]) || outer.messages[0].role !== "user" || typeof outer.messages[0].content !== "string") { throw new SaiClassifierRefusal("image-budget-unavailable"); } const inner: unknown = JSON.parse(outer.messages[0].content); if (!record(inner) || typeof inner.state !== "string" || !record(inner.questions) || inspectClassifierMaterial(outer) !== "ok") throw new SaiClassifierRefusal("image-budget-unavailable"); } catch (error) { if (error instanceof SaiClassifierRefusal) throw error; throw new SaiClassifierRefusal("image-budget-unavailable"); } } /** The System One body a chat completion carries as its message content, or undefined. */ function systemOneBody(completion: unknown): Record | undefined { const choices = record(completion) ? completion.choices : undefined; const message = Array.isArray(choices) && record(choices[0]) ? choices[0].message : undefined; const content = record(message) ? message.content : undefined; if (typeof content !== "string") return undefined; try { const body: unknown = JSON.parse(content); return record(body) ? body : undefined; } catch { return undefined; } } /** The amount the account was charged for this response, or undefined when the gateway did not say. */ function billedCost(headers: Headers): number | undefined { if ((headers.get(CACHE_HIT_HEADER) ?? "").trim() !== "") return 0; const reported = headers.get(COST_HEADER); if (reported === null || reported.trim() === "") return undefined; const cost = Number(reported); return Number.isFinite(cost) && cost >= 0 ? cost : undefined; } /** A transport abstention with no request: the same error result upstream would produce, without sending. */ function imageRejectedResult(model: ClassifierModel, errorMessage: string, failure?: SaiClassifierFailure): ServicePricedClassifierResult { return { ...(failure ? { saiFailure: failure } : {}), api: model.api, provider: model.provider, model: model.id, answers: {}, stopReason: "error", errorMessage, timestamp: Date.now(), }; } /** Read only a small JSON error envelope and recognize explicit context error codes, never message text. */ async function isContextOverflow(response: Response): Promise { const reader = response.clone().body?.getReader(); if (!reader) return false; const chunks: Uint8Array[] = []; let size = 0; try { for (;;) { const { value, done } = await reader.read(); if (done) break; size += value.byteLength; if (size > 8 * 1024) return false; chunks.push(value); } const body: unknown = JSON.parse(Buffer.concat(chunks).toString("utf8")); const error = record(body) && record(body.error) ? body.error : body; if (!record(error)) return false; return [error.code, error.type].some((code) => code === "context_length_exceeded" || code === "context_window_exceeded"); } catch { return false; } finally { // The original response remains for upstream; waiting for a tee cancellation could deadlock. void reader.cancel().catch(() => {}); } } /** Retry-After is scheduling metadata, not provider text; cap it to a safe integer. */ function retryAfterMs(headers: Headers): number | undefined { const millis = headers.get("retry-after-ms"); const after = headers.get("retry-after"); let delay: number; if (millis !== null && millis.trim() !== "") delay = Number(millis); else if (after !== null && after.trim() !== "") { const seconds = Number(after); delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(after) - Date.now(); } else return undefined; if (Number.isNaN(delay) || delay < 0) return undefined; return Math.min(Math.ceil(delay), Number.MAX_SAFE_INTEGER); } export const classifyTokenIn = async ( model: ClassifierModel, context: ClassifierContext, options?: ClassifierOptions, ): Promise => { let billed: number | undefined; let normalized: ReturnType | undefined; let malformedResponse = false; let failure: SaiClassifierFailure | undefined; const requestFetch = options?.fetch ?? globalThis.fetch; // Direct registry calls must not bypass the shared gate: refuse an invalid or unsupported set // atomically, before anything is sent or clipped. const supplied = context.images ?? []; if (supplied.length > 0) { if (!model.input.includes("image")) return imageRejectedResult(model, "TokenIn classifier model does not accept images"); const validation = validateSaiImages(supplied); if (!validation.ok) return imageRejectedResult(model, saiImageRejectionMessage(validation.reason)); return imageRejectedResult(model, "Image token accounting is unavailable; no request was sent", { kind: "local", reason: "image-budget-unavailable" }); } // Upstream's parser rejects the whole batch on a missing/type-confused answer. Give it no // questions to parse, but restore the original wire questions before the caller's payload hook. // The adapter validates the actual response against the original typed questions below. const { images: _inState, ...textContext } = context; const wireQuestions = Object.fromEntries(Object.entries(context.questions).map(([id, question]) => [id, question.type === "bool" ? { ...question, type: "noul", criteria: { ...question.criteria } } : question.type === "score" ? { ...question, criteria: [...question.criteria] } : { ...question, criteria: { ...question.criteria } }, ])); const result: ServicePricedClassifierResult = await classifySystemOne(model, { ...textContext, questions: {} }, { ...options, onPayload: async (payload, requestModel) => { try { if (!record(payload)) throw new Error("TokenIn classifier requires an object payload"); const fullPayload = { ...payload, questions: wireQuestions }; const transformed = await options?.onPayload?.(fullPayload, requestModel); return chatBody(transformed === undefined ? fullPayload : transformed); } catch (error) { if (error instanceof SaiClassifierRefusal) failure = { kind: "local", reason: error.reason }; throw error; } }, fetch: async (input, init) => { // Reset per attempt: a retry must not inherit pricing from another response. billed = undefined; normalized = undefined; malformedResponse = false; failure = undefined; try { const url = new URL(input instanceof Request ? input.url : String(input)); url.pathname = url.pathname.replace(/\/systemone$/u, "/chat/completions"); let response: Response; try { assertClassifierBody(init?.body); response = await requestFetch(url, init); } catch (error) { if (!(error instanceof SaiClassifierRefusal)) failure = { kind: "network" }; throw error; } if (!response.ok) { const delay = retryAfterMs(response.headers); failure = { kind: "http", httpStatus: response.status, ...(delay === undefined ? {} : { retryAfterMs: delay }), ...(response.status === 400 && await isContextOverflow(response) ? { contextOverflow: true } : {}), }; return response; } billed = billedCost(response.headers); // An unreadable envelope becomes an empty body: upstream reports the missing answers once, // instead of this layer failing the attempt and having it retried (and billed) again. const body = systemOneBody(await response.json().catch(() => undefined)) ?? {}; if (record(body.answers)) normalized = normalizeClassifierAnswers(context.questions, body.answers, "wire"); else malformedResponse = true; return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } }); } catch (error) { if (error instanceof SaiClassifierRefusal) failure = { kind: "local", reason: error.reason }; throw error; } }, }); if (result.stopReason === "error" || result.stopReason === "aborted") { if (failure) result.saiFailure = failure; // Upstream's normalized string may include provider bodies or arbitrary transport errors. result.errorMessage = "TokenIn classifier request failed"; } if (result.stopReason === "error" && malformedResponse) result.malformedResponse = true; if (result.stopReason === "stop" && normalized) { // pi-ai's choice/score types require confidence, but a service may omit it; preserve that // omission rather than fabricating a value to satisfy the upstream declaration. result.answers = normalized.answers as ClassifierResult["answers"]; if (Object.keys(normalized.rejected).length > 0) result.answerRejections = normalized.rejected; if (Object.keys(normalized.answers).length === 0 && result.answerRejections) { result.stopReason = "error"; result.errorMessage = "TokenIn classifier returned no valid answers"; } } if (result.usage) { const { cost: _catalogEstimate, ...counts } = result.usage; result.unpricedUsage = counts; delete result.usage; } if (billed !== undefined) result.reportedCost = billed; return result; };