/** * TypeSafe and OpenRouter use the `{ model, state, questions }` body. Images * join `state` without replacing it (see {@link stateWithImages}): TypeSafe as * references to a top-level `assets` map, OpenRouter as `image_url` data URL * parts. Cloudflare optionally accepts image data URLs in its envelope. * * - `typesafe`: `POST {baseUrl}/v1/systemone` — TypeSafe and every System One * server (OpenJEV, Kev, self-hosted Clef, OpenCode Zen, Command Code). * - `openrouter-decisions`: `POST {baseUrl}/decisions` under OpenRouter's * `/api/alpha`; usage adds the billed `cost` in USD. * - `workers-ai-systemone`: Cloudflare Workers AI's `POST {baseUrl}/{model}` * (`/client/v4/accounts//ai/run/@cf/cloudflare/clef`). The body * names the model by its last path segment (`clef`), adds a top-level array * of base64 data URLs when images are supplied, and returns Cloudflare's * `{ result, success, errors }` envelope. The credential is a Cloudflare * API token plus the account id the URL needs. * - `liquid-systemone`: Liquid AI's `POST {baseUrl}/decisions/v1/systemone`. * Same body, verbatim model, plus a top-level array of base64 data URLs * when images are supplied. * - `nace-systemone`: Nace's Drex at `POST {baseUrl}/v1/systemone`. The * `liquid-systemone` body: Drex rejects TypeSafe's `assets` map, so images * go in a top-level array of base64 data URLs. Drex also rejects any text * holding a base64 media data URL, so each one in `state`, `instructions` or * `criteria` is replaced by a description (see {@link describeMediaDataUrls}). */ import { parseCloudflareCredential } from "@oh-my-pi-zen/pi-catalog/wire/cloudflare"; import { $env, isRecord } from "@oh-my-pi-zen/pi-utils"; import * as AIError from "../error"; import { embeddedImageDataUrl } from "../images/shared"; import type { ImageContent } from "../types"; import { type BoxQuestion, type JsonValue, type JudgmentRequest, type Question, type Questions, tokenUsage, } from "./types"; import { type DecodedJudgment, finiteNumber, type JudgmentImageAssets, type JudgmentWire } from "./wire"; function decodeSystemOne(payload: unknown): DecodedJudgment | undefined { if (!isRecord(payload) || !isRecord(payload.answers)) return undefined; const usage = isRecord(payload.usage) ? payload.usage : {}; return { model: typeof payload.model === "string" ? payload.model : undefined, answers: payload.answers, refused: [], usage: tokenUsage(finiteNumber(usage.input_tokens), finiteNumber(usage.output_tokens), finiteNumber(usage.cost)), }; } /** * `state` with `items` added and every original value kept: arrays gain them at the end, objects * under the first free `__omp_judgment_images[_N]` key, and a string root becomes `[state, ...items]`. */ function stateWithImages(state: JudgmentRequest["state"], items: readonly JsonValue[]): JudgmentRequest["state"] { if (Array.isArray(state)) return [...state, ...items]; if (!isRecord(state)) return [state, ...items]; const baseKey = "__omp_judgment_images"; let key = baseKey; for (let suffix = 1; Object.hasOwn(state, key); suffix++) key = `${baseKey}_${suffix}`; return { ...state, [key]: items }; } function* jsonStrings(value: unknown): Generator { if (typeof value === "string") yield value; else if (Array.isArray(value)) for (const item of value) yield* jsonStrings(item); else if (isRecord(value)) for (const key in value) yield* jsonStrings(value[key]); } /** * The subset of `assets` that `request` references. TypeSafe substitutes a reference wherever it * occurs as text in a string of `state`, a question's `instructions`, or its `criteria` (never in * object keys) and rejects a request carrying an unreferenced asset. */ export function referencedImageAssets(request: JudgmentRequest, assets: JudgmentImageAssets): JudgmentImageAssets { const pending = new Set(Object.keys(assets)); const visit = (value: unknown): void => { for (const text of jsonStrings(value)) { if (pending.size === 0) return; for (const reference of pending) if (text.includes(reference)) pending.delete(reference); } }; visit(request.state); for (const id in request.questions) { const question = request.questions[id]; visit(question.instructions); if (question.type !== "box") visit(question.criteria); } // `fromEntries` defines own properties, so a `__proto__` reference stays an ordinary key. return Object.fromEntries(Object.entries(assets).filter(([reference]) => !pending.has(reference))); } interface SystemOneImageAsset { type: "image"; format: "png" | "jpeg"; data_base64: string; } function encodeSystemOneImage(image: ImageContent): SystemOneImageAsset { const mimeType = image.mimeType.toLowerCase(); return { type: "image", format: mimeType === "image/png" ? "png" : "jpeg", data_base64: Buffer.from(image.data, "base64").toString("base64"), }; } type WireQuestion = Exclude | Omit; /** `questions` as System One takes them: a box question's `image` only names the image its edges measure locally. */ function wireQuestions(questions: Questions): Record { const wire: Record = {}; for (const id in questions) { const question = questions[id]; wire[id] = question.type === "box" ? { type: "box", instructions: question.instructions } : question; } return wire; } /** A System One dialect posting the verbatim request to `path` under the base URL with the key as bearer. */ function systemOneWire(path: string): JudgmentWire { return { target: (baseUrl, _model, key) => ({ url: `${baseUrl}${path}`, token: key }), encode: (request, model) => ({ state: request.state, model, questions: wireQuestions(request.questions) }), decode: decodeSystemOne, }; } /** TypeSafe's own route, shared by every System One server. */ export const typesafeWire: JudgmentWire = { ...systemOneWire("/v1/systemone"), images: { mimeTypes: ["image/png", "image/jpeg"] }, supportsImageAssets: true, imagesRequireOptIn: true, imageHeaders: { "TypeSafe-Danger-Zone-Images": "true" }, ignoresImageDetail: true, boxes: true, encode(request, model, imageAssets) { const suppliedAssets = Object.entries(imageAssets ?? {}); const images = request.images ?? []; if (suppliedAssets.length === 0 && images.length === 0) { return { state: request.state, model, questions: wireQuestions(request.questions) }; } const assets: Record = Object.fromEntries( suppliedAssets.map(([reference, image]) => [reference, encodeSystemOneImage(image)]), ); const references: string[] = []; for (const image of images) { let reference = crypto.randomUUID(); while (Object.hasOwn(assets, reference)) reference = crypto.randomUUID(); assets[reference] = encodeSystemOneImage(image); references.push(reference); } return { state: references.length ? stateWithImages(request.state, references) : request.state, model, questions: wireQuestions(request.questions), assets, }; }, }; /** OpenRouter Decisions: System One body under `/api/alpha`, with image parts preserved in state. */ export const openrouterDecisionsWire: JudgmentWire = { ...systemOneWire("/decisions"), images: {}, imagesRequireOptIn: true, boxes: true, encode(request, model) { const imageParts: JsonValue[] = (request.images ?? []).map(image => ({ type: "image_url", image_url: { url: embeddedImageDataUrl(image), ...(image.detail !== undefined ? { detail: image.detail === "original" ? "auto" : image.detail } : {}), }, })); return { state: imageParts.length ? stateWithImages(request.state, imageParts) : request.state, model, questions: wireQuestions(request.questions), }; }, }; /** Cloudflare Workers AI decision models (Clef) through the account-scoped `ai/run` route. */ export const workersAiWire: JudgmentWire = { // Clef's `images` extension: https://developers.cloudflare.com/workers-ai/models/clef/ images: { mimeTypes: ["image/png", "image/jpeg", "image/webp"], maxCount: 4, maxBytes: 4 * 1024 * 1024, maxTotalBytes: 8 * 1024 * 1024, maxPixels: 16_000_000, }, target(baseUrl, model, key) { const credential = parseCloudflareCredential(key); const accountId = credential?.accountId ?? $env.CLOUDFLARE_ACCOUNT_ID?.trim(); if (!accountId) { throw new AIError.ConfigurationError( "Cloudflare account ID is required: log in to Cloudflare Workers AI again or set CLOUDFLARE_ACCOUNT_ID", ); } return { url: `${baseUrl.replace("", accountId)}/${model}`, token: credential?.token ?? key }; }, encode: (request, model) => ({ state: request.state, model: model.slice(model.lastIndexOf("/") + 1), questions: request.questions, ...(request.images?.length ? { images: request.images.map(embeddedImageDataUrl) } : {}), }), decode: payload => (isRecord(payload) ? decodeSystemOne(payload.result) : undefined), }; /** The verbatim body plus, when images are supplied, a top-level array of their base64 data URLs. */ function encodeWithImageDataUrls(request: JudgmentRequest, model: string): Record { return { state: request.state, model, questions: wireQuestions(request.questions), ...(request.images?.length ? { images: request.images.map(embeddedImageDataUrl) } : {}), }; } /** Liquid AI decision models (`d1.1`, `d1-ultra`). */ export const liquidWire: JudgmentWire = { ...systemOneWire("/decisions/v1/systemone"), // https://docs.liquid.ai/lfm/models/d1.md#image-requirements. The documented 4.5 MB cap covers the // whole JSON body; 3 MB of images is 4 MB as base64, leaving room for state and questions. images: { mimeTypes: ["image/jpeg", "image/png", "image/webp", "image/gif"], maxCount: 8, maxTotalBytes: 3_000_000, maxTotalPatches: 10_000, }, encode: encodeWithImageDataUrls, }; /** * A base64 data URL of an image, audio, video or PDF type, as Drex finds it in any string of `state`, * `instructions` or `criteria` (case-insensitive, anywhere in the string, even with an empty payload) to * answer 422 `is media (image/png data URL)`. Other types, such as `text/plain`, pass. */ const MEDIA_DATA_URL = /data:((?:image|audio|video)\/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*|application\/pdf)(?:;[^;,\s]*)*;base64,[A-Za-z0-9+/=_-]*/gi; /** `value` with each media data URL in its strings, never its keys, replaced by `[ base64 data URL]`. */ function describeMediaDataUrls(value: unknown): unknown { if (typeof value === "string") return value.replace(MEDIA_DATA_URL, "[$1 base64 data URL]"); if (Array.isArray(value)) return value.map(describeMediaDataUrls); if (!isRecord(value)) return value; // `fromEntries` defines own properties, so a `__proto__` key stays an ordinary key. return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, describeMediaDataUrls(item)])); } /** Nace Drex decision models (`drex-max-v1.1`, `drex-max-finance`, `drex-v2.0`). */ export const naceWire: JudgmentWire = { ...systemOneWire("/v1/systemone"), // https://console.nace.ai/docs/drex/api-reference/systemone: at most 8 images of at most 40 megapixels, // with state, questions and images serialized together at most 4 MiB; 3 MB of images is 4 MB as base64, // leaving room for state and questions. images: { mimeTypes: ["image/png", "image/jpeg", "image/webp"], maxCount: 8, maxTotalBytes: 3_000_000, maxPixels: 40_000_000, }, encode: (request, model) => encodeWithImageDataUrls( { ...request, state: describeMediaDataUrls(request.state) as JudgmentRequest["state"], questions: describeMediaDataUrls(request.questions) as Questions, }, model, ), };