/** * Media you can evaluate: images, documents, audio, and video as the subject of an evaluation, judged * by checks that read the media themselves (format, size, dimensions, aspect, duration) and by a * model that is shown them. */ import { type JudgeClient } from '../evals/judge.js'; import type { EvaluationContext, Evaluator, MediaPart } from '../types/evaluate.js'; export { type MediaDuration, readMediaDuration } from '../utils/media-duration.js'; /** What kind of medium something is. */ export type MediaSubjectKind = 'image' | 'audio' | 'video' | 'document' | 'other'; /** A medium read for judging: what it is, and as much of it as could be read. */ export interface ResolvedMedia { /** The part it was read from. */ part: MediaPart; /** Its kind, from its part type or its MIME type. */ kind: MediaSubjectKind; /** Its MIME type: the one its bytes say, else the one declared. */ mimeType?: string; /** Its bytes, when they could be read. */ bytes?: Uint8Array; /** Where it is, when only an address is known. */ url?: string; /** A file path it names. */ path?: string; /** A transcript standing in for audio. */ transcript?: string; /** Width in pixels, from an image's header or the asset's own record. */ width?: number; /** Height in pixels. */ height?: number; /** * Duration in milliseconds, read from the file's own headers: WAV, MP4 and MOV, WebM and Matroska, * MP3, Ogg (Vorbis and Opus), and FLAC. */ durationMs?: number; /** Why its bytes could not be read, when they could not. */ unreadable?: string; } /** Where media judged by reference are read from. */ export interface MediaResolveOptions { /** Reads stored assets: an `AssetStore` and the tenant whose assets they are. */ assets?: { store: { get(assetId: string, tenantId: string): unknown; }; tenantId: string; }; /** Reads a file a part names by path, such as `fs.promises.readFile`. Without it, paths are not read. */ readFile?: (path: string) => Promise; /** Fetches a medium a part names by URL. Without it, nothing is fetched. */ fetch?: typeof globalThis.fetch; /** The most bytes read of one medium. Defaults to 20 MiB. */ maxBytes?: number; } /** Which media an evaluator judges. */ export interface MediaSelection { /** * Media in the output, the default; in the example's inputs; or both. With nothing in the output, * `output` falls back to the media the run's trace says it produced. */ from?: 'output' | 'input' | 'all'; /** Picks the media itself instead, from the whole evaluation context. */ select?: (context: EvaluationContext) => MediaPart[] | Promise; } /** * The media an evaluation is about: those in the output, or, with none there, those the run's trace * says it produced; or those it was given. */ export declare function mediaOf(context: EvaluationContext, from?: MediaSelection['from']): MediaPart[]; /** Reads one medium: its kind, its bytes where it can, and what its header says. */ export declare function resolveMedia(part: MediaPart, options?: MediaResolveOptions): Promise; /** A range a measurement must fall in. */ export interface MediaRange { /** The smallest allowed. */ min?: number; /** The largest allowed. */ max?: number; } /** Options for `mediaChecks()`. Only the checks given are made. */ export interface MediaCheckOptions extends MediaResolveOptions, MediaSelection { /** Prefixes every score's key. Defaults to `media`. */ key?: string; /** How many media there must be. Defaults to at least one. */ count?: MediaRange; /** The kinds allowed, such as `['image']`. */ kinds?: MediaSubjectKind[]; /** The MIME types allowed, with `*` for any subtype, as `image/*`. */ mimeTypes?: string[]; /** The size in bytes allowed. */ bytes?: MediaRange; /** The width in pixels allowed. */ width?: MediaRange; /** The height in pixels allowed. */ height?: MediaRange; /** Width over height, within a tolerance (0.02 by default), such as `{ ratio: 16 / 9 }`. */ aspectRatio?: { ratio: number; tolerance?: number; }; /** The duration in milliseconds allowed, for audio or video whose headers say it. */ durationMs?: MediaRange; /** A medium whose measurement could not be read passes or fails that check. Defaults to `fail`. */ unknown?: 'pass' | 'fail'; } /** * Checks the media an evaluation is about, reading them: how many there are, and of each its kind, * MIME type, size, dimensions, aspect ratio, and duration, as the options ask. Each check is a score * of its own, `media.format` for instance: the share of media that pass it. `media` itself is the * share that pass every one, and 0 when the count is wrong. */ export declare function mediaChecks(options?: MediaCheckOptions): Evaluator; /** Options for `mediaJudge()`. */ export interface MediaJudgeOptions extends MediaResolveOptions, MediaSelection { /** The client the judge model is called through, such as a `NexusAI`. */ client: JudgeClient; /** A model that can see the media: a vision model for images and documents, an audio one for audio. */ model: string; /** What good media look like, in plain language. */ rubric: string; /** The score's key. Defaults to `media-judge`. */ key?: string; /** * What the media were made for, shown to the judge beside them. Defaults to the example's inputs as * text. */ brief?: (context: EvaluationContext) => string | undefined; /** The scoring range. Defaults to 0 to 1. */ range?: [number, number]; /** The score at or above which media pass when the judge does not say. Defaults to the middle. */ passThreshold?: number; /** Sampling temperature. Defaults to 0. */ temperature?: number; /** Output token limit. Defaults to 512. */ maxTokens?: number; /** Replaces the instruction that asks for JSON with a score, a verdict, and a rationale. */ systemPrompt?: string; /** The most media shown at once. Defaults to 8. */ maxMedia?: number; /** With nothing to judge, a score of 0, or no score. Defaults to `fail`. */ whenMissing?: 'fail' | 'skip'; /** Application data sent with every judge request. */ metadata?: Record; } /** * Judges the media an evaluation is about with a model that is shown them: each image, document, * audio clip, or video goes to the model as itself, read from bytes, base64, a URL, a path, or an * asset store, beside the rubric and the brief, and the model answers with a score. A stored asset is * read and sent as bytes, since a provider cannot read the store. */ export declare function mediaJudge(options: MediaJudgeOptions): Evaluator;