import type { OperationLifecycleLike, ProviderCallContext } from '../types/lifecycle.js'; import type { CompletionRequest, DocumentContent, ImageContent } from '../types/messages.js'; import type { NexusResponse, ResponseCost } from '../types/response.js'; /** What a media provider can be asked to do. */ export type MediaTask = 'read' | 'describe' | 'answer'; /** A medium to understand: an image or a document part, holding bytes, base64, a URL, or an asset. */ export type MediaInput = ImageContent | DocumentContent; /** The kinds of media there are: `image` or `document`. */ export type MediaKind = MediaInput['type']; /** Reading a medium's text. */ export interface MediaReadRequest { /** The image or document. */ media: MediaInput; /** Only these pages, numbered from 0, on a provider that can choose. */ pages?: readonly number[]; /** The provider to use. Defaults to the manager's for reading. */ provider?: string; /** The tenant it is for: its credentials, regions, and budget apply. */ tenantId?: string; /** Stops the call. */ signal?: AbortSignal; } /** Describing an image, or a document's look. */ export interface MediaDescribeRequest { /** The image or document. */ media: MediaInput; /** A sentence, or a paragraph. Defaults to `brief`. */ detail?: 'brief' | 'detailed'; /** The provider to use. */ provider?: string; /** The tenant it is for: its credentials, regions, and budget apply. */ tenantId?: string; /** Stops the call. */ signal?: AbortSignal; } /** Asking a question about one medium or several. */ export interface MediaAnswerRequest { /** The media the question is about. */ media: MediaInput | readonly MediaInput[]; /** The question. */ question: string; /** The provider to use. */ provider?: string; /** The tenant it is for: its credentials, regions, and budget apply. */ tenantId?: string; /** Stops the call. */ signal?: AbortSignal; } /** One page of a medium's text. An image is one page. */ export interface MediaPage { /** The page's number, from 0. */ index: number; /** Its text, as Markdown where the provider keeps structure such as tables and headings. */ text: string; /** Its width and height, in pixels, when the provider says. */ width?: number; /** Its height, in pixels, when the provider says. */ height?: number; /** Pictures on the page, by the provider's id, with their box when it gives one. */ images?: Array<{ id: string; box?: { left: number; top: number; right: number; bottom: number; }; }>; } /** What a call used, where the provider reports it. */ export interface MediaUsage { /** Input tokens, for a model. */ inputTokens?: number; /** Output tokens, for a model. */ outputTokens?: number; /** Pages processed, for an OCR service. */ pages?: number; } /** What every media result carries: who produced it, and what it used and cost. */ export interface MediaResultMeta { /** The provider that answered. */ provider: string; /** The model it used. */ model: string; /** What the call used. */ usage?: MediaUsage; /** What it cost, when the provider or the registry prices it. */ cost?: ResponseCost; } /** A medium's text, page by page. */ export interface MediaText extends MediaResultMeta { /** The pages, in order. */ pages: MediaPage[]; /** Every page's text, joined by a blank line. */ text: string; } /** A description of a medium. */ export interface MediaDescription extends MediaResultMeta { /** A caption, a sentence or a paragraph as asked. */ caption: string; /** Short alternative text, for a screen reader. */ altText: string; /** A few words naming what it shows. */ tags: string[]; } /** An answer about media. */ export interface MediaAnswer extends MediaResultMeta { /** The answer. */ answer: string; /** The pages, numbered from 0, it rests on, when the provider says. */ pages?: number[]; } /** What a media provider supports. */ export interface MediaProviderCapabilities { /** The tasks it performs. */ tasks: readonly MediaTask[]; /** The kinds of media it takes. */ inputs: readonly MediaKind[]; } /** Identifies a media provider and what it supports. */ export interface MediaProviderInfo { /** Its name, which results carry. */ name: string; /** What it supports. */ capabilities: MediaProviderCapabilities; } /** What every media call receives: how to stop, and the request's id. */ export interface MediaCallContext { /** Stops the call. */ signal: AbortSignal; /** The request's id, for logs and traces. */ requestId: string; /** The tenant it is for. */ tenantId?: string; /** * The tenant's credential for a provider, with a credential resolver: an adapter sends it in place * of the key it was configured with. */ credential?: ProviderCallContext['credential']; } /** A media understanding backend: a vision model, an OCR service, or a stand-in. Each task is optional. */ export interface MediaProvider { /** What it is and supports. */ readonly info: MediaProviderInfo; /** Reads a medium's text. */ read?(request: MediaReadRequest, context: MediaCallContext): Promise; /** Describes a medium. */ describe?(request: MediaDescribeRequest, context: MediaCallContext): Promise; /** Answers a question about media. */ answer?(request: MediaAnswerRequest, context: MediaCallContext): Promise; } /** Raised for a media request no provider can serve, or a medium a provider cannot read. */ export declare class MediaError extends Error { /** A stable code: `MEDIA_NO_PROVIDER`, `MEDIA_UNSUPPORTED`, or `MEDIA_INVALID`. */ readonly code: 'MEDIA_NO_PROVIDER' | 'MEDIA_UNSUPPORTED' | 'MEDIA_INVALID'; constructor(message: string, /** A stable code: `MEDIA_NO_PROVIDER`, `MEDIA_UNSUPPORTED`, or `MEDIA_INVALID`. */ code: 'MEDIA_NO_PROVIDER' | 'MEDIA_UNSUPPORTED' | 'MEDIA_INVALID'); } /** Configuration for `MediaManager`. */ export interface MediaConfig { /** Providers, by name. */ providers?: Record; /** The provider each task uses when a request names none. Otherwise the first that can. */ defaults?: Partial>; } /** What the manager runs on. */ export interface MediaRuntime { /** * Runs every call as one operation of a client's lifecycle, family `media`: pass `ai.lifecycle`, so * it is authorized, budgeted, and audited as every other call is. A vision model reached through * the client already is; this covers a provider that calls its service directly, such as OCR. */ lifecycle?: OperationLifecycleLike; } /** * Media understanding behind one handle: each task goes to the provider named, the task's default, * or the first provider that performs it on that kind of medium. * * ```ts * const media = new MediaManager({ * providers: { ocr: new MistralOcrProvider({ apiKey }), vision: modelMediaProvider(ai, { model: 'gpt-5.4' }) }, * defaults: { read: 'ocr', describe: 'vision', answer: 'vision' }, * }); * const { pages } = await media.read({ media: { type: 'document', source: { buffer: pdf } } }); * ``` */ export declare class MediaManager { private readonly config; private readonly runtime; private readonly providers; constructor(config?: MediaConfig, runtime?: MediaRuntime); /** Registers a provider under a name. Returns the manager, for chaining. */ registerMediaProvider(name: string, provider: MediaProvider): this; /** Whether a provider is registered. */ hasMediaProvider(name: string): boolean; /** Every registered provider's name. */ listMediaProviders(): string[]; /** Reads a medium's text, page by page. */ read(request: MediaReadRequest): Promise; /** Describes a medium: a caption, alternative text, and tags. */ describe(request: MediaDescribeRequest): Promise; /** Answers a question about one medium or several. */ answer(request: MediaAnswerRequest): Promise; private run; private route; } /** Anything that completes a request, such as a `NexusAI` client. */ export interface CompletionClientLike { /** Runs one completion. */ complete(request: CompletionRequest): Promise; } /** Options for `modelMediaProvider()`. */ export interface ModelMediaProviderOptions { /** The vision model, which must take the media's kinds. */ model: string; /** The provider's name in results. Defaults to `model`. */ name?: string; /** The kinds of media the model takes. Defaults to images and documents. */ inputs?: readonly MediaKind[]; /** The most tokens a reading may produce. Defaults to 8,192. */ maxTokens?: number; } /** * Any vision model as a media provider, through a client's own `complete()`: so the call is routed, * authorized, budgeted, traced, and priced as every completion is. It asks for JSON, and reads a * plain answer as one page, or one answer, when a model gives one. */ export declare function modelMediaProvider(client: CompletionClientLike, options: ModelMediaProviderOptions): MediaProvider; /** Media to check a provider with, and what its text should contain. */ export interface MediaConformanceFixtures { /** An image the provider should read and describe. */ image?: MediaInput; /** A document the provider should read. */ document?: MediaInput; /** A pattern the read text must match, such as a word printed on the media. */ expectText?: RegExp; /** A question about the media, and a pattern its answer must match. */ question?: { ask: string; expect: RegExp; }; } /** How a provider did against the media contract. */ export interface MediaConformanceReport { /** The provider. */ provider: string; /** Checks that passed. */ passed: string[]; /** Checks that failed, each with why. */ failures: string[]; } /** * Checks a provider against the media contract: every task it declares returns the right shape, and * the text it reads holds what the fixtures say. Never throws; failures are listed. */ export declare function runMediaConformance(provider: MediaProvider, fixtures: MediaConformanceFixtures): Promise;