import { Registry } from "./registry"; import { ENCODE_METHOD, DECODE_METHOD } from "../utils/encoder_symbols"; import type { AdkEncodableSnapshot } from "./encodable"; import type { MediaReader } from "../contracts/media_reader"; /** * The set of supported media kinds. * * @remarks * Modality coverage is asymmetric across providers. The framework defines no * `supportedModalities` field — how a battery handles a modality it cannot natively render is * the battery author's call (see `unsupportedMediaPolicy` on the OpenAI Chat Completions * battery). */ export declare const MediaKind: readonly [ "image", "audio", "video", "document" ]; /** * Union of all recognised media kind identifier strings. */ export type MediaKind = (typeof MediaKind)[number]; /** * Provenance axis. *Who is the framework willing to vouch for as the source of these bytes?* * * @remarks * Mirrors `RetrievableTrustTier` deliberately — same vocabulary, same question: * *did this content come from a place the agent should treat as authoritative?* * * - `'first-party'` — deployer-vetted bytes (tool output the operator authored, signed * internal assets). * - `'third-party-public'` — open-web fetches, public APIs, public corpora. * - `'third-party-private'` — user uploads, partner APIs, private corpora. */ export declare const MediaTrustTier: readonly [ "first-party", "third-party-public", "third-party-private" ]; export type MediaTrustTier = (typeof MediaTrustTier)[number]; /** * Modality-hazard axis. *How dangerous is it to let the model decode these bytes?* * * @remarks * Orthogonal to provenance — a first-party trusted PDF can still carry hidden text layers; a * third-party-public raw image can still be encoded as opaque pixels with adversarial * perturbations. * * - `'inert'` — bytes the model never decodes as instructions (e.g. a handle that is never * inlined into the prompt). * - `'extractable-instructions'` — text-bearing media: PDFs, screenshots with UI text, documents. * Hazard is OCR / embedded-text-layer reads. * - `'opaque-perceptual'` — raw vision/audio/video the model encodes directly. Hazard is * steganographic LSB prompts, adversarial perturbations, ultrasonic audio — invisible to any * pre-screen. * * See `/the-loop/trust-tiers/media` and its research sub-page `/the-loop/trust-tiers/media/research`. */ export declare const MediaModalityHazard: readonly [ "inert", "extractable-instructions", "opaque-perceptual" ]; export type MediaModalityHazard = (typeof MediaModalityHazard)[number]; /** * Per-entry shape stored in a {@link Media}'s `stash` register. * * @remarks * Each entry carries its own trust tier so render code can route derived text (OCR, captions, * transcripts) through its own envelope independent of the parent media. How a battery or * middleware assigns those entry-level tiers is the implementor's call — the primitive contract * does not enforce a "downgrade derived interpretation from possibly-adversarial bytes" policy. */ export interface MediaStashEntry { /** The value of the entry — any serialisable shape the consumer wants to store. */ value: unknown; /** Trust tier for this specific entry; routed independently of the parent media. */ trustTier: MediaTrustTier; /** Optional pointer to the parent Media id this entry was derived from. */ derivedFromMedia?: string; } /** * Plain input object supplied to {@link Media} at construction time. * * @remarks * Validated against `rawMediaSchema` before the `Media` instance is created. */ export interface RawMedia { /** * Stable unique identifier for this media instance. Required for strict symmetry with * `Message.id` and `ToolCall.id`. When omitted, a fresh UUIDv6 is assigned at construction * time. */ id?: string; /** The media kind. See {@link MediaKind}. */ kind: MediaKind; /** The MIME type of the underlying bytes. */ mimeType: string; /** Filename used by providers that key on it (e.g. OpenAI `file.filename`). */ filename: string; /** Re-openable byte source. See {@link @nhtio/adk!MediaReader}. */ reader: MediaReader; /** * Trust tier declared at construction time. Required — there is NO default. * See {@link MediaTrustTier}. */ trustTier: MediaTrustTier; /** * Modality hazard declared at construction time. Required — there is NO default. * See {@link MediaModalityHazard}. */ modalityHazard: MediaModalityHazard; /** Optional provenance pointer (URL, tool name, etc.) for audit / events. */ source?: string; /** * Free-form per-instance metadata register. Middleware pipelines append to this — typically * with a text description, transcript, caption, or alt-text — so downstream code that cannot * consume the media natively has a model-readable fallback. No keys are reserved by the * framework. Defaults to `{}`. */ stash?: Record; } /** * Shape returned by {@link Media.toJSON}. Metadata-only — bytes and the reader are stripped so * naive event/log serialisation never materialises bytes. */ /** The plain-object, JSON-safe form of a {@link Media} produced by {@link Media.toJSON}. */ export interface SerializedMedia { /** Stable identifier for this media asset. */ id: string; /** High-level modality of the asset (e.g. image, audio, document). */ kind: MediaKind; /** MIME type of the underlying bytes (e.g. `image/png`). */ mimeType: string; /** Original or suggested file name for the asset. */ filename: string; /** Optional provenance string (URL, path, or other origin marker). */ source?: string; /** Trust tier governing how the asset's content is framed to the model. */ trustTier: MediaTrustTier; /** Whether the modality can carry hidden instructions (`extractable-instructions`) or is opaque-perceptual. */ modalityHazard: MediaModalityHazard; /** Adapter-scoped side-channel data keyed by name (e.g. provider upload handles). */ stash: Record; /** Size of the underlying bytes in bytes, when known. */ byteLength?: number; } /** * Lazy, re-openable view over a binary asset (image, audio, video, document). * * @remarks * Dual-peer to {@link @nhtio/adk!Tokenizable} (silo) and {@link @nhtio/adk!SpooledArtifact} * (handle). Wraps a {@link @nhtio/adk!MediaReader} contract — the framework owns the contract, the * implementor owns the storage backend. Bytes are reached only through the reader; the primitive * itself never inlines bytes. * * Construction requires `trustTier` and `modalityHazard` — the framework refuses to guess * provenance or decoding hazard. Ergonomic factories ({@link Media.userAttachment}, * {@link Media.toolGenerated}, {@link Media.retrievedPublic}, {@link Media.retrievedPrivate}) * force the labelling decision at the call site without becoming defaults on the bare * constructor. */ export declare class Media { #private; /** * Validator schema that accepts a {@link RawMedia} object. */ static schema: import("@nhtio/validation").ObjectSchema; /** * The set of recognised media kinds. Exposed for downstream schemas that need to discriminate * on `kind`. */ static MediaKind: readonly [ "image", "audio", "video", "document" ]; /** * The set of recognised trust tiers. */ static MediaTrustTier: readonly [ "first-party", "third-party-public", "third-party-private" ]; /** * The set of recognised modality hazards. */ static MediaModalityHazard: readonly [ "inert", "extractable-instructions", "opaque-perceptual" ]; /** * Returns `true` if `value` is a {@link Media} instance. * * @remarks * Uses {@link @nhtio/adk!isInstanceOf} for cross-realm safety. * * @param value - The value to test. * @returns `true` when `value` is a {@link Media} instance. */ static isMedia(value: unknown): value is Media; /** Stable unique identifier. */ readonly id: string; /** Media kind. */ readonly kind: MediaKind; /** MIME type of the underlying bytes. */ readonly mimeType: string; /** Filename surfaced to providers that key on it. */ readonly filename: string; /** Optional provenance pointer. */ readonly source: string | undefined; /** Trust tier declared at construction time. */ readonly trustTier: MediaTrustTier; /** Modality hazard declared at construction time. */ readonly modalityHazard: MediaModalityHazard; /** Mutable per-instance metadata register; middleware pipelines append to this. */ readonly stash: Registry; /** * @param raw - The raw media input validated against `rawMediaSchema`. * @throws {@link @nhtio/adk/exceptions!E_INVALID_INITIAL_MEDIA_VALUE} when `raw` does not satisfy the schema. * @throws {@link @nhtio/adk/exceptions!E_NOT_A_MEDIA_READER} when `raw.reader` does not implement {@link @nhtio/adk!MediaReader}. */ constructor(raw: RawMedia); /** * Re-opens the underlying byte source and returns a fresh ReadableStream. * * @returns A drainable `ReadableStream` over the underlying bytes. */ stream(): Promise>; /** * Returns the total number of bytes in the underlying data, or `undefined` if unknown. * * @returns The byte length, or `undefined` when the underlying source cannot report it. */ byteLength(): Promise; /** * Drains the reader's stream and returns the underlying bytes as a single `Uint8Array`. * * @remarks * Convenience for callers that need the full buffer (e.g. inline base64 encoding). Forces * full materialisation — large assets should be piped through {@link Media.stream} instead. */ asBytes(): Promise; /** * Drains the reader's stream and returns the underlying bytes as a base64 string. * * @remarks * Cross-environment: prefers Node's `Buffer.from(buf).toString('base64')` when available; * otherwise chunk-encodes through `btoa` with a 0x8000-byte window to avoid stack overflow * on large buffers. */ asBase64(): Promise; /** * Returns the metadata-only serialisation of this Media. Bytes and the reader are stripped * so naive event/log serialisation never materialises bytes. * * @remarks * Implementations that have cheap, already-cached `byteLength` may opt to include it; this * default implementation omits it to preserve the "lazy by default" invariant. Consumers that * need byteLength on the serialised payload should call `await media.byteLength()` and merge * the result. */ toJSON(): SerializedMedia; /** * Serialise this Media into an `@nhtio/encoder` snapshot — the **handle**, never the bytes. * * @remarks * Emits every metadata field plus the reader's {@link @nhtio/adk!ReaderDescriptor} (via its * `describe()` method). The bytes are not inlined: decode re-binds the reader from the descriptor * through a registered resolver. Throws {@link @nhtio/adk!E_READER_NOT_DESCRIBABLE} when the backing * reader cannot describe itself (e.g. a `fromWebFile` Blob reader) — there is no serialisable handle to * write, and silently dropping the reader would decode into a handle pointing at nothing. * * @returns A snapshot consumed by {@link Media.[DECODE_METHOD]} / the encoder. * @throws {@link @nhtio/adk!E_READER_NOT_DESCRIBABLE} when the reader has no `describe()`. */ [ENCODE_METHOD](): AdkEncodableSnapshot; /** * Reconstruct a {@link Media} from an {@link Media.[ENCODE_METHOD]} snapshot. * * @remarks * Re-binds the reader from the captured descriptor through the registered resolver * ({@link @nhtio/adk!resolveMediaReader}), then re-validates via the normal constructor. Throws * {@link @nhtio/adk!E_NO_READER_RESOLVER} when no resolver is registered for the descriptor's tag. * * @param data - The snapshot produced by {@link Media.[ENCODE_METHOD]}. * @returns A fully-validated {@link Media} backed by a freshly-resolved reader. */ static [DECODE_METHOD](data: AdkEncodableSnapshot): Media; /** * Factory: constructs a {@link Media} representing a user-supplied attachment. * * @remarks * Pre-fills `trustTier: 'third-party-private'` and derives `modalityHazard` from `kind` * (`document` → `'extractable-instructions'`; everything else → `'opaque-perceptual'`). * Use the bare constructor when the conservative kind→hazard mapping is wrong for your case. */ static userAttachment(args: { id?: string; kind: MediaKind; mimeType: string; filename: string; reader: MediaReader; source?: string; stash?: Record; }): Media; /** * Factory: constructs a {@link Media} produced by a first-party tool. * * @remarks * Pre-fills `trustTier: 'first-party'` and derives `modalityHazard` from `kind`. */ static toolGenerated(args: { id?: string; kind: MediaKind; mimeType: string; filename: string; reader: MediaReader; source?: string; stash?: Record; }): Media; /** * Factory: constructs a {@link Media} retrieved from a public third-party source. * * @remarks * Pre-fills `trustTier: 'third-party-public'` and derives `modalityHazard` from `kind`. */ static retrievedPublic(args: { id?: string; kind: MediaKind; mimeType: string; filename: string; reader: MediaReader; source?: string; stash?: Record; }): Media; /** * Factory: constructs a {@link Media} retrieved from a private third-party source. * * @remarks * Pre-fills `trustTier: 'third-party-private'` and derives `modalityHazard` from `kind`. */ static retrievedPrivate(args: { id?: string; kind: MediaKind; mimeType: string; filename: string; reader: MediaReader; source?: string; stash?: Record; }): Media; } /** * Returns `true` if `value` is a {@link Media} instance. * * @remarks * Module-level convenience alias for {@link Media.isMedia}. Uses {@link @nhtio/adk!isInstanceOf} for * cross-realm safety. */ export declare const isMedia: (value: unknown) => value is Media;