import type * as ort from 'onnxruntime-web'; import { RefObject } from 'react'; /** * Per-class NMS — boxes are suppressed only by other boxes of the same class. * * Mirrors `torchvision.ops.batched_nms`. * * @param boxes Flat array of length `4 * N` in xyxy order. * @param scores Detection score per box, length `N`. * @param idxs Class index per box, length `N`. Boxes with different `idxs` * never suppress each other. * @param iouThreshold IoU threshold for suppression within a class. * @returns Indices of kept boxes, sorted by descending score across all * classes. Survivors from different classes that are tied on score are * ordered lowest-index first — an explicit tie-break, because the order the * per-class loop emits them in is an implementation detail (here, `Map` * insertion order; in Python, sorted class order). */ export declare function batchedNms(boxes: Float32Array, scores: Float32Array, idxs: Int32Array, iouThreshold: number): Int32Array; /** * Axis-aligned bounding box in absolute pixel coordinates (xyxy format). * * Coordinates refer to the original input image (before any internal resize), * so callers can map detections back onto their source image without any * additional bookkeeping. */ export declare class BoundingBox { readonly x1: number; readonly y1: number; readonly x2: number; readonly y2: number; constructor(x1: number, y1: number, x2: number, y2: number); /** Box width in pixels (clamped to non-negative). */ get width(): number; /** Box height in pixels (clamped to non-negative). */ get height(): number; /** Box area in pixels squared. */ get area(): number; /** The box as `[x1, y1, x2, y2]` in absolute pixels (Ultralytics-style). */ get xyxy(): readonly [number, number, number, number]; /** * The box as `[cx, cy, w, h]` with `(cx, cy)` at the center. * * Matches Ultralytics' `boxes.xywh` and YOLO's native head format. For the * top-left `[x, y, w, h]` convention, use {@link asXywh}. */ get xywh(): readonly [number, number, number, number]; /** * The box as `[x1, y1, x2, y2]` normalized to `[0, 1]`. * * @param origShape `[height, width]` of the source image, in pixels. */ xyxyn(origShape: readonly [number, number]): readonly [number, number, number, number]; /** * The box as `[cx, cy, w, h]` normalized to `[0, 1]`. * * @param origShape `[height, width]` of the source image, in pixels. */ xywhn(origShape: readonly [number, number]): readonly [number, number, number, number]; /** Returns `[x1, y1, x2, y2]`. */ asXyxy(): readonly [number, number, number, number]; /** * Returns `[x, y, width, height]` with `(x, y)` at the **top-left**. * * Note: this is the top-left convention. Ultralytics' `xywh` getter uses * **center** coordinates — for that, read {@link xywh}. */ asXywh(): readonly [number, number, number, number]; /** Returns `[x1, y1, x2, y2]` truncated to integers, useful for slicing arrays. */ asIntXyxy(): readonly [number, number, number, number]; } /** * Bulk numpy-style view of detected boxes for a single image. * * Mirrors Ultralytics' `Boxes` interface. Coordinates in {@link xyxy} and * {@link xywh} are absolute pixels in the original image; the `*n` variants * are normalized to `[0, 1]` using `origShape`. */ export declare class Boxes { readonly xyxy: Float32Array; readonly cls: Int32Array; readonly conf: Float32Array; readonly origShape: readonly [number, number]; /** * @param xyxy Flat array of length `4 * N` in `[x1, y1, x2, y2, ...]` order. * @param cls One class index per box, length `N`. * @param conf One confidence per box, length `N`. * @param origShape `[height, width]` of the original image. */ constructor(xyxy: Float32Array, cls: Int32Array, conf: Float32Array, origShape: readonly [number, number]); /** Number of detected boxes. */ get length(): number; /** `[N, 4]` shape of the `xyxy` view. */ get shape(): readonly [number, number]; /** Boxes as `[N, 4]` `[cx, cy, w, h]` flat array in absolute pixels. */ get xywh(): Float32Array; /** Boxes as `[N, 4]` `[x1, y1, x2, y2]` normalized to `[0, 1]`. */ get xyxyn(): Float32Array; /** Boxes as `[N, 4]` `[cx, cy, w, h]` normalized to `[0, 1]`. */ get xywhn(): Float32Array; /** * Concatenated `[N, 6]` array of `[x1, y1, x2, y2, conf, cls]`. * * Matches Ultralytics' `boxes.data`. */ get data(): Float32Array; } /** A classified camera error with a human-readable, English message. */ export declare interface CameraStreamError { kind: CameraStreamErrorKind; message: string; } /** Classified reason a camera stream could not be acquired. */ export declare type CameraStreamErrorKind = "unsupported" | "permission-denied" | "no-camera" | "in-use" | "insecure" | "unknown"; /** Lifecycle status of the camera stream. */ export declare type CameraStreamStatus = "idle" | "loading" | "ready" | "error"; /** * Infer how many classes a classification head emits. * * A classifier declares `(B, nc)`, so the count is the last static axis. * * @param shape Declared shape of the model's first output. * @returns The class count, or `null` when the last axis is dynamic or absent. */ export declare function classificationNumClasses(shape: DeclaredShape): number | null; /** * Output of an image classification inference. */ export declare interface ClassificationResult { readonly classId: number; readonly className: string; readonly confidence: number; /** Alias for `classId` (Ultralytics-style). */ readonly cls: number; /** Alias for `className`. */ readonly name: string; /** Alias for `confidence` (Ultralytics-style). */ readonly conf: number; /** The original input image as an HWC RGB uint8 array. */ readonly image: RGBImage; /** * Probabilities per class, sorted in descending order. The first entry * mirrors `classId`, `className`, and `confidence`. When `topK` was passed * to `predict`, the array is truncated to that length. */ readonly probabilities: readonly ClassProbability[]; } /** * Per-image classification envelope (Ultralytics-style `Results`). */ export declare class ClassificationResults { readonly probs: Probs; readonly result: ClassificationResult; readonly names: Readonly>; readonly origImg: RGBImage; readonly origShape: readonly [number, number]; readonly path: string | null; readonly speed: Readonly; constructor(probs: Probs, result: ClassificationResult, names: Readonly>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly); /** Top-1 class index (Ultralytics-style alias). */ get cls(): number; /** Top-1 confidence (Ultralytics-style alias). */ get conf(): number; /** Top-1 class name. */ get name(): string; /** Per-class probability list, sorted descending (legacy field). */ get probabilities(): readonly ClassificationResult["probabilities"][number][]; } /** * Image classifier wrapping an ONNX model with ImageNet-style preprocessing. * * `predict()` returns `Promise` (length 1 for a * single image), mirroring Ultralytics' API. The envelope exposes a `probs` * collection (`top1`, `top1conf`, `top5`, `top5conf`, `data`) plus the * legacy per-class probability list with names already resolved. * * Defaults: 224×224 RGB input, `float32` normalized with ImageNet mean/std, * NCHW layout, batch size 1, softmax applied to the raw output. * * @example * ```typescript * const clf = await Classifier.create("/models/resnet50.onnx", { * labels: ["tench", "goldfish", ...] // 1000 ImageNet labels * }); * const r = (await clf.predict("/images/dog.jpg"))[0]; * console.log(r.cls, r.conf, r.name); * console.log(r.probs.top5, r.probs.top5conf); * ``` */ export declare class Classifier extends VisionTask { private readonly _labels; private readonly _names; private readonly _inputSize; private readonly _mean; private readonly _std; private readonly _applySoftmax; private readonly _normalization; private constructor(); /** * Whether a softmax is applied to the model's output before ranking. * * Resolved once at construction. Worth reading when confidences look * compressed: a second softmax over an already-normalized vector leaves the * ordering intact and the numbers meaningless, which is invisible to any check * that only looks at the predicted class. */ get appliesSoftmax(): boolean; /** * Which preprocessing this classifier applies to every image. * * One of the {@link Normalization} preset names, or `"custom"` when the caller * supplied `mean`/`std` directly. Worth reading when a model underperforms: * feeding a classifier a differently prepared tensor than it was trained on * degrades it without throwing anything, so "what does this assume" is the * first question. */ get normalization(): string; private _pipelineCache; /** * Run the model once on a zero tensor, paying one-time costs up front. * * The first inference of a session is not representative: WebGPU compiles its * shaders on it and the WASM backend faults in its arenas. Calling this while * a loading spinner is still up moves that cost somewhere the user is already * waiting — which matters most for a classifier running as the last step of * an on-device analysis, where the delay lands right before the answer shows. * * @param runs How many warm-up inferences to run. One is enough for WASM; * WebGPU sometimes settles on the second. */ warmup(runs?: number): Promise; /** * The fused preprocessing pipeline, built on first use. * * Lazily, because constructing it reserves the output buffer: a task built in * an environment without a canvas implementation stays constructible, and only * fails if it is actually asked to preprocess something. */ private get _pipeline(); /** * Load the model, resolve labels, and settle the preprocessing. * * @param model The model source — a URL, or the bytes. * @param options Labels, input size, normalization, and session options. * @throws {RangeError} If `normalization` names an unknown preset, or names * one while `mean`/`std` are also given. */ static create(model: ModelSource, options?: ClassifierOptions): Promise; /** Class labels indexed by class id. */ get labels(): readonly string[]; /** Class id → class name dict (matches Ultralytics' `model.names`). */ get names(): Readonly>; /** * The `[width, height]` this task preprocesses to. * * Resolved at creation time from the model's graph when it declares a static * input, so reading it back tells you the resolution inference really runs at * — not merely what was requested. */ get inputSize(): readonly [number, number]; /** Number of classes the model can predict. */ get numClasses(): number; /** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */ call(image: ImageInput, options?: ClassifierPredictOptions): Promise; /** Run classification on a single image. */ predict(image: ImageInput, options?: ClassifierPredictOptions): Promise; private _preprocess; private _postprocess; } export declare interface ClassifierOptions extends OrtSessionOptions { /** * Class label spec — see {@link resolveLabels}. * * Optional: when omitted, the names the export baked into the model are used * (Ultralytics writes them as `names` in the metadata map). Only when the * model carries none does this fall back to generated `class_` labels. * Passing a spec always wins, for a model whose names are wrong or absent. */ readonly labels?: LabelSpec; /** * Number of classes the model can predict. * * Optional: inferred from the classification head's declared output shape * `(B, nc)`. Pass it to validate that the supplied labels match the model. */ readonly numClasses?: number; /** * Model input `[width, height]` in pixels. * * Only used when the model's graph leaves its spatial axes dynamic: a graph * that declares a static size always wins, since that is the only shape ONNX * Runtime will accept. Defaults to `[224, 224]`. */ readonly inputSize?: readonly [number, number]; /** * Which preprocessing this model expects — see {@link Normalization}. * * Defaults to `"auto"`, which reads the model's own export metadata and picks * `"ultralytics"` (raw `[0, 1]`) for an Ultralytics classification head and * `"imagenet"` for everything else. */ readonly normalization?: Normalization; /** Per-channel RGB mean, overriding the preset. Defaults to the preset's. */ readonly mean?: readonly [number, number, number]; /** Per-channel RGB standard deviation, overriding the preset. */ readonly std?: readonly [number, number, number]; /** * Whether the model's output still needs a softmax. * * Left undefined (the default), this reads the model's metadata and answers * `false` for an Ultralytics classification export, whose graph already ends * in one — applying a second softmax to a probability vector keeps the * ranking but flattens the confidences, so the top-1 stays right while every * number attached to it is wrong. Detection covers that family; for any other * model that already emits probabilities, pass `false` explicitly. */ readonly applySoftmax?: boolean; } export declare interface ClassifierPredictOptions { /** * If set, the per-class probability list in `results[0].result.probabilities` * is truncated to the top-K entries. The bulk `probs` view always exposes * the full vector. */ readonly topK?: number; } /** * Probability assigned to a single class for a classification prediction. * * `cls` / `name` / `conf` are Ultralytics-style aliases populated alongside * the verbose `classId` / `className` / `probability` fields. */ export declare interface ClassProbability { readonly classId: number; readonly className: string; readonly probability: number; /** Alias for `classId` (Ultralytics-style). */ readonly cls: number; /** Alias for `className`. */ readonly name: string; /** Alias for `probability` (Ultralytics-style). */ readonly conf: number; } /** COCO 2017 80-class labels in canonical class-id order. */ export declare const COCO_CLASSES: readonly string[]; /** * Mean BT.709 luminance (`0.2126*R + 0.7152*G + 0.0722*B`) of a decoded frame, * scaled to `0..255`. See {@link LuminanceSource} for what counts as one. * * The source is downsampled so its longest edge is at most * {@link LUMINANCE_SAMPLE_MAX_EDGE} before pixels are read. The 2D context is * created with `willReadFrequently` so repeated sampling (live feedback) stays * on the fast path. * * Pass `reusableCanvas` to avoid allocating a fresh canvas every frame in a hot * loop; when omitted a one-shot detached canvas is created. * * @param source - the decoded frame to sample. * @param reusableCanvas - optional canvas reused across frames to avoid GC churn. * @returns The mean luminance in `0..255`, or `0` when the source is unloaded * (zero-sized) or a 2D context is unavailable. */ export declare function computeImageLuminance(source: LuminanceSource, reusableCanvas?: HTMLCanvasElement): number; /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * The contract a fused pipeline carries inside its own `.onnx` file. * * A pipeline built by the Python SDK's `ort_vision_sdk.compose` is a single * graph that already contains the detector, the crop-and-resize bridge and the * classifier. Everything the runtime needs to drive it — the letterbox * resolution, the crop resolution, where the crops come from, how many * detections the graph emits, the class names of both stages — was decided at * fusion time and written into the model's metadata. * * This module reads it back. It is the browser half of a contract whose other * half lives in `ort_vision_sdk/fusion.py`: the same keys, the same encodings, * the same fallbacks. A pipeline fused once therefore runs identically in * Python and in a browser tab, off the same file. * * Building a pipeline stays a Python-side build step — there is no ONNX * protobuf writer here, and there is no reason for one: fusing is something you * do once next to your export pipeline, not in a page load. */ /** * Which tensor the bridge crops the detected boxes out of. * * - `"detector_input"`: the letterboxed tensor already fed to the detector. * The fused graph then has a **single** image input, but a small object is * cropped out of its downscaled copy. * - `"original"`: a second, full-resolution image input. The bridge undoes the * letterbox transform in-graph and crops at native resolution. Two tensors to * feed, still one session and one model load. */ export declare type CropSource = "detector_input" | "original"; /** Name reported when the caller supplied `mean`/`std` directly. */ export declare const CUSTOM_NORMALIZATION = "custom"; /** * One declared dimension: a number when the graph pins it, `null` when the * dimension is symbolic (dynamic). */ export declare type DeclaredDim = number | null; /** A declared input/output shape, dynamic axes appearing as `null`. */ export declare type DeclaredShape = readonly DeclaredDim[]; /** * Convert ORT value metadata into declared shapes. * * @param metadata Metadata as reported by `InferenceSession.inputMetadata`, or * `undefined` on ORT builds that predate it (added in onnxruntime 1.21). * @returns One shape per value, in declaration order. Non-tensor values and * builds without metadata yield empty shapes, which read as "nothing * declared" everywhere downstream. */ export declare function declaredShapesFrom(metadata: readonly ort.InferenceSession.ValueMetadata[] | undefined): readonly DeclaredShape[]; export declare interface DecodedAnchors { /** Indices into the original `numAnchors` axis, in descending confidence order. */ readonly anchorIndices: Int32Array; /** `[k, 4]` boxes in original-image pixel coords, flat row-major xyxy. */ readonly boxesXyxy: Float32Array; /** Predicted class id per survivor. */ readonly classIds: Int32Array; /** Confidence per survivor. */ readonly confidences: Float32Array; } export declare interface DecodedDetection { readonly bbox: BoundingBox; readonly classId: number; readonly confidence: number; } export declare interface DecodedSegmentation { readonly bbox: BoundingBox; readonly classId: number; readonly confidence: number; /** Binary mask cropped to `bbox`. Width/height match `bbox.asIntXyxy()` extents. */ readonly mask: Mask; } /** * Decode an anchor-free YOLO detection output into a list of detections. * * Works for **YOLOv8 / v9 / v10 / v11 / v12** detect heads. * * Expected raw shape: `[1, 4 + numClasses, N]`. `numClasses` is inferred * from the channel count. */ export declare function decodeYolo(output: Float32Array, outputDims: readonly number[], options: DecodeYoloOptions): DecodedDetection[]; /** * Shared YOLO per-anchor decode used by both detection and segmentation * (v8 / v9 / v10 / v11 / v12). * * Only the first `4 + numClasses` channels are read; later channels (e.g. * mask coefficients) are ignored — callers can fetch them via the returned * {@link DecodedAnchors.anchorIndices}. * * @param data Flat per-anchor output, length `channels * numAnchors`. * @param dims Dims as reported by ORT, e.g. `[1, 84, 8400]` (det) or * `[1, 116, 8400]` (seg). The leading batch dim must be 1. */ export declare function decodeYoloAnchors(data: Float32Array, dims: readonly number[], options: DecodeYoloAnchorsOptions): DecodedAnchors; export declare interface DecodeYoloAnchorsOptions { /** Number of class-score channels following the 4 box channels. */ readonly numClasses: number; readonly originalWidth: number; readonly originalHeight: number; readonly padLeft: number; readonly padTop: number; readonly scale: number; readonly confThreshold: number; readonly iouThreshold: number; readonly maxDetections: number; } export declare interface DecodeYoloOptions { readonly originalWidth: number; readonly originalHeight: number; readonly padLeft: number; readonly padTop: number; readonly scale: number; readonly confThreshold: number; readonly iouThreshold: number; readonly maxDetections: number; } /** * Decode YOLO segmentation raw outputs into a list of segmented instances. * * Compatible with YOLOv8-seg / YOLOv11-seg. * * @param perAnchorData Flat `output0`, length `(4 + numClasses + numMaskCoefs) * numAnchors`. * @param perAnchorDims Dims as reported by ORT, e.g. `[1, 116, 8400]`. * @param prototypeData Flat `output1`, length `numMaskCoefs * maskH * maskW`. * @param prototypeDims Dims as reported by ORT, e.g. `[1, 32, 160, 160]`. */ export declare function decodeYoloSeg(perAnchorData: Float32Array, perAnchorDims: readonly number[], prototypeData: Float32Array, prototypeDims: readonly number[], options: DecodeYoloSegOptions): DecodedSegmentation[]; export declare interface DecodeYoloSegOptions { readonly numClasses: number; /** Model input `[width, height]` (post-letterbox). */ readonly inputWidth: number; readonly inputHeight: number; /** Original image `[width, height]`. */ readonly originalWidth: number; readonly originalHeight: number; /** Letterbox horizontal padding in input-tensor pixels. */ readonly padLeft: number; /** Letterbox vertical padding in input-tensor pixels. */ readonly padTop: number; /** Letterbox scale factor. */ readonly scale: number; readonly confThreshold: number; readonly iouThreshold: number; readonly maxDetections: number; /** Probability cutoff applied to the soft mask. Defaults to `0.5`. */ readonly maskThreshold?: number; } /** * Default execution provider preference order for browser ORT. * * `webgpu` is tried first when available; ORT-Web falls back to `wasm` * automatically when WebGPU is not supported by the browser or device. */ export declare const DEFAULT_PROVIDERS: readonly string[]; /** * Resolve a labels specification into an ordered array of class names. * * @throws {@link LabelMapError} if the spec is invalid, the preset is unknown, * or the resolved length disagrees with `numClasses`. */ /** * Pick the fallback label spec for a model that declares no class names. * * The COCO preset is the right default for a stock YOLO export and an * impossible one for anything else: it names exactly 80 classes, so handing it * to a 3-class head makes {@link resolveLabels} throw `Resolved 80 labels but * the model has 3 classes` and the task cannot be created at all. A custom * model without baked-in `names` is an ordinary thing to have — it should come * up as `class_0`, `class_1`, ..., not as a failure. * * @param numClasses Classes the model predicts, or `undefined` when the output * shape does not say. * @returns `"coco"` when the preset can describe the model, otherwise `null`, * which makes {@link resolveLabels} generate `class_N` names. */ export declare function defaultLabels(numClasses: number | undefined): LabelSpec; /** * Detector and classifier running as a single ONNX model. * * Everything the pipeline needs to know about itself — the resolution to * letterbox to, whether it wants the full-resolution image as well, whether its * classifier output still needs a softmax, the class names of both stages — was * written into the file at fusion time and is read back here. Nothing is * restated on the JavaScript side, so nothing can drift out of step with the * Python side that built it. * * @example * ```typescript * const pipeline = await DetectClassify.create("/models/pipeline.onnx"); * const result = (await pipeline.predict("/images/flock.jpg"))[0]; * for (const detection of result) { * console.log(detection.name, detection.conf, detection.classification?.name); * } * ``` */ export declare class DetectClassify extends VisionTask { private readonly _spec; private readonly _labels; private readonly _names; private readonly _classifierLabels; private readonly _classifierNames; private readonly _raiseOnEmpty; private constructor(); private _pipelineCache; /** * Run the model once on zero-filled inputs, paying one-time costs up front. * * Worth more here than on a single-stage task: a fused pipeline is two models * plus the bridge in one graph, so the first inference compiles shaders for * all of it. Calling this while a loading spinner is still up moves that cost * somewhere the user is already waiting. * * @param runs How many warm-up inferences to run. One is enough for WASM; * WebGPU sometimes settles on the second. */ warmup(runs?: number): Promise; /** * The fused preprocessing pipeline, built on first use. * * Lazily, because constructing it allocates canvases: a pipeline built in an * environment without a canvas implementation stays constructible, and only * fails if it is actually asked to preprocess something. */ private get _pipeline(); /** * Load a fused pipeline and resolve both label spaces. * * @param model The fused `.onnx` — a URL, an `ArrayBuffer`, or bytes. * @param options Label overrides plus the usual session options. * @throws {@link FusionError} when the model carries no pipeline metadata, * i.e. it is a plain detector or classifier rather than something * `ort_vision_sdk.compose` produced. */ static create(model: ModelSource, options?: DetectClassifyOptions): Promise; /** The pipeline configuration recorded in the model at fusion time. */ get spec(): FusionSpec; /** The `[width, height]` the detection stage runs at. */ get inputSize(): readonly [number, number]; /** Detector class labels indexed by class id. */ get labels(): readonly string[]; /** Detector class id → class name (matches Ultralytics' `model.names`). */ get names(): Readonly>; /** Classifier class labels indexed by class id. */ get classifierLabels(): readonly string[]; /** Classifier class id → class name. */ get classifierNames(): Readonly>; /** * Alias for {@link predict} — call the pipeline like a torch `nn.Module`. * * Use as `pipeline.call(img)` since JavaScript class instances are not * callable; for direct invocation, prefer `pipeline.predict(img)`. */ call(image: ImageInput, options?: DetectClassifyPredictOptions): Promise; /** * Run the pipeline on a single image. * * The returned envelope carries a {@link Speed} breakdown in `speed`. Its * `inference` figure covers detection *and* classification, since the * pipeline runs them as one graph and no boundary between them is observable * from outside. */ predict(image: ImageInput, options?: DetectClassifyPredictOptions): Promise; /** * Letterbox the image and build the graph's feeds. * * The detector input runs through {@link LetterboxPipeline}, which fuses the * resize, the padding and the HWC-to-CHW float conversion into one * `drawImage` plus one readback, and reuses its output buffer between frames. * That buffer goes straight to ONNX Runtime, so `_pipeline.release()` must not * be called until the run resolves. * * A pipeline fused with `cropSource: "original"` also takes the untouched * image as a second input, plus the scale and padding of the letterbox — that * is what lets the graph undo the letterbox transform internally and crop at * native resolution instead of from the downscaled copy. That one is **not** * letterboxed by definition, so it does not go through the fused path. */ private _preprocess; /** * Map one letterboxed xyxy row back onto the original image. * * The graph always reports boxes in the detector's letterboxed pixel space, * whichever crop source it was fused with, so both sources agree here. */ private _toOriginal; /** * Turn one row of the classifier output into a result object. * * @param row The output row for this detection. * @param image The crop the row describes, carried so callers can display * what was classified. * @param k Optional truncation of the probability list. */ private _classify; } export declare interface DetectClassifyOptions extends OrtSessionOptions { /** * Class label spec for the **detection** stage — see {@link resolveLabels}. * Defaults to the names recorded at fusion time, falling back to the COCO * 80-class preset when the fusion recorded none. */ readonly labels?: LabelSpec; /** * Class label spec for the **classification** stage. Defaults to the recorded * names, falling back to generated `class_` names. */ readonly classifierLabels?: LabelSpec; /** * If `true`, a run that finds nothing throws {@link NoDetectionsError} * instead of returning an empty envelope. Default `false`, because looking * and finding nothing is a successful inference. Turn it on when an empty * result means the surrounding pipeline should stop rather than carry on with * zero rows. Can be overridden per `predict` call. */ readonly raiseOnEmpty?: boolean; } export declare interface DetectClassifyPredictOptions { /** * Drop detections scoring below this. The graph's own NMS threshold was fixed * at fusion time and cannot be lowered here — this only filters further. */ readonly confThreshold?: number; /** If set, keep only detections whose detector `classId` is in this list. */ readonly classes?: readonly number[]; /** Truncate each detection's `classification.probabilities` to its top-k entries. */ readonly topK?: number; /** Override the constructor's `raiseOnEmpty` setting for this call. */ readonly raiseOnEmpty?: boolean; } /** * Per-image envelope for a fused detect→classify pipeline. * * Structurally a {@link DetectionResults} with a second class map: every * detection it yields carries a populated `classification`, and the two stages * have their own, unrelated label spaces — a detector that finds `sheep` * feeding a classifier that answers `famacha_3` shares no class ids with it. * Merging them into one `names` record would make `cls` and * `classification.cls` look comparable when they are not. * * ```typescript * const result = (await pipeline.predict("flock.jpg"))[0]; * for (const detection of result) { * console.log(detection.name, detection.conf, detection.classification?.name); * } * ``` */ export declare class DetectClassifyResults implements Iterable { readonly boxes: Boxes; readonly detections: readonly DetectionResult[]; readonly names: Readonly>; readonly classifierNames: Readonly>; readonly origImg: RGBImage; readonly origShape: readonly [number, number]; readonly path: string | null; readonly speed: Readonly; constructor(boxes: Boxes, detections: readonly DetectionResult[], names: Readonly>, classifierNames: Readonly>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly); /** Number of surviving detections. */ get length(): number; /** Index into the per-instance detections. */ get(index: number): DetectionResult | undefined; [Symbol.iterator](): Iterator; } /** * Infer how many classes a YOLO detection/segmentation head emits. * * Such a head declares `(B, 4 + nc, N)` — four box coordinates stacked above one * score per class, over `N` candidate anchors. `N` is in the thousands and the * batch is 1, so the channel axis is the smallest static axis above 1. * * @param shape Declared shape of the model's first output. * @returns The class count, or `null` when the shape leaves it undeterminable — * fully dynamic, or too small to hold boxes plus at least one class. */ export declare function detectionNumClasses(shape: DeclaredShape): number | null; /** * Single detected object produced by an object-detection model. */ export declare interface DetectionResult { readonly classId: number; readonly className: string; readonly confidence: number; readonly bbox: BoundingBox; /** Alias for `classId` (Ultralytics-style). */ readonly cls: number; /** Alias for `className`. */ readonly name: string; /** Alias for `confidence` (Ultralytics-style). */ readonly conf: number; /** Alias for `bbox` (Ultralytics-style). */ readonly box: BoundingBox; /** * The original image cropped to `bbox`, HWC RGB uint8. Empty boxes * (zero area) yield a zero-sized `RGBImage`. */ readonly croppedImage: RGBImage; /** * What a second, classification stage predicted **for this crop** — * populated only by {@link DetectClassify}, and `null` for a plain detector. * * Kept as its own field rather than folded into `classId`/`className` * because the two answers are different questions: the detector says *what * kind of object this is*, the classifier says *which sub-category the object * belongs to*, and collapsing them would lose one of the two. */ readonly classification?: ClassificationResult | null; } /** * Per-image detection envelope (Ultralytics-style `Results`). * * Iterating yields per-instance {@link DetectionResult} entries, so legacy * code that did `for (const d of detector.predict(img))` only needs an * extra `[0]` to bridge: * * ```typescript * for (const d of (await detector.predict(img))[0]) { * console.log(d.cls, d.conf, d.box.xyxy); * } * ``` * * For numpy-style bulk access, use the `boxes` collection. */ export declare class DetectionResults implements Iterable { readonly boxes: Boxes; readonly detections: readonly DetectionResult[]; readonly names: Readonly>; readonly origImg: RGBImage; readonly origShape: readonly [number, number]; readonly path: string | null; readonly speed: Readonly; constructor(boxes: Boxes, detections: readonly DetectionResult[], names: Readonly>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly); /** Number of surviving detections. */ get length(): number; /** Index into the per-instance detections. */ get(index: number): DetectionResult | undefined; [Symbol.iterator](): Iterator; } /** * Object detector for anchor-free YOLO ONNX models (v8/v9/v10/v11/v12). * * `predict()` returns `Promise` (length 1 for a single * image), mirroring Ultralytics' `YOLO("img.jpg")`. Iterate the envelope for * per-instance dataclasses, or use the bulk `boxes` view (`.xyxy`, `.xywh`, * `.xyxyn`, `.xywhn`, `.cls`, `.conf`). * * @example * ```typescript * const det = await Detector.create("/models/yolov8n.onnx"); * const results = await det.predict("/images/street.jpg"); * const r = results[0]; * console.log(r.boxes.xyxy, r.boxes.cls, r.boxes.conf, r.names); * for (const d of r) { * console.log(d.cls, d.conf, d.box.xyxy); * } * ``` */ export declare class Detector extends VisionTask { private readonly _head; private readonly _labels; private readonly _names; private readonly _inputSize; private readonly _confThreshold; private readonly _iouThreshold; private readonly _maxDetections; private readonly _raiseOnEmpty; private constructor(); private _pipelineCache; /** * Run the model once on a zero-filled tensor, paying one-time costs up front. * * The first inference of a session is not representative: WebGPU compiles its * shaders on it and the WASM backend faults in its arenas, which on a phone * can turn the first frame into seconds while every later frame is tens of * milliseconds. Calling this while a loading spinner is still up moves that * cost somewhere the user is already waiting. * * @param runs How many warm-up inferences to run. One is enough for WASM; * WebGPU sometimes settles on the second. */ warmup(runs?: number): Promise; /** * The fused preprocessing pipeline, built on first use. * * Lazily, because constructing it allocates canvases: a task built in an * environment without a canvas implementation stays constructible, and only * fails if it is actually asked to preprocess something. */ private get _pipeline(); /** Load the model and resolve labels. */ static create(model: ModelSource, options?: DetectorOptions): Promise; /** The decoder family used to interpret the model's output. */ get head(): DetectorHead; /** Class labels indexed by class id. */ get labels(): readonly string[]; /** Class id → class name dict (matches Ultralytics' `model.names`). */ get names(): Readonly>; /** * The `[width, height]` this task preprocesses to. * * Resolved at creation time from the model's graph when it declares a static * input, so reading it back tells you the resolution inference really runs at * — not merely what was requested. */ get inputSize(): readonly [number, number]; /** Number of classes the model predicts. */ get numClasses(): number; /** * Alias for {@link predict} — call the detector like a torch `nn.Module`. * * Use as `det.call(img)` since JavaScript class instances are not callable; * for direct invocation, prefer `det.predict(img)`. The full * {@link DetectorPredictOptions} (including `classes`) is supported. */ call(image: ImageInput, options?: DetectorPredictOptions): Promise; /** * Run detection on a single image. * * The returned envelope carries a {@link Speed} breakdown in `speed`, * mirroring Ultralytics' `results[0].speed`. */ predict(image: ImageInput, options?: DetectorPredictOptions): Promise; /** * Letterbox and pack the image into the tensor the model expects. * * Runs through {@link LetterboxPipeline}, which fuses the resize, the * padding and the HWC-to-CHW float conversion into one `drawImage` plus one * readback loop, and reuses its output buffer between frames. The buffer is * handed straight to ONNX Runtime, so {@link _pipeline.release} must not be * called until the run resolves. */ private _preprocess; private _buildResult; private _buildBoxes; } /** * Decoder family for the model's detection head. * * - `"yolo"`: anchor-free YOLO head with output shape `[1, 4 + nc, N]` — * covers YOLOv8, v9, v10, v11, v12, v26 detect exports. * * The SDK does **not** auto-detect the head from the model — the caller is * responsible for picking a head that matches their export. Future families * (v5/v6/v7 with `[1, N, 5+nc]`) will be added as new literal members. */ export declare type DetectorHead = "yolo"; export declare interface DetectorOptions extends OrtSessionOptions { /** * Decoder family for the detection head. Default `"yolo"` covers * YOLOv8/v9/v10/v11/v12/v26. */ readonly head?: DetectorHead; /** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */ readonly labels?: LabelSpec; /** Number of classes — used to validate the supplied labels. */ readonly numClasses?: number; /** * Model input `[width, height]` in pixels for letterboxing. * * Only used when the model's graph leaves its spatial axes dynamic: a graph * that declares a static size always wins, since that is the only shape ONNX * Runtime will accept. Defaults to `[640, 640]`. */ readonly inputSize?: readonly [number, number]; /** Default minimum class score to keep a candidate. */ readonly confThreshold?: number; /** Default IoU threshold for non-maximum suppression. */ readonly iouThreshold?: number; /** Maximum number of detections per image. */ readonly maxDetections?: number; /** * If `true`, a run that finds nothing throws {@link NoDetectionsError} * instead of returning an empty envelope. Default `false`, because looking * and finding nothing is a successful inference. Turn it on when an empty * result means the surrounding pipeline should stop rather than carry on with * zero rows. Can be overridden per `predict` call. */ readonly raiseOnEmpty?: boolean; } export declare interface DetectorPredictOptions { /** Override the default confidence threshold. */ readonly confThreshold?: number; /** Override the default IoU threshold. */ readonly iouThreshold?: number; /** * If set, keep only detections whose `classId` is in this list. * Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`. */ readonly classes?: readonly number[]; /** Override the constructor's `raiseOnEmpty` setting for this call. */ readonly raiseOnEmpty?: boolean; } /** * Narrow a provider list to the ones this browser can actually offer. * * ORT-Web exposes no equivalent of Node's `session.getProviders()`, so there is * no way to ask which provider a session ended up on. What the browser *does* * answer is whether the underlying API exists at all — no `navigator.gpu`, or no * adapter behind it, means `webgpu` was never going to run — and that covers the * case that actually bites: a page asking for WebGPU on a device without it, * silently getting WASM, and being several times slower than intended with no * error to point at. * * This is best-effort by construction. A provider that survives here can still * fail inside ORT for a reason the browser does not surface (a missing shader * feature, an exhausted device), so a surviving entry means "not ruled out", * not "confirmed running". Anything this function does not know how to test is * kept rather than dropped: guessing a provider away would be worse than * admitting ignorance about it. * * @param requested Providers in preference order, already resolved. * @returns The subset that is not ruled out, in the same order. */ export declare function detectProviders(requested: readonly string[]): Promise; /** * Convert an HWC BGR uint8 buffer (OpenCV layout) to the SDK's HWC RGB. * * Use when you receive image bytes from `cv2.imencode` over the wire and * want to feed them to the SDK without going through a canvas decode. * * @param bgr Flat BGR Uint8Array of length `width * height * 3`. */ export declare function fromCv2(bgr: Uint8Array, width: number, height: number): RGBImage; /** Geometry of a letterbox, plus the planar tensor data it produced. */ export declare interface FusedLetterboxResult { /** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */ readonly data: Float32Array; /** Factor applied to the original image (`< 1` if downscaled). */ readonly scale: number; /** Horizontal padding in pixels. */ readonly padLeft: number; /** Vertical padding in pixels. */ readonly padTop: number; /** * Whether {@link data} is the pipeline's reusable buffer. * * `true` means the next {@link LetterboxPipeline.run} overwrites it, so a * caller keeping the values past its own inference has to copy them. */ readonly reused: boolean; } /** Planar tensor data produced by {@link ResizePipeline}. */ export declare interface FusedResizeResult { /** CHW float32, normalized, length `3 * targetHeight * targetWidth`. */ readonly data: Float32Array; /** * Whether {@link data} is the pipeline's reusable buffer. * * `true` means the next {@link ResizePipeline.run} overwrites it, so a caller * keeping the values past its own inference has to copy them. */ readonly reused: boolean; } /** Value of the `ovs.kind` metadata key for a detector→classifier pipeline. */ export declare const FUSION_KIND_DETECT_CLASSIFY = "detect_classify"; /** * Raised when a model cannot be driven as a fused detect→classify pipeline. * * In the browser this means the file carries no `ovs.*` metadata — it is a * plain detector or classifier rather than something `ort_vision_sdk.compose` * produced — or the graph is missing an output the pipeline contract requires. * Building a pipeline is a Python-side build step; the browser only runs one. */ export declare class FusionError extends OrtVisionError { } /** Everything a fused pipeline declares about how it must be driven. */ export declare interface FusionSpec { /** Pipeline family. Only `"detect_classify"` exists today. */ readonly kind: string; /** `[width, height]` the detector stage expects — the resolution to letterbox to. */ readonly inputSize: readonly [number, number]; /** `[width, height]` every crop is resampled to inside the graph. */ readonly cropSize: readonly [number, number]; /** Which tensor the crops are taken from. */ readonly cropSource: CropSource; /** * Fixed number of rows `K` every output carries, surplus zero-padded and * counted by {@link OUTPUT_NUM_DETECTIONS}. `null` means the graph emits * exactly as many rows as survived NMS. */ readonly maxDetections: number | null; /** Score threshold baked into the graph's NMS node. */ readonly confThreshold: number; /** IoU threshold baked into the graph's NMS node. */ readonly iouThreshold: number; /** Whether the classifier stage emits logits that still need a softmax. */ readonly applySoftmax: boolean; /** Detector class names in class-id order, or `null` when the fusion recorded none. */ readonly detectorNames: readonly string[] | null; /** Classifier class names in class-id order, or `null`. */ readonly classifierNames: readonly string[] | null; /** Version of `ort-vision-sdk` that produced the file. */ readonly sdkVersion: string; /** Whether driving this pipeline requires feeding the full-resolution input. */ readonly needsSourceImage: boolean; } /** Mean that leaves an image untouched — what an Ultralytics classifier expects. */ export declare const IDENTITY_MEAN: readonly [number, number, number]; /** Deviation that leaves an image untouched — what an Ultralytics classifier expects. */ export declare const IDENTITY_STD: readonly [number, number, number]; /** Anything {@link loadImage} accepts as an image input. */ export declare type ImageInput = string | Blob | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | ImageData | RGBImage; /** Raised when an input image cannot be decoded into the canonical format. */ export declare class ImageLoadError extends OrtVisionError { } /** Per-channel RGB mean of ImageNet, the torchvision preprocessing convention. */ export declare const IMAGENET_MEAN: readonly [number, number, number]; /** Per-channel RGB standard deviation of ImageNet. */ export declare const IMAGENET_STD: readonly [number, number, number]; /** Raised when ONNX Runtime fails while executing a model. */ export declare class InferenceError extends OrtVisionError { } /** Name of the fused graph's letterboxed detector input, `[1, 3, H, W]` float32 in `[0, 1]`. */ export declare const INPUT_IMAGE = "images"; /** Name of the `[2]` float32 `[padLeft, padTop]`. Only with `cropSource === "original"`. */ export declare const INPUT_PAD = "letterbox_pad"; /** Name of the `[1]` float32 letterbox scale factor. Only with `cropSource === "original"`. */ export declare const INPUT_SCALE = "letterbox_scale"; /** Name of the full-resolution input. Present only when `cropSource === "original"`. */ export declare const INPUT_SOURCE = "source_image"; /** * Whether a measured luminance clears a brightness threshold. * * `threshold` is intentionally required — a sensible value is * application-specific (it depends on the model, the lighting the model was * trained on, and the acceptable false-reject rate), so the SDK does not bake * in a default. * * @param luminance - measured mean luminance in `0..255`. * @param threshold - minimum acceptable luminance in `0..255`. * @returns `true` when `luminance >= threshold`. */ export declare function isLuminanceAcceptable(luminance: number, threshold: number): boolean; /** * Whether a metadata map came out of `YOLO(...).export(format="onnx")`. * * Every Ultralytics export stamps `author` and `task` into its metadata, and the * pair is unambiguous: `"Ultralytics"` plus `"classify"` is a classification head * from that codebase and nothing else. * * @param metadata The model's custom metadata map. */ export declare function isUltralyticsClassifier(metadata: Readonly>): boolean; /** Raised when class labels cannot be resolved from the supplied spec. */ export declare class LabelMapError extends OrtVisionError { } /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Class label resolution: presets, lists, dicts, or auto-generated. * * Tasks call {@link resolveLabels} once at construction time to turn whatever * the caller passed (preset name, array, dict, or `null`) into an ordered * array of class names indexed by class id. * * In the browser there is no filesystem, so this module does not load labels * from a path — fetch the file yourself and pass an array. */ /** * Anything accepted by {@link resolveLabels}. * * - `string[]` / `readonly string[]`: explicit names indexed by class id. * - `Record`: sparse mapping (gaps filled with `class_`). * - `string`: a preset name (e.g. `"coco"`). * - `null` / `undefined`: auto-generate `class_0` ... `class_{numClasses-1}`. */ export declare type LabelSpec = readonly string[] | Record | string | null | undefined; /** * Resize preserving aspect ratio, padding to `(targetWidth, targetHeight)` * with a constant fill color. * * Standard YOLO preprocessing — returning `scale` and `padLeft`/`padTop` * lets callers map detections back to the original image coordinates. */ export declare function letterbox(image: RGBImage, targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]): LetterboxResult; /** * Reusable letterbox → tensor pipeline for one target resolution. * * Holds a target canvas and an output buffer across calls, so a steady stream * of frames at the same size allocates nothing. Create one per task, not per * frame. */ export declare class LetterboxPipeline { private readonly _targetWidth; private readonly _targetHeight; private readonly _fill; private readonly _target; private readonly _targetContext; private readonly _buffer; private _source; private _sourceContext; /** * @param targetWidth Model input width in pixels. * @param targetHeight Model input height in pixels. * @param fill RGB padding colour; defaults to YOLO grey. */ constructor(targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]); /** The `[width, height]` this pipeline letterboxes into. */ get targetSize(): readonly [number, number]; /** * Letterbox an image and write it as planar float32. * * The returned buffer is reused between calls unless a previous result is * still checked out — {@link release} marks it free again. A second `run` * before the first is released allocates a fresh buffer rather than * corrupting it, so concurrent `predict()` calls on one task stay correct at * the cost of the allocation they were trying to avoid. A buffer that was * detached by a consumer that transferred it is replaced rather than written * into — see {@link ReusableBuffer}. * * @param image Source image in the SDK's canonical HWC RGB layout. */ run(image: RGBImage): FusedLetterboxResult; /** * Mark the reusable buffer free again. * * Call it once the tensor built from a {@link run} result has been handed to * ONNX Runtime and the run has resolved — after that the values are inside * the WASM heap and the buffer can be overwritten. */ release(): void; /** * Grow the scratch source canvas to fit an image, reusing it when possible. * * A canvas is only reallocated when a frame arrives at a different size than * the last one, which for a camera or video source is never after the first. * * @param width Source width in pixels. * @param height Source height in pixels. */ private _ensureSource; } export declare interface LetterboxResult { /** The padded image at the target size. */ readonly image: RGBImage; /** The factor applied to the original image (`< 1` if downscaled). */ readonly scale: number; /** Horizontal padding in pixels (left side; right side has the same or +1). */ readonly padLeft: number; /** Vertical padding in pixels (top side). */ readonly padTop: number; } /** * Letterbox an image into planar float32 without keeping any state. * * The allocation-free path is {@link LetterboxPipeline}; this is the one-shot * form, for a caller who wants the fused behaviour without owning a pipeline. * * @param image Source image in the SDK's canonical HWC RGB layout. * @param targetWidth Model input width in pixels. * @param targetHeight Model input height in pixels. * @param fill RGB padding colour; defaults to YOLO grey. */ export declare function letterboxToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]): FusedLetterboxResult; /** * Load an image from any supported source into a HWC uint8 RGB array. * * @throws {@link ImageLoadError} if the source cannot be decoded or has an unsupported shape. */ export declare function loadImage(source: ImageInput): Promise; /** * Error raised when a captured frame is too dark to be analysed reliably. * Carries the measured luminance and the threshold it failed so callers can * surface actionable feedback. */ export declare class LowLuminanceError extends Error { /** Measured mean luminance, `0..255`. */ readonly luminance: number; /** Threshold that was checked against, `0..255`. */ readonly threshold: number; /** * @param luminance - the measured mean luminance in `0..255`. * @param threshold - the threshold the measurement failed to reach. */ constructor(luminance: number, threshold: number); } /** * Longest edge (in pixels) the source is downsampled to before sampling. * Averaging over a small downsample is statistically equivalent for a * brightness threshold and orders of magnitude faster than reading every pixel * of a full-resolution camera frame. */ export declare const LUMINANCE_SAMPLE_MAX_EDGE = 256; /** * Drawable source we can sample luminance from. * * The list tracks what `CanvasRenderingContext2D.drawImage` accepts and we can * read a pixel size off, which is what the implementation actually needs. * `ImageBitmap` matters for the decode-downscaled path: `createImageBitmap(blob, * { resizeWidth })` is how a caller avoids materialising a full-resolution * phone photo, and the frame it hands back is the frame whose brightness has to * be checked. */ export declare type LuminanceSource = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | ImageBitmap | OffscreenCanvas; /** * Single-channel binary or grayscale mask, laid out row-major. * * `data.length` must equal `width * height`. For binary masks, values are * `0` (background) or `255` (foreground); soft masks may use the full * `[0, 255]` range. */ export declare class Mask { readonly data: Uint8Array; readonly width: number; readonly height: number; constructor(data: Uint8Array, width: number, height: number); } /** * Per-instance binary masks for a single image. * * Each mask is cropped to its instance's bounding box. To paint masks onto * a full-image canvas, use `xyxy[i]` as the top-left target. */ export declare class Masks { readonly data: ReadonlyArray<{ readonly data: Uint8Array; readonly width: number; readonly height: number; }>; readonly xyxy: Float32Array; readonly origShape: readonly [number, number]; /** * @param data Per-instance binary masks (`Mask` objects from `types.ts`). * @param xyxy Flat `[N, 4]` of bounding-box coordinates in original pixels. * @param origShape `[height, width]` of the original image. */ constructor(data: ReadonlyArray<{ readonly data: Uint8Array; readonly width: number; readonly height: number; }>, xyxy: Float32Array, origShape: readonly [number, number]); /** Number of instance masks. */ get length(): number; /** `[N]` shape of the masks collection. */ get shape(): readonly [number]; [Symbol.iterator](): Iterator<{ readonly data: Uint8Array; readonly width: number; readonly height: number; }>; } /** * Namespace for every metadata key the fusion writes. * * Namespaced on purpose: the detector's own Ultralytics metadata (`names`, * `task`, `imgsz`) is carried over into the fused model, and an un-prefixed key * would either collide with it or be mistaken for it. */ export declare const METADATA_PREFIX = "ovs."; /** Raised when an ONNX model cannot be loaded into an inference session. */ export declare class ModelLoadError extends OrtVisionError { } /** * Read the class names an export baked into the model metadata. * * Ultralytics writes `names` as the Python `repr` of a `dict[int, str]` — e.g. * `"{0: 'deworm', 1: 'not_deworm'}"`. The value is parsed structurally (never * evaluated), and anything unparseable, non-`dict`, or not keyed by contiguous * integers from zero is rejected whole rather than half-applied: a partial name * map would silently mislabel predictions. * * @param metadata A model's custom metadata map. * @returns Class names in class-id order, or `null` when the model carries no * usable `names` entry. */ export declare function modelNames(metadata: Readonly> | undefined): readonly string[] | null; /** Anything `InferenceSession.create` accepts. */ export declare type ModelSource = string | ArrayBufferLike | Uint8Array; /** * Greedy non-maximum suppression on axis-aligned bounding boxes. * * Mirrors `torchvision.ops.nms` (keeps boxes with the highest score, drops * any subsequent box whose IoU exceeds the threshold). * * @param boxes Flat array of length `4 * N` in xyxy order: `[x1,y1,x2,y2, ...]`. * @param scores Detection score per box, length `N`. * @param iouThreshold Boxes with IoU above this threshold relative to a kept box are suppressed. * @returns Indices of kept boxes, in descending score order. Boxes tied on * score are visited lowest-index first, so the survivor of a tie is * deterministic and matches both `torchvision` and the Python SDK. */ export declare function nms(boxes: Float32Array, scores: Float32Array, iouThreshold: number): Int32Array; /** * Raised when a detection task finds nothing and was asked to treat that as an error. * * Only raised when the caller opts in with `raiseOnEmpty: true`. The default * stays an empty result, because "the model looked and found nothing" is a * successful inference, not a failure — a photo of an empty field is a valid * photo. What the flag is for is the opposite situation: a pipeline step whose * *precondition* is that something is there, where an empty result means the * caller should stop rather than quietly carry on with zero rows. * * "Nothing was detected" and "nothing was confident enough" are the same * condition here, since the confidence threshold is what decides what counts as * a detection in the first place. */ export declare class NoDetectionsError extends OrtVisionError { } /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Which preprocessing a classifier expects its input to have had. * * A classifier is trained on a specific tensor, and feeding it a differently * prepared one degrades it silently — no exception, no warning, just worse * predictions. The two families this SDK sees most disagree completely: * torchvision-style models want the ImageNet mean and deviation subtracted and * divided out, while an Ultralytics classification head consumes raw `[0, 1]`. * * Guessing wrong is not detectable from the outside, but it is not a guess: an * Ultralytics export stamps `author` and `task` into its own metadata, and every * task in this SDK already reads that map for the class names. * * Mirrors `ort_vision_sdk.normalization` in the Python SDK; the two must agree, * because the same model file is driven by both. */ /** * Which preprocessing the classifier expects its input to have had. * * - `"auto"` (the default wherever it is accepted) reads the model's own export * metadata and picks. An Ultralytics classification head gets * `"ultralytics"`; anything else gets `"imagenet"`. * - `"imagenet"` subtracts the ImageNet mean and divides by the ImageNet * deviation — the torchvision convention. * - `"ultralytics"` leaves the image in `[0, 1]`. Ultralytics' own classifier * applies no mean/std at all, so anything else feeds it images it never saw * in training. * - `"none"` is the same arithmetic as `"ultralytics"` — identity — under a * name that says "this model wants raw `[0, 1]`" rather than naming a vendor. */ export declare type Normalization = "auto" | "imagenet" | "ultralytics" | "none"; /** * Convert a uint8 image to a normalized float32 array (HWC layout preserved). * * Applies `(pixel * scale - mean) / std` channel-wise. */ export declare function normalize(image: RGBImage, mean: readonly [number, number, number], std: readonly [number, number, number], scale?: number): Float32Array; /** * Wrap an ONNX Runtime Web `InferenceSession` with convenient metadata access. * * The wrapper exposes input/output names and the shapes the graph declares, * manages execution-provider selection, provides a typed {@link OrtSession.run} * method, and releases the native session through {@link OrtSession.release}. */ export declare class OrtSession { private readonly _session; /** * Execution providers this session is expected to run on. * * The requested list narrowed to what this browser can actually offer — a * `webgpu` entry survives only where an adapter exists. Best-effort: ORT-Web * exposes no way to ask which provider a session ended up on, so an entry * here means "not ruled out", not "confirmed". See * {@link requestedProviders} for what was asked for. * * When nothing survives — a caller asking for `webgpu` alone on a device * without it — this falls back to {@link FALLBACK_PROVIDER}, which ORT-Web * can always run. Handing ORT the unsatisfiable list instead makes * `InferenceSession.create` reject with "no available backend found", so the * page gets no inference at all rather than the slow-but-working fallback * the `console.warn` describes. Measured in a real Chromium, where * `navigator.gpu` exists but yields no adapter. */ readonly providers: readonly string[]; private readonly _metadata; /** * Execution providers that were asked for, after defaults were applied. * * Kept separate because ORT-Web falls back silently: a page that asks for * `webgpu` on a device without it runs on WASM and is told nothing. */ readonly requestedProviders: readonly string[]; private constructor(); /** * Load an ONNX model into an ORT inference session. * * The metadata map is read **before** the session is built, and that order is * load-bearing on memory-constrained devices. ORT copies the model into its * WASM heap and then allocates the graph and the weights on top of that copy; * a `readModelMetadata` call placed after `InferenceSession.create` keeps the * JavaScript-side buffer reachable across the whole build, so a 5 MB model * costs 5 MB of JS heap plus 5 MB of WASM heap plus the weights at the same * instant. Reading first makes the buffer collectable as soon as ORT has copied * it — on a phone that was the difference between a session and * `Can't create a session. failed to allocate a buffer of size N`. * * @param model Either a URL string, or a `Uint8Array`/`ArrayBuffer` containing the model bytes. * @param options Provider list, pass-through `SessionOptions`, and whether to * read the model's metadata map (see {@link OrtSessionOptions.readMetadata}). * @throws {@link ModelLoadError} if the model cannot be loaded. */ static create(model: ModelSource, options?: OrtSessionOptions): Promise; /** Names of the model's inputs, in declaration order. */ get inputNames(): readonly string[]; /** Name of the first (and usually only) input. */ get inputName(): string; /** Names of the model's outputs, in declaration order. */ get outputNames(): readonly string[]; /** * Shapes the graph declares for its inputs, in declaration order. * * Dynamic (symbolic) axes appear as `null`. Empty shapes mean the runtime * reported no metadata — either a non-tensor input, or an `onnxruntime-web` * older than 1.21, which predates input metadata. */ get inputShapes(): readonly DeclaredShape[]; /** * Shape the graph declares for its first input, dynamic axes as `null`. * * Empty when the runtime reports no metadata for it. */ get inputShape(): DeclaredShape; /** * Shapes the graph declares for its outputs, in declaration order. * * Dynamic (symbolic) axes appear as `null`. Reading them is how a task can * tell how many classes a head emits without being told. */ get outputShapes(): readonly DeclaredShape[]; /** * Shape the graph declares for its first output, dynamic axes as `null`. * * Empty when the runtime reports no metadata for it. */ get outputShape(): DeclaredShape; /** * The model's custom metadata map — `names`, `task`, `imgsz`, ... for an * Ultralytics export. * * Read from the model's bytes at load time, since the runtime does not expose * it. Empty when the session was created with `readMetadata: false`, from a * URL that could not be fetched here, or from a model carrying no metadata. */ get metadata(): Readonly>; /** * Release the native session and free its memory. * * Call it when a session is discarded while the page lives on — rebuilding a * task at a different input size, swapping in a newer model. A failure from * the runtime is ignored: a session being torn down has nothing left to fail * at, and the caller is already moving on. */ release(): Promise; /** The underlying `onnxruntime-web` session, for advanced use cases. */ get raw(): ort.InferenceSession; /** * Run inference and return all outputs. * * @param feeds Map of input name to `ort.Tensor`. Keys must match {@link inputNames}. * @throws {@link InferenceError} if ORT raises any error during execution. */ run(feeds: Record): Promise>; } export declare interface OrtSessionOptions { /** * Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}. * * Naming one explicitly also opts into a `console.warn` when this browser * cannot offer it, instead of falling back in silence. */ readonly providers?: readonly string[]; /** Optional ORT session options forwarded to `InferenceSession.create`. */ readonly sessionOptions?: ort.InferenceSession.SessionOptions; /** * Whether to read the model's custom metadata map (`names`, `task`, `imgsz`). * Defaults to `true`. * * The runtime does not expose that map, so it is read from the file itself — * which means a URL model is fetched here and handed to ORT as bytes instead * of letting ORT fetch it. That is the same single download either way, and * it is what lets a task resolve its labels off the model. Set to `false` to * keep the URL path untouched and leave {@link OrtSession.metadata} empty. * * `false` is also the escape hatch when a device cannot afford the bytes: the * fetched buffer is dropped before ORT builds the graph (see * {@link OrtSession.create}), but ORT's own load path still keeps the model out * of reach of anything the SDK holds. A session built this way resolves its * input size from the graph as usual — only the class names are lost, so a * caller taking this route has to pass `labels` itself. */ readonly readMetadata?: boolean; } /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Exceptions raised by the SDK. * * All exceptions inherit from {@link OrtVisionError}, so callers can catch * the base class to handle any SDK-originated failure uniformly. */ export declare class OrtVisionError extends Error { constructor(message: string, options?: ErrorOptions); } /** * Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels. * * A file fused by `ort-vision-sdk` 0.9.0 or later reports the box that was * actually classified: clamped to the image the crop came from, exactly as * RoiAlign received it. Older files report the raw box, so one that ran off the * frame draws a rectangle wider than the region the classifier saw. */ export declare const OUTPUT_BOXES = "boxes"; /** Name of the `[K]` int64 detector-class output. */ export declare const OUTPUT_CLASSES = "classes"; /** Name of the `[1]` int64 output holding how many of the `K` rows are real. */ export declare const OUTPUT_NUM_DETECTIONS = "num_detections"; /** Name of the `[K, numClassifierClasses]` float32 classifier output, one row per box. */ export declare const OUTPUT_PROBS = "probs"; /** Name of the `[K]` float32 detection-confidence output. */ export declare const OUTPUT_SCORES = "scores"; /** * Parse a `repr`-encoded `dict[int, str]` class map. * * Split out of {@link modelNames} because the same encoding is reused by a * fused pipeline, which carries one class map per stage and therefore cannot * store both under the single `names` key Ultralytics uses. * * @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`. * @returns Class names in class-id order, or `null` when the value is missing, * unparseable, not a `dict`, or not keyed by contiguous integers from zero. */ export declare function parseNames(encoded: string | undefined): readonly string[] | null; /** * Top-k classification probabilities for a single image. * * Mirrors Ultralytics' `Probs` interface. */ export declare class Probs { readonly data: Float32Array; /** @param data `[numClasses]` per-class probabilities, indexed by class id. */ constructor(data: Float32Array); /** Number of classes. */ get length(): number; /** `[numClasses]` shape of the underlying vector. */ get shape(): readonly [number]; /** Index of the most probable class. */ get top1(): number; /** Probability of the top-1 class. */ get top1conf(): number; /** * Indices of the top-5 most probable classes, descending. * * The array is the memoised selection itself, not a copy — reading it twice * hands back the same object. Treat it as read-only: writing into it edits * what every later read of this `Probs` returns. */ get top5(): Int32Array; /** * Probabilities of the top-5 classes, descending. * * Shares the memoised selection with {@link Probs.top5}, under the same * read-only caveat. */ get top5conf(): Float32Array; private _cache; /** * Memoised {@link Probs._topK}. * * `top5` and `top5conf` are separate getters over the same selection, so a * caller reading both would otherwise pay for it twice — once per frame, in * a camera loop. The probabilities a `Probs` was built from do not change, * so the result is computed once per `k` and kept. * * @param k - How many classes to select. * @returns The cached selection for that `k`. */ private _top; /** * Select the `k` highest probabilities without ordering the rest. * * A full sort to read five entries out of a thousand-class vector costs * O(n log n) plus an index array the size of the vector; keeping `k` slots * ordered by insertion and scanning once costs O(n·k) with no allocation * beyond the result. Measured on 1000 classes over 2000 iterations, 134.5 µs * against 1.5 µs for the same output. * * Ties keep the lower class index first, matching the stable sort this * replaced: a candidate only displaces an entry it is strictly greater * than. That is also what keeps this selection in step with the Python * SDK's `np.argsort(-data, kind="stable")`, where the sort is C and the * cost this avoids does not arise. * * @param k - How many classes to select. * @returns Indices and probabilities, descending by probability. */ private _topK; } /** Raised when a requested execution provider is not available. */ export declare class ProviderNotAvailableError extends OrtVisionError { } /** * Read a pipeline spec out of a model's custom metadata. * * Individual malformed entries fall back to the value a fusion would have used * by default — a single bad float is not a reason to reject an otherwise * loadable pipeline. A malformed resolution is fatal, because there is no safe * default for one. * * @param metadata A model's custom metadata map, as read by * {@link readModelMetadata}. * @returns The decoded spec, or `null` when the model is not a fused pipeline — * it carries no `ovs.kind` entry, or one naming a pipeline kind this version * does not know how to drive. */ export declare function readFusionSpec(metadata: Readonly> | undefined): FusionSpec | null; /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Read the metadata an exporter baked into a `.onnx` file. * * `onnxruntime-web` exposes input/output metadata but **not** the model's * custom metadata map, which is where Ultralytics writes `names`, `task` and * `imgsz`. The Python SDK gets it for free from * `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the * only way to the same information is to read it out of the file, so this * module walks just enough of the ModelProto wire format to collect * `metadata_props`. * * It never throws and never allocates unbounded: a truncated, hostile or * simply unexpected file yields an empty map, and every caller treats that as * "the model says nothing", falling back to what it was given. */ /** * Collect a model's custom metadata map straight out of its bytes. * * @param model The `.onnx` file contents. * @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an * Ultralytics export — or an empty object when the file carries none or * cannot be walked. */ export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly>; /** * Turn an empty result into an error, when the caller asked for that. * * Shared by every task that can come back with nothing — {@link Detector}, * {@link Segmenter} and {@link DetectClassify} — so the three agree on when * they throw and on what the message says. The message names the two settings * that decide the outcome, because "no detections" on its own leaves the reader * unable to tell a blank image from a threshold set too high. * * @param count How many detections survived every filter. * @param options The flag for this call, the threshold actually applied (after * any per-call override), the class allowlist if one narrowed the search, and * the source path when the input was one. * @throws {@link NoDetectionsError} when the flag is set and `count` is zero. */ export declare function requireDetections(count: number, options: { readonly raiseOnEmpty: boolean; readonly confThreshold: number; readonly classes: readonly number[] | undefined; readonly path: string | null; }): void; /** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */ export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage; /** * Reusable stretch-resize → normalized tensor pipeline for one target size. * * The classification counterpart of {@link LetterboxPipeline}. A classifier * stretches to the model's square input instead of letterboxing into it — no * padding, no scale to invert later, because nothing is mapped back onto the * source image afterwards. That difference is why it cannot simply reuse the * letterbox path. * * What it does share is the technique. The composable route * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two * `Float32Array`s and walks each end to end on every call: about 1.4 MB of * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone * near its memory ceiling can least afford it. Here one `drawImage` resizes, * and one loop reads the resulting RGBA and writes normalized planar float32 * into a buffer held across calls. * * Create one per task, not per frame. */ export declare class ResizePipeline { private readonly _targetWidth; private readonly _targetHeight; private readonly _mean; private readonly _std; private readonly _buffer; private _target; private _targetContext; private _source; private _sourceContext; /** * @param targetWidth Model input width in pixels. * @param targetHeight Model input height in pixels. * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift. * @param std Per-channel RGB standard deviation. Defaults to no scaling. */ constructor(targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]); /** The `[width, height]` this pipeline resizes into. */ get targetSize(): readonly [number, number]; /** * Resize an image to the target size and write it as normalized planar float32. * * An image that already arrives at the target size skips the canvas entirely * and is read straight out of its packed RGB — which is both faster and what * keeps the result identical to `resize()`, whose own fast path returns the * input untouched. * * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when * a consumer detached it by transferring the tensor. * * @param image Source image in the SDK's canonical HWC RGB layout. */ run(image: RGBImage): FusedResizeResult; /** * Mark the reusable buffer free again. * * Call it once the tensor built from a {@link run} result has been handed to * ONNX Runtime and the run has resolved — after that the values are inside * the WASM heap and the buffer can be overwritten. */ release(): void; /** * Build the target canvas on first use. * * Lazily, so a pipeline constructed where no canvas implementation exists * (a Node test, a worker without OffscreenCanvas) only fails if it is asked * to resize something. */ private _ensureTarget; /** Grow the scratch source canvas to fit an image, reusing it when possible. */ private _ensureSource; } /** * Resize an image into normalized planar float32 without keeping any state. * * The allocation-free path is {@link ResizePipeline}; this is the one-shot * form, for a caller who wants the fused behaviour without owning a pipeline. * * @param image Source image in the SDK's canonical HWC RGB layout. * @param targetWidth Model input width in pixels. * @param targetHeight Model input height in pixels. * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift. * @param std Per-channel RGB standard deviation. Defaults to no scaling. */ export declare function resizeToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]): FusedResizeResult; /** What {@link resolveNormalization} settled on. */ export declare interface ResolvedNormalization { /** The preset name, or `"custom"` when the caller supplied the values. */ readonly name: string; /** Per-channel mean to subtract. */ readonly mean: readonly [number, number, number]; /** Per-channel deviation to divide by. */ readonly std: readonly [number, number, number]; } /** * Decide the input size a task will preprocess to. * * Precedence is graph → caller → fallback. The graph wins over an explicit * `inputSize` because a static shape is not a preference, it is what ORT will * accept: honoring the caller there would only turn a fixable mismatch into a * failed run. A disagreement is a configuration bug in the caller, so it is * reported through `console.warn` instead of being swallowed. * * @param options Graph shape, requested size and per-task fallback. * @returns The `[width, height]` to preprocess to. */ export declare function resolveInputSize(options: ResolveInputSizeOptions): readonly [number, number]; export declare interface ResolveInputSizeOptions { /** Declared shape of the model's image input, from {@link declaredShapesFrom}. */ readonly graphShape?: DeclaredShape; /** Size the caller asked for, if any. */ readonly requested?: readonly [number, number]; /** Size to use when neither the graph nor the caller pins one. */ readonly fallback: readonly [number, number]; } export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[]; export declare interface ResolveLabelsOptions { /** * Expected number of classes. * * - When `spec` is `null`/`undefined`, this is required to auto-generate names. * - When `spec` is provided, it validates that the resolved length matches. */ readonly numClasses?: number; } /** * Settle which `mean`/`std` to apply, and what to call the choice. * * Explicit `mean`/`std` always win — they are the escape hatch for a model whose * preprocessing neither preset describes. Anything they leave open falls back to * the preset, so passing only a `mean` does not silently reset the deviation * to 1. * * Warns (via `console.warn`) when the model is an Ultralytics export and the * supplied values are not the identity it was trained with. Nothing fails in * that case: the prediction has the right shape and is simply less accurate, * which is exactly why it is worth saying out loud. * * @param metadata The model's custom metadata map, read to detect the family. * @param options The preset asked for, plus any explicit `mean`/`std`. * @throws {RangeError} If `normalization` is not a known preset, or names one * while `mean`/`std` are also supplied — two answers to the same question. */ export declare function resolveNormalization(metadata: Readonly>, options?: { readonly normalization?: Normalization; readonly mean?: readonly [number, number, number]; readonly std?: readonly [number, number, number]; }): ResolvedNormalization; /** * Resolve the execution providers to pass to `InferenceSession.create`. * * @param requested Explicit provider list in preference order; `undefined` returns the default. */ export declare function resolveProviders(requested?: readonly string[]): string[]; /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Public output types returned by the SDK's vision tasks. * * These types form the contract between the SDK and its callers. They mirror * the Python `ort-vision-sdk` output dataclasses 1-to-1. * * Naming is intentionally compatible with the Ultralytics / torchvision idiom * (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported * from those projects keeps working with minimal edits. The original verbose * names (`classId`, `className`, `confidence`, `bbox`) are still populated for * backwards compatibility. */ /** * HWC RGB uint8 image — the canonical image format used across the SDK. * * `data.length` must equal `width * height * 3`. The buffer is laid out row * by row, top-to-bottom, with each pixel as `[R, G, B]`. */ export declare class RGBImage { readonly data: Uint8Array; readonly width: number; readonly height: number; constructor(data: Uint8Array, width: number, height: number); } /** * Single segmented instance produced by an instance-segmentation model. * * Mirrors {@link DetectionResult} and adds the per-instance binary mask * plus a "ready-to-display" background-removed crop. */ export declare interface SegmentationResult { readonly classId: number; readonly className: string; readonly confidence: number; readonly bbox: BoundingBox; /** Alias for `classId` (Ultralytics-style). */ readonly cls: number; /** Alias for `className`. */ readonly name: string; /** Alias for `confidence` (Ultralytics-style). */ readonly conf: number; /** Alias for `bbox` (Ultralytics-style). */ readonly box: BoundingBox; /** * Binary mask cropped to `bbox`. Values are `0` (background) or `255` * (foreground). Empty boxes yield a zero-sized `Mask`. */ readonly mask: Mask; /** * The original image cropped to `bbox` with background pixels (where * `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized * `RGBImage`. */ readonly segmentedImage: RGBImage; } /** * Per-image instance-segmentation envelope (Ultralytics-style `Results`). * * Iterating yields per-instance {@link SegmentationResult} entries. `boxes` * and `masks` mirror Ultralytics' bulk-array views. */ export declare class SegmentationResults implements Iterable { readonly boxes: Boxes; readonly masks: Masks; readonly detections: readonly SegmentationResult[]; readonly names: Readonly>; readonly origImg: RGBImage; readonly origShape: readonly [number, number]; readonly path: string | null; readonly speed: Readonly; constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly); /** Number of surviving instances. */ get length(): number; /** Index into the per-instance results. */ get(index: number): SegmentationResult | undefined; [Symbol.iterator](): Iterator; } /** * Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...). * * The model is expected to expose two outputs: * * 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor * predictions (boxes, class scores, mask coefficients). * 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks. * * `predict()` returns `Promise` (length 1 for a * single image), mirroring Ultralytics' API. The envelope exposes: * * - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`). * - `masks`: per-instance binary masks cropped to each box. * - per-instance {@link SegmentationResult} via iteration. * * @example * ```typescript * const seg = await Segmenter.create("/models/yolov8n-seg.onnx"); * const r = (await seg.predict("/images/street.jpg"))[0]; * for (const inst of r) { * console.log(inst.cls, inst.conf, inst.box.xyxy); * } * ``` */ export declare class Segmenter extends VisionTask { private readonly _head; private readonly _labels; private readonly _names; private readonly _inputSize; private readonly _confThreshold; private readonly _iouThreshold; private readonly _maxDetections; private readonly _maskThreshold; private readonly _raiseOnEmpty; private constructor(); private _pipelineCache; /** * Run the model once on a zero-filled tensor, paying one-time costs up front. * * The first inference of a session is not representative: WebGPU compiles its * shaders on it and the WASM backend faults in its arenas, which on a phone * can turn the first frame into seconds while every later frame is tens of * milliseconds. Calling this while a loading spinner is still up moves that * cost somewhere the user is already waiting. * * @param runs How many warm-up inferences to run. One is enough for WASM; * WebGPU sometimes settles on the second. */ warmup(runs?: number): Promise; /** * The fused preprocessing pipeline, built on first use. * * Lazily, because constructing it allocates canvases: a task built in an * environment without a canvas implementation stays constructible, and only * fails if it is actually asked to preprocess something. */ private get _pipeline(); /** Load the model and resolve labels. */ static create(model: ModelSource, options?: SegmenterOptions): Promise; /** The decoder family used to interpret the model's output. */ get head(): SegmenterHead; /** Class labels indexed by class id. */ get labels(): readonly string[]; /** Class id → class name dict (matches Ultralytics' `model.names`). */ get names(): Readonly>; /** * The `[width, height]` this task preprocesses to. * * Resolved at creation time from the model's graph when it declares a static * input, so reading it back tells you the resolution inference really runs at * — not merely what was requested. */ get inputSize(): readonly [number, number]; /** Number of classes the model predicts. */ get numClasses(): number; /** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */ call(image: ImageInput, options?: SegmenterPredictOptions): Promise; /** Run instance segmentation on a single image. */ predict(image: ImageInput, options?: SegmenterPredictOptions): Promise; private _preprocess; private _splitOutputs; private _buildResult; private _buildBoxes; private _buildMasks; } /** * Decoder family for the segmentation head. * * - `"yolo-seg"`: YOLO instance-segmentation head with two outputs — * `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]` * prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg. * * The SDK does **not** auto-detect this — the caller is responsible for * picking a head that matches their export. */ export declare type SegmenterHead = "yolo-seg"; export declare interface SegmenterOptions extends OrtSessionOptions { /** * Decoder family for the segmentation head. Default `"yolo-seg"` covers * YOLOv8-seg/v11-seg/v26-seg. */ readonly head?: SegmenterHead; /** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */ readonly labels?: LabelSpec; /** Number of classes — used to validate the supplied labels. */ readonly numClasses?: number; /** * Model input `[width, height]` in pixels for letterboxing. * * Only used when the model's graph leaves its spatial axes dynamic: a graph * that declares a static size always wins, since that is the only shape ONNX * Runtime will accept. Defaults to `[640, 640]`. */ readonly inputSize?: readonly [number, number]; /** Default minimum class score to keep a candidate. */ readonly confThreshold?: number; /** Default IoU threshold for non-maximum suppression. */ readonly iouThreshold?: number; /** Maximum number of instances per image. */ readonly maxDetections?: number; /** * If `true`, a run that finds nothing throws {@link NoDetectionsError} * instead of returning an empty envelope. Default `false`, because looking * and finding nothing is a successful inference. Turn it on when an empty * result means the surrounding pipeline should stop rather than carry on with * zero rows. Can be overridden per `predict` call. */ readonly raiseOnEmpty?: boolean; /** Probability cutoff applied to soft masks. Defaults to `0.5`. */ readonly maskThreshold?: number; } export declare interface SegmenterPredictOptions { readonly confThreshold?: number; readonly iouThreshold?: number; /** * If set, keep only instances whose `classId` is in this list. * Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`. */ readonly classes?: readonly number[]; /** Override the constructor's `raiseOnEmpty` setting for this call. */ readonly raiseOnEmpty?: boolean; } /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Classification head postprocessing — softmax + top-k. */ /** Apply numerically-stable softmax to a 1-D vector of logits. */ export declare function softmax(logits: Float32Array | readonly number[]): Float32Array; /** * Read the spatial input size out of a declared NCHW shape. * * @param shape The declared shape of the model's image input. * @returns `[width, height]` in pixels, or `null` when the shape is not 4D or * leaves either spatial axis dynamic — in which case the model accepts more * than one resolution and there is nothing to correct. */ export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null; /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */ /** * Per-stage timing for a single `predict()` call. * * Populates the `speed` field every `Results` envelope carries, mirroring * Ultralytics' `results[0].speed`. All values are milliseconds measured with * `performance.now()`. */ /** * Stage durations of one inference, in milliseconds. * * `preprocess`, `inference` and `postprocess` are the three keys Ultralytics * reports, measured over the same boundaries. `load` is specific to this SDK: * `predict()` accepts a URL, `Blob` or DOM element and decodes it internally, * so the fetch/decode cost would otherwise be invisible — and on a cold cache * it dominates everything else. */ export declare interface Speed { /** Fetching and decoding the input into an `RGBImage`. */ load: number; /** Letterbox/resize, normalization and tensor packing. */ preprocess: number; /** The ONNX Runtime forward pass. */ inference: number; /** Decoding raw outputs into results (NMS, mask assembly, top-k). */ postprocess: number; } /** * Accumulate stage durations while a `predict()` call runs. * * Each `stage()` call closes the previous stage: the elapsed time since the * last boundary is attributed to the name given. This keeps the call sites * free of paired start/stop bookkeeping and guarantees the four stages tile * the whole call without gaps. */ export declare class SpeedTimer { private _last; private readonly _speed; constructor(); /** * Attribute the time elapsed since the previous boundary to `stage`. * * @param stage Which stage just finished. */ stage(stage: keyof Speed): void; /** * The accumulated durations. * * @returns The `speed` object to hand to the `Results` envelope. */ speed(): Speed; } /** * Transpose interleaved HWC data to planar CHW layout. * * @param hwc Source array of length `width * height * channels`. */ export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array; /** * Convert the SDK's HWC RGB image to an HWC BGR `Uint8Array` (OpenCV layout). * * Useful for round-tripping data to a Python OpenCV consumer. */ export declare function toCv2(image: RGBImage): Uint8Array; /** Convert a uint8 image to a `Float32Array` in `[0, 1]` (HWC layout preserved). */ export declare function toFloat32(image: RGBImage, scale?: number): Float32Array; /** Wrap a Float32 buffer into an `ort.Tensor`. */ export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor; /** * Return the top-k entries of a 1-D probability vector, sorted descending. * * @param k Number of entries to return; `null` returns all entries. */ export declare function topK(probabilities: Float32Array, k: number | null): TopKResult; export declare interface TopKResult { readonly indices: Int32Array; readonly values: Float32Array; } /** * Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`. * * Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW, * `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that * don't require ImageNet normalization. * * @returns CHW `Float32Array` of length `width * height * 3`. */ export declare function toTensor(image: RGBImage): Float32Array; /** * Acquire a `MediaStream` via `getUserMedia`, attach it to a `