/** * @license * Copyright 2021 Google LLC. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============================================================================= */ import * as handPoseDetection from '@tensorflow-models/hand-pose-detection'; import type { Hand, HandDetector, Keypoint } from '@tensorflow-models/hand-pose-detection'; import * as tf from '@tensorflow/tfjs'; /** * Number of keypoints returned by MediaPipe Hands (wrist + 5 fingers × 4 joints) */ export declare const NUM_HAND_KEYPOINTS = 21; /** * Size of the feature vector: 21 keypoints × (x, y) normalised to [0, 1] */ export declare const HAND_FEATURE_SIZE: number; /** Default input image size used by hand models (same as PoseNet) */ export declare const DEFAULT_IMAGE_SIZE = 257; /** * The metadata to describe the model's creation, * includes the labels associated with the classes * and versioning information from training. */ export interface Metadata { tfjsVersion: string; tmVersion?: string; packageVersion: string; packageName: string; modelName?: string; timeStamp?: string; imageSize?: number; labels: string[]; userMetadata?: unknown; modelSettings: Partial; } export interface HandModelSettings { maxHands: number; modelType: 'lite' | 'full'; detectorModelUrl?: string; landmarkModelUrl?: string; } export declare type ClassifierInputSource = handPoseDetection.HandDetectorInput; export declare const MAX_HANDS = 2; export declare const MULTI_HAND_FEATURE_SIZE: number; /** * Computes the probabilities of the top K classes given logits by sorting * the softmax probabilities. */ export declare function getTopKClasses(labels: string[], logits: tf.Tensor, topK?: number): Promise; export declare class CustomHandPose { model: tf.LayersModel; handDetector: HandDetector; protected _metadata: Metadata; constructor(model: tf.LayersModel, handDetector: HandDetector, metadata: Partial); getMetadata(): Metadata; getLabels(): string[]; getLabel(index: number): string; /** * Run hand detection on the input image. * Returns the first detected hand, its flattened feature array, and the * full list of detected hands. * * The feature array is 42 floats: 21 keypoints × (x/imageWidth, y/imageHeight). * All values are in [0, 1]. If no hand is detected the array is all zeros. * * @param image Input image compatible with MediaPipe Hands * @param flipHorizontal Whether to mirror the result (e.g. webcam stream) * @param staticImageMode Whether to treat the input as a static image (default false). If true, the internal state of the MediaPipe graph is reset between calls so each image is processed independently. If false, the graph maintains state across calls which can improve detection on video streams but may cause inconsistent results on static images. */ estimateHand(image: ClassifierInputSource, flipHorizontal?: boolean, staticImageMode?: boolean): Promise<{ hand: Hand | null; handOutput: Float32Array; allHands: Hand[]; allHandOutputs: Float32Array[]; jointHandOutput: Float32Array; }>; handOutputsToJointArray(allHandOutputs: Float32Array[]): Float32Array; /** * Flatten and normalise hand keypoints into a fixed-size Float32Array. * Each keypoint's x is divided by imageWidth, y by imageHeight → [0, 1]. * * @param keypoints 21 keypoints from a detected hand * @param imageWidth Width of the source image in pixels * @param imageHeight Height of the source image in pixels */ handOutputsToArray(keypoints: Keypoint[], imageWidth: number, imageHeight: number): Float32Array; /** * Given a hand feature vector, return the full probability distribution * across all classes. * @param handOutput Float32Array of size HAND_FEATURE_SIZE */ predict(handOutput: Float32Array): Promise; /** * Given a hand feature vector, return the top-K class predictions. * @param handOutput Float32Array of size HAND_FEATURE_SIZE * @param maxPredictions Maximum number of top predictions (default 3) */ predictTopK(handOutput: Float32Array, maxPredictions?: number): Promise; dispose(): void; } /** * Create and initialise a MediaPipe Hands detector using the TFJS runtime. */ export declare function loadHandDetector(config?: Partial): Promise; /** * Load a trained CustomHandPose model from a URL checkpoint. * @param checkpoint URL of the model.json file * @param metadata Optional metadata URL or Metadata object */ export declare function load(checkpoint: string, metadata?: string | Metadata): Promise; /** * Load a trained CustomHandPose model from local File objects (browser). * @param json model.json File * @param weights model.weights.bin File * @param metadata metadata.json File */ export declare function loadFromFiles(json: File, weights: File, metadata: File): Promise;