import * as Comlink from "comlink"; import { NormalizedLandmark, NormalizedLandmarkList } from "@mediapipe/holistic"; import { Nullable, Vector3 } from "@babylonjs/core"; import { KeysMatching, ReadonlyKeys } from "../helper/utils"; import { TransformNodeTreeNode } from "v3d-core-realbits/dist/src/importer/babylon-vrm-loader/src"; import { CloneableResults, FilteredLandmarkVector, CloneableFMResults } from "../helper/landmark"; import { BoneOptions } from "../v3d-web"; export declare class PoseKeyPoints { top_face_oval: FilteredLandmarkVector; left_face_oval: FilteredLandmarkVector; bottom_face_oval: FilteredLandmarkVector; right_face_oval: FilteredLandmarkVector; left_eye_top: FilteredLandmarkVector; left_eye_bottom: FilteredLandmarkVector; left_eye_inner: FilteredLandmarkVector; left_eye_outer: FilteredLandmarkVector; left_eye_inner_secondary: FilteredLandmarkVector; left_eye_outer_secondary: FilteredLandmarkVector; left_iris_top: FilteredLandmarkVector; left_iris_bottom: FilteredLandmarkVector; left_iris_left: FilteredLandmarkVector; left_iris_right: FilteredLandmarkVector; right_eye_top: FilteredLandmarkVector; right_eye_bottom: FilteredLandmarkVector; right_eye_inner: FilteredLandmarkVector; right_eye_outer: FilteredLandmarkVector; right_eye_inner_secondary: FilteredLandmarkVector; right_eye_outer_secondary: FilteredLandmarkVector; right_iris_top: FilteredLandmarkVector; right_iris_bottom: FilteredLandmarkVector; right_iris_left: FilteredLandmarkVector; right_iris_right: FilteredLandmarkVector; mouth_top_first: FilteredLandmarkVector; mouth_top_second: FilteredLandmarkVector; mouth_top_third: FilteredLandmarkVector; mouth_bottom_first: FilteredLandmarkVector; mouth_bottom_second: FilteredLandmarkVector; mouth_bottom_third: FilteredLandmarkVector; mouth_left: FilteredLandmarkVector; mouth_right: FilteredLandmarkVector; } export type PosesKey = keyof Omit | ReadonlyKeys>; export declare class Poses { static readonly FACE_MESH_CONNECTIONS: import("@mediapipe/holistic").LandmarkConnectionArray[]; static readonly HAND_BASE_ROOT_NORMAL: Vector3; private static readonly HAND_POSITION_SCALING; private static readonly IRIS_MP_X_RANGE; private static readonly IRIS_MP_Y_RANGE; private static readonly IRIS_BJS_X_RANGE; private static readonly IRIS_BJS_Y_RANGE; private static readonly BLINK_RATIO_LOW; private static readonly BLINK_RATIO_HIGH; private static readonly MOUTH_MP_RANGE_LOW; private static readonly MOUTH_MP_RANGE_HIGH; private static readonly EYE_WIDTH_BASELINE; private static readonly MOUTH_WIDTH_BASELINE; private static readonly LR_FACE_DIRECTION_RANGE; private _boneOptions; updateBoneOptions(value: BoneOptions): void; private readonly _boneRotationUpdateFn; private bonesHierarchyTree; cloneableInputResults: Nullable; cloneableFMInputResults: Nullable; inputPoseLandmarks: NormalizedLandmarkList; private poseLandmarks; private worldPoseLandmarks; cloneablePoseLandmarks: NormalizedLandmarkList; inputFaceLandmarks: NormalizedLandmarkList; private faceLandmarks; private _faceMeshLandmarkIndexList; get faceMeshLandmarkIndexList(): number[][]; private _faceMeshLandmarkList; get faceMeshLandmarkList(): NormalizedLandmarkList[]; private leftWristOffset; inputLeftHandLandmarks: NormalizedLandmarkList; private leftHandLandmarks; cloneableLeftHandLandmarks: NormalizedLandmarkList; private leftHandNormal; private rightWristOffset; inputRightHandLandmarks: NormalizedLandmarkList; private rightHandLandmarks; cloneableRightHandLandmarks: NormalizedLandmarkList; private rightHandNormal; private leftFootNormal; private rightFootNormal; private leftFootBasisRotation; private rightFootBasisRotation; private _keyPoints; get keyPoints(): PoseKeyPoints; private _blinkBase; private _leftBlinkArr; private _rightBlinkArr; private _faceNormal; get faceNormal(): NormalizedLandmark; private _headQuaternion; private _initBoneRotations; private _boneRotations; private textEncoder; private _leftHandNormals; get leftHandNormals(): NormalizedLandmarkList; private _rightHandNormals; get rightHandNormals(): NormalizedLandmarkList; private _poseNormals; get poseNormals(): NormalizedLandmarkList; midHipPos: Nullable; midHipInitOffset: Nullable; midHipOffset: FilteredLandmarkVector; constructor(boneOptions: BoneOptions, boneRotationUpdateFn?: ((data: Uint8Array) => void) & Comlink.ProxyMarked); /** * One time operation to set bones hierarchy from VRMManager * @param tree root node of tree */ setBonesHierarchyTree(tree: TransformNodeTreeNode, forceReplace?: boolean): void; /** * All MediaPipe inputs have the following conventions: * - Left-right mirrored (selfie mode) * - Face towards -Z (towards camera) by default * TODO: interpolate results to 60 FPS. * @param results Result object from MediaPipe Holistic */ process(results: CloneableResults): void; processFaceMesh(results: CloneableFMResults): void; resetBoneRotations(sendResult?: boolean): void; private filterBoneRotations; private getKeyPoints; private calcFaceBones; private calcIrisNormal; private calcFMIrisNormal; private calcExpressions; private calcFMExpressions; private calcPoseBones; /** * thisKey: key in _boneRotations * prevQuaternion: Parent cumulated rotation quaternion * firstQuaternion: Rotation quaternion calculated without applying X rotation * normal: A normal pointing to local -y * thisBasis: basis on this node after prevQuaternion is applied */ private applyXRotationWithChild; private calcWristBones; private calcHandBones; private calcFeetBones; private preProcessResults; private preProcessFMResults; private preProcessLandmarks; private toCloneableLandmarks; private filterFaceLandmarks; private lRLinkWeights; private lRLink; private lRLinkVector; private lRLinkQuaternion; private initHandBoneRotations; private initBoneRotations; private static normalFromVertices; private applyQuaternionChain; private shallUpdateArm; private shallUpdateLegs; private pushBoneRotationBuffer; } export declare const poseWrapper: { poses: typeof Poses; }; export default Poses;