/** * MIT License * * Copyright (c) 2024 Mika Suominen * adapted by Marc Buils for digipair-xr * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ import { Quaternion } from 'three'; declare class TalkingHead { [key: string]: any; /** * Avatar. * @typedef {Object} Avatar * @property {string} [body] Body form 'M' or 'F' * @property {string} [lipsyncLang] Lip-sync language, e.g. 'fi', 'en' * @property {string} [avatarMood] Initial mood. * @property {boolean} [avatarMute] If true, muted. * @property {numeric} [avatarIdleEyeContact] Eye contact while idle [0,1] * @property {numeric} [avatarIdleHeadMove] Eye contact while idle [0,1] * @property {numeric} [avatarSpeakingEyeContact] Eye contact while speaking [0,1] * @property {numeric} [avatarSpeakingHeadMove] Eye contact while speaking [0,1] * @property {Object[]} [modelDynamicBones] Config for Dynamic Bones feature */ /** * Callback when new subtitles have been written to the DOM node. * @callback subtitlesfn * @param {Object} node DOM node */ /** * Callback when the speech queue processes this marker item. * @callback markerfn */ /** * Audio object. * @typedef {Object} Audio * @property {ArrayBuffer|ArrayBuffer[]} audio Audio buffer or array of buffers * @property {string[]} words Words * @property {number[]} wtimes Starting times of words * @property {number[]} wdurations Durations of words * @property {string[]} [visemes] Oculus lip-sync viseme IDs * @property {number[]} [vtimes] Starting times of visemes * @property {number[]} [vdurations] Durations of visemes * @property {string[]} [markers] Timed callback functions * @property {number[]} [mtimes] Starting times of markers */ /** * Lip-sync object. * @typedef {Object} Lipsync * @property {string[]} visemes Oculus lip-sync visemes * @property {number[]} times Starting times in relative units * @property {number[]} durations Durations in relative units */ /** * @constructor * @param {Object} node DOM element of the avatar * @param {Object} [opt=null] Global/default options */ constructor(opt?: any); /** * Helper that returns the parameter or, if it is a function, its return value. * @param {Any} x Parameter * @return {Any} Value */ valueFn(x: () => any): any; /** * Helper to deep copy and edit an object. * @param {Object} x Object to copy and edit * @param {function} [editFn=null] Callback function for editing the new object * @return {Object} Deep copy of the object. */ deepCopy(x: any, editFn?: any): any; /** * Convert a Base64 MP3 chunk to ArrayBuffer. * @param {string} chunk Base64 encoded chunk * @return {ArrayBuffer} ArrayBuffer */ b64ToArrayBuffer(chunk: string | string[]): ArrayBuffer; /** * Concatenate an array of ArrayBuffers. * @param {ArrayBuffer[]} bufs Array of ArrayBuffers * @return {ArrayBuffer} Concatenated ArrayBuffer */ concatArrayBuffers(bufs: string | any[]): any; /** * Convert PCM buffer to AudioBuffer. * NOTE: Only signed 16bit little endian supported. * @param {ArrayBuffer} buf PCM buffer * @return {AudioBuffer} AudioBuffer */ pcmToAudioBuffer(buf: any): any; /** * Convert internal notation to THREE objects. * NOTE: All rotations are converted to quaternions. * @param {Object} p Pose * @return {Object} A new pose object. */ propsToThreeObjects(p: ArrayLike | { [s: string]: unknown; }): any; /** * Clear 3D object. * @param {Object} obj Object */ clearThree(obj: { children: string | any[]; remove: (arg0: any) => void; geometry: { dispose: () => void; }; material: { [x: string]: { dispose: () => void; }; dispose?: any; }; }): void; /** * Loader for 3D avatar model. * @param {string} avatar Avatar object with 'url' property to GLTF/GLB file. * @param {progressfn} [onprogress=null] Callback for progress */ showAvatar(element: any, gltf: any, avatar: any): Promise; /** * Get view names. * @return {string[]} Supported view names. */ getViewNames(): string[]; /** * Get current view. * @return {string} View name. */ getView(): any; /** * Fit 3D object to the view. * @param {string} [view=null] Camera view. If null, reset current view * @param {Object} [opt=null] Options */ setView(view: string, opt?: any): void; /** * Update avatar pose. * @param {number} t High precision timestamp in ms. */ updatePoseBase(t: number): void; /** * Update avatar pose deltas */ updatePoseDelta(): void; /** * Update morph target values. * @param {number} dt Delta time in ms. */ updateMorphTargets(dt: number): void; /** * Get given pose as a string. * @param {Object} pose Pose * @param {number} [prec=1000] Precision used in values * @return {string} Pose as a string */ getPoseString(pose: ArrayLike | { [s: string]: unknown; }, prec?: number): string; /** * Return pose template property taking into account mirror pose and gesture. * @param {string} key Property key * @return {Quaternion|Vector3} Position or rotation */ getPoseTemplateProp(key: string): any; /** * Change body weight from current leg to another. * @param {Object} p Pose properties * @return {Object} Mirrored pose. */ mirrorPose(p: ArrayLike | { [s: string]: unknown; }): any; /** * Create a new pose. * @param {Object} template Pose template * @param {numeric} [ms=2000] Transition duration in ms * @return {Object} A new pose object. */ poseFactory(template: { props: any; standing: any; }, ms?: number): { template: { props: any; standing: any; }; props: any; }; /** * Set a new pose and start transition timer. * @param {Object} template Pose template, if null update current pose * @param {number} [ms=2000] Transition time in milliseconds */ setPoseFromTemplate(template: { lying: any; standing: any; } | null, ms?: number): void; /** * Get morph target value. * @param {string} mt Morph target * @return {number} Value */ getValue(mt: string | number): any; /** * Set morph target value. * @param {string} mt Morph target * @param {number} val Value * @param {number} [ms=null] Transition time in milliseconds. */ setValue(mt: string, val: number, ms?: null): void; /** * Get mood names. * @return {string[]} Mood names. */ getMoodNames(): string[]; /** * Get current mood. * @return {string[]} Mood name. */ getMood(): any; /** * Set mood. * @param {string} s Mood name. */ setMood(s: any): void; /** * Get morph target names. * @return {string[]} Morph target names. */ getMorphTargetNames(): string[]; /** * Get baseline value for the morph target. * @param {string} mt Morph target name * @return {number} Value, null if not in baseline */ getBaselineValue(mt: string): any; /** * Set baseline for morph target. * @param {string} mt Morph target name * @param {number} val Value, null if to be removed from baseline */ setBaselineValue(mt: string, val: number | null): void; /** * Get fixed value for the morph target. * @param {string} mt Morph target name * @return {number} Value, null if not fixed */ getFixedValue(mt: string): any; /** * Fix morph target. * @param {string} mt Morph target name * @param {number} val Value, null if to be removed */ setFixedValue(mt: string, val: number | null, ms?: null): void; /** * Create a new animation based on an animation template. * @param {Object} t Animation template * @param {number} [loop=false] Number of loops, false if not looped * @param {number} [scaleTime=1] Scale template times * @param {number} [scaleValue=1] Scale template values * @return {Object} New animation object. */ animFactory(t: { name?: string; dt?: any[] | number[] | any[] | number[][]; vs?: { bodyRotateX: number[]; bodyRotateY: number[]; eyesRotateX: number[]; eyesRotateY: number[]; browInnerUp: number[][]; mouthLeft: number[][]; mouthRight: number[][]; eyeContact: number[]; headMove: number[]; } | { bodyRotateX: number[]; bodyRotateY: number[]; eyesRotateX: number[]; eyesRotateY: number[]; browInnerUp: number[][]; mouthLeft: number[][]; mouthRight: number[][]; eyeContact: number[]; headMove: number[]; } | { headRotateY: number[]; headRotateX: any[]; headRotateZ: number[]; } | { headRotateY: any[]; headRotateX: any[]; headRotateZ: (number | null)[]; eyeLookInLeft: (number | null)[]; eyeLookOutLeft: (number | null)[]; eyeLookInRight: (number | null)[]; eyeLookOutRight: (number | null)[]; eyeContact: number[]; } | { eyeContact: number[]; } | { moveto: ({ duration: number; props: { 'LeftHand.quaternion': Quaternion; 'RightHand.quaternion': Quaternion; }; } | { duration: number; props: {}; })[]; }; delay?: number; hasOwnProperty?: any; mood?: any; }, loop?: any, scaleTime?: number, scaleValue?: number): any; /** * Calculate the correct value based on a given time using the given function. * @param {number[]} vstart Start value * @param {number[]} vend End value * @param {number[]} tstart Start time * @param {number[]} tend End time * @param {number[]} t Current time * @param {function} [fun=null] Ease in/out function, null = linear * @return {number} Value based on the given time. */ valueAnimationSeq(vstart: any, vend: any, tstart: number, tend: number, t: number, fun?: any): number; /** * Return gaussian distributed random value between start and end with skew. * @param {number} start Start value * @param {number} end End value * @param {number} [skew=1] Skew * @param {number} [samples=5] Number of samples, 1 = uniform distribution. * @return {number} Gaussian random value. */ gaussianRandom(...args: any[]): any; /** * Create a sigmoid function. * @param {number} k Sharpness of ease. * @return {function} Sigmoid function. */ sigmoidFactory(k: number): (t: number) => number; /** * Convert value from one range to another. * @param {number} value Value * @param {number[]} r1 Source range * @param {number[]} r2 Target range * @return {number} Scaled value */ convertRange(value: number, r1: any[], r2: number[]): number; /** * Animate the avatar. * @param {number} t High precision timestamp in ms. */ animate(t: number): void; /** * Reset all the visemes */ resetLips(): void; /** * Preprocess text for tts/lipsync, including: * - convert symbols/numbers to words * - filter out characters that should be left unspoken * @param {string} s Text * @param {string} lang Language * @return {string} Pre-processsed text. */ lipsyncPreProcessText(s: string, lang: string | number): any; /** * Convert words to Oculus LipSync Visemes. * @param {string} word Word * @param {string} lang Language * @return {Lipsync} Lipsync object. */ lipsyncWordsToVisemes(word: string, lang: string | number): any; /** * Add text to the speech queue. * @param {string} s Text. * @param {Options} [opt=null] Text-specific options for lipsync/TTS language, voice, rate and pitch, mood and mute * @param {subtitlesfn} [onsubtitles=null] Callback when a subtitle is written * @param {number[][]} [excludes=null] Array of [start, end] index arrays to not speak */ speakText(s: any, opt?: any, onsubtitles?: null, excludes?: any): void; /** * Add emoji to speech queue. * @param {string} em Emoji. */ speakEmoji(em: string | number): Promise; /** * Add a break to the speech queue. * @param {numeric} t Duration in milliseconds. */ speakBreak(t: any): Promise; /** * Callback when speech queue processes this marker. * @param {markerfn} onmarker Callback function. */ speakMarker(onmarker: any): Promise; /** * Play background audio. * @param {string} url URL for the audio, stop if null. */ playBackgroundAudio(url: string | URL | Request): Promise; /** * Stop background audio. */ stopBackgroundAudio(): void; /** * Setup the convolver node based on an impulse. * @param {string} [url=null] URL for the impulse, dry impulse if null */ setReverb(url?: null): Promise; /** * Set audio gain. * @param {number} speech Gain for speech, if null do not change * @param {number} [background=null] Gain for background audio, if null do not change * @param {number} [fadeSecs=0] Gradual exponential fade in/out time in seconds */ setMixerGain(speech: number | null, background?: null, fadeSecs?: number): void; /** * Add audio to the speech queue. * @param {Audio} r Audio message. * @param {Options} [opt=null] Text-specific options for lipsyncLang * @param {subtitlesfn} [onsubtitles=null] Callback when a subtitle is written */ speakAudio(r: { words: string | any[]; wtimes: any[]; wdurations: any[]; visemes: string | any[]; vtimes: any[]; vdurations: any[]; markers: string | any[]; mtimes: any[]; audio: any; }, opt?: any, onsubtitles?: null): void; /** * Play audio playlist using Web Audio API. * @param {boolean} [force=false] If true, forces to proceed */ playAudio(force?: boolean): Promise; /** * Take the next queue item from the speech queue, convert it to text, and * load the audio file. * @param {boolean} [force=false] If true, forces to proceed (e.g. after break) */ startSpeaking(force?: boolean): Promise; /** * Speak from tts result. */ speakFromTTS(line: any, data: any): Promise; /** * Pause speaking. */ pauseSpeaking(): void; /** * Stop speaking and clear the speech queue. */ stopSpeaking(): void; /** * Make eye contact. * @param {number} t Time in milliseconds */ makeEyeContact(t: any): void; /** * Look ahead. * @param {number} t Time in milliseconds */ lookAhead(t: any): void; /** * Turn head and eyes to look at the camera. * @param {number} t Time in milliseconds */ lookAtCamera(t: number): void; /** * Turn head and eyes to look at the point (x,y). * @param {number} x X-coordinate relative to visual viewport * @param {number} y Y-coordinate relative to visual viewport * @param {number} t Time in milliseconds */ lookAt(x: any, y: any, t: any): void; /** * Set the closest hand to touch at (x,y). * @param {number} x X-coordinate relative to visual viewport * @param {number} y Y-coordinate relative to visual viewport * @return {Boolean} If true, (x,y) touch the avatar */ touchAt(x: number, y: number): boolean; /** * Talk with hands. * @param {number} [delay=0] Delay in milliseconds * @param {number} [prob=1] Probability of hand movement */ speakWithHands(delay?: number, prob?: number): void; /** * Get slowdown. * @return {numeric} Slowdown factor. */ getSlowdownRate(k: any): any; /** * Set slowdown. * @param {numeric} k Slowdown factor. */ setSlowdownRate(k: any): void; /** * Start animation cycle. */ start(): void; /** * Stop animation cycle. */ stop(): void; /** * Start listening incoming audio. * @param {AnalyserNode} analyzer Analyzer node for incoming audio * @param {Object} [opt={}] Options * @param {function} [onchange=null] Callback function for start */ startListening(analyzer: any, opt?: any, onchange?: null): void; /** * Stop animation cycle. */ stopListening(): void; /** * Play RPM/Mixamo animation clip. * @param {string|Object} url URL to animation file FBX * @param {progressfn} [onprogress=null] Callback for progress * @param {number} [dur=10] Duration in seconds, but at least once * @param {number} [ndx=0] Index of the clip * @param {number} [scale=0.01] Position scale factor */ playAnimation(url: string, onprogress?: null, dur?: number, ndx?: number, scale?: number): Promise; /** * Stop running animations. */ stopAnimation(): void; /** * Play RPM/Mixamo pose. * @param {string|Object} url Pose name | URL to FBX * @param {progressfn} [onprogress=null] Callback for progress * @param {number} [dur=5] Duration of the pose in seconds * @param {number} [ndx=0] Index of the clip * @param {number} [scale=0.01] Position scale factor */ playPose(url: string, onprogress?: null, dur?: number, ndx?: number, scale?: number): Promise; /** * Stop the pose. (Functionality is the same as in stopAnimation.) */ stopPose(): void; /** * Play a gesture, which is either a hand gesture, an emoji animation or their * combination. * @param {string} name Gesture name * @param {number} [dur=3] Duration of the gesture in seconds * @param {boolean} [mirror=false] Mirror gesture * @param {number} [ms=1000] Transition time in milliseconds */ playGesture(...args: any[]): void; /** * Stop the gesture. * @param {number} [ms=1000] Transition time in milliseconds */ stopGesture(ms?: number): void; /** * Cyclic Coordinate Descent (CCD) Inverse Kinematic (IK) algorithm. * Adapted from: * https://github.com/mrdoob/three.js/blob/master/examples/jsm/animation/CCDIKSolver.js * @param {Object} ik IK configuration object * @param {Vector3} [target=null] Target coordinate, if null return to template * @param {Boolean} [relative=false] If true, target is relative to root * @param {numeric} [d=null] If set, apply in d milliseconds */ ikSolve(ik: { iterations?: any; root: any; effector: any; links: any; }, target?: any, relative?: boolean, d?: any): void; } export { TalkingHead };