import { AnimationAction, AnimationClip, AnimationMixer, Audio, AudioListener, AudioLoader, Euler, Object3D, Quaternion, QuaternionKeyframeTrack, Vector3, VectorKeyframeTrack } from "three"; import { isDevEnvironment } from "../../engine/debug/index.js"; import { Context } from "../../engine/engine_setup.js"; import type { Constructor } from "../../engine/engine_types.js"; import { getParam, resolveUrl } from "../../engine/engine_utils.js"; import { Animator } from "../Animator.js" import { AudioSource } from "../AudioSource.js"; import { GameObject } from "../Component.js"; import type { PlayableDirector } from "./PlayableDirector.js"; import { SignalReceiver } from "./SignalAsset.js"; import * as Models from "./TimelineModels.js"; import type { TimelineBuilder } from "./TimelineBuilder.js"; import { AnimationUtils } from "../../engine/engine_animation.js"; const debug = getParam("debugtimeline"); /** * A TrackHandler is responsible for evaluating a specific type of timeline track. * A timeline track can be an animation track, audio track, signal track, control track etc and is controlled by a {@link PlayableDirector}. */ export abstract class TimelineTrackHandler { director!: PlayableDirector; track!: Models.TrackModel; get muted(): boolean { return this.track.muted; } set muted(val: boolean) { if (val !== this.track.muted) { this.track.muted = val; this.onMuteChanged?.call(this); } } *forEachClip(backwards: boolean = false): IterableIterator { if (!this.track?.clips) return; if (backwards) { for (let i = this.track.clips.length - 1; i >= 0; i--) { yield this.track.clips[i]; } } else { for (const clip of this.track.clips) { yield clip; } } } onEnable?(); onDisable?(); onDestroy?(); abstract evaluate(time: number); onMuteChanged?(); onPauseChanged?(); /** invoked when PlayableDirectory playmode state changes (paused, playing, stopped) */ onStateChanged?(isPlaying: boolean); getClipTime(time: number, model: Models.ClipModel) { return model.clipIn + (time - model.start) * model.timeScale; } getClipTimeNormalized(time: number, model: Models.ClipModel) { return (time - model.start) / model.duration; } evaluateWeight(time: number, index: number, models: Array, isActive: boolean = true) { if (index < 0 || index >= models.length) return 0; const model = models[index]; if (isActive || time >= model.start && time <= model.end) { let weight = 1; const isBlendingWithNext = false; // this blending with next clips is already baked into easeIn/easeOut // if (allowBlendWithNext && index + 1 < models.length) { // const next = models[index + 1]; // const nextWeight = (time - next.start) / (model.end - next.start); // isBlendingWithNext = nextWeight > 0; // weight = 1 - nextWeight; // } if (model.easeInDuration > 0) { const easeIn = Math.min((time - model.start) / model.easeInDuration, 1); weight *= easeIn; } if (model.easeOutDuration > 0 && !isBlendingWithNext) { const easeOut = Math.min((model.end - time) / model.easeOutDuration, 1); weight *= easeOut; } return weight; } return 0; } } class AnimationClipOffsetData { clip: AnimationClip; rootPositionOffset?: Vector3; rootQuaternionOffset?: Quaternion; get hasOffsets(): boolean { return this.rootPositionOffset !== undefined || this.rootQuaternionOffset !== undefined; } // not necessary rootStartPosition?: Vector3; rootEndPosition?: Vector3; rootStartQuaternion?: Quaternion; rootEndQuaternion?: Quaternion; constructor(action: AnimationAction) { const clip = action.getClip(); this.clip = clip; const root = action.getRoot(); const rootPositionTrackName = root.name + ".position"; const rootRotationTrackName = root.name + ".quaternion"; if (debug) console.log(clip.name, clip.tracks, rootPositionTrackName); for (const track of clip.tracks) { if (track.times.length <= 0) continue; if (track.name.endsWith(rootPositionTrackName)) { this.rootStartPosition = new Vector3().fromArray(track.values, 0); this.rootEndPosition = new Vector3().fromArray(track.values, track.values.length - 3); this.rootPositionOffset = this.rootEndPosition.clone().sub(this.rootStartPosition); if (debug) console.log(this.rootPositionOffset); // this.rootPositionOffset.set(0, 0, 0); } else if (track.name.endsWith(rootRotationTrackName)) { this.rootStartQuaternion = new Quaternion().fromArray(track.values, 0); this.rootEndQuaternion = new Quaternion().fromArray(track.values, track.values.length - 4); this.rootQuaternionOffset = this.rootEndQuaternion.clone().multiply(this.rootStartQuaternion); if (debug) { const euler = new Euler().setFromQuaternion(this.rootQuaternionOffset); console.log("ROT", euler); } } } } } // TODO: add support for clip clamp modes (loop, pingpong, clamp) export class TimelineAnimationTrack extends TimelineTrackHandler { /** @internal */ models: Array = []; /** @internal */ trackOffset?: Models.TrackOffset; /** The object that is being animated. */ target?: Object3D; /** The AnimationMixer, should be shared with the animator if an animator is bound */ mixer?: AnimationMixer; clips: Array = []; actions: Array = []; /** * You can use the weight to blend the timeline animation tracks with multiple animation tracks on the same object. * @default 1 */ weight: number = 1; /** holds data/info about clips differences */ private _actionOffsets: Array = []; private _didBind: boolean = false; private _animator: Animator | null = null; onDisable() { // if this track is disabled we need to stop the currently active actions this.mixer?.stopAllAction(); } onDestroy() { this.director.context.animations.unregisterAnimationMixer(this.mixer); } // Using this callback instead of onEnable etc // because we want to re-enable the animator when the director is at the end and wrap mode is set to none // in which case the director is stopped (but not disabled) // which means we want to notify the object that it's not animated anymore // and the animator can then take over onStateChanged() { if (this._animator) { // We can not check the *isPlaying* state here because the timeline might be paused and evaluated by e.g. ScrollFollow AnimationUtils.setObjectAnimated(this._animator.gameObject, this, this.director.enabled && this.director.weight > 0); } } createHooks(clipModel: Models.AnimationClipModel, clip) { if (clip.tracks?.length <= 0) { console.warn("No tracks in AnimationClip", clip); return; } // we only want to hook into the binding of the root object let foundPositionTrack: boolean = false; let foundRotationTrack: boolean = false; const parts = clip.tracks.find(t => t.name.includes(".position") || t.name.includes(".quaternion"))?.name.split("."); if (parts) { const rootName = parts[parts.length - 2]; const positionTrackName = rootName + ".position"; const rotationTrackName = rootName + ".quaternion"; for (const t of clip.tracks) { if (!foundPositionTrack && t.name.endsWith(positionTrackName)) { foundPositionTrack = true; this.createPositionInterpolant(clip, clipModel, t); } else if (!foundRotationTrack && t.name.endsWith(rotationTrackName)) { foundRotationTrack = true; this.createRotationInterpolant(clip, clipModel, t); } } } // ensure we always have a position and rotation track so we can apply offsets in interpolator // TODO: this currently assumes that there is only one root always that has offsets so it only does create the interpolator for the first track which might be incorrect. In general it would probably be better if we would not create additional tracks but apply the offsets for these objects elsewhere!? if (!foundPositionTrack || !foundRotationTrack) { const root = this.mixer?.getRoot() as Object3D; const track = clip.tracks[0]; const indexOfProperty = track.name.lastIndexOf("."); const baseName = track.name.substring(0, indexOfProperty); const objName = baseName.substring(baseName.lastIndexOf(".") + 1); const targetObj = root.getObjectByName(objName); // TODO can't animate unnamed objects which use GUID as name this way, need scene.getObjectByProperty('uuid', objectName); // This should be right but needs testing: // const parsedPath = PropertyBinding.parseTrackName(track.name); // const targetObj = PropertyBinding.findNode(root, parsedPath.nodeName); if (targetObj) { if (!foundPositionTrack) { const trackName = baseName + ".position"; if (debug) console.warn("Create position track", objName, targetObj); // apply initial local position so it doesnt get flipped or otherwise changed const track = new VectorKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 0, 0, 0]); clip.tracks.push(track); this.createPositionInterpolant(clip, clipModel, track); } else if (!foundRotationTrack) { const trackName = clip.tracks[0].name.substring(0, indexOfProperty) + ".quaternion"; if (debug) console.warn("Create quaternion track", objName, targetObj); const track = new QuaternionKeyframeTrack(trackName, [0, clip.duration], [0, 0, 0, 1, 0, 0, 0, 1]); clip.tracks.push(track); this.createRotationInterpolant(clip, clipModel, track); } } } } bind() { if (this._didBind) return; this._didBind = true; if (debug) console.log(this.models); // the object being animated if (this.mixer) this.target = this.mixer.getRoot() as Object3D; else console.warn("No mixer was assigned to animation track") for (const action of this.actions) { const off = new AnimationClipOffsetData(action); this._actionOffsets.push(off); } if (this.target) { // We need to disable the animator component in case it also animates // which overrides the timeline this._animator = GameObject.getComponent(this.target, Animator) ?? null; if (this._animator) { AnimationUtils.setObjectAnimated(this._animator.gameObject, this, true); } } // Clip Offsets for (const model of this.models) { const clipData = model.asset as Models.AnimationClipModel; const pos = clipData.position as any; const rot = clipData.rotation as any; if (pos && pos.x !== undefined) { if (!pos.isVector3) { clipData.position = new Vector3(pos.x, pos.y, pos.z); } if (!rot.isQuaternion) { clipData.rotation = new Quaternion(rot.x, rot.y, rot.z, rot.w); } } } this.ensureTrackOffsets(); } private ensureTrackOffsets() { if (this.trackOffset) { const pos = this.trackOffset.position as any; if (pos) { if (!pos.isVector3) { this.trackOffset.position = new Vector3(pos.x, pos.y, pos.z); } } const rot = this.trackOffset.rotation as any; if (rot) { if (!rot.isQuaternion) { this.trackOffset.rotation = new Quaternion(rot.x, rot.y, rot.z, rot.w); } } } } private _useclipOffsets: boolean = true; private _totalOffsetPosition: Vector3 = new Vector3(); private _totalOffsetRotation: Quaternion = new Quaternion(); private _totalOffsetPosition2: Vector3 = new Vector3(); private _totalOffsetRotation2: Quaternion = new Quaternion(); private _summedPos = new Vector3(); private _tempPos = new Vector3(); private _summedRot = new Quaternion(); private _tempRot = new Quaternion(); private _clipRotQuat = new Quaternion(); evaluate(time: number) { if (this.track.muted) return; if (!this.mixer) return; this.bind(); // if (this._animator && this.director.isPlaying && this.director.weight > 0) this._animator.enabled = false; this._totalOffsetPosition.set(0, 0, 0); this._totalOffsetRotation.set(0, 0, 0, 1); this._totalOffsetPosition2.set(0, 0, 0); this._totalOffsetRotation2.set(0, 0, 0, 1); let activeClips = 0; let blend: number = 0; let didPostExtrapolate = false; let didPreExtrapolate = false; // The total weight is used to blend with the animator controller active states let totalWeight = 0; for (let i = 0; i < this.clips.length; i++) { const model = this.models[i]; const action = this.actions[i]; const clipModel = model.asset as Models.AnimationClipModel; action.weight = 0; const isInTimeRange = time >= model.start && time <= model.end; const preExtrapolation: Models.ClipExtrapolation = model.preExtrapolationMode; const postExtrapolation: Models.ClipExtrapolation = model.postExtrapolationMode; const nextClip = i < this.clips.length - 1 ? this.models[i + 1] : null; let isActive = isInTimeRange; let doPreExtrapolate = false; if (!isActive && !didPostExtrapolate && model.end < time && postExtrapolation !== Models.ClipExtrapolation.None) { // use post extrapolate if its the last clip of the next clip has not yet started if (!nextClip || nextClip.start > time) { isActive = true; didPostExtrapolate = true; } } else if (i == 0 && !isActive && !didPreExtrapolate && model.start > time && preExtrapolation !== Models.ClipExtrapolation.None) { if (!nextClip || nextClip.start < time) { isActive = true; doPreExtrapolate = true; didPreExtrapolate = true; } } if (isActive) { // const clip = this.clips[i]; let weight = this.weight; weight *= this.evaluateWeight(time, i, this.models, isActive); weight *= this.director.weight; let handleLoop = isInTimeRange; if (doPreExtrapolate) { switch (preExtrapolation) { case Models.ClipExtrapolation.Hold: // Nothing to do break; case Models.ClipExtrapolation.Loop: // TODO: this is not correct yet time += model.start; handleLoop = true; break; default: time += model.start; handleLoop = true; break; } } // TODO: handle clipIn again let t = this.getClipTime(time, model); let loops = 0; const duration = clipModel.duration; // This is the actual duration of the clip in the timeline (with clipping and scale) // const clipDuration = (model.end - model.start) * model.timeScale; if (doPreExtrapolate) { if (preExtrapolation === Models.ClipExtrapolation.Hold) { t = 0; } } if (handleLoop) { if (clipModel.loop) { // const t0 = t - .001; loops += Math.floor(t / (duration + .000001)); while (t > duration) { t -= duration; } } } else if (!isInTimeRange) { if (didPostExtrapolate) { switch (postExtrapolation) { case Models.ClipExtrapolation.Hold: t = this.getClipTime(model.end, model); break; case Models.ClipExtrapolation.Loop: t %= duration; break; case Models.ClipExtrapolation.PingPong: const loops = Math.floor(t / duration); const invert = loops % 2 !== 0; t %= duration; if (invert) t = duration - t; break; } } } if (model.reversed === true) action.time = action.getClip().duration - t; else action.time = t; action.timeScale = 0; const effectiveWeight = Math.max(0, weight); action.weight = effectiveWeight; totalWeight += effectiveWeight; action.clampWhenFinished = false; if (!action.isRunning()) action.play(); // console.log(action.time, action.weight); if (this._useclipOffsets) { const totalPosition = activeClips == 0 ? this._totalOffsetPosition : this._totalOffsetPosition2; const totalRotation = activeClips == 0 ? this._totalOffsetRotation : this._totalOffsetRotation2; if (activeClips < 1) blend = 1 - weight; activeClips += 1; const summedPos = this._summedPos.set(0, 0, 0); const tempPos = this._tempPos.set(0, 0, 0); const summedRot = this._summedRot.identity(); const tempRot = this._tempRot.identity(); const clipOffsetRot = clipModel.rotation as Quaternion; if (clipOffsetRot) { this._clipRotQuat.identity(); this._clipRotQuat.slerp(clipOffsetRot, weight); } const offsets = this._actionOffsets[i]; if (offsets.hasOffsets) { for (let i = 0; i < loops; i++) { if (offsets.rootPositionOffset) tempPos.copy(offsets.rootPositionOffset); else tempPos.set(0, 0, 0); tempPos.applyQuaternion(summedRot); if (this._clipRotQuat) tempPos.applyQuaternion(this._clipRotQuat); if (offsets.rootQuaternionOffset) { // console.log(new Euler().setFromQuaternion(offsets.rootQuaternionOffset).y.toFixed(2)); tempRot.copy(offsets.rootQuaternionOffset); summedRot.multiply(tempRot); } summedPos.add(tempPos); } } if (this._clipRotQuat) totalRotation.multiply(this._clipRotQuat); totalRotation.multiply(summedRot); if (clipModel.position) summedPos.add(clipModel.position as Vector3); totalPosition.add(summedPos); } } } if (this._useclipOffsets) { this._totalOffsetPosition.lerp(this._totalOffsetPosition2, blend); this._totalOffsetRotation.slerp(this._totalOffsetRotation2, blend); } if (this["__mixerError"] === undefined && (debug || isDevEnvironment()) && this._animator?.runtimeAnimatorController?.mixer && this.mixer !== this._animator?.runtimeAnimatorController?.mixer) { this["__mixerError"] = true; console.error("AnimationTrack mixer is not shared with the animator controller - this might result in the timeline to not animate properly. Please report a bug to the Needle Engine team!", this); } if (this._animator?.runtimeAnimatorController) { // If the Timeline is running then the timeline track takes control over the animatorcontroller // we calculate the weight left for the animatorcontroller actions const weightLeft = Math.max(0, 1 - totalWeight); this._animator?.runtimeAnimatorController?.update(weightLeft); } else { this.mixer.update(time); } } private createRotationInterpolant(_clip: AnimationClip, _clipModel: Models.AnimationClipModel, track: any) { const createInterpolantOriginal = track.createInterpolant.bind(track); const quat: Quaternion = new Quaternion(); this.ensureTrackOffsets(); const trackOffsetRot: Quaternion | null = this.trackOffset?.rotation as Quaternion; track.createInterpolant = () => { const createdInterpolant: any = createInterpolantOriginal(); const interpolate = createdInterpolant.evaluate.bind(createdInterpolant); // console.log(interpolate); createdInterpolant.evaluate = (time) => { const res = interpolate(time); quat.set(res[0], res[1], res[2], res[3]); quat.premultiply(this._totalOffsetRotation); // console.log(new Euler().setFromQuaternion(quat).y.toFixed(2)); if (trackOffsetRot) quat.premultiply(trackOffsetRot); if (this.director.animationCallbackReceivers) { for (const rec of this.director.animationCallbackReceivers) { rec?.onTimelineRotation?.call(rec, this.director, this.target!, time, quat); } } res[0] = quat.x; res[1] = quat.y; res[2] = quat.z; res[3] = quat.w; return res; }; return createdInterpolant; } } private createPositionInterpolant(clip: AnimationClip, clipModel: Models.AnimationClipModel, track: any) { const createInterpolantOriginal = track.createInterpolant.bind(track); const currentPosition: Vector3 = new Vector3(); this.ensureTrackOffsets(); const trackOffsetRot: Quaternion | null = this.trackOffset?.rotation as Quaternion; const trackOffsetPos: Vector3 | null = this.trackOffset?.position as Vector3; let startOffset: Vector3 | null | undefined = undefined; track.createInterpolant = () => { const createdInterpolant: any = createInterpolantOriginal(); const evaluate = createdInterpolant.evaluate.bind(createdInterpolant); createdInterpolant.evaluate = (time) => { const res = evaluate(time); currentPosition.set(res[0], res[1], res[2]); if (clipModel.removeStartOffset) { if (startOffset === undefined) { startOffset = null; startOffset = this._actionOffsets.find(a => a.clip === clip)?.rootStartPosition?.clone(); } else if (startOffset?.isVector3) { currentPosition.sub(startOffset); } } currentPosition.applyQuaternion(this._totalOffsetRotation); currentPosition.add(this._totalOffsetPosition); // apply track offset if (trackOffsetRot) currentPosition.applyQuaternion(trackOffsetRot); if (trackOffsetPos) { // flipped unity X currentPosition.x -= trackOffsetPos.x; currentPosition.y += trackOffsetPos.y; currentPosition.z += trackOffsetPos.z; } if (this.director.animationCallbackReceivers) { for (const rec of this.director.animationCallbackReceivers) { rec?.onTimelinePosition?.call(rec, this.director, this.target!, time, currentPosition); } } res[0] = currentPosition.x; res[1] = currentPosition.y; res[2] = currentPosition.z; return res; }; return createdInterpolant; } } } const muteAudioTracks = getParam("mutetimeline"); declare type AudioClipModel = Models.ClipModel & { _didTriggerPlay: boolean }; /** * Handles audio playback for a timeline audio track. * * **Runtime mutation:** The track model is read fresh every frame during `evaluate()`. * You can mutate `track.volume`, `clip.start`, `clip.end`, `clip.asset.volume` etc. * at any time — changes take effect on the next frame without rebuilding the timeline. * * **Audio stopping:** Audio clips are automatically stopped when: * - Timeline time moves outside a clip's `[start, end]` range (e.g. jumping or normal playback advancing past a clip) * - The track is muted (via `muted = true`) * - The director is stopped (`director.stop()`) * - The director is paused (`director.pause()`) * - The director is disabled or destroyed */ export class TimelineAudioTrack extends TimelineTrackHandler { models: Array = []; listener!: AudioListener; audio: Array