import { AxesHelper, Box3, Cache, Object3D, Vector2, Vector3 } from "three"; import { isDevEnvironment } from "../engine/debug/index.js"; import { AnimationUtils } from "../engine/engine_animation.js"; import { addComponent } from "../engine/engine_components.js"; import { Context } from "../engine/engine_context.js"; import { destroy } from "../engine/engine_gameobject.js"; import { Gizmos } from "../engine/engine_gizmos.js"; import { getLoader } from "../engine/engine_gltf.js"; import { BlobStorage } from "../engine/engine_networking_blob.js"; import { PreviewHelper } from "../engine/engine_networking_files.js"; import { generateSeed, InstantiateIdProvider } from "../engine/engine_networking_instantiate.js"; import { serializable } from "../engine/engine_serialization_decorator.js"; import { fitObjectIntoVolume, getBoundingBox, placeOnSurface } from "../engine/engine_three_utils.js"; import { IGameObject, Model, Vec3 } from "../engine/engine_types.js"; import { getParam, setParamWithoutReload } from "../engine/engine_utils.js"; import { determineMimeTypeFromExtension } from "../engine/engine_utils_format.js"; import { Animation } from "./Animation.js"; import { Behaviour } from "./Component.js"; import { EventList } from "./EventList.js"; /** * Debug mode can be enabled with the URL parameter `?debugdroplistener`, which * logs additional information during drag and drop events and visualizes hit points. */ const debug = getParam("debugdroplistener"); /** * Events dispatched by the DropListener component * @enum {string} */ export enum DropListenerEvents { /** * Dispatched when a file is dropped into the scene. The detail of the event is the {@link File} that was dropped. * The event is called once for each dropped file. */ FileDropped = "file-dropped", /** * Dispatched when a new object is added to the scene. The detail of the event contains {@link DropListenerOnDropArguments} for the content that was added. */ ObjectAdded = "object-added", } /** * Context information for a drop operation */ declare type DropContext = { /** Position where the file was dropped in screen coordinates */ screenposition: Vector2; /** URL of the dropped content, if applicable */ url?: string, /** File object of the dropped content, if applicable */ file?: File; /** 3D position where the content should be placed */ point?: Vec3; /** Size dimensions for the content */ size?: Vec3; } /** * Network event arguments passed between clients when using the DropListener with networking */ export declare type DropListenerNetworkEventArguments = { /** Unique identifier of the sender */ guid: string, /** Name of the dropped object */ name: string, /** URL or array of URLs to the dropped content */ url: string | string[], /** Worldspace point where the object was placed in the scene */ point: Vec3; /** Bounding box size */ size: Vec3; /** MD5 hash of the content for verification */ contentMD5: string; } /** * Arguments provided to handlers when an object is dropped or added to the scene */ export declare type DropListenerOnDropArguments = { /** The DropListener component that processed the drop event */ sender: DropListener, /** The root object added to the scene */ object: Object3D, /** The complete model with all associated data */ model: Model, /** MD5 hash of the content for verification */ contentMD5: string; /** The original dropped URL or File object */ dropped: URL | File | undefined; } /** * CustomEvent dispatched when an object is added to the scene via the DropListener */ class DropListenerAddedEvent extends CustomEvent { /** * Creates a new added event with the provided details * @param detail Information about the added object */ constructor(detail: T) { super(DropListenerEvents.ObjectAdded, { detail }); } } /** * Key name used for blob storage parameters */ const blobKeyName = "blob"; /** The DropListener component is used to listen for drag and drop events in the browser and add the dropped files to the scene * It can be used to allow users to drag and drop glTF files into the scene to add new objects. * * If {@link useNetworking} is enabled, the DropListener will automatically synchronize dropped files to other connected clients. * Enable {@link fitIntoVolume} to automatically scale dropped objects to fit within the volume defined by {@link fitVolumeSize}. * * The following events are dispatched by the DropListener: * - **object-added** - dispatched when a new object is added to the scene * - **file-dropped** - dispatched when a file is dropped into the scene * * @example * ```typescript * import { DropListener, DropListenerEvents } from "@needle-tools/engine"; * * const dropListener = new DropListener(); * * gameObject.addComponent(dropListener); * dropListener.on(DropListenerEvents.FileDropped, (evt) => { * console.log("File dropped", evt.detail); * const file = evt.detail as File; * }); * * dropListener.on(DropListenerEvents.ObjectAdded, (evt) => { * console.log("Object added", evt.detail); * const gltf = evt.detail as GLTF; * }); * ``` * * @category Asset Management * @group Components */ export class DropListener extends Behaviour { /** * When enabled, the DropListener will automatically synchronize dropped files to other connected clients. * When a file is dropped locally, it will be uploaded to blob storage and the URL will be shared with other clients. */ @serializable() useNetworking: boolean = true; /** * When assigned, the DropListener will only accept files that are dropped on this specific object. * This allows creating designated drop zones in your scene. */ @serializable(Object3D) dropArea?: Object3D; /** * When enabled, dropped objects will be automatically scaled to fit within the volume defined by fitVolumeSize. * Useful for ensuring dropped models appear at an appropriate scale. * @default false */ @serializable() fitIntoVolume: boolean = false; /** * Defines the dimensions of the volume that dropped objects will be scaled to fit within. * Only used when fitIntoVolume is enabled. */ @serializable(Vector3) fitVolumeSize = new Vector3(1, 1, 1); /** * When enabled, dropped objects will be positioned at the point where the cursor hit the scene. * When disabled, objects will be placed at the origin of the DropListener. * @default true */ @serializable() placeAtHitPosition: boolean = true; /** * Event list that gets invoked after a file has been successfully added to the scene. * Receives {@link DropListenerOnDropArguments} containing the added object and related information. * @event object-added * @example * ```typescript * dropListener.onDropped.addEventListener((evt) => { * console.log("Object added", evt.model); * }); */ @serializable(EventList) onDropped: EventList = new EventList(); /** @internal */ onEnable(): void { this.context.renderer.domElement.addEventListener("dragover", this.onDrag); this.context.renderer.domElement.addEventListener("drop", this.onDrop); window.addEventListener("paste", this.handlePaste); this.context.connection.beginListen("droplistener", this.onNetworkEvent) } /** @internal */ onDisable(): void { this.context.renderer.domElement.removeEventListener("dragover", this.onDrag); this.context.renderer.domElement.removeEventListener("drop", this.onDrop); window.removeEventListener("paste", this.handlePaste); this.context.connection.stopListen("droplistener", this.onNetworkEvent); } /** * Loads a file from the given URL and adds it to the scene. */ loadFromURL(url: string, data?: { point?: Vec3, size?: Vec3 }) { this.addFromUrl(url, { screenposition: new Vector2(), point: data?.point, size: data?.size, }, true); } /** * Forgets all previously added objects. * The droplistener will then not be able to remove previously added objects. */ forgetObjects() { this.removePreviouslyAddedObjects(false); } /** * Handles network events received from other clients containing information about dropped objects * @param evt Network event data containing object information, position, and content URL */ private onNetworkEvent = (evt: DropListenerNetworkEventArguments) => { if (!this.useNetworking) { if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", evt); return; } if (evt.guid?.startsWith(this.guid)) { const url = evt.url; console.debug("[DropListener] Received networked event", evt); if (url) { if (Array.isArray(url)) { for (const _url of url) { this.addFromUrl(_url, { screenposition: new Vector2(), point: evt.point, size: evt.size, }, true); } } else { this.addFromUrl(url, { screenposition: new Vector2(), point: evt.point, size: evt.size }, true); } } } } /** * Handles clipboard paste events and processes them as potential URL drops * Only URLs are processed by this handler, and only when editing is allowed * @param evt The paste event */ private handlePaste = (evt: Event) => { if (this.context.connection.allowEditing === false) return; if (evt.defaultPrevented) return; const clipboard = navigator.clipboard; clipboard.readText() .then(value => { if (value) { const isUrl = value.startsWith("http") || value.startsWith("https") || value.startsWith("blob"); if (isUrl) { const ctx = { screenposition: new Vector2(this.context.input.mousePosition.x, this.context.input.mousePosition.y) }; if (this.testIfIsInDropArea(ctx)) this.addFromUrl(value, ctx, false); } } }) .catch(console.warn); } /** * Handles drag events over the renderer's canvas * Prevents default behavior to enable drop events * @param evt The drag event */ private onDrag = (evt: DragEvent) => { if (this.context.connection.allowEditing === false) return; // necessary to get drop event evt.preventDefault(); } /** * Processes drop events to add files to the scene * Handles both file drops and text/URL drops * @param evt The drop event */ private onDrop = async (evt: DragEvent) => { if (this.context.connection.allowEditing === false) return; if (debug) console.log(evt); if (!evt?.dataTransfer) return; // If the event is marked as handled for droplisteners then ignore it if (evt["droplistener:handled"]) return; evt.preventDefault(); const ctx: DropContext = { screenposition: new Vector2(evt.offsetX, evt.offsetY) }; if (this.dropArea) { const res = this.testIfIsInDropArea(ctx); if (res === false) return; } // Don't stop propagation because this will break e.g. the RemoteSkybox drop // evt.stopImmediatePropagation(); // Mark the event handled for droplisteners evt["droplistener:handled"] = true; const items = evt.dataTransfer.items; if (!items) return; const files: File[] = []; for (const ite in items) { const it = items[ite]; if (it.kind === "file") { const file = it.getAsFile(); if (!file) continue; files.push(file); } else if (it.kind === "string" && it.type == "text/plain") { it.getAsString(str => { this.addFromUrl(str, ctx, false); }); } } if (files.length > 0) { await this.addDroppedFiles(files, ctx); } } /** * Processes a dropped or pasted URL and tries to load it as a 3D model * Handles special cases like GitHub URLs and Polyhaven asset URLs * @param url The URL to process * @param ctx Context information about where the drop occurred * @param isRemote Whether this URL was shared from a remote client * @returns The added object or null if loading failed */ private async addFromUrl(url: string, ctx: DropContext, isRemote: boolean) { if (debug) console.log("dropped url", url); try { if (url.startsWith("https://github.com/")) { // make raw.githubusercontent.com url const parts = url.split("/"); const user = parts[3]; const repo = parts[4]; const branch = parts[6]; const path = parts.slice(7).join("/"); url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${path}`; } else if (url.startsWith("https://polyhaven.com/a")) { url = tryResolvePolyhavenAssetUrl(url); } if (!url) return null; // Ignore dropped images const lowercaseUrl = url.toLowerCase(); if (lowercaseUrl.endsWith(".hdr") || lowercaseUrl.endsWith(".hdri") || lowercaseUrl.endsWith(".exr") || lowercaseUrl.endsWith(".png") || lowercaseUrl.endsWith(".jpg") || lowercaseUrl.endsWith(".jpeg")) { return null; } // TODO: if the URL is invalid this will become a problem this.removePreviouslyAddedObjects(); // const binary = await fetch(url).then(res => res.arrayBuffer()); const res = await FileHelper.loadFileFromURL(new URL(url), { guid: this.guid, context: this.context, parent: this.gameObject, point: ctx.point, size: ctx.size, }); if (res && this._addedObjects.length <= 0) { ctx.url = url; const obj = this.addObject(res, ctx, isRemote); return obj; } } catch (_) { console.warn("String is not a valid URL", url); } return null; } private _abort: AbortController | null = null; /** * Processes dropped files, loads them as 3D models, and handles networking if enabled * Creates an abort controller to cancel previous uploads if new files are dropped * @param fileList Array of dropped files * @param ctx Context information about where the drop occurred */ private async addDroppedFiles(fileList: Array, ctx: DropContext) { if (debug) console.log("Add files", fileList) if (!Array.isArray(fileList)) return; if (!fileList.length) return; this.deleteDropEvent(); this.removePreviouslyAddedObjects(); setParamWithoutReload(blobKeyName, null); // Create an abort controller for the current drop operation this._abort?.abort("New files dropped"); this._abort = new AbortController(); for (const file of fileList) { if (!file) continue; if (file.type.startsWith("image/")) { // Ignore dropped images if (debug) console.warn("Ignoring dropped image file", file.name, file.type); continue; } else if (file.name.endsWith(".bin")) { // Ignore dropped binary files if (debug) console.warn("Ignoring dropped binary file", file.name, file.type); continue; } console.debug("Load file " + file.name + " + " + file.type); const res = await FileHelper.loadFile(file, this.context, { guid: this.guid }); if (res) { this.dispatchEvent(new CustomEvent(DropListenerEvents.FileDropped, { detail: file })); ctx.file = file; const obj = this.addObject(res, ctx, false); // handle uploading the dropped object and networking the event if (obj && this.context.connection.isConnected && this.useNetworking) { console.debug("Uploading dropped file to blob storage"); BlobStorage.upload(file, { abort: this._abort?.signal, }) .then(upload => { // check if the upload was successful and if the object should still be visible if (upload?.download_url && this._addedObjects.includes(obj)) { // setParamWithoutReload(blobKeyName, upload.key); this.sendDropEvent(upload.download_url, obj, res.contentMD5); } }) .catch(console.warn); } // we currently only support dropping one file break; } } } /** Previously added objects */ private readonly _addedObjects = new Array(); private readonly _addedModels = new Array(); /** * Removes all previously added objects from the scene * @param doDestroy When true, destroys the objects; when false, just clears the references */ private removePreviouslyAddedObjects(doDestroy: boolean = true) { if (doDestroy) { for (const prev of this._addedObjects) { if (prev.parent === this.gameObject) { destroy(prev, true, true); } } } this._addedObjects.length = 0; this._addedModels.length = 0; } /** * Adds a loaded model to the scene with proper positioning and scaling. * Handles placement based on component settings and raycasting. * If {@link fitIntoVolume} is enabled, the object will be scaled to fit within the volume defined by {@link fitVolumeSize}. * @param data The loaded model data and content hash * @param ctx Context information about where the drop occurred * @param isRemote Whether this object was shared from a remote client * @returns The added object or null if adding failed */ private addObject(data: { model: Model, contentMD5: string }, ctx: DropContext, isRemote: boolean): Object3D | null { const { model, contentMD5 } = data; if (debug) console.log(`Dropped ${this.gameObject.name}`, model); if (!model?.scene) { console.warn("No object specified to add to scene", model); return null; } this.removePreviouslyAddedObjects(); const obj = model.scene; // use attach to ignore the DropListener scale (e.g. if the parent object scale is not uniform) this.gameObject.attach(obj); obj.position.set(0, 0, 0); obj.quaternion.identity(); this._addedObjects.push(obj); this._addedModels.push(model); const volume = new Box3().setFromCenterAndSize(new Vector3(0, this.fitVolumeSize.y * .5, 0).add(this.gameObject.worldPosition), this.fitVolumeSize); if (debug) Gizmos.DrawWireBox3(volume, 0x0000ff, 5); if (this.fitIntoVolume) { fitObjectIntoVolume(obj, volume, { position: !this.placeAtHitPosition }); } if (this.placeAtHitPosition && ctx && ctx.screenposition) { obj.visible = false; // < don't raycast on the placed object const rc = this.context.physics.raycast({ screenPoint: this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone()) }); obj.visible = true; if (rc && rc.length > 0) { for (const hit of rc) { const pos = hit.point.clone(); if (debug) console.log("Place object at hit", hit); placeOnSurface(obj, pos); break; } } } AnimationUtils.assignAnimationsFromFile(model, { createAnimationComponent: obj => addComponent(obj, Animation) }); const evt = new DropListenerAddedEvent({ sender: this, gltf: model, model: model, object: obj, contentMD5: contentMD5, dropped: ctx.file || (ctx.url ? new URL(ctx.url) : undefined), }); this.dispatchEvent(evt); this.onDropped?.invoke(evt.detail); // send network event if (!isRemote && ctx.url?.startsWith("http") && this.context.connection.isConnected && obj) { this.sendDropEvent(ctx.url, obj, contentMD5); } return obj; } /** * Sends a network event to other clients about a dropped object * Only triggered when networking is enabled and the connection is established * @param url The URL to the content that was dropped * @param obj The object that was added to the scene * @param contentmd5 The content hash for verification */ private async sendDropEvent(url: string, obj: Object3D, contentmd5: string) { if (!this.useNetworking) { if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", url); return; } if (this.context.connection.isConnected) { console.debug("Sending drop event \"" + obj.name + "\"", url); const bounds = getBoundingBox([obj]); const evt: DropListenerNetworkEventArguments = { name: obj.name, guid: this.guid, url, point: obj.worldPosition.clone(), size: bounds.getSize(new Vector3()), contentMD5: contentmd5, }; this.context.connection.send("droplistener", evt); } } /** * Deletes remote state for this DropListener's objects * Called when new files are dropped to clean up previous state */ private deleteDropEvent() { this.context.connection.sendDeleteRemoteState(this.guid); } /** * Tests if a drop event occurred within the designated drop area if one is specified * @param ctx The drop context containing screen position information * @returns True if the drop is valid (either no drop area is set or the drop occurred inside it) */ private testIfIsInDropArea(ctx: DropContext): boolean { if (this.dropArea) { const screenPoint = this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone()); const hits = this.context.physics.raycast({ targets: [this.dropArea], screenPoint, recursive: true, testObject: obj => { // Ignore hits on the already added objects, they don't count as part of the dropzone if (this._addedObjects.includes(obj)) return false; return true; } }); if (!hits.length) { if (isDevEnvironment()) console.log(`Dropped outside of drop area for DropListener \"${this.name}\".`); return false; } } return true; } } /** * Attempts to convert a Polyhaven website URL to a direct glTF model download URL * @param urlStr The original Polyhaven URL * @returns The direct download URL for the glTF model if it's a valid Polyhaven asset URL, otherwise returns the original URL */ function tryResolvePolyhavenAssetUrl(urlStr: string) { if (!urlStr.startsWith("https://polyhaven.com/")) return urlStr; // Handle dropping polyhaven image url const baseUrl = "https://dl.polyhaven.org/file/ph-assets/Models/gltf/4k/"; const url = new URL(urlStr); const path = url.pathname; const name = path.split("/").pop(); const assetUrl = `${baseUrl}${name}/${name}_4k.gltf`; console.log("Resolved polyhaven asset url", urlStr, "→", assetUrl); // TODO: need to resolve textures properly return assetUrl; } /** * Helper namespace for loading files and models from various sources */ namespace FileHelper { /** * Loads and processes a File object into a 3D model * @param file The file to load (supported formats: gltf, glb, fbx, obj, usdz, vrm) * @param context The application context * @param args Additional arguments including a unique guid for instantiation * @returns Promise containing the loaded model and its content hash, or null if loading failed */ export async function loadFile(file: File, context: Context, args: { guid: string }): Promise<{ model: Model, contentMD5: string } | null> { // first load it locally const seed = args.guid; const prov = new InstantiateIdProvider(seed); const blob = new Blob([file], { type: file.type || determineMimeTypeFromExtension(file.name) || undefined }); const objectUrl = URL.createObjectURL(blob); const model = await getLoader().loadSync(context, objectUrl, file.name, prov).catch(err => { console.error(`Failed to load file "${file.name}" (${file.type}):`, err); return null; }) URL.revokeObjectURL(objectUrl); // clean up the object URL if (model) { return new Promise((resolve, _reject) => { const reader = new FileReader() reader.readAsArrayBuffer(file); reader.onloadend = async (_ev: ProgressEvent) => { const content = reader.result as ArrayBuffer; const hash = BlobStorage.hashMD5(content); return resolve({ model, contentMD5: hash }); }; }); } else { console.warn(`Failed to load "${file.name}" (${file.type})`); return null; } } // return new Promise((resolve, _reject) => { // }); // } /** * Loads a 3D model from a URL with progress visualization * @param url The URL to load the model from * @param args Arguments including context, parent object, and optional placement information * @returns Promise containing the loaded model and its content hash, or null if loading failed */ export async function loadFileFromURL(url: URL, args: { guid: string, context: Context, parent: Object3D, point?: Vec3, size?: Vec3 }): Promise<{ model: Model, contentMD5: string } | null> { return new Promise(async (resolve, _reject) => { const prov = new InstantiateIdProvider(args.guid); const urlStr = url.toString(); if (debug) Gizmos.DrawWireSphere(args.point!, .1, 0xff0000, 3); const preview = PreviewHelper.addPreview({ guid: args.guid, parent: args.parent, position: args?.point, size: args?.size, }); const model = await getLoader().loadSync(args.context, urlStr, urlStr, prov, prog => { preview.onProgress(prog.loaded / prog.total); }).catch(console.warn); if (model) { const binary = await fetch(urlStr).then(res => res.arrayBuffer()); const hash = BlobStorage.hashMD5(binary); if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000); else PreviewHelper.removePreview(args.guid); resolve({ model, contentMD5: hash }); } else { if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000); else PreviewHelper.removePreview(args.guid); console.warn("Unsupported file type: " + url.toString()); } }); } }