import '../../../engine/engine_shims.js'; import { AnimationClip, Bone, BufferAttribute, BufferGeometry, Color, DoubleSide, InterleavedBufferAttribute, LinearFilter, Material, MathUtils, Matrix4, Mesh, MeshBasicMaterial, MeshPhysicalMaterial, MeshStandardMaterial, Object3D, OrthographicCamera, PerspectiveCamera, PlaneGeometry, Quaternion, RGBAFormat, Scene, ShaderMaterial, SkinnedMesh, SRGBColorSpace, Texture, Uniform, UnsignedByteType, Vector3, Vector4, WebGLRenderer, WebGLRenderTarget } from 'three'; import * as fflate from 'three/examples/jsm/libs/fflate.module.js'; import { VERSION } from "../../../engine/engine_constants.js"; import type { OffscreenCanvasExt } from '../../../engine/engine_shims.js'; import { Progress } from '../../../engine/engine_time_utils.js'; // import { BehaviorExtension } from '../../api.js'; import type { IUSDExporterExtension } from './Extension.js'; import type { AnimationExtension } from './extensions/Animation.js'; import { BehaviorExtension } from './extensions/behavior/Behaviour.js'; import type { PhysicsExtension } from './extensions/behavior/PhysicsExtension.js'; import {buildNodeMaterial} from './extensions/NodeMaterialConverter.js'; type MeshPhysicalNodeMaterial = import("three/src/materials/nodes/MeshPhysicalNodeMaterial.js").default; function makeNameSafe( str ) { // Remove characters that are not allowed in USD ASCII identifiers str = str.replace( /[^a-zA-Z0-9_]/g, '' ); // If name doesn't start with a-zA-Z_, add _ to the beginning – required by USD if ( !str.match( /^[a-zA-Z_]/ ) ) str = '_' + str; return str; } function makeDisplayNameSafe(str) { str = str.replace("\"", "\\\""); return str; } // TODO check if this works when bones in the skeleton are completely unordered function findCommonAncestor(objects: Object3D[]): Object3D | null { if (objects.length === 0) return null; const ancestors = objects.map((obj) => { const objAncestors = new Array(); while (obj.parent) { objAncestors.unshift(obj.parent); obj = obj.parent; } return objAncestors; }); //@ts-ignore – findLast seems to be missing in TypeScript types pre-5.x const commonAncestor = ancestors[0].findLast((ancestor) => { return ancestors.every((a) => a.includes(ancestor)); }); return commonAncestor || null; } function findStructuralNodesInBoneHierarchy(bones: Array) { const commonAncestor = findCommonAncestor(bones); // find all structural nodes – parents of bones that are not bones themselves const structuralNodes = new Set(); for ( const bone of bones ) { let current = bone.parent; while ( current && current !== commonAncestor ) { if ( !bones.includes(current) ) { structuralNodes.add(current); } current = current.parent; } } return structuralNodes; } declare type USDObjectTransform = { position: Vector3 | null; quaternion: Quaternion | null; scale: Vector3 | null; } const PositionIdentity = new Vector3(); const QuaternionIdentity = new Quaternion(); const ScaleIdentity = new Vector3(1,1,1); // #region USDObject type USDObjectEventType = "serialize" & ({} & string); class USDObject { static USDObject_export_id = 0; uuid: string; name: string; /** If no type is provided, type is chosen automatically (Xform or Mesh) */ type?: string; /** MaterialBindingAPI and SkelBindingAPI are handled automatically, extra schemas can be added here */ extraSchemas: string[] = []; displayName?: string; visibility?: "inherited" | "invisible"; // defaults to "inherited" in USD getMatrix(): Matrix4 { if (!this.transform) return new Matrix4(); const { position, quaternion, scale } = this.transform; const matrix = new Matrix4(); matrix.compose(position || PositionIdentity, quaternion || QuaternionIdentity, scale || ScaleIdentity); return matrix; } setMatrix( value ) { if (!value || !(value instanceof Matrix4)) { this.transform = null; return; } const position = new Vector3(); const quaternion = new Quaternion(); const scale = new Vector3(); value.decompose(position, quaternion, scale); this.transform = { position, quaternion, scale }; } /** @deprecated Use `transform`, or `getMatrix()` if you really need the matrix */ get matrix() { return this.getMatrix(); } /** @deprecated Use `transform`, or `setMatrix()` if you really need the matrix */ set matrix( value ) { this.setMatrix( value ); } transform: USDObjectTransform | null = null; private _isDynamic: boolean; get isDynamic() { return this._isDynamic; } private set isDynamic( value ) { this._isDynamic = value; } geometry: BufferGeometry | null; material: MeshStandardMaterial | MeshBasicMaterial | Material | MeshPhysicalNodeMaterial | null; // usdMaterial?: USDMaterial; camera: PerspectiveCamera | OrthographicCamera | null; parent: USDObject | null; skinnedMesh: SkinnedMesh | null; children: Array = []; animations: AnimationClip[] | null; _eventListeners: Record; // these are for tracking which xformops are needed needsTranslate: boolean = false; needsOrient: boolean = false; needsScale: boolean = false; static createEmptyParent( object: USDObject ) { const emptyParent = new USDObject( MathUtils.generateUUID(), object.name + '_empty_' + ( USDObject.USDObject_export_id ++ ), object.transform ); const parent = object.parent; if (parent) parent.add( emptyParent ); emptyParent.add( object ); emptyParent.isDynamic = true; object.transform = null; return emptyParent; } static createEmpty() { const empty = new USDObject( MathUtils.generateUUID(), 'Empty_' + ( USDObject.USDObject_export_id ++ ) ); empty.isDynamic = true; return empty; } constructor( id, name, transform: USDObjectTransform | null = null, mesh: BufferGeometry | null = null, material: MeshStandardMaterial | MeshBasicMaterial | MeshPhysicalNodeMaterial | Material | null = null, camera: PerspectiveCamera | OrthographicCamera | null = null, skinnedMesh: SkinnedMesh | null = null, animations: AnimationClip[] | null = null ) { this.uuid = id; this.name = makeNameSafe( name ); this.displayName = name; if (!transform) this.transform = null; else this.transform = { position: transform.position?.clone() || null, quaternion: transform.quaternion?.clone() || null, scale: transform.scale?.clone() || null }; this.geometry = mesh; this.material = material; this.camera = camera; this.parent = null; this.children = []; this._eventListeners = {} as Record; this._isDynamic = false; this.skinnedMesh = skinnedMesh; this.animations = animations; } is( obj ) { if ( ! obj ) return false; return this.uuid === obj.uuid; } isEmpty() { return ! this.geometry; } clone() { const clone = new USDObject( MathUtils.generateUUID(), this.name, this.transform, this.geometry, this.material ); clone.isDynamic = this.isDynamic; return clone; } deepClone() { const clone = this.clone(); for ( const child of this.children ) { if ( !child ) continue; clone.add( child.deepClone() ); } return clone; } getPath() { let current = this.parent; let path = this.name; while ( current ) { // StageRoot has a special path right now since there's additional Xforms for encapsulation. // Better would be to actually model them as part of our object graph, but they're written separately, // so currently we don't and instead work around that here. const currentName = current.parent ? current.name : (current.name + "/Scenes/Scene"); path = currentName + '/' + path; current = current.parent; } return ''; } add( child ) { if ( child.parent ) { child.parent.remove( child ); } child.parent = this; this.children.push( child ); } remove( child ) { const index = this.children.indexOf( child ); if ( index >= 0 ) { if ( child.parent === this ) child.parent = null; this.children.splice( index, 1 ); } } addEventListener( evt : USDObjectEventType, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) { if ( ! this._eventListeners[ evt ] ) this._eventListeners[ evt ] = []; this._eventListeners[ evt ].push( listener ); } removeEventListener( evt, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) { if ( ! this._eventListeners[ evt ] ) return; const index = this._eventListeners[ evt ].indexOf( listener ); if ( index >= 0 ) { this._eventListeners[ evt ].splice( index, 1 ); } } onSerialize( writer, context ) { const listeners = this._eventListeners[ 'serialize' ]; if ( listeners ) listeners.forEach( listener => listener( writer, context ) ); } } // #region USDMaterial // class MaterialInput { // name: string; // } class USDMaterial { static USDMaterial_id = 0; readonly material: Material; readonly id: number; name: string; isOverride: boolean = false; isInstanceable: boolean = false; constructor( material: Material ) { this.material = material; this.id = USDMaterial.USDMaterial_id ++; this.name = makeNameSafe( material.name || 'Material_' + this.id ); } readonly inputs: Record = {}; // addInput( name: string, value: "" ) { // } serialize( writer: USDWriter, _context: USDZExporterContext ) { const name = this.name; writer.appendLine( `def Material "${this.name}" ${name ?`( displayName = "${makeDisplayNameSafe(name)}" )` : ''}` ); writer.beginBlock(); writer.closeBlock(); // def Material "${materialName}" ${material.name ?`( // displayName = "${material.name}" // )` : ''} // { // token outputs:mtlx:surface.connect = ${materialRoot}/${materialName}/Occlusion.outputs:out> // def Shader "Occlusion" // { // uniform token info:id = "${mode}" // token outputs:out // } // }`; } } // #region USDDocument class USDDocument extends USDObject { stageLength: number; get isDocumentRoot() { return true; } get isDynamic() { return false; } constructor() { super(undefined, 'StageRoot', null, null, null, null); this.children = []; this.stageLength = 200; } add( child: USDObject ) { child.parent = this; this.children.push( child ); } remove( child: USDObject ) { const index = this.children.indexOf( child ); if ( index >= 0 ) { if ( child.parent === this ) child.parent = null; this.children.splice( index, 1 ); } } traverse( callback: ( object: USDObject ) => void, current: USDObject | null = null ) { if ( current !== null ) callback( current ); else current = this; if ( current.children ) { for ( const child of current.children ) { this.traverse( callback, child ); } } } findById( uuid: string ) { let found = false; function search( current: USDObject ): USDObject | undefined { if ( found ) return undefined; if ( current.uuid === uuid ) { found = true; return current; } if ( current.children ) { for ( const child of current.children ) { if (!child) continue; const res = search( child ); if ( res ) return res; } } return undefined; } return search( this ); } buildHeader( _context: USDZExporterContext ) { const animationExtension = _context.extensions?.find( ext => ext?.extensionName === 'animation' ) as AnimationExtension | undefined; const behaviorExtension = _context.extensions?.find( ext => ext?.extensionName === 'Behaviour' ) as BehaviorExtension | undefined; const physicsExtension = _context.extensions?.find( ext => ext?.extensionName === 'Physics' ) as PhysicsExtension | undefined; const startTimeCode = animationExtension?.getStartTimeCode() ?? 0; const endTimeCode = animationExtension?.getEndTimeCode() ?? 0; let comment = ""; const registeredClips = animationExtension?.registeredClips; if (registeredClips) { for ( const clip of registeredClips ) { comment += `\t# Animation: ${clip.name}, start=${animationExtension.getStartTimeByClip(clip) * 60}, length=${clip.duration * 60}\n`; } } const comments = comment; return `#usda 1.0 ( customLayerData = { string creator = "Needle Engine ${VERSION}" dictionary Needle = { bool animations = ${animationExtension ? 1 : 0} bool interactive = ${behaviorExtension ? 1 : 0} bool physics = ${physicsExtension ? 1 : 0} bool quickLookCompatible = ${_context.quickLookCompatible ? 1 : 0} } } defaultPrim = "${makeNameSafe( this.name )}" metersPerUnit = 1 upAxis = "Y" startTimeCode = ${startTimeCode} endTimeCode = ${endTimeCode} timeCodesPerSecond = 60 framesPerSecond = 60 doc = """Generated by Needle Engine USDZ Exporter ${VERSION}""" ${comments} ) `; } } const newLine = '\n'; const materialRoot = ' = []; quickLookCompatible: boolean; exportInvisible: boolean; materials: Map; textures: TextureMap; files: { [path: string]: Uint8Array | [Uint8Array, fflate.ZipOptions] | null | any } document: USDDocument; output: string; animations: AnimationClip[]; constructor( root: Object3D | null | undefined, exporter: USDZExporter, options: { extensions?: Array, quickLookCompatible: boolean, exportInvisible: boolean, } ) { this.root = root || undefined; this.exporter = exporter; this.quickLookCompatible = options.quickLookCompatible; this.exportInvisible = options.exportInvisible; if ( options.extensions ) this.extensions = options.extensions; this.materials = new Map(); this.textures = {}; this.files = {}; this.document = new USDDocument(); this.output = ''; this.animations = []; } makeNameSafe( str ) { return makeNameSafe( str ); } } /**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_anchoring_type) */ export type Anchoring = "plane" | "image" | "face" | "none" /**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_planeanchoring_alignment) */ export type Alignment = "horizontal" | "vertical" | "any"; type USDZExporterOptions = { ar: { anchoring: { type: Anchoring }, planeAnchoring: { alignment: Alignment }, } quickLookCompatible: boolean; extensions: Array; maxTextureSize: number; exportInvisible: boolean; } const getDefaultExporterOptions : () => USDZExporterOptions = () => { return { ar: { anchoring: { type: 'plane' }, planeAnchoring: { alignment: 'horizontal' } }, quickLookCompatible: false, extensions: [], maxTextureSize: 4096, exportInvisible: false } } // #region USDZExporter class USDZExporter { debug: boolean; pruneUnusedNodes: boolean; sceneAnchoringOptions: USDZExporterOptions = getDefaultExporterOptions(); extensions: Array = []; keepObject?: (object: Object3D) => boolean; beforeWritingDocument?: () => void; constructor() { this.debug = false; this.pruneUnusedNodes = true; } async parse(scene: Object3D | null | undefined, options: USDZExporterOptions = getDefaultExporterOptions()) { // clone options to avoid modifying the original object options = Object.assign({}, options ); this.sceneAnchoringOptions = options; const context = new USDZExporterContext( scene, this, options ); this.extensions = context.extensions; const files = context.files; const modelFileName = 'model.usda'; // model file should be first in USDZ archive so we init it here files[ modelFileName ] = null; const materials = context.materials; const textures = context.textures; Progress.report('export-usdz', "Invoking onBeforeBuildDocument"); await invokeAll( context, 'onBeforeBuildDocument' ); Progress.report('export-usdz', "Done onBeforeBuildDocument"); Progress.report('export-usdz', "Reparent bones to common ancestor"); // Find all skeletons and reparent them to their skelroot / armature / uppermost bone parent. // This may not be correct in all cases. const reparentings: Array<{ object: Object3D, originalParent: Object3D | null, newParent: Object3D }> = []; const allReparentingObjects = new Set(); scene?.traverse(object => { if (!options.exportInvisible && !object.visible) return; if (object instanceof SkinnedMesh) { const bones = object.skeleton.bones as Bone[]; const commonAncestor = findCommonAncestor(bones); if (commonAncestor) { const newReparenting = { object, originalParent: object.parent, newParent: commonAncestor }; reparentings.push( newReparenting ); // keep track of which nodes are important for skeletal export consistency allReparentingObjects.add(newReparenting.object.uuid); if (newReparenting.newParent) allReparentingObjects.add(newReparenting.newParent.uuid); if (newReparenting.originalParent) allReparentingObjects.add(newReparenting.originalParent.uuid); } } }); for ( const reparenting of reparentings ) { const { object, originalParent, newParent } = reparenting; newParent.add( object ); } Progress.report('export-usdz', "Traversing hierarchy"); if (scene) traverse( scene, context.document, context, this.keepObject); // Root object should have identity matrix // so that root transformations don't end up in the resulting file. // if (context.document.children?.length > 0) // context.document.children[0]?.matrix.identity(); //.multiply(new Matrix4().makeRotationY(Math.PI)); Progress.report('export-usdz', "Invoking onAfterBuildDocument"); await invokeAll( context, 'onAfterBuildDocument' ); // At this point, we know all animated objects, all skinned mesh objects, and all objects targeted by behaviors. // We can prune all empty nodes (no geometry or material) depth-first. // This avoids unnecessary export of e.g. animated bones as nodes when they have no children // (for example, a sword attached to a hand still needs that entire hierarchy exported) const behaviorExt = context.extensions.find( ext => ext.extensionName === 'Behaviour' ) as BehaviorExtension | undefined; const allBehaviorTargets = behaviorExt?.getAllTargetUuids() ?? new Set(); // Prune pass. Depth-first removal of nodes that don't affect the outcome of the scene. if (this.pruneUnusedNodes) { const options = { allBehaviorTargets, debug: false, boneReparentings: allReparentingObjects, quickLookCompatible: context.quickLookCompatible, }; if (this.debug) logUsdHierarchy(context.document, "Hierarchy BEFORE pruning", options); prune( context.document, options ); if (this.debug) logUsdHierarchy(context.document, "Hierarchy AFTER pruning"); } else if (this.debug) { console.log("Pruning of empty nodes is disabled. This may result in a larger USDZ file."); } Progress.report('export-usdz', { message: "Parsing document", autoStep: 10 }); await parseDocument( context, options ); Progress.report("export-usdz", "Invoking onAfterSerialize"); await invokeAll( context, 'onAfterSerialize' ); // repair the parenting again for ( const reparenting of reparentings ) { const { object, originalParent, newParent } = reparenting; if (originalParent) originalParent.add( object ); } // Moved into parseDocument callback for proper defaultPrim encapsulation // context.output += buildMaterials( materials, textures, options.quickLookCompatible ); // callback for validating after all export has been done context.exporter?.beforeWritingDocument?.(); const header = context.document.buildHeader( context ); const final = header + '\n' + context.output; // full output file if ( this.debug ) console.debug( final ); files[ modelFileName ] = fflate.strToU8( final ); context.output = ''; Progress.report("export-usdz", { message: "Exporting textures", autoStep: 10 }); Progress.start("export-usdz-textures", { parentScope: "export-usdz", logTimings: false }); const decompressionRenderer = new WebGLRenderer( { antialias: false, alpha: true, premultipliedAlpha: false, preserveDrawingBuffer: true } ); const textureCount = Object.keys(textures).length; Progress.report("export-usdz-textures", { totalSteps: textureCount * 3, currentStep: 0 }); const convertTexture = async (id: string) => { const textureData = textures[ id ]; const texture = textureData.texture; const isRGBA = formatsWithAlphaChannel.includes( texture.format ); // Change: we need to always read back the texture now, otherwise the unpremultiplied workflow doesn't work. let img: ImageReadbackResult = { imageData: texture.image as ImageData }; Progress.report("export-usdz-textures", { message: "read back texture", autoStep: true }); const anyColorScale = textureData.scale !== undefined && textureData.scale.x !== 1 && textureData.scale.y !== 1 && textureData.scale.z !== 1 && textureData.scale.w !== 1; // @ts-ignore if ( texture.isCompressedTexture || texture.isRenderTargetTexture || anyColorScale ) { img = await decompressGpuTexture( texture, options.maxTextureSize, decompressionRenderer, textureData.scale ); } Progress.report("export-usdz-textures", { message: "convert texture to canvas", autoStep: true }); const canvas = await imageToCanvasUnpremultiplied( img.imageBitmap || img.imageData, options.maxTextureSize ).catch( err => { console.error("Error converting texture to canvas", texture, err); }); if ( canvas ) { Progress.report("export-usdz-textures", { message: "convert canvas to blob", autoStep: true }); const blob = await canvas.convertToBlob( {type: isRGBA ? 'image/png' : 'image/jpeg', quality: 0.95 } ); files[ `textures/${id}.${isRGBA ? 'png' : 'jpg'}` ] = new Uint8Array( await blob.arrayBuffer() ); } else { console.warn( 'Can`t export texture: ', texture ); } }; for ( const id in textures ) { await convertTexture( id ); } decompressionRenderer.dispose(); // 64 byte alignment // https://github.com/101arrowz/fflate/issues/39#issuecomment-777263109 Progress.end("export-usdz-textures"); let offset = 0; for ( const filename in files ) { const file = files[ filename ]; const headerSize = 34 + filename.length; offset += headerSize; const offsetMod64 = offset & 63; if ( offsetMod64 !== 4 ) { const padLength = 64 - offsetMod64; const padding = new Uint8Array( padLength ); files[ filename ] = [ file, { extra: { 12345: padding } } ]; } offset = file.length; } Progress.report("export-usdz", "zip archive"); return fflate.zipSync( files, { level: 0 } ); } } // #endregion // #region traverse function traverse( object: Object3D, parentModel: USDObject, context: USDZExporterContext, keepObject?: (object: Object3D) => boolean ) { if (!context.exportInvisible && !object.visible) return; let model: USDObject | undefined = undefined; let geometry: BufferGeometry | undefined = undefined; let material: Material | Material[] | undefined = undefined; const transform: USDObjectTransform = { position: object.position, quaternion: object.quaternion, scale: object.scale }; if (object.position.x === 0 && object.position.y === 0 && object.position.z === 0) transform.position = null; if (object.quaternion.x === 0 && object.quaternion.y === 0 && object.quaternion.z === 0 && object.quaternion.w === 1) transform.quaternion = null; if (object.scale.x === 1 && object.scale.y === 1 && object.scale.z === 1) transform.scale = null; if (object instanceof Mesh || object instanceof SkinnedMesh) { geometry = object.geometry; material = object.material; } // API for an explicit choice to discard this object – for example, a geometry that should not be exported, // but childs should still be exported. if (keepObject && !keepObject(object)) { geometry = undefined; material = undefined; } if ( (object instanceof Mesh || object instanceof SkinnedMesh) && material && typeof material === 'object' && (material instanceof MeshStandardMaterial || material instanceof MeshBasicMaterial || // material instanceof MeshPhysicalNodeMaterial || (material as any).isMeshPhysicalNodeMaterial || (material instanceof Material && material.type === "MeshLineMaterial"))) { const name = getObjectId( object ); const skinnedMeshObject = object instanceof SkinnedMesh ? object : null; model = new USDObject( object.uuid, name, transform, geometry, material as any, undefined, skinnedMeshObject, object.animations ); } else if ( object instanceof PerspectiveCamera || object instanceof OrthographicCamera ) { const name = getObjectId( object ); model = new USDObject( object.uuid, name, transform, undefined, undefined, object ); } else { const name = getObjectId( object ); model = new USDObject( object.uuid, name, transform, undefined, undefined, undefined, undefined, object.animations ); } if ( model ) { model.displayName = object.userData?.name || object.name; model.visibility = object.visible ? undefined : "invisible"; if ( parentModel ) { parentModel.add( model ); } parentModel = model; if ( context.extensions ) { for ( const ext of context.extensions ) { if ( ext.onExportObject ) ext.onExportObject.call( ext, object, model, context ); } } } else { const name = getObjectId( object ); const empty = new USDObject( object.uuid, name, { position: object.position, quaternion: object.quaternion, scale: object.scale } ); if ( parentModel ) { parentModel.add( empty ); } parentModel = empty; } for ( const ch of object.children ) { traverse( ch, parentModel, context, keepObject ); } } // #endregion function logUsdHierarchy( object: USDObject, prefix: string, ...extraLogObjects: any[] ) { const item = {}; let itemCount = 0; function collectItem( object: USDObject, current) { itemCount++; let name = object.displayName || object.name; name += " (" + object.uuid + ")"; const hasAny = object.geometry || object.material || object.camera || object.skinnedMesh; if (hasAny) { name += " (" + (object.geometry ? "geo, " : "") + (object.material ? "mat, " : "") + (object.camera ? "cam, " : "") + (object.skinnedMesh ? "skin, " : "") + ")"; } current[name] = {}; const props = { object }; if (object.material) props['mat'] = true; if (object.geometry) props['geo'] = true; if (object.camera) props['cam'] = true; if (object.skinnedMesh) props['skin'] = true; current[name]._self = props; for ( const child of object.children ) { if (child) { collectItem(child, current[name]); } } } collectItem(object, item); console.log(prefix + " (" + itemCount + " nodes)", item, ...extraLogObjects); } function prune ( object: USDObject, options : { allBehaviorTargets: Set, debug: boolean, boneReparentings: Set, quickLookCompatible: boolean, } ) { let allChildsWerePruned = true; const prunedChilds = new Array(); const keptChilds = new Array(); if (object.children.length === 0) { allChildsWerePruned = true; } else { const childs = [...object.children]; for ( const child of childs ) { if (child) { const childWasPruned = prune(child, options); if (options.debug) { if (childWasPruned) prunedChilds.push(child); else keptChilds.push(child); } allChildsWerePruned = allChildsWerePruned && childWasPruned; } } } // check if this object is referenced by any behavior const isBehaviorSourceOrTarget = options.allBehaviorTargets.has(object.uuid); // check if this object has any material or geometry const isVisible = object.geometry || object.material || (object.camera && !options.quickLookCompatible) || object.skinnedMesh || false; // check if this object is part of any reparenting const isBoneReparenting = options.boneReparentings.has(object.uuid); const canBePruned = allChildsWerePruned && !isBehaviorSourceOrTarget && !isVisible && !isBoneReparenting; if (canBePruned) { if (options.debug) console.log("Pruned object:", (object.displayName || object.name) + " (" + object.uuid + ")", { isVisible, isBehaviorSourceOrTarget, allChildsWerePruned, isBoneReparenting, object, prunedChilds, keptChilds }); object.parent?.remove(object); } else { if (options.debug) console.log("Kept object:", (object.displayName || object.name) + " (" + object.uuid + ")", { isVisible, isBehaviorSourceOrTarget, allChildsWerePruned, isBoneReparenting, object, prunedChilds, keptChilds }); } // if it has no children and is not a behavior source or target, and is not visible, prune it return canBePruned; } // #region parseDocument async function parseDocument( context: USDZExporterContext, options: USDZExporterOptions ) { Progress.start("export-usdz-resources", "export-usdz"); const resources: Array<() => void> = []; for ( const child of context.document.children ) { addResources( child, context, resources ); } // addResources now only collects promises for better progress reporting. // We are resolving them here and reporting progress on that: const total = resources.length; for (let i = 0; i < total; i++) { Progress.report("export-usdz-resources", { totalSteps: total, currentStep: i }); await new Promise((resolve, _reject) => { resources[i](); resolve(); }); } Progress.end("export-usdz-resources"); const writer = new USDWriter(); const arAnchoringOptions = context.exporter.sceneAnchoringOptions.ar; writer.beginBlock( `def Xform "${context.document.name}"` ); writer.beginBlock( `def Scope "Scenes" ( kind = "sceneLibrary" )` ); writer.beginBlock( `def Xform "Scene"`, '(', false); writer.appendLine( `apiSchemas = ["Preliminary_AnchoringAPI"]` ); writer.appendLine( `customData = {`); writer.appendLine( ` bool preliminary_collidesWithEnvironment = 0` ); writer.appendLine( ` string sceneName = "Scene"`); writer.appendLine( `}` ); writer.appendLine( `sceneName = "Scene"` ); writer.closeBlock( ')' ); writer.beginBlock(); writer.appendLine( `token preliminary:anchoring:type = "${arAnchoringOptions.anchoring.type}"` ); if (arAnchoringOptions.anchoring.type === 'plane') writer.appendLine( `token preliminary:planeAnchoring:alignment = "${arAnchoringOptions.planeAnchoring.alignment}"` ); // bit hacky as we don't have a callback here yet. Relies on the fact that the image is named identical in the ImageTracking extension. if (arAnchoringOptions.anchoring.type === 'image') writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = ` ); writer.appendLine(); const count = (object: USDObject | null) => { if (!object) return 0; let total = 1; for ( const child of object.children ) total += count( child ); return total; } const totalXformCount = count(context.document); Progress.start("export-usdz-xforms", "export-usdz"); Progress.report("export-usdz-xforms", { totalSteps: totalXformCount, currentStep: 1 }); for ( const child of context.document.children ) { buildXform( child, writer, context ); } Progress.end("export-usdz-xforms"); Progress.report("export-usdz", "invoke onAfterHierarchy"); await invokeAll( context, 'onAfterHierarchy', writer ); writer.closeBlock(); writer.closeBlock(); // TODO property use context/writer instead of string concat Progress.report('export-usdz', "Building materials"); const result = buildMaterials( context.materials, context.textures, options.quickLookCompatible ); writer.appendLine(result); writer.closeBlock(); Progress.report("export-usdz", "write to string") context.output += writer.toString(); } // #endregion // #region addResources function addResources( object: USDObject | null, context: USDZExporterContext, resources: Array<() => void>) { if ( !object ) return; const geometry = object.geometry; const material = object.material; if ( geometry ) { if ( material && ( 'isMeshStandardMaterial' in material && material.isMeshStandardMaterial || 'isMeshBasicMaterial' in material && material.isMeshBasicMaterial || material.type === "MeshLineMaterial" ) ) { // TODO convert unlit to lit+emissive const geometryFileName = 'geometries/' + getGeometryName(geometry, object.name) + '.usda'; if ( ! ( geometryFileName in context.files ) ) { const action = () => { const meshObject = buildMeshObject( geometry, object.skinnedMesh?.skeleton?.bones, context.quickLookCompatible ); context.files[ geometryFileName ] = buildUSDFileAsString( meshObject, context); }; resources.push(action); } } else { console.warn( 'NeedleUSDZExporter: Unsupported material type (USDZ only supports MeshStandardMaterial)', material?.name ); } } if ( material ) { if ( context.materials.get( material.uuid ) === undefined ) { context.materials[ material.uuid ] = material; } } for ( const ch of object.children ) { addResources( ch, context, resources ); } } async function invokeAll( context: USDZExporterContext, name: string, writer: USDWriter | null = null ) { if ( context.extensions ) { for ( const ext of context.extensions ) { if ( !ext ) continue; if ( typeof ext[ name ] === 'function' ) { const method = ext[ name ]; const res = method.call( ext, context, writer ); if(res instanceof Promise) { await res; } } } } } // #endregion // #region GPU utils let _renderer: WebGLRenderer | null = null; let renderTarget: WebGLRenderTarget | null = null; let fullscreenQuadGeometry: PlaneGeometry | null; let fullscreenQuadMaterial: ShaderMaterial | null; let fullscreenQuad: Mesh | null; declare type ImageReadbackResult = { imageData: ImageData; imageBitmap?: ImageBitmap; } /** Reads back a texture from the GPU (can be compressed, a render texture, or anything), optionally applies RGBA colorScale to it, and returns CPU data for further usage. * Note that there are WebGL / WebGPU rules preventing some use of data between WebGL contexts. */ async function decompressGpuTexture( texture, maxTextureSize = Infinity, renderer: WebGLRenderer | null = null, colorScale: Vector4 | undefined = undefined): Promise { if ( ! fullscreenQuadGeometry ) fullscreenQuadGeometry = new PlaneGeometry( 2, 2, 1, 1 ); if ( ! fullscreenQuadMaterial ) fullscreenQuadMaterial = new ShaderMaterial( { uniforms: { blitTexture: new Uniform( texture ), flipY: new Uniform( false ), scale: new Uniform( new Vector4( 1, 1, 1, 1 ) ), }, vertexShader: ` varying vec2 vUv; uniform bool flipY; void main(){ vUv = uv; if (flipY) vUv.y = 1. - vUv.y; gl_Position = vec4(position.xy * 1.0,0.,.999999); }`, fragmentShader: ` uniform sampler2D blitTexture; uniform vec4 scale; varying vec2 vUv; void main(){ gl_FragColor = vec4(vUv.xy, 0, 1); #ifdef IS_SRGB gl_FragColor = sRGBTransferOETF( texture2D( blitTexture, vUv) ); #else gl_FragColor = texture2D( blitTexture, vUv); #endif gl_FragColor.rgba *= scale.rgba; }` } ); // update uniforms const uniforms = fullscreenQuadMaterial.uniforms; uniforms.blitTexture.value = texture; uniforms.flipY.value = false; uniforms.scale.value = new Vector4( 1, 1, 1, 1 ); if ( colorScale !== undefined ) uniforms.scale.value.copy( colorScale ); fullscreenQuadMaterial.defines.IS_SRGB = texture.colorSpace == SRGBColorSpace; fullscreenQuadMaterial.needsUpdate = true; if ( ! fullscreenQuad ) { fullscreenQuad = new Mesh( fullscreenQuadGeometry, fullscreenQuadMaterial ); fullscreenQuad.frustumCulled = false; } const _camera = new PerspectiveCamera(); const _scene = new Scene(); _scene.add( fullscreenQuad ); if ( ! renderer ) { renderer = _renderer = new WebGLRenderer( { antialias: false, alpha: true, premultipliedAlpha: false, preserveDrawingBuffer: true } ); } const width = Math.min( texture.image.width, maxTextureSize ); const height = Math.min( texture.image.height, maxTextureSize ); // dispose render target if the size is wrong if ( renderTarget && ( renderTarget.width !== width || renderTarget.height !== height ) ) { renderTarget.dispose(); renderTarget = null; } if ( ! renderTarget ) { renderTarget = new WebGLRenderTarget( width, height, { format: RGBAFormat, type: UnsignedByteType, minFilter: LinearFilter, magFilter: LinearFilter } ); } renderer.setRenderTarget( renderTarget ); renderer.setSize( width, height ); renderer.clear(); renderer.render( _scene, _camera ); if ( _renderer ) { _renderer.dispose(); _renderer = null; } const buffer = new Uint8ClampedArray( renderTarget.width * renderTarget.height * 4 ); renderer.readRenderTargetPixels( renderTarget, 0, 0, renderTarget.width, renderTarget.height, buffer ); const imageData = new ImageData( buffer, renderTarget.width, renderTarget.height, undefined ); const bmp = await createImageBitmap( imageData, { premultiplyAlpha: "none" } ); return { imageData, imageBitmap: bmp }; } /** Checks if the given image is of a type with readable data and width/height */ function isImageBitmap( image ) { return ( typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement ) || ( typeof HTMLCanvasElement !== 'undefined' && image instanceof HTMLCanvasElement ) || ( typeof OffscreenCanvas !== 'undefined' && image instanceof OffscreenCanvas ) || ( typeof ImageBitmap !== 'undefined' && image instanceof ImageBitmap ); } /** This method uses a 'bitmaprenderer' context and doesn't do any pixel manipulation. * This way, we can keep the alpha channel as it was, but we're losing the ability to do pixel manipulations or resize operations. */ async function imageToCanvasUnpremultiplied( image: ImageBitmapSource & { width: number, height: number }, maxTextureSize = 4096) { const scale = maxTextureSize / Math.max( image.width, image.height ); const width = image.width * Math.min( 1, scale ); const height = image.height * Math.min( 1, scale ); const canvas = new OffscreenCanvas( width, height ); const settings: ImageBitmapOptions = { premultiplyAlpha: "none" }; if (image.width !== width) settings.resizeWidth = width; if (image.height !== height) settings.resizeHeight = height; const imageBitmap = await createImageBitmap(image, settings); const ctx = canvas.getContext("bitmaprenderer") as ImageBitmapRenderingContext | null; if (ctx) { ctx.transferFromImageBitmap(imageBitmap); } return canvas as OffscreenCanvasExt; } /** This method uses a '2d' canvas context for pixel manipulation, and can apply a color scale or Y flip to the given image. * Unfortunately, canvas always uses premultiplied data, and thus images with low alpha values (or multiplying by a=0) will result in black pixels. */ async function imageToCanvas( image: HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap, color: Vector4 | undefined = undefined, flipY = false, maxTextureSize = 4096 ) { if ( isImageBitmap( image ) ) { // max. canvas size on Safari is still 4096x4096 const scale = maxTextureSize / Math.max( image.width, image.height ); const canvas = new OffscreenCanvas( image.width * Math.min( 1, scale ), image.height * Math.min( 1, scale ) ); const context = canvas.getContext( '2d', { alpha: true, premultipliedAlpha: false } ) as OffscreenCanvasRenderingContext2D; if (!context) throw new Error('Could not get canvas 2D context'); if ( flipY === true ) { context.translate( 0, canvas.height ); context.scale( 1, - 1 ); } context.drawImage( image, 0, 0, canvas.width, canvas.height ); // Currently only used to apply opacity scale since QuickLook and usdview don't support that yet if ( color !== undefined ) { const r = color.x; const g = color.y; const b = color.z; const a = color.w; const imagedata = context.getImageData( 0, 0, canvas.width, canvas.height ); const data = imagedata.data; for ( let i = 0; i < data.length; i += 4 ) { data[ i + 0 ] = data[ i + 0 ] * r; data[ i + 1 ] = data[ i + 1 ] * g; data[ i + 2 ] = data[ i + 2 ] * b; data[ i + 3 ] = data[ i + 3 ] * a; } context.putImageData( imagedata, 0, 0 ); } return canvas as OffscreenCanvasExt; } else { throw new Error( 'NeedleUSDZExporter: No valid image data found. Unable to process texture.' ); } } // const PRECISION = 7; function buildHeader() { return `#usda 1.0 ( customLayerData = { string creator = "Needle Engine USDZExporter" } metersPerUnit = 1 upAxis = "Y" ) `; } function buildUSDFileAsString( dataToInsert, _context: USDZExporterContext ) { let output = buildHeader(); output += dataToInsert; return fflate.strToU8( output ); } function getObjectId( object ) { return object.name.replace( /[-<>\(\)\[\]§$%&\/\\\=\?\,\;]/g, '' ) + '_' + object.id; } function getBoneName(bone: Object3D) { return makeNameSafe(bone.name || 'bone_' + bone.uuid); } function getGeometryName(geometry: BufferGeometry, _fallbackName: string) { // Using object names here breaks instancing... // So we removed fallbackName again. Downside: geometries don't have nice names... // A better workaround would be that we actually name them on glTF import (so the geometry has a name, not the object) return makeNameSafe(geometry.name || 'Geometry') + "_" + geometry.id; } function getMaterialName(material: Material) { return makeNameSafe(material.name || 'Material') + "_" + material.id; } function getPathToSkeleton(bone: Object3D, assumedRoot: Object3D) { let path = getBoneName(bone); let current = bone.parent; while ( current && current !== assumedRoot ) { path = getBoneName(current) + '/' + path; current = current.parent; } return path; } // #endregion // #region XForm export function buildXform( model: USDObject | null, writer: USDWriter, context: USDZExporterContext ) { if ( model == null) return; Progress.report("export-usdz-xforms", { message: "buildXform " + model.displayName || model.name, autoStep: true }); // const matrix = model.matrix; const transform = model.transform; const geometry = model.geometry; const material = model.material; const camera = model.camera; const name = model.name; if ( model.animations ) { for ( const animation of model.animations ) { context.animations.push( animation ) } } // const transform = buildMatrix( matrix ); /* if ( matrix.determinant() < 0 ) { console.warn( 'NeedleUSDZExporter: USDZ does not support negative scales', name ); } */ const isSkinnedMesh = geometry && geometry.isBufferGeometry && geometry.attributes.skinIndex !== undefined && geometry.attributes.skinIndex.count > 0; const objType = isSkinnedMesh ? 'SkelRoot' : 'Xform'; const _apiSchemas = new Array(); // Specific case: the material is white unlit, the mesh has vertex colors, so we can // export as displayColor directly const isUnlitDisplayColor = material && material instanceof MeshBasicMaterial && material.color && material.color.r === 1 && material.color.g === 1 && material.color.b === 1 && !material.map && material.opacity === 1 && geometry?.attributes.color; if (geometry?.attributes.color && !isUnlitDisplayColor) { console.warn("NeedleUSDZExporter: Geometry has vertex colors. Vertex colors will only be shown in QuickLook for unlit materials with white color and no texture. Otherwise, they will be ignored.", model.displayName); } writer.appendLine(); if ( geometry ) { writer.beginBlock( `def ${objType} "${name}"`, "(", false ); // NE-4084: To use the doubleSided workaround with skeletal meshes we'd have to // also emit extra data for jointIndices etc., so we're skipping skinned meshes here. if (context.quickLookCompatible && material && material.side === DoubleSide && !isSkinnedMesh) writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@`); else writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@`); if (!isUnlitDisplayColor) _apiSchemas.push("MaterialBindingAPI"); if (isSkinnedMesh) _apiSchemas.push("SkelBindingAPI"); } else if ( camera && !context.quickLookCompatible) writer.beginBlock( `def Camera "${name}"`, "(", false ); else if ( model.type !== undefined) writer.beginBlock( `def ${model.type} "${name}"` ); else // if (model.type === undefined) writer.beginBlock( `def Xform "${name}"`, "(", false); if (model.type === undefined) { if (model.extraSchemas?.length) _apiSchemas.push(...model.extraSchemas); if (_apiSchemas.length) writer.appendLine(`prepend apiSchemas = [${_apiSchemas.map(s => `"${s}"`).join(', ')}]`); } if (model.displayName) writer.appendLine(`displayName = "${makeDisplayNameSafe(model.displayName)}"`); if ( camera || model.type === undefined) { writer.closeBlock( ")" ); writer.beginBlock(); } if ( geometry && material ) { if (!isUnlitDisplayColor) { const materialName = getMaterialName(material); writer.appendLine( `rel material:binding = ` ); } // Turns out QuickLook / RealityKit doesn't support the doubleSided attribute, so we // work around that by emitting additional indices above, and then we shouldn't emit the attribute either as geometry is // already doubleSided then. if (!context.quickLookCompatible && material.side === DoubleSide ) { // double-sided is a mesh property in USD, we can apply it as `over` here writer.beginBlock( `over "Geometry" `); writer.appendLine( `uniform bool doubleSided = 1` ); writer.closeBlock(); } } let haveWrittenAnyXformOps = false; if ( isSkinnedMesh ) { writer.appendLine( `rel skel:skeleton = ` ); writer.appendLine( `rel skel:animationSource = `); haveWrittenAnyXformOps = false; // writer.appendLine( `matrix4d xformOp:transform = ${buildMatrix(new Matrix4())}` ); // always identity / in world space } else if (model.type === undefined) { if (transform) { haveWrittenAnyXformOps = haveWrittenAnyXformOps || (transform.position !== null || transform.quaternion !== null || transform.scale !== null); if (transform.position) { model.needsTranslate = true; writer.appendLine( `double3 xformOp:translate = (${fn(transform.position.x)}, ${fn(transform.position.y)}, ${fn(transform.position.z)})` ); } if (transform.quaternion) { model.needsOrient = true; writer.appendLine( `quatf xformOp:orient = (${fn(transform.quaternion.w)}, ${fn(transform.quaternion.x)}, ${fn(transform.quaternion.y)}, ${fn(transform.quaternion.z)})` ); } if (transform.scale) { model.needsScale = true; writer.appendLine( `double3 xformOp:scale = (${fn(transform.scale.x)}, ${fn(transform.scale.y)}, ${fn(transform.scale.z)})` ); } } } if (model.visibility !== undefined) writer.appendLine(`token visibility = "${model.visibility}"`); if ( camera && !context.quickLookCompatible) { if ( 'isOrthographicCamera' in camera && camera.isOrthographicCamera ) { writer.appendLine( `float2 clippingRange = (${camera.near}, ${camera.far})` ); writer.appendLine( `float horizontalAperture = ${( ( Math.abs( camera.left ) + Math.abs( camera.right ) ) * 10 ).toPrecision( PRECISION )}` ); writer.appendLine( `float verticalAperture = ${( ( Math.abs( camera.top ) + Math.abs( camera.bottom ) ) * 10 ).toPrecision( PRECISION )}` ); writer.appendLine( 'token projection = "orthographic"' ); } else if ( 'isPerspectiveCamera' in camera && camera.isPerspectiveCamera) { writer.appendLine( `float2 clippingRange = (${camera.near.toPrecision( PRECISION )}, ${camera.far.toPrecision( PRECISION )})` ); writer.appendLine( `float focalLength = ${camera.getFocalLength().toPrecision( PRECISION )}` ); writer.appendLine( `float focusDistance = ${camera.focus.toPrecision( PRECISION )}` ); writer.appendLine( `float horizontalAperture = ${camera.getFilmWidth().toPrecision( PRECISION )}` ); writer.appendLine( 'token projection = "perspective"' ); writer.appendLine( `float verticalAperture = ${camera.getFilmHeight().toPrecision( PRECISION )}` ); } } if ( model.onSerialize ) { model.onSerialize( writer, context ); } // after serialization, we know which xformops to actually define here: if (model.type === undefined) { // TODO only write the necessary ones – this isn't trivial though because we need to know // if some of them are animated, and then we need to include those. // Best approach would likely be to write xformOpOrder _after_ onSerialize // and keep track of what was written in onSerialize (e.g. model.needsTranslate = true) const ops = new Array(); if (model.needsTranslate) ops.push('"xformOp:translate"'); if (model.needsOrient) ops.push('"xformOp:orient"'); if (model.needsScale) ops.push('"xformOp:scale"'); if (ops.length) writer.appendLine( `uniform token[] xformOpOrder = [${ops.join(', ')}]` ); } if ( model.children ) { writer.appendLine(); for ( const ch of model.children ) { buildXform( ch, writer, context ); } } writer.closeBlock(); } function fn( num:number ): string { return Number.isInteger(num) ? num.toString() : num.toFixed( 10 ); } function buildMatrix( matrix ) { const array = matrix.elements; return `( ${buildMatrixRow( array, 0 )}, ${buildMatrixRow( array, 4 )}, ${buildMatrixRow( array, 8 )}, ${buildMatrixRow( array, 12 )} )`; } function buildMatrixRow( array, offset ) { return `(${fn( array[ offset + 0 ] )}, ${fn( array[ offset + 1 ] )}, ${fn( array[ offset + 2 ] )}, ${fn( array[ offset + 3 ] )})`; } // #region Mesh function buildMeshObject( geometry: BufferGeometry, bonesArray: Bone[] = [], quickLookCompatible: boolean = true) { const mesh = buildMesh( geometry, bonesArray, quickLookCompatible ); return ` def "Geometry" ${mesh} `; } function buildMesh( geometry: BufferGeometry, bones: Bone[] = [], quickLookCompatible: boolean = true) { const name = 'Geometry'; const attributes = geometry.attributes; const count = attributes.position.count; const hasBones = bones && bones.length > 0; // We need to sort bones and all skinning data by path – // Neither glTF nor three.js care, but in USD they must be sorted // since the array defines the virtual hierarchy and is evaluated in that order const sortedBones: Array<{bone: Object3D, index: number}> = []; const indexMapping: number[] = []; let sortedSkinIndex = new Array(); let sortedSkinIndexAttribute: BufferAttribute | InterleavedBufferAttribute | null = attributes.skinIndex; let bonesArray = ""; if (hasBones) { const uuidsFound:string[] = []; for (const bone of bones ) { // if (bone.parent!.type !== 'Bone') { sortedBones.push({bone: bone, index: bones.indexOf(bone)}); uuidsFound.push(bone.uuid); } } let maxSteps = 10_000; while (uuidsFound.length < bones.length && maxSteps-- > 0) { for (const sortedBone of sortedBones) { const children = sortedBone.bone.children as Bone[]; for (const childBone of children){ if (uuidsFound.indexOf(childBone.uuid) === -1 && bones.indexOf(childBone) !== -1){ sortedBones.push({bone: childBone, index: bones.indexOf(childBone)}); uuidsFound.push(childBone.uuid); } } } } if (maxSteps <= 0) console.error("Failed to sort bones in skinned mesh", sortedBones, bones, uuidsFound); // add structural nodes to the list of bones for (const structuralNode of findStructuralNodesInBoneHierarchy(bones) ) { sortedBones.push( { bone: structuralNode, index: sortedBones.length } ); } // sort bones by path – need to be sorted in the same order as during mesh export const assumedRoot = sortedBones[0].bone.parent!; sortedBones.sort((a, b) => getPathToSkeleton(a.bone, assumedRoot) > getPathToSkeleton(b.bone, assumedRoot) ? 1 : -1); bonesArray = sortedBones.map( x => "\"" + getPathToSkeleton(x.bone, assumedRoot) + "\"" ).join( ', ' ); // TODO we can probably skip the expensive attribute re-ordering if the bones were already in a correct order. // That doesn't mean that they are strictly sorted by path – just that all parents strictly need to come first. // build index mapping for (const i in sortedBones) { indexMapping[sortedBones[i].index] = parseInt(i); } // remap skin index attributes const skinIndex = attributes.skinIndex; sortedSkinIndex = new Array(); for ( let i = 0; i < skinIndex.count; i ++ ) { const x = skinIndex.getX( i ); const y = skinIndex.getY( i ); const z = skinIndex.getZ( i ); const w = skinIndex.getW( i ); sortedSkinIndex.push( indexMapping[x], indexMapping[y], indexMapping[z], indexMapping[w] ); } // turn it back into an attribute so the rest of the code doesn't need to learn a new thing sortedSkinIndexAttribute = new BufferAttribute( new Uint16Array( sortedSkinIndex ), 4 ); } const isSkinnedMesh = attributes.skinWeight && attributes.skinIndex; return ` { def Mesh "${name}" ${isSkinnedMesh ? `( prepend apiSchemas = ["SkelBindingAPI"] )` : ''} { int[] faceVertexCounts = [${buildMeshVertexCount( geometry )}] int[] faceVertexIndices = [${buildMeshVertexIndices( geometry )}] ${attributes.normal || quickLookCompatible ? // in QuickLook, normals are required, otherwise double-sided rendering doesn't work. `normal3f[] normals = [${buildVector3Array( attributes.normal, count )}] ( interpolation = "vertex" )` : '' } point3f[] points = [${buildVector3Array( attributes.position, count )}] ${attributes.uv ? `texCoord2f[] primvars:st = [${buildVector2Array( attributes.uv, count, true )}] ( interpolation = "vertex" )` : '' } ${attributes.uv1 ? buildCustomAttributeAccessor('st1', attributes.uv1) : '' } ${attributes.uv2 ? buildCustomAttributeAccessor('st2', attributes.uv2) : '' } ${attributes.uv3 ? buildCustomAttributeAccessor('st3', attributes.uv3) : '' } ${isSkinnedMesh ? `matrix4d primvars:skel:geomBindTransform = ( (1, 0, 0, 0), (0, 1, 0, 0), (0, 0, 1, 0), (0, 0, 0, 1) ) ( elementSize = 1 interpolation = "constant" )` : '' } ${attributes.skinIndex ? `int[] primvars:skel:jointIndices = [${buildVector4Array( sortedSkinIndexAttribute, true )}] ( elementSize = 4 interpolation = "vertex" )` : '' } ${attributes.skinWeight ? `float[] primvars:skel:jointWeights = [${buildVector4Array( attributes.skinWeight )}] ( elementSize = 4 interpolation = "vertex" )` : '' } ${attributes.color ? `color3f[] primvars:displayColor = [${buildVector3Array( attributes.color, count )}] ( interpolation = "vertex" )` : '' } uniform token subdivisionScheme = "none" } } ${quickLookCompatible ? ` # This is a workaround for QuickLook/RealityKit not supporting the doubleSided attribute. We're adding a second # geometry definition here, that uses the same mesh data but appends extra faces with reversed winding order. def "${name}_doubleSided" ( prepend references = ) { over "Geometry" { int[] faceVertexCounts = [${buildMeshVertexCount( geometry ) + ", " + buildMeshVertexCount( geometry )}] int[] faceVertexIndices = [${buildMeshVertexIndices( geometry ) + ", " + buildMeshVertexIndices( geometry, true )}] } } ` : '' } `; } function buildMeshVertexCount( geometry: BufferGeometry ) { const count = geometry.index !== null ? geometry.index.count : geometry.attributes.position.count; return Array( Math.floor(count / 3) ).fill( 3 ).join( ', ' ); } function buildMeshVertexIndices( geometry: BufferGeometry, reverseWinding: boolean = false ) { const index = geometry.index; const array: Array = []; if ( index !== null ) { for ( let i = 0; i < index.count; i ++ ) { let val = i; if ( reverseWinding ) val = i % 3 === 0 ? i + 2 : i % 3 === 2 ? i - 2 : i; array.push( index.getX( val ) ); } } else { const length = geometry.attributes.position.count; for ( let i = 0; i < length; i ++ ) { let val = i; if ( reverseWinding ) val = i % 3 === 0 ? i + 2 : i % 3 === 2 ? i - 2 : i; array.push( val ); } } return array.join( ', ' ); } /** Returns a string with the correct attribute declaration for the given attribute. Could have 2,3,4 components. */ function buildCustomAttributeAccessor( primvarName: string, attribute: BufferAttribute | InterleavedBufferAttribute ) { const count = attribute.itemSize; switch (count) { case 2: // TODO: Check if we want to flip Y here as well. We do that for texcoords, but the data in UV1..N could be intended for other purposes. return `texCoord2f[] primvars:${primvarName} = [${buildVector2Array( attribute, count, true )}] ( interpolation = "vertex" )`; case 3: return `texCoord3f[] primvars:${primvarName} = [${buildVector3Array( attribute, count )}] ( interpolation = "vertex" )`; case 4: return `double4[] primvars:${primvarName} = [${buildVector4Array2( attribute, count )}] ( interpolation = "vertex" )`; default: console.warn('USDZExporter: Attribute with ' + count + ' components are currently not supported. Results may be undefined for ' + primvarName + '.'); return ''; } } function buildVector3Array( attribute, count ) { if ( attribute === undefined ) { console.warn( 'USDZExporter: A mesh attribute is missing and will be set with placeholder data. The result may look incorrect.' ); return Array( count ).fill( '(0, 0, 1)' ).join( ', ' ); } const array: Array = []; for ( let i = 0; i < attribute.count; i ++ ) { const x = attribute.getX( i ); const y = attribute.getY( i ); const z = attribute.getZ( i ); array.push( `(${x.toPrecision( PRECISION )}, ${y.toPrecision( PRECISION )}, ${z.toPrecision( PRECISION )})` ); } return array.join( ', ' ); } function buildVector4Array2( attribute, count ) { if ( attribute === undefined ) { console.warn( 'USDZExporter: Attribute is missing. Results may be undefined.' ); return Array( count ).fill( '(0, 0, 0, 0)' ).join( ', ' ); } const array: Array = []; for ( let i = 0; i < attribute.count; i ++ ) { const x = attribute.getX( i ); const y = attribute.getY( i ); const z = attribute.getZ( i ) || 0; const w = attribute.getW( i ) || 0; array.push( `(${x.toPrecision( PRECISION )}, ${y.toPrecision( PRECISION )}, ${z.toPrecision( PRECISION )}, ${w.toPrecision( PRECISION )})` ); } return array.join( ', ' ); } function buildVector4Array( attribute, ints = false ) { const array: Array = []; for ( let i = 0; i < attribute.count; i ++ ) { const x = attribute.getX( i ); const y = attribute.getY( i ); const z = attribute.getZ( i ); const w = attribute.getW( i ); array.push( `${ints ? x : x.toPrecision( PRECISION )}` ); array.push( `${ints ? y : y.toPrecision( PRECISION )}` ); array.push( `${ints ? z : z.toPrecision( PRECISION )}` ); array.push( `${ints ? w : w.toPrecision( PRECISION )}` ); } return array.join( ', ' ); } function buildVector2Array( attribute: BufferAttribute | InterleavedBufferAttribute, count: number, flipY: boolean = false ) { if ( attribute === undefined ) { console.warn( 'USDZExporter: UVs missing.' ); return Array( count ).fill( '(0, 0)' ).join( ', ' ); } const array: Array = []; for ( let i = 0; i < attribute.count; i ++ ) { const x = attribute.getX( i ); let y = attribute.getY( i ); if (flipY) y = 1 - y; array.push( `(${x.toPrecision( PRECISION )}, ${y.toPrecision( PRECISION )})` ); } return array.join( ', ' ); } // #endregion // #region Materials function buildMaterials( materials: Map, textures: TextureMap, quickLookCompatible = false ) { const array: Array = []; for ( const uuid in materials ) { const material = materials[ uuid ]; array.push( buildMaterial( material, textures, quickLookCompatible ) ); } return ` def "Materials" { ${array.join( '' )} }`; } /** Slot of the exported texture. Some slots (e.g. normal, opacity) require additional processing. */ declare type MapType = 'diffuse' | 'normal' | 'occlusion' | 'opacity' | 'roughness' | 'emissive' | 'metallic' | 'transmission'; function texName(tex: Texture) { // If we have a source, we only want to use the source's id, not the texture's id // to avoid exporting the same underlying data multiple times. return makeNameSafe(tex.name) + '_' + (tex.source?.id ?? tex.id); } function buildTexture( texture: Texture, mapType: MapType, quickLookCompatible: boolean, textures: TextureMap, material, usedUVChannels, color: Color | undefined = undefined, opacity: number | undefined = undefined ) { const name = texName(texture); // File names need to be unique; so when we're baking color and/or opacity into a texture we need to keep track of that. // TODO This currently results in textures potentially being exported multiple times that are actually identical! // const colorHex = color ? color.getHexString() : undefined; // const colorId = ( color && colorHex != 'ffffff' ? '_' + color.getHexString() : '' ); const id = name + ( opacity !== undefined && opacity !== 1.0 ? '_' + opacity : '' ); // Seems neither QuickLook nor usdview support scale/bias on .a values, so we need to bake opacity multipliers into // the texture. This is not ideal. const opacityIsAppliedToTextureAndNotAsScale = quickLookCompatible && opacity !== undefined && opacity !== 1.0; const scaleToApply = opacityIsAppliedToTextureAndNotAsScale ? new Vector4(1, 1, 1, opacity) : undefined; // Treat undefined opacity as 1 if (opacity === undefined) opacity = 1.0; // When we're baking opacity into the texture, we shouldn't apply it as scale as well – so we set it to 1. if (opacityIsAppliedToTextureAndNotAsScale) opacity = 1.0; // sanitize scaleToApply.w === 0 - this isn't working right now, seems the PNG can't be read back properly. if (scaleToApply && scaleToApply.w <= 0.05) scaleToApply.w = 0.05; textures[ id ] = { texture, scale: scaleToApply }; const uv = texture.channel > 0 ? 'st' + texture.channel : 'st'; usedUVChannels.add(texture.channel); const isRGBA = formatsWithAlphaChannel.includes( texture.format ); const WRAPPINGS = { 1000: 'repeat', // RepeatWrapping 1001: 'clamp', // ClampToEdgeWrapping 1002: 'mirror' // MirroredRepeatWrapping }; const repeat = texture.repeat.clone(); const offset = texture.offset.clone(); const rotation = texture.rotation; // rotation is around the wrong point. after rotation we need to shift offset again so that we're rotating around the right spot const xRotationOffset = Math.sin(rotation); const yRotationOffset = Math.cos(rotation); // texture coordinates start in the opposite corner, need to correct offset.y = 1 - offset.y - repeat.y; // turns out QuickLook is buggy and interprets texture repeat inverted. // Apple Feedback: FB10036297 and FB11442287 if ( quickLookCompatible ) { // This is NOT correct yet in QuickLook, but comes close for a range of models. // It becomes more incorrect the bigger the offset is // sanitize repeat values to avoid NaNs if (repeat.x === 0) repeat.x = 0.0001; if (repeat.y === 0) repeat.y = 0.0001; offset.x = offset.x / repeat.x; offset.y = offset.y / repeat.y; offset.x += xRotationOffset / repeat.x; offset.y += yRotationOffset - 1; } else { // results match glTF results exactly. verified correct in usdview. offset.x += xRotationOffset * repeat.x; offset.y += (1 - yRotationOffset) * repeat.y; } const materialName = getMaterialName(material); const needsTextureTransform = ( repeat.x != 1 || repeat.y != 1 || offset.x != 0 || offset.y != 0 || rotation != 0 ); const textureTransformInput = `${materialRoot}/${materialName}/${'uvReader_' + uv}.outputs:result>`; const textureTransformOutput = `${materialRoot}/${materialName}/Transform2d_${mapType}.outputs:result>`; const needsTextureScale = mapType !== 'normal' && (color && (color.r !== 1 || color.g !== 1 || color.b !== 1 || opacity !== 1)) || false; const needsNormalScaleAndBias = mapType === 'normal'; const normalScale = material instanceof MeshStandardMaterial ? (material.normalScale ? material.normalScale.x * 2 : 2) : 2; const normalScaleValueString = normalScale.toFixed( PRECISION ); const normalBiasString = (-1 * (normalScale / 2)).toFixed( PRECISION ); const normalBiasZString = (1 - normalScale).toFixed( PRECISION ); return ` ${needsTextureTransform ? `def Shader "Transform2d_${mapType}" ( sdrMetadata = { string role = "math" } ) { uniform token info:id = "UsdTransform2d" float2 inputs:in.connect = ${textureTransformInput} float2 inputs:scale = ${buildVector2( repeat )} float2 inputs:translation = ${buildVector2( offset )} float inputs:rotation = ${(rotation / Math.PI * 180).toFixed( PRECISION )} float2 outputs:result } ` : '' } def Shader "${name}_${mapType}" { uniform token info:id = "UsdUVTexture" asset inputs:file = @textures/${id}.${isRGBA ? 'png' : 'jpg'}@ token inputs:sourceColorSpace = "${ texture.colorSpace === 'srgb' ? 'sRGB' : 'raw' }" float2 inputs:st.connect = ${needsTextureTransform ? textureTransformOutput : textureTransformInput} ${needsTextureScale ? ` float4 inputs:scale = (${color ? color.r + ', ' + color.g + ', ' + color.b : '1, 1, 1'}, ${opacity}) ` : `` } ${needsNormalScaleAndBias ? ` float4 inputs:scale = (${normalScaleValueString}, ${normalScaleValueString}, ${normalScaleValueString}, 1) float4 inputs:bias = (${normalBiasString}, ${normalBiasString}, ${normalBiasZString}, 0) ` : `` } token inputs:wrapS = "${ WRAPPINGS[ texture.wrapS ] }" token inputs:wrapT = "${ WRAPPINGS[ texture.wrapT ] }" float outputs:r float outputs:g float outputs:b float3 outputs:rgb ${material.transparent || material.alphaTest > 0.0 ? 'float outputs:a' : ''} }`; } function buildMaterial( material: MeshBasicMaterial, textures: TextureMap, quickLookCompatible = false ) { const materialName = getMaterialName(material); // TODO: refactor to USDMaterial class // const usdMaterial = new USDMaterial(material); // Special case: occluder material // Supported on iOS 18+ and visionOS 1+ const isShadowCatcherMaterial = material.colorWrite === false || (material.userData?.isShadowCatcherMaterial || material.userData?.isLightBlendMaterial); if (isShadowCatcherMaterial) { // Two options here: // - ND_realitykit_occlusion_surfaceshader (non-shadow receiving) // - ND_realitykit_shadowreceiver_surfaceshader (shadow receiving) const mode = (material.userData.isLightBlendMaterial || material.userData.isShadowCatcherMaterial) ? "ND_realitykit_shadowreceiver_surfaceshader" : "ND_realitykit_occlusion_surfaceshader"; return ` def Material "${materialName}" ${material.name ?`( displayName = "${material.name}" )` : ''} { token outputs:mtlx:surface.connect = ${materialRoot}/${materialName}/Occlusion.outputs:out> def Shader "Occlusion" { uniform token info:id = "${mode}" token outputs:out } }`; } // https://graphics.pixar.com/usd/docs/UsdPreviewSurface-Proposal.html const pad = ' '; const inputs: Array = []; const samplers: Array = []; const usedUVChannels: Set = new Set(); if ((material as any).isMeshPhysicalNodeMaterial === true) { return buildNodeMaterial(material as unknown as MeshPhysicalNodeMaterial, materialName, textures); } let effectiveOpacity = ( material.transparent || material.alphaTest ) ? material.opacity : 1; let haveConnectedOpacity = false; let haveConnectedOpacityThreshold = false; if ( material instanceof MeshPhysicalMaterial && material.transmission !== undefined) { // TODO does not help when a roughnessMap is used effectiveOpacity *= (1 - material.transmission * (1 - (material.roughness * 0.5))); } if ( material.map ) { inputs.push( `${pad}color3f inputs:diffuseColor.connect = ${materialRoot}/${materialName}/${texName(material.map)}_diffuse.outputs:rgb>` ); // Enforce alpha hashing in QuickLook for unlit materials if (material instanceof MeshBasicMaterial && material.transparent && material.alphaTest == 0.0 && quickLookCompatible) { inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/${materialName}/${texName(material.map)}_diffuse.outputs:a>` ); haveConnectedOpacity = true; // see below – QuickLook applies alpha hashing instead of pure blending when // both opacity and opacityThreshold are connected inputs.push( `${pad}float inputs:opacityThreshold = ${0.0000000001}` ); haveConnectedOpacityThreshold = true; } else if ( material.transparent ) { inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/${materialName}/${texName(material.map)}_diffuse.outputs:a>` ); haveConnectedOpacity = true; } else if ( material.alphaTest > 0.0 ) { inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/${materialName}/${texName(material.map)}_diffuse.outputs:a>` ); haveConnectedOpacity = true; inputs.push( `${pad}float inputs:opacityThreshold = ${material.alphaTest}` ); haveConnectedOpacityThreshold = true; } samplers.push( buildTexture( material.map, 'diffuse', quickLookCompatible, textures, material, usedUVChannels, material.color, effectiveOpacity ) ); } else { inputs.push( `${pad}color3f inputs:diffuseColor = ${buildColor( material.color )}` ); } if ( material.alphaHash && quickLookCompatible) { // Seems we can do this to basically enforce alpha hashing / dithered transparency in QuickLook – // works completely different in usdview though... if (haveConnectedOpacityThreshold) { console.warn('Opacity threshold for ' + material.name + ' was already connected. Skipping alphaHash opacity threshold.'); } else { inputs.push( `${pad}float inputs:opacityThreshold = 0.0000000001` ); haveConnectedOpacityThreshold = true; } } if ( material.aoMap ) { inputs.push( `${pad}float inputs:occlusion.connect = ${materialRoot}/${materialName}/${texName(material.aoMap)}_occlusion.outputs:r>` ); samplers.push( buildTexture( material.aoMap, 'occlusion', quickLookCompatible, textures, material, usedUVChannels ) ); } if ( material.alphaMap ) { inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/${materialName}/${texName(material.alphaMap)}_opacity.outputs:r>` ); // TODO this is likely not correct and will prevent blending from working correctly – masking will be used instead inputs.push( `${pad}float inputs:opacityThreshold = 0.0000000001` ); haveConnectedOpacity = true; haveConnectedOpacityThreshold = true; samplers.push( buildTexture( material.alphaMap, 'opacity', quickLookCompatible, textures, material, usedUVChannels, new Color( 1, 1, 1 ), effectiveOpacity ) ); } else { // opacity will always be connected if we have a map if (haveConnectedOpacity) { console.warn('Opacity for ' + material.name + ' was already connected. Skipping default opacity.'); } else { inputs.push( `${pad}float inputs:opacity = ${effectiveOpacity}` ); haveConnectedOpacity = true; } if ( material.alphaTest > 0.0 ) { if (haveConnectedOpacityThreshold) { console.warn('Opacity threshold for ' + material.name + ' was already connected. Skipping default opacity threshold.'); } else { inputs.push( `${pad}float inputs:opacityThreshold = ${material.alphaTest}` ); haveConnectedOpacityThreshold = true; } } } if ( material instanceof MeshStandardMaterial ) { if ( material.emissiveMap ) { inputs.push( `${pad}color3f inputs:emissiveColor.connect = ${materialRoot}/${materialName}/${texName(material.emissiveMap)}_emissive.outputs:rgb>` ); const color = material.emissive.clone(); color.multiplyScalar( material.emissiveIntensity ); samplers.push( buildTexture( material.emissiveMap, 'emissive', quickLookCompatible, textures, material, usedUVChannels, color ) ); } else if ( material.emissive?.getHex() > 0 ) { const color = material.emissive.clone(); color.multiplyScalar( material.emissiveIntensity ); inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( color )}` ); } else { // We don't need to export (0,0,0) as emissive color // inputs.push( `${pad}color3f inputs:emissiveColor = (0, 0, 0)` ); } if ( material.normalMap ) { inputs.push( `${pad}normal3f inputs:normal.connect = ${materialRoot}/${materialName}/${texName(material.normalMap)}_normal.outputs:rgb>` ); samplers.push( buildTexture( material.normalMap, 'normal', quickLookCompatible, textures, material, usedUVChannels ) ); } if ( material.roughnessMap && material.roughness === 1 ) { inputs.push( `${pad}float inputs:roughness.connect = ${materialRoot}/${materialName}/${texName(material.roughnessMap)}_roughness.outputs:g>` ); samplers.push( buildTexture( material.roughnessMap, 'roughness', quickLookCompatible, textures, material, usedUVChannels ) ); } else { inputs.push( `${pad}float inputs:roughness = ${material.roughness !== undefined ? material.roughness : 1 }` ); } if ( material.metalnessMap && material.metalness === 1 ) { inputs.push( `${pad}float inputs:metallic.connect = ${materialRoot}/${materialName}/${texName(material.metalnessMap)}_metallic.outputs:b>` ); samplers.push( buildTexture( material.metalnessMap, 'metallic', quickLookCompatible, textures, material, usedUVChannels ) ); } else { inputs.push( `${pad}float inputs:metallic = ${material.metalness !== undefined ? material.metalness : 0 }` ); } } if ( material instanceof MeshPhysicalMaterial ) { inputs.push( `${pad}float inputs:clearcoat = ${material.clearcoat}` ); inputs.push( `${pad}float inputs:clearcoatRoughness = ${material.clearcoatRoughness}` ); inputs.push( `${pad}float inputs:ior = ${material.ior}` ); if ( !material.transparent && ! (material.alphaTest > 0.0) && material.transmissionMap) { inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/${materialName}/${texName(material.transmissionMap)}_transmission.outputs:r>` ); samplers.push( buildTexture( material.transmissionMap, 'transmission', quickLookCompatible, textures, material, usedUVChannels ) ); } } // check if usedUVChannels contains any data besides exactly 0 or 1 if (usedUVChannels.size > 2) { console.warn('USDZExporter: Material ' + material.name + ' uses more than 2 UV channels. Currently, only UV0 and UV1 are supported.'); } else if (usedUVChannels.size === 2) { // check if it's exactly 0 and 1 if (!usedUVChannels.has(0) || !usedUVChannels.has(1)) { console.warn('USDZExporter: Material ' + material.name + ' uses UV channels other than 0 and 1. Currently, only UV0 and UV1 are supported.'); } } return ` def Material "${materialName}" ${material.name ?`( displayName = "${makeDisplayNameSafe(material.name)}" )` : ''} { token outputs:surface.connect = ${materialRoot}/${materialName}/PreviewSurface.outputs:surface> def Shader "PreviewSurface" { uniform token info:id = "UsdPreviewSurface" ${inputs.join( '\n' )} int inputs:useSpecularWorkflow = ${material instanceof MeshBasicMaterial ? '1' : '0'} token outputs:surface } ${samplers.length > 0 ? ` ${usedUVChannels.has(0) ? ` def Shader "uvReader_st" { uniform token info:id = "UsdPrimvarReader_float2" string inputs:varname = "st" float2 inputs:fallback = (0.0, 0.0) float2 outputs:result } ` : ''} ${usedUVChannels.has(1) ? ` def Shader "uvReader_st1" { uniform token info:id = "UsdPrimvarReader_float2" string inputs:varname = "st1" float2 inputs:fallback = (0.0, 0.0) float2 outputs:result } ` : ''} ${samplers.join( '\n' )}` : ''} }`; } // #endregion function buildColor( color ) { return `(${color.r}, ${color.g}, ${color.b})`; } function buildVector2( vector ) { return `(${ vector.x }, ${ vector.y })`; } const formatsWithAlphaChannel = [ // uncompressed formats with alpha channel 1023, // RGBAFormat // compressed formats with alpha channel 33777, // RGBA_S3TC_DXT1_Format 33778, // RGBA_S3TC_DXT3_Format 33779, // RGBA_S3TC_DXT5_Format 35842, // RGBA_PVRTC_4BPPV1_Format 35843, // RGBA_PVRTC_2BPPV1_Format 37496, // RGBA_ETC2_EAC_Format 37808, // RGBA_ASTC_4x4_Format 37809, // RGBA_ASTC_5x4_Format 37810, // RGBA_ASTC_5x5_Format 37811, // RGBA_ASTC_6x5_Format 37812, // RGBA_ASTC_6x6_Format 37813, // RGBA_ASTC_8x5_Format 37814, // RGBA_ASTC_8x6_Format 37815, // RGBA_ASTC_8x8_Format 37816, // RGBA_ASTC_10x5_Format 37817, // RGBA_ASTC_10x6_Format 37818, // RGBA_ASTC_10x8_Format 37819, // RGBA_ASTC_10x10_Format 37820, // RGBA_ASTC_12x10_Format 37821, // RGBA_ASTC_12x12_Format 36492, // RGBA_BPTC_Format ]; // #region exports export { buildMatrix, decompressGpuTexture, findStructuralNodesInBoneHierarchy, formatsWithAlphaChannel, getBoneName, getMaterialName as getMaterialNameForUSD, getPathToSkeleton, imageToCanvas, imageToCanvasUnpremultiplied, makeNameSafe as makeNameSafeForUSD, USDDocument, fn as usdNumberFormatting, USDObject, USDWriter, USDZExporter, USDZExporterContext, };