import { Engine, Matrix, Texture2D, TextureFormat, TextureWrapMode } from "oasis-engine"; import { PaladinAdapter } from "../../adapter/PaladinAdapter"; import { ARConfig, ARMode, LocalizeConfig, PixelFormat, RequestState } from "../AREnum"; import { IARCtrl } from "../IARCtrl"; import { ARFeatureMode, ARFeatureSupport, ARSlamMode, EasyARFrame, LocalizeError, TargetInstance, TargetStatus } from "./Type"; export class ARAndroid implements IARCtrl { private _engine: Engine; /** 是否已经初始化ExpAR */ private initialized: boolean = false; /** 实例与数据 */ private curARSession: any; private expARLocalizer: any; private expARRTCT: any; /** 当前模式 */ private _curARMode: ARMode; private _curARFrame: EasyARFrame; private _curLocalizerRes: any; // 上次请求 localizer 的时间戳 private _requestTime: number = 0; // localizer 的请求状态(来保证无论间隔或者网络状态如何设置,都在上次结果返回后才请求) private _requestState = RequestState.notInit; // 请求 localizer 的间隔(ms) private _requestInv: number = 800; // AR 支持功能模块的映射 private arSupportMap: Record = {}; private _markerURL: string; /** @internal */ _worldMatrix: Matrix = new Matrix(); /** @internal */ _projectMatrix: Matrix = new Matrix(); /** @internal */ _textureProject: Matrix = new Matrix(); /** @internal */ _textureY: Texture2D; /** @internal */ _textureUV: Texture2D; create(config: ARConfig, success: Function, fail?: Function): void { const createInstance = () => { this._curARMode = config.mode; this.canIUse(ARFeatureMode.motionTrack, () => { let trackingMode: ARSlamMode; switch (this._curARMode) { case ARMode.Camera: trackingMode = ARSlamMode.VIO; break; case ARMode.Dof3: case ARMode.Dof6: trackingMode = ARSlamMode.LARGE_SCALE; break; case ARMode.Localizer: trackingMode = ARSlamMode.LARGE_SCALE; break; } const arConfig = { mode: ARFeatureMode.motionTrack, trackingMode: trackingMode, viewWidth: config.pixelWidth, viewHeight: config.pixelHeight, nearPlane: config.near, farPlane: config.far }; if (this.curARSession) { // 销毁旧的 // @ts-ignore ExpAREngine.destroyInstance(this.curARSession); } switch (this._curARMode) { case ARMode.Camera: case ARMode.Dof3: case ARMode.Dof6: // 判断是否支持 Track try { // @ts-ignore this.curARSession = ExpAREngine.createInstance(arConfig); success && success(); } catch (error) { fail && fail(); } break; case ARMode.Localizer: // 判断是否支持 Localizer try { // @ts-ignore this.curARSession = ExpAREngine.createInstance(arConfig); // @ts-ignore this.expARLocalizer = ExpAREngine.createInstance({ mode: ARFeatureMode.localizer, apiKey: LocalizeConfig.ApiKey, apiSecret: LocalizeConfig.ApiSecret, appId: LocalizeConfig.AppId }); // @ts-ignore this.expARRTCT = ExpAREngine.createInstance({ mode: ARFeatureMode.rtct }); success && success(); } catch (error) { fail && fail(); } break; case ARMode.ImageTrack: // 判断是否支持 imageTrack try { // @ts-ignore this.curARSession = ExpAREngine.createInstance({ mode: "imageTrack", imageTrackMode: "quality", imageTrackSimultaneousNum: 1, viewWidth: config.pixelWidth, viewHeight: config.pixelHeight, nearPlane: config.near, farPlane: config.far }); this._markerURL = config.markerURL; success && success(); } catch (error) { fail && fail(); } break; default: break; } }); }; if (this.initialized) { createInstance(); } else { PaladinAdapter.loadPlugin("ExpAR"); // @ts-ignore if (!ExpARLoader) { console.info("[MY] onReady: expARLoader not defined"); fail && fail(); } else { // @ts-ignore ExpARLoader.load({ success: () => { // 异步初始化ExpAR,对Android来说包含加载动态Bundle并load so过程,有可能失败 // @ts-ignore ExpAREngine.init({ success: () => { console.log("ARControl.init:success"); this.initialized = true; createInstance(); }, fail: () => { console.log("ARControl.init:fail"); fail && fail(); } }); }, fail: () => { fail && fail(); } }); } } } private offscreenCanvas: any = null; /** * 开始跑 AR 帧 */ start(success?: Function, fail?: Function): void { try { switch (this._curARMode) { case ARMode.Localizer: this.expARLocalizer && this.expARLocalizer.start(); this._requestState = RequestState.idle; case ARMode.Camera: case ARMode.Dof3: case ARMode.Dof6: this.curARSession && this.curARSession.start(); break; case ARMode.ImageTrack: this.curARSession && this.curARSession.start(); PaladinAdapter.loadImage(this._markerURL) .then((img) => { // 加载完毕 if (!this.offscreenCanvas) { // @ts-ignore this.offscreenCanvas = my.createCanvas(img.width, img.height); } if (this.offscreenCanvas.width !== img.width || this.offscreenCanvas.height !== img.height) { this.offscreenCanvas.width = img.width; this.offscreenCanvas.height = img.height; } const context: CanvasRenderingContext2D = this.offscreenCanvas.getContext("2d"); context.drawImage(img, 0, 0); const imageData = context.getImageData(0, 0, img.width, img.height); const identifier = "face"; this.curARSession.loadImageTarget({ // image target配置 name: identifier, uid: "TESTUID" + identifier, meta: "TESTMETA", scale: 1, // image数据 data: imageData.data, pixelFormat: "RGBA8888", width: imageData.width, height: imageData.height, complete: (result) => { if (result.success) { console.log("loadImageTarget success", result); } else { console.log("loadImageTarget fail", result); } } }); }) .catch((reason) => { console.log("loadImageError", reason); }); break; default: break; } success && success(); } catch (error) { fail && fail(); } } /** * 帧开始的时候执行 */ update(): void { if (this.curARSession) { const arFrame = (this._curARFrame = this.curARSession.peekFrame()); if (arFrame) { this.updateTexture(); // Localizer 模式下需要 requestLocalizer // Dof3, Dof6, Localizer 模式都需要矫正相机的位置和 Project // Camera 模式只需要更新纹理即可 switch (this._curARMode) { case ARMode.Localizer: this.requestLocalizer(arFrame); case ARMode.Dof3: case ARMode.Dof6: this.updatePosition(); break; case ARMode.ImageTrack: this.updateImageTrack(); this.updatePosition(); break; default: break; } this.disposeFrame(); } } } /** * 销毁的逻辑 */ destroy(): void { // 销毁帧数据 if (this._curARFrame) { this.curARSession.disposeFrame(this._curARFrame); this._curARFrame = null; } // 销毁 Instance if (this.curARSession) { //@ts-ignore ExpAREngine.destroyInstance(this.curARSession); this.curARSession = null; } } /** * 检查是否可以使用 AR 的特定模式 * @param featureMode - 功能模式 */ canIUse(featureMode: string, successFun: Function, failFun?: Function) { const mapValue = this.arSupportMap[featureMode]; switch (mapValue) { case ARFeatureSupport.enable: console.log("ARControl.canIUse:success"); successFun(); break; case ARFeatureSupport.disable: console.log("ARControl.canIUse:fail"); failFun && failFun(); break; default: // @ts-ignore ExpAREngine.isSupported({ mode: featureMode, complete: (result: any) => { console.log("ExpAR isSupported:" + result["isSupported"]); if (result.isSupported) { this.arSupportMap[featureMode] = ARFeatureSupport.enable; successFun(); } else { this.arSupportMap[featureMode] = ARFeatureSupport.disable; failFun && failFun(); } } }); break; } } onShow(): void { // 如果压后台的时刻正在运行,就 pause // @ts-ignore this.initialized && ExpAREngine.pause(); } onHide(): void { // 如果压后台前正在运行,就 resume // @ts-ignore this.initialized && ExpAREngine.resume(); } /** * Create ARAndroid. * @param engine - Engine */ constructor(engine: Engine) { this._engine = engine; } /** * 请求 Localizer * @param arFrame - 帧数据 */ private requestLocalizer(arFrame: EasyARFrame) { const timestamp = Date.now(); // 1.是 Localizer 模式 // 2.trackingStatus === 1 // 3.上次请求结果已经返回 // 4.间隔时间已过 if ( this._curARMode === ARMode.Localizer && arFrame.camera.trackingStatus === 1 && this._requestState === RequestState.idle && timestamp - this._requestTime >= this._requestInv ) { this._requestTime = timestamp; this._requestState = RequestState.waiting; this.expARLocalizer.resolve({ frame: arFrame, complete: (localizerResult) => { if (localizerResult && localizerResult.localizeStatus === LocalizeError.success) { this._curLocalizerRes = localizerResult; this.expARRTCT.insertData({ timestamp: arFrame.timestamp, localCameraTransform: arFrame.camera.transform, mapPose: localizerResult.pose }); } this._requestState = RequestState.idle; } }); } } /** * 根据 AR 帧信息更新相机的世界矩阵和投影矩阵 * @param cameraWorldMatrix - 为了减少 new 缓存的相机世界矩阵 * @param projectionMatrix - 为了减少 new 缓存的相机投影矩阵 * @returns */ private updatePosition(): void { const { camera, timestamp } = this._curARFrame; switch (this._curARMode) { case ARMode.Localizer: // Update camera by localizer. if (this._curLocalizerRes && camera.trackingStatus != -1) { var pose_every_frame = this.expARRTCT.getPoseInMap({ timestamp: timestamp, motionTrackingStatus: camera.trackingStatus, localCameraTransform: camera.transform }); this._worldMatrix.copyFromArray(pose_every_frame).transpose().invert(); this._projectMatrix.copyFromArray(camera.projection).transpose(); } break; case ARMode.Dof6: case ARMode.Dof3: // Update camera by track. if (camera.trackingStatus != -1) { this._worldMatrix.copyFromArray(camera.transform).transpose(); this._projectMatrix.copyFromArray(camera.projection).transpose(); } break; case ARMode.ImageTrack: this._projectMatrix.copyFromArray(camera.projection).transpose(); break; default: break; } } /** * 更新图片识别 */ private updateImageTrack() { const { results } = this._curARFrame; const map = []; let resTarget: TargetInstance; if (results && results.length > 0) { for (var i = 0; i < results.length; i++) { var targetInstances = results[i].targetInstances; for (var j = 0; j < targetInstances.length; j++) { var targetIns = targetInstances[j]; if (targetIns.status === TargetStatus.Tracked) { this._worldMatrix.copyFromArray(targetIns.pose).transpose().invert(); return; } } } } } /** * 更新背景的视频流 * @param - arTexture */ private updateTexture() { const { image } = this._curARFrame; // 调整这个 Plane 的宽高 if (image) { const { pixelWidth, pixelHeight, rawData } = image; // 更新 UV 矩阵 // 更新 旋转矩阵 // 更新捕获的屏幕 if (!this._textureY) { // 初始化 textureFrameY 和 textureFrameUV // prettier-ignore this._textureY = new Texture2D(this._engine, pixelWidth, pixelHeight, TextureFormat.Alpha8, false); // prettier-ignore this._textureUV = new Texture2D(this._engine, pixelWidth / 2, pixelHeight / 2, TextureFormat.LuminanceAlpha, false); // prettier-ignore this._textureY.wrapModeU = this._textureY.wrapModeV = this._textureUV.wrapModeU = this._textureUV.wrapModeV = TextureWrapMode.Clamp; } // 更新 UV 矩阵 this._textureProject.copyFromArray(image.imageProjection).transpose(); const byteLength = rawData.byteLength; const frameSize = pixelWidth * pixelHeight; switch (image.format) { case PixelFormat.YUV_I420: this._textureY.setPixelBuffer(new Uint8Array(rawData, 0, frameSize)); this._textureUV.setPixelBuffer(new Uint8Array(rawData, frameSize, byteLength - frameSize)); break; case PixelFormat.YUV_NV12: this._textureY.setPixelBuffer(new Uint8Array(rawData, 0, frameSize)); this._textureUV.setPixelBuffer(new Uint8Array(rawData, frameSize, byteLength - frameSize)); break; case PixelFormat.YUV_NV21: this._textureY.setPixelBuffer(new Uint8Array(rawData, 0, frameSize)); this._textureUV.setPixelBuffer(new Uint8Array(rawData, frameSize, byteLength - frameSize)); break; case PixelFormat.YUV_YV12: this._textureY.setPixelBuffer(new Uint8Array(rawData, 0, frameSize)); this._textureUV.setPixelBuffer(new Uint8Array(rawData, frameSize, byteLength - frameSize)); break; default: break; } } } /** * 帧结束的时候执行 */ private disposeFrame(): void { this._curARFrame && this.curARSession.disposeFrame(this._curARFrame); this._curARFrame = null; } }