import { Engine, Matrix, Texture2D, TextureFormat, TextureWrapMode } from "oasis-engine"; import { ARConfig, ARMode, RequestState, LocalizeConfig } from "../AREnum"; import { PoseFusion } from "./PoseFusion"; import { ARSessionFrame, ARSessionTrackingMode } from "./Type"; import { Mat4 } from "./math"; import { IARCtrl } from "../IARCtrl"; export class ARIos implements IARCtrl { /** @internal */ _worldMatrix: Matrix = new Matrix(); /** @internal */ _projectMatrix: Matrix = new Matrix(); /** @internal */ _textureProject: Matrix = new Matrix(); /** @internal */ _textureY: Texture2D; /** @internal */ _textureUV: Texture2D; private _engine: Engine; private _initialized: boolean = false; /** 当前模式 */ private _curARMode: ARMode; private _curTrackMode: ARSessionTrackingMode; private _curARSession: any; private _viewWidth: number; private _viewHeight: number; private _requestState: RequestState; private _curARFrame: ARSessionFrame; /** 上次请求 Localize 的时间 */ private _lastLocalizeTime: number = 0; /** 请求 Localize 的结果 */ private _curLocalizerRes: any; /** 相机近平面 */ private _near: number; /** 相机远平面 */ private _far: number; create(config: ARConfig, success: Function, fail: Function): void { const createInstance = () => { this._curARMode = config.mode; this._viewWidth = config.pixelWidth; this._viewHeight = config.pixelHeight; this._near = config.near; this._far = config.far; switch (this._curARMode) { case ARMode.Localizer: case ARMode.Dof6: this._curTrackMode = ARSessionTrackingMode.WorldTracking; break; case ARMode.Dof3: this._curTrackMode = ARSessionTrackingMode.OrientationTracking; break; case ARMode.Camera: this._curTrackMode = ARSessionTrackingMode.CAMERA; break; default: break; } // 追踪模式是否支持 const trackMode = this._curTrackMode; //@ts-ignore if (ARSession.isSupported({ mode: trackMode })) { // @ts-ignore ARSession.createSession({ mode: trackMode, success: (session) => { this._curARSession = session; success && success(); }, fail: () => { fail && fail(); } }); } else { console.error("不支持:" + trackMode); fail && fail(); } }; if (this._initialized) { createInstance(); } else { // 支付宝小游戏需要主动load // @ts-ignore if (my.LoadPlugin) { // @ts-ignore console.log("Load ARSession Plugin: " + my.LoadPlugin("ARSession")); } // @ts-ignore if (ARSession) { this._initialized = true; createInstance(); } else { console.error("支付宝版本有误,未包含ARSession !"); fail && fail(); } } } /** * 帧处理函数 */ update(): void { const arFrame = this._curARFrame; if (arFrame) { this.updateTexture(); // Localizer 模式下需要 requestLocalizer // Dof3, Dof6, Localizer 模式都需要矫正相机的位置和 Project // Camera 模式只需要更新纹理即可 switch (this._curARMode) { case ARMode.Localizer: this.requestLocalizer(arFrame); case ARMode.Dof3: case ARMode.Dof6: this.updatePosition(); break; default: break; } this.disposeFrame(); } } /** * 启动 * @returns */ start(success?: Function, fail?: Function): void { let result = false; if (this._curARSession) { this._requestState = RequestState.idle; const param = { vw: this._viewWidth, vh: this._viewHeight }; switch (this._curTrackMode) { case ARSessionTrackingMode.WorldTracking: result = this._curARSession.start(JSON.stringify(param)); break; case ARSessionTrackingMode.OrientationTracking: result = this._curARSession.start(JSON.stringify(param)); break; case ARSessionTrackingMode.CAMERA: result = this._curARSession.start(JSON.stringify(param)); break; default: break; } } else { console.error("start失败,请检查是否创建实例"); } if (result) { success && success(); this._curARSession.onARFrame((arFrame: ARSessionFrame) => { this._curARFrame = arFrame; }); } else { fail && fail(); } } // 无需验签 static requestPoseFushion(param: Array, callback: Function): void { // @ts-ignore ExpAREngine.poseFushion({ url: LocalizeConfig.PoseFushionUrl, params: JSON.stringify(param), complete: (result) => { var res = JSON.parse(result); if (res.status === 200 && res.data) { var result = JSON.parse(res.data); callback(result); } else { callback({}); } } }); } destroy(): void { // @ts-ignore ARSession.removeSession(); this._curARSession = null; } onShow(): void {} onHide(): void {} /** * Create IOSAndroid. * @param engine - Engine */ constructor(engine: Engine) { this._engine = engine; } /** * 请求定位信息 * @param arFrame - 当前视频流帧数据 */ private requestLocalizer(arFrame: ARSessionFrame): void { var timestamp = Date.now(); if ( this._curARMode === ARMode.Localizer && this._requestState === RequestState.idle && timestamp - this._lastLocalizeTime >= 800 ) { this._lastLocalizeTime = timestamp; this._requestState = RequestState.waiting; // 相机原始YUV数据 var pixels = arFrame.capturedImage; var width = arFrame.width; var height = arFrame.height; // 相机内参 var intrinsics = arFrame.camera.intrinsics; var fx = intrinsics[0]; var fy = intrinsics[4]; var cx = intrinsics[6]; var cy = intrinsics[7]; var cameraParam = [fx, fy, cx, cy]; var params = { appId: LocalizeConfig.AppId, apiKey: LocalizeConfig.ApiKey, timestamp: timestamp, cameraParam: cameraParam, apiSecret: LocalizeConfig.ApiSecret }; //@ts-ignore ExpAREngine.localize({ url: LocalizeConfig.LocalizeUrl, params: JSON.stringify(params), image: pixels, width: width, height: height, complete: (res) => { var result = JSON.parse(res); // console.log("localize", res); if (result.status === 200) { var localizerResult = JSON.parse(result.data); if (localizerResult && localizerResult.statusCode === 0) { this._curLocalizerRes = localizerResult.result[0]; // 列主序 转 行主序 var transform = []; for (let i = 0; i < 4; i++) { transform[i] = arFrame.camera.transform[4 * i]; transform[i + 4] = arFrame.camera.transform[4 * i + 1]; transform[i + 8] = arFrame.camera.transform[4 * i + 2]; transform[i + 12] = arFrame.camera.transform[4 * i + 3]; } PoseFusion.getInstance().insertData(transform, this._curLocalizerRes.pose, arFrame.timestamp); } } this._requestState = RequestState.idle; } }); } } /** * 根据 AR 帧信息更新相机的世界矩阵和投影矩阵 * @returns */ private updatePosition(): void { const { _curARFrame: arFrame } = this; const { camera } = arFrame; camera.transform[12] /= 100; camera.transform[13] /= 100; camera.transform[14] /= 100; switch (this._curARMode) { case ARMode.Localizer: // Update camera by localizer. if (this._curLocalizerRes && camera.trackingState == "normal") { // Mat4 var vm = new Mat4().set(camera.transform).getInverse(); // Oasis Matrix // var vm = new Matrix().copyFromArray(camera.transform).invert(); var pose = PoseFusion.getInstance().getPoseInMap(vm); if (pose) { this._worldMatrix.copyFromArray(pose); // cameraWorldMatrix.copyFromArray(pose.elements); if (this._engine.hasEvent("success_identify")) this._engine.dispatch("success_identify"); } else { this._worldMatrix.copyFromArray(camera.transform); } this._projectMatrix.copyFromArray(camera.projection).transpose(); } break; case ARMode.Dof6: case ARMode.Dof3: // Update camera by track. if (camera.trackingState === "normal") { this._worldMatrix.copyFromArray(camera.transform); this._projectMatrix.copyFromArray(camera.projection).transpose(); } break; default: break; } // IOS 需要矫正一下 near 和 far const { _near: near, _far: far } = this; this._projectMatrix.elements[10] = (near + far) / (near - far); this._projectMatrix.elements[14] = (2 * near * far) / (near - far); } /** * 更新背景的视频流 */ private updateTexture() { // 同步三维空间的背景 const { capturedImage, width, height, capturedImageMatrix } = this._curARFrame; // 调整这个 Plane 的宽高 if (capturedImage) { // 更新 UV 矩阵 // 更新 旋转矩阵 // 更新捕获的屏幕 if (!this._textureY) { // 初始化 textureFrameY 和 textureFrameUV // prettier-ignore this._textureY = new Texture2D(this._engine, width, height, TextureFormat.Alpha8, false); // prettier-ignore this._textureUV = new Texture2D(this._engine, width / 2, height / 2, TextureFormat.LuminanceAlpha, false); // prettier-ignore this._textureY.wrapModeU = this._textureY.wrapModeV = this._textureUV.wrapModeU = this._textureUV.wrapModeV = TextureWrapMode.Clamp; } this._textureProject.copyFromArray(capturedImageMatrix); const byteLength = capturedImage.byteLength; const frameSize = width * height; this._textureY.setPixelBuffer(new Uint8Array(capturedImage, 0, frameSize)); this._textureUV.setPixelBuffer(new Uint8Array(capturedImage, frameSize, byteLength - frameSize)); } } /** * 帧结束的时候执行 */ private disposeFrame(): void { this._curARFrame = null; } }