import {int} from "aws-sdk/clients/datapipeline"; import {AllFileContext, FileComplete, ImageComplete, UploadResult, VideoComplete} from "../Model"; import {FileException, ImageException, VedioExceptionModel} from "./ExceptionUtil"; import UGCPublish, {VideoPublishParam} from "../index"; import CryptVideoUploadTask from "../CryptUploadTack/CryptVideoUploadTask"; import FFmpeg from "../node-ffmpeg"; import fs from "fs"; import path from "path"; import OSS from 'ali-oss'; import {CryptUploadDispatcher} from "../CryptUploadTack/CryptUploadDispatcher"; import chackCache, {CheckBucket} from "../StreamCache/CheckCache"; import {ApplyUploadRequest} from "../BackendServerModel"; import uuid from 'uuid'; import {GetCoverImage, TransformImage} from "./ImageUtils"; import {SencondPassBucket} from "../StreamCache/SencondPassCache"; import {EnctyptObject, enctyptSdk, getPathDigest} from "../Crypt/Encrypt"; import {stringToUint8Array} from "./TransferFunction"; import S3 from "aws-sdk/clients/s3"; import {FileSencondPassBucket} from "../StreamCache/FileSencondPassCache"; /** * 流程管理工具箱 * 每次流程结束会调用 VerifUtils 或者 流程是否成功进行校验 * */ export class ProcessUtil { /** 统一参数流程 */ /** 获取参数流程 */ public async getAllFileContextParam(uploadTaskType: string, file: string, _objectKeyPrefix: string, client,bucket, allFileInfo: AllFileContext): Promise { let verifContext: Map = new Map([ ["bucket",false], ["mediaId", false], ["mediaType", false], ["file", false], ["fileSize", false], ["fileExt", false], ["today", false], ["timeData", false], ["objectKeyPartial", false], ["client", false], ["digest", false], ["uploadTaskType", false] ]); for (let i: int = 0; i <= 3; i++) { /** bucket*/ if (verifContext.get("bucket") == false) { allFileInfo.bucket = bucket; } /** 获取文件大小*/ if (verifContext.get("file") == false) { allFileInfo.file = file; } /** 获取文件大小*/ if (verifContext.get("fileSize") == false) { allFileInfo.fileSize = fs.statSync(file).size; } /** 获取源文件摘要信息 */ if (verifContext.get("digest") == false) { allFileInfo.digest = new Uint8Array(getPathDigest(file)); } /** 获取文件后缀当然他有可能是空的 (除了文件上传 其他上传不允许为空)*/ if (verifContext.get("fileExt") == false) { allFileInfo.fileExt = path.extname(file).toLowerCase(); } /** 获取当前系统的时间对象*/ if (verifContext.get("today") == false) { allFileInfo.today = new Date(); } /** 获取日期时间戳 *//** 获取对象key uri 拼接信息 */ if (verifContext.get("timeData") == false || verifContext.get("objectKeyPartial") == false) { if (allFileInfo.today == undefined) { allFileInfo.today = new Date(); } let timeData = CryptUploadDispatcher._timeUtil.getTimeData(allFileInfo.today); if (timeData != undefined) { allFileInfo.timeData = timeData; } allFileInfo.objectKeyPartial = `${_objectKeyPrefix}/${allFileInfo.timeData.y}/${allFileInfo.timeData.m}/${allFileInfo.timeData.d}/`; } /** 获取客户端信息*/ if (verifContext.get("client") == false) { allFileInfo.client = await client.getS3orOss(); } if (verifContext.get("uploadTaskType") == false) { allFileInfo.uploadTaskType = uploadTaskType; } if (UGCPublish.verifUtil.verifGetAllFileContext(allFileInfo, verifContext) == true) { return {allFileInfo: allFileInfo, exception: true}; } if (i == 3) { /** 注意 如果不是文件上传的话 后缀名不能为空 */ if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let videoComplete: VideoComplete = { Exception: VedioExceptionModel.E_GET_FILE_INFO_AND_CLIENT_INFOEXCEPTION }; //VideoComplete; return {Complete: videoComplete, exception: false};//videoComplete; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let imageComplete: ImageComplete = { Exception: ImageException.E_GET_FILE_INFO_AND_CLIENT_INFOEXCEPTION }; //ImageComplete[]; return {Complete: imageComplete, exception: false};//videoComplete; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let fileComplete: FileComplete = { Exception: FileException.E_GET_FILE_INFO_AND_CLIENT_INFOEXCEPTION }; //FileComplete; return {Complete: fileComplete, exception: false};//videoComplete; } } } return null; } /** 获取续传信息 */ public async getCheckPassInfo(file: string): Promise{ let checkpoint: CheckBucket = await CryptUploadDispatcher._checkPass.CheckPassQuery(file); /** 若不存在则直接返回为空 */ if (checkpoint == undefined){ return null; } if (UGCPublish.verifUtil.verifCheckPassInfo(checkpoint) != true){ //等待上传完成直接删除 await chackCache.build().then((obj)=>{ obj.delete(file); }); }; return checkpoint; } /** 构建上传参数 */ public async getMakeApplyUploadRequest(_uploadTask,_minor_used,allFileInfo,file, _region, _bucket): Promise{ /** 状态为0,未上传,则开始上传*/ let serverType = _minor_used ? 2 : 3; //console.log("allFileInfo.uploadTaskType:",allFileInfo.uploadTaskType); for (let i:int = 0;i <= 3; i++) { let applyRequest:ApplyUploadRequest = _uploadTask.makeApplyUploadRequest(allFileInfo.objectKeyPartial, file, _region, _bucket, allFileInfo.fileExt, serverType); if(UGCPublish.verifUtil.verifMakeApplyUploadRequest(_uploadTask,applyRequest,allFileInfo) == true){ return {applyRequest:applyRequest,exception:true}; }; if (i == 3){ /** 注意 如果不是文件上传的话 后缀名不能为空 */ if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let videoComplete: VideoComplete; //VideoComplete; videoComplete.Exception = VedioExceptionModel.E_GET_MAKE_UPLOAD_REQUEST; return {Complete:videoComplete,exception:false};//videoComplete; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let imageComplete: ImageComplete[]; //ImageComplete[]; imageComplete[0].Exception = ImageException.E_GET_MAKE_UPLOAD_REQUEST; return {Complete:imageComplete,exception:false}; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let fileComplete = { //FileComplete; Exception:FileException.E_GET_MAKE_UPLOAD_REQUEST, } return {Complete:fileComplete,exception:false}; } } } return {Complete:null,exception:false}; } /** 给上下文添加信息 */ public async applyUploadRequestAddAllFileInfo(checkpoint,allFileInfo,applyRequest) { if (checkpoint != null) { //let checkData = await CryptUploadDispatcher._checkPass.MakeCheckPass(this.getObjectKeyPartial(checkpoint.object_key), checkpoint, applyRequest); allFileInfo.mediaId = checkpoint.media_id; } else { allFileInfo.mediaId = uuid.v4(); } if (undefined == allFileInfo.duration || 0 == allFileInfo.duration) { applyRequest.applyRequest.StorageDuration = 0; } else if (allFileInfo.duration > 0) { applyRequest.applyRequest.StorageDuration = allFileInfo.duration; } else if (allFileInfo.duration < 0) { applyRequest.applyRequest.StorageDuration = -1; } allFileInfo.mediaType = applyRequest.applyRequest.MediaType; allFileInfo.mediaInfo = JSON.stringify(applyRequest.applyRequest.MediaInfo); allFileInfo.categoryId = applyRequest.applyRequest.CategoryId; allFileInfo.duration = applyRequest.applyRequest.StorageDuration; for (let i: int = 0; i <= 3; i++) { if (UGCPublish.verifUtil.verifUuid(allFileInfo) == true) { break; } if (i == 3) { /** 注意 如果不是文件上传的话 后缀名不能为空 */ if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let videoComplete: any; //VideoComplete; videoComplete.Exception = VedioExceptionModel.E_GET_UUID_EXCEPTION; return {Complete: videoComplete, exception: false};//videoComplete; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let imageComplete: any; //ImageComplete[]; imageComplete[0].Exception = ImageException.E_GET_UUID_EXCEPTION; return {Complete: imageComplete, exception: false}; } else if (allFileInfo.uploadTaskType == "IUploadTask") { /** 这里设置异常参数 */ let fileComplete: any; //FileComplete; fileComplete.Exception = FileException.E_GET_UUID_EXCEPTION; return {Complete: fileComplete, exception: false}; } } } return {allFileInfo: allFileInfo, exception: true}; } /** 视频流程控制 (影响后续流程继续执行并无法进行错误处理所有返回异常参数) */ public async getVedioParamProcess(param:VideoPublishParam): Promise{ let task: CryptVideoUploadTask; /** 获取视频信息 */ for (let i:int = 0;i <= 3; i++) { task = await this.getVedioParam(param); /** 如果参数没有异常问题则直接跳出这个函数 */ if (UGCPublish.verifUtil.verifGetVedioParam(task)){ return task; break; } if (i == 3){ //console.log("获取视频信息失败"); return null; } } return null; } /** 读取视频信息 */ private async getVedioParam(param: VideoPublishParam): Promise{ let task = new CryptVideoUploadTask; /** 获取参数信息 */ let video = await new FFmpeg(param.video, UGCPublish._ffmpegPath); let metadata: any = video.metadata; let duration = metadata.duration.seconds; let resolution = metadata.video.resolution; let rotate = metadata.video.rotate; task.ffmpegPath = UGCPublish._ffmpegPath; task.duration = duration; /** 判断视频首帧的方向 */ task.width = (rotate == 90 || rotate == 270) ? resolution.h : resolution.w; task.height = (rotate == 90 || rotate == 270) ? resolution.w : resolution.h; return task; } /** 获取封面objectkey */ public async getCoverKey(objectKeyPartial,mediaId) : Promise{ let arr = objectKeyPartial.split('/'); arr[0] = 'images'; arr[arr.length - 1] = mediaId + '.jpg'; /** 封面objectkey */ return arr.join('/'); } /** 获取解压之后的封面文件地址 */ public async getCoverFile(ffmpegPath, file): Promise { for (let i: int = 0; i <= 3; i++){ let coverFile = await GetCoverImage(ffmpegPath, file); if (UGCPublish.verifUtil.verifGetCoverFile(coverFile) == true){ return coverFile; } } return null; } /** 获取视频秒传信息 */ public async getVedioSencodePassInfo(statusCp,allFileInfo){ /** 查找图片秒速状态缓存 */ let recordbucket = []; let result = []; let streamNeedUpload = [false]; if (statusCp == undefined) { let ret = await CryptUploadDispatcher._sencondPass.SencondPassQuery(allFileInfo.mediaType, allFileInfo.digest.toString(), allFileInfo.today); //console.log("SencondPassQuery",ret); if (ret != null) { recordbucket[0] = {Complete: ret.StreamBucket, exception: FileException.E_NULL_Exception}; } /** 只是验证一次 如果参数无效则直接删除*/ //console.log("开始验证秒传信息",recordbucket[0]); if (recordbucket[0].Complete != null) { if (UGCPublish.verifUtil.verifSencodePassInfo(recordbucket[0].Complete) != true) { /** 获取秒传信息失败则直接删除 */ let db = await CryptUploadDispatcher._cacheManager.getVedioSencondPassCache(); streamNeedUpload[0] = false; await db.delete(allFileInfo.digest.toString()); }else { //console.log("验证成功") streamNeedUpload[0] = true; } } if (streamNeedUpload[0] == true){ return CryptUploadDispatcher._sencondPass.SencondPassResult(recordbucket,allFileInfo.mediaType); } return null; } } /** 获取文件秒传信息 */ public async getFileSencodePassInfo(statusCp,allFileInfo){ let recordbucket = []; let result = [] ; let streamNeedUpload = [false]; if (statusCp == undefined) { await CryptUploadDispatcher._sencondPass.SencondPassQuery(allFileInfo.mediaType,allFileInfo.digest.toString(), allFileInfo.today).then((ret) => { if (ret != null ) { recordbucket[0]= {Complete:ret.StreamBucket,exception:FileException.E_NULL_Exception}; streamNeedUpload[0] = ret.boolean; } }); } //console.log("recordbucket[0]",recordbucket[0]); if (streamNeedUpload[0] == true) { if (null == recordbucket[0].Complete || recordbucket[0].Complete == undefined) { return null; }else { result = CryptUploadDispatcher._sencondPass.SencondPassResult(recordbucket,allFileInfo.mediaType); //console.log("result",result); if (result == null) { return null; } } } /** 只是验证一次 如果参数无效则直接删除*/ if ( result[0] != null && result[0] != undefined ) { if (UGCPublish.verifUtil.verifSencodePassInfo(recordbucket[0].Complete) != true) { /** 获取秒传信息失败则直接删除 */ let db = await CryptUploadDispatcher._cacheManager.getVedioSencondPassCache(); await db.delete(allFileInfo.file); } } return result; } /** 获取图片秒传信息 */ public async getImageSencodePassInfo(statusCp,allFileInfo,bufArr,object) { let recordbucket = []; let streamNeedUpload = []; let verifList = []; let resultIndex = []; let result = []; for (let i = 0; i < bufArr.length; i++) { verifList[i] = false; } let uploaded: Array = []; for (let j = 0; j <= 3; j++) { /** 只有当满足所有参数的时候 */ for (let i = 0; i < bufArr.length; i++) { if (verifList[i] == undefined || verifList[i] == false) { if (statusCp == undefined) { await CryptUploadDispatcher._sencondPass.SencondPassQuery( allFileInfo.mediaType, new Uint8Array(getPathDigest(bufArr[i].file)).toString(), allFileInfo.today).then((ret) => { console.log("获取秒传数据成功",ret.StreamBucket); recordbucket[i] = {Complete:ret.StreamBucket,exception:ImageException.E_NULL_Exception}; streamNeedUpload[i] = ret.boolean; }); } //console.log("recordbucket[i]",recordbucket[i]); /** 查询到本地存在数据 */ if (recordbucket[i].Complete != undefined) { /** 若获取成功则验证秒传参数是否完整 */ if (UGCPublish.verifUtil.verifSencodePassInfo(recordbucket[i].Complete) != true) { console.log("验证信息失败"); /** 若数据存在异常就删除他 */ await CryptUploadDispatcher._cacheManager.getImageSencondPassCache().then((db)=>{ db.delete(getPathDigest(bufArr[i].file).toString()); }); verifList[i] = false; }else { console.log("成功获取到秒传数据"); /** 数据完整则进行赋值 */ let resultIndex = CryptUploadDispatcher._sencondPass.SencondPassResult(recordbucket[i], allFileInfo.mediaType); if (resultIndex[0] != undefined) { result[i] = resultIndex[0]; } verifList[i] = true; } } if (verifList[i] == false){ console.log("执行了一次传输"); /** 若某个不存在则执行重新上传逻辑 */ let extName = bufArr[i].name == 'Original' ? allFileInfo.fileExt : '.jpg'; let imageKeys = `${allFileInfo.objectKeyPartial}${allFileInfo.mediaId}_${bufArr[i].width}x${bufArr[i].height}${extName}`; let isGif = allFileInfo.fileExt == '.gif'; /** 创建流数据 */ let fileStream = fs.readFileSync(bufArr[i].file); /** 创建加密对象 */ let enctyptObject = enctyptSdk(fileStream); resultIndex[i] = await object.UploadImage(allFileInfo.client, allFileInfo.bucket, imageKeys, fileStream.length, enctyptObject, allFileInfo.objectKeyPartial.split("/")[0], isGif, (progress) => { let bytes = 0; uploaded.forEach(value => { bytes += value }); allFileInfo.listener.call(allFileInfo.listener, { Total: allFileInfo.totalSize, Uploaded: bytes }); }); //console.log("resultIndex",resultIndex[i]); if (null == resultIndex[i]) { verifList[i] = false; continue; } verifList[i] = true; resultIndex[i].Error = ImageException.E_NULL_Exception; resultIndex[i].digest = new Uint8Array(getPathDigest(bufArr[i].file)).toString(); result[i] = resultIndex[i]; } } } /** 这里验证一下所有参数是否完整 */ let verifIsAllTrue = true; for (let item = 0;item<= bufArr.length;item++){ if (verifList[item] == false){ verifIsAllTrue = false; } } if (verifIsAllTrue == true){ break; } if (j == 3){ return [{ Error:ImageException.E_UPLOAD_EXCEPTION }]; } } return result; } /** 获取视频信息的key 之前已经验证过了 这里就不重复验证了 */ public async getVedioKey(statusCp,allFileInfo){ return `${allFileInfo.objectKeyPartial}${allFileInfo.mediaId}.mp4`; } /** 获取文件信息的key*/ public async getFileKey(statusCp,allFileInfo) { return statusCp ? statusCp.object_key : `${allFileInfo.objectKeyPartial}${allFileInfo.mediaId}${allFileInfo.fileExt}`; } /** 获取加密对象 */ public async getEnctyptObject(allFileInfo,statusCp){ /** 用来判断是否插入执行续传语句*/ let sequelInsert: boolean = true; let enctyptObject: EnctyptObject = undefined; //console.log("allFileInfo.digest:",allFileInfo.digest); if (allFileInfo.digest != undefined && allFileInfo.digest.length == 32) { if (statusCp != undefined) { /** 获取缓存中的密钥*/ let key_value = stringToUint8Array(statusCp.key_value); let iv_value = stringToUint8Array(statusCp.iv_value); /** 验证摘要信息 */ if (UGCPublish.verifUtil.verifCacheCache(key_value, iv_value) == true) { /** 开始执行续传逻辑 */ //console.log("filePath", allFileInfo.file) /** 创建流数据 */ let fileStream = fs.readFileSync(allFileInfo.file); allFileInfo.fileSize = fileStream.length; /** 创建加密对象 */ enctyptObject = enctyptSdk(fileStream, key_value, iv_value); sequelInsert = false; return {Complete: enctyptObject, exception: 0}; } else { /** 获取到到数据如果存在问题则删除不使用 并删除他 */ enctyptObject = undefined; (await CryptUploadDispatcher._cacheManager.getCheckCache()).delete(allFileInfo.digest.toString()); } } } /** 创建加密对象 */ if (enctyptObject == undefined && allFileInfo.file != undefined){ //console.log("获取续传信息失败开始重新加密对象"); for (let i = 0; i <= 3; i++) { /** 创建流数据 */ let fileStream = fs.readFileSync(allFileInfo.file); allFileInfo.videoSize = fileStream.length; enctyptObject = enctyptSdk(fileStream); if (UGCPublish.verifUtil.verifEnctyptInfo(enctyptObject) == true){ return {Complete:enctyptObject,exception:0}; } if (i==3){ return {Complete:null,exception:VedioExceptionModel.E_GET_ENCTYUPT_E_GET_UUID_EXCEPTION}; } } } return {Complete:null,exception:VedioExceptionModel.E_GET_ENCTYUPT_E_GET_UUID_EXCEPTION}; } /** 上传视频 */ public async uploadVedioAndCover(object,s3, coverKey,videoKey,sequelInsert, allFileInfo, enctyptObjectVideo,statusCp,listener) { /** 上传视频 */ let promise0 = await object.UploadVideo(s3, allFileInfo.bucket, videoKey, sequelInsert, allFileInfo.mediaId, allFileInfo.file, allFileInfo.digest, allFileInfo.videoSize, enctyptObjectVideo, statusCp, listener); /** 创建流数据 */ let fileStreamImage = fs.readFileSync(allFileInfo.coverFile); let enctyptObjectImage = enctyptSdk(fileStreamImage, new Uint8Array(enctyptObjectVideo.key), enctyptObjectVideo.iv); //console.log("aaa",coverKey); /** 上传封面 */ let promise1 =await object.UploadImage(s3, allFileInfo.bucket, coverKey, fileStreamImage.length, enctyptObjectImage, false, (progress) => { }); /** 对上传后的信息进行验证 */ return [{Complete: promise0, exception: 0}, {Complete: promise1, exception: 0}] } /** 上传文件 */ public async uploadFile(object, allFileInfo, enctyptObject,statusCp,listener) { let result = []; //for (let i = 0;i <= 3; i++) { /** 判断并开始进行上传 */ if (allFileInfo.client instanceof S3) { //执行S3的上传方法 //console.log("开始S3加密上传"); result = await object.fileUploadS3Task(allFileInfo.client, allFileInfo.bucket, allFileInfo.objectKey[0], allFileInfo.enctyptObject.ciphertext, allFileInfo.fileSize, listener); } else if (allFileInfo.client instanceof OSS) { //执行OSS的上传方法 //console.log("开始OSS加密上传",allFileInfo.digest,); result = await object.fileUploadOSSTask(allFileInfo.client, allFileInfo.digest, allFileInfo.bucket, allFileInfo.objectKey[0], enctyptObject, allFileInfo.file, allFileInfo.fileSize, statusCp, allFileInfo.mediaId, listener); } else { throw 'makeUploadTask client object is wrong.'; } if (UGCPublish.verifUtil.verifUploadInfoFile(result[0]) == true) { //console.log("上传成功 直接返回参数"); //等待上传完成直接删除 await CryptUploadDispatcher._cacheManager.getCheckCache().then((obj) => { obj.delete(allFileInfo.digest.toString()); }); return [{ Complete: result, exception: 0 }] } return [ { Complete: null, exception: FileException.E_UPLOAD_EXCEPTION } ]; } /** 压缩图片信息 */ public async transformImage(ffmpegPath, file, _resolutions) { let bufArr = []; for (let i = 0; i <= 3; i++) { /** 压缩和转换 */ bufArr = await TransformImage(ffmpegPath, file, _resolutions); //console.log("bufArr",bufArr); /** 验证参数 是否转换成功 是否获取到参数 */ if (UGCPublish.verifUtil.verifTransformImage(bufArr,_resolutions) == true) { //console.log("验证压缩通过"); return [ { Complete: bufArr, exception: ImageException.E_NULL_Exception } ]; } if (i == 3) { return [ { Complete: null, exception: ImageException.E_TRANSFORM_IMAGE_EXCEPTION } ]; } } return [ { Complete: null, exception: ImageException.E_TRANSFORM_IMAGE_EXCEPTION } ]; } private getObjectKeyPartial(objectKey: string): string { let index = objectKey.lastIndexOf('/'); if (index > 0) { return objectKey.substring(0, index); } else { return ''; } } }