import { Functions } from "firebase/functions" import mime from "mime-types" import fs, { createWriteStream } from "fs" import fetch from "@adobe/node-fetch-retry" import https from "https" import { GenericBar } from "cli-progress" import { ETagWithPartNumber, ChunkWithUrl, TemporaryParticipantContributionData } from "../types/index" import { commonTerms } from "./constants" import { completeMultiPartUpload, generateGetObjectPreSignedUrl, generatePreSignedUrlsParts, openMultiPartUpload, temporaryStoreCurrentContributionMultiPartUploadId, temporaryStoreCurrentContributionUploadedChunkData } from "./functions" /** * Return the bucket name based on ceremony prefix. * @param ceremonyPrefix - the ceremony prefix. * @param ceremonyPostfix - the ceremony postfix. * @returns */ export const getBucketName = (ceremonyPrefix: string, ceremonyPostfix: string): string => `${ceremonyPrefix}${ceremonyPostfix}` /** * Get chunks and signed urls related to an object that must be uploaded using a multi-part upload. * @param cloudFunctions - the Firebase Cloud Functions service instance. * @param bucketName - the name of the ceremony artifacts bucket (AWS S3). * @param objectKey - the unique key to identify the object inside the given AWS S3 bucket. * @param localFilePath - the local path where the artifact will be downloaded. * @param uploadId - the unique identifier of the multi-part upload. * @param configStreamChunkSize - size of each chunk into which the artifact is going to be splitted (nb. will be converted in MB). * @param [ceremonyId] - the unique identifier of the ceremony. * @returns Promise> - the chunks with related pre-signed url. */ export const getChunksAndPreSignedUrls = async ( cloudFunctions: Functions, bucketName: string, objectKey: string, localFilePath: string, uploadId: string, configStreamChunkSize: number, ceremonyId?: string ): Promise> => { // Prepare a new stream to read the file. const stream = fs.createReadStream(localFilePath, { highWaterMark: configStreamChunkSize * 1024 * 1024 // convert to MB. }) // Split in chunks. const chunks = [] for await (const chunk of stream) chunks.push(chunk) // Check if the file is not empty. if (!chunks.length) throw new Error("Unable to split an empty file into chunks.") // Request pre-signed url generation for each chunk. const preSignedUrls: Array = await generatePreSignedUrlsParts( cloudFunctions, bucketName, objectKey, uploadId, chunks.length, ceremonyId ) // Map pre-signed urls with corresponding chunks. return chunks.map((val1, index) => ({ partNumber: index + 1, chunk: val1, preSignedUrl: preSignedUrls[index] })) } /** * Forward the request to upload each single chunk of the related ceremony artifact. * @param chunksWithUrls > - the array containing each chunk mapped with the corresponding pre-signed urls. * @param contentType - the content type of the ceremony artifact. * @param cloudFunctions - the Firebase Cloud Functions service instance. * @param ceremonyId - the unique identifier of the ceremony. * @param alreadyUploadedChunks Array - the temporary information about the already uploaded chunks. * @param logger - an optional logger to show progress. * @returns >> - the completed (uploaded) chunks information. */ export const uploadParts = async ( chunksWithUrls: Array, contentType: string | false, cloudFunctions?: Functions, ceremonyId?: string, alreadyUploadedChunks?: Array, logger?: GenericBar ): Promise> => { // Keep track of uploaded chunks. const uploadedChunks: Array = alreadyUploadedChunks || [] // if we were passed a logger, start it if (logger) logger.start(chunksWithUrls.length, 0) // Loop through remaining chunks. for (let i = alreadyUploadedChunks ? alreadyUploadedChunks.length : 0; i < chunksWithUrls.length; i += 1) { // Consume the pre-signed url to upload the chunk. // @ts-ignore const response = await fetch(chunksWithUrls[i].preSignedUrl, { retryOptions: { retryInitialDelay: 500, // 500 ms. socketTimeout: 60000, // 60 seconds. retryMaxDuration: 300000 // 5 minutes. }, method: "PUT", body: chunksWithUrls[i].chunk, headers: { "Content-Type": contentType.toString(), "Content-Length": chunksWithUrls[i].chunk.length.toString() }, agent: new https.Agent({ keepAlive: true }) }) // Verify the response. if (response.status !== 200 || !response.ok) throw new Error( `Unable to upload chunk number ${i}. Please, terminate the current session and retry to resume from the latest uploaded chunk.` ) // Extract uploaded chunk data. const chunk = { ETag: response.headers.get("etag") || undefined, PartNumber: chunksWithUrls[i].partNumber } uploadedChunks.push(chunk) // Temporary store uploaded chunk data to enable later resumable contribution. // nb. this must be done only when contributing (not finalizing). if (!!ceremonyId && !!cloudFunctions) await temporaryStoreCurrentContributionUploadedChunkData(cloudFunctions, ceremonyId, chunk) // increment the count on the logger if (logger) logger.increment() } return uploadedChunks } /** * Upload a ceremony artifact to the corresponding bucket. * @notice this method implements the multi-part upload using pre-signed urls, optimal for large files. * Steps: * 0) Check if current contributor could resume a multi-part upload. * 0.A) If yes, continue from last uploaded chunk using the already opened multi-part upload. * 0.B) Otherwise, start creating a new multi-part upload. * 1) Generate a pre-signed url for each (remaining) chunk of the ceremony artifact. * 2) Consume the pre-signed urls to upload chunks. * 3) Complete the multi-part upload. * @param cloudFunctions - the Firebase Cloud Functions service instance. * @param bucketName - the name of the ceremony artifacts bucket (AWS S3). * @param objectKey - the unique key to identify the object inside the given AWS S3 bucket. * @param localPath - the local path where the artifact will be downloaded. * @param configStreamChunkSize - size of each chunk into which the artifact is going to be splitted (nb. will be converted in MB). * @param [ceremonyId] - the unique identifier of the ceremony (used as a double-edge sword - as identifier and as a check if current contributor is the coordinator finalizing the ceremony). * @param [temporaryDataToResumeMultiPartUpload] - the temporary information necessary to resume an already started multi-part upload. * @param logger - an optional logger to show progress. */ export const multiPartUpload = async ( cloudFunctions: Functions, bucketName: string, objectKey: string, localFilePath: string, configStreamChunkSize: number, ceremonyId?: string, temporaryDataToResumeMultiPartUpload?: TemporaryParticipantContributionData, logger?: GenericBar ) => { // The unique identifier of the multi-part upload. let multiPartUploadId: string = "" // The list of already uploaded chunks. let alreadyUploadedChunks: Array = [] // Step (0). if (temporaryDataToResumeMultiPartUpload && !!temporaryDataToResumeMultiPartUpload.uploadId) { // Step (0.A). multiPartUploadId = temporaryDataToResumeMultiPartUpload.uploadId alreadyUploadedChunks = temporaryDataToResumeMultiPartUpload.chunks } else { // Step (0.B). // Open a new multi-part upload for the ceremony artifact. multiPartUploadId = await openMultiPartUpload(cloudFunctions, bucketName, objectKey, ceremonyId) // Store multi-part upload identifier on document collection. if (ceremonyId) // Store Multi-Part Upload ID after generation. await temporaryStoreCurrentContributionMultiPartUploadId(cloudFunctions, ceremonyId!, multiPartUploadId) } // Step (1). const chunksWithUrlsZkey = await getChunksAndPreSignedUrls( cloudFunctions, bucketName, objectKey, localFilePath, multiPartUploadId, configStreamChunkSize, ceremonyId ) // Step (2). const partNumbersAndETagsZkey = await uploadParts( chunksWithUrlsZkey, mime.lookup(localFilePath), // content-type. cloudFunctions, ceremonyId, alreadyUploadedChunks, logger ) // Step (3). await completeMultiPartUpload( cloudFunctions, bucketName, objectKey, multiPartUploadId, partNumbersAndETagsZkey, ceremonyId ) } /** * Download an artifact from S3 (only for authorized users) * @param cloudFunctions Firebase cloud functions instance. * @param bucketName Name of the bucket where the artifact is stored. * @param storagePath Path to the artifact in the bucket. * @param localPath Path to the local file where the artifact will be saved. */ export const downloadCeremonyArtifact = async ( cloudFunctions: Functions, bucketName: string, storagePath: string, localPath: string ) => { // Request pre-signed url to make GET download request. const getPreSignedUrl = await generateGetObjectPreSignedUrl(cloudFunctions, bucketName, storagePath) // Make fetch to get info about the artifact. // @ts-ignore const response = await fetch(getPreSignedUrl) if (response.status !== 200 && !response.ok) throw new Error( `There was an erorr while downloading the object ${storagePath} from the bucket ${bucketName}. Please check the function inputs and try again.` ) const content: any = response.body // Prepare stream. const writeStream = createWriteStream(localPath) // Write chunk by chunk. for await (const chunk of content) { // Write chunk. writeStream.write(chunk) } } /** * Get R1CS file path tied to a particular circuit of a ceremony in the storage. * @notice each R1CS file in the storage must be stored in the following path: `circuits//`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeR1csFilename - the complete R1CS filename (name + ext). * @returns - the storage path of the R1CS file. */ export const getR1csStorageFilePath = (circuitPrefix: string, completeR1csFilename: string): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${completeR1csFilename}` /** * Get WASM file path tied to a particular circuit of a ceremony in the storage. * @notice each WASM file in the storage must be stored in the following path: `circuits//`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeWasmFilename - the complete WASM filename (name + ext). * @returns - the storage path of the WASM file. */ export const getWasmStorageFilePath = (circuitPrefix: string, completeWasmFilename: string): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${completeWasmFilename}` /** * Get PoT file path in the storage. * @notice each PoT file in the storage must be stored in the following path: `pot/`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param completePotFilename - the complete PoT filename (name + ext). * @returns - the storage path of the PoT file. */ export const getPotStorageFilePath = (completePotFilename: string): string => `${commonTerms.foldersAndPathsTerms.pot}/${completePotFilename}` /** * Get zKey file path tied to a particular circuit of a ceremony in the storage. * @notice each zKey file in the storage must be stored in the following path: `circuits//contributions/`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeZkeyFilename - the complete zKey filename (name + ext). * @returns - the storage path of the zKey file. */ export const getZkeyStorageFilePath = (circuitPrefix: string, completeZkeyFilename: string): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${commonTerms.collections.contributions.name}/${completeZkeyFilename}` /** * Get verification key file path tied to a particular circuit of a ceremony in the storage. * @notice each verification key file in the storage must be stored in the following path: `circuits//`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeVerificationKeyFilename - the complete verification key filename (name + ext). * @returns - the storage path of the verification key file. */ export const getVerificationKeyStorageFilePath = ( circuitPrefix: string, completeVerificationKeyFilename: string ): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${completeVerificationKeyFilename}` /** * Get verifier contract file path tied to a particular circuit of a ceremony in the storage. * @notice each verifier contract file in the storage must be stored in the following path: `circuits//`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeVerifierContractFilename - the complete verifier contract filename (name + ext). * @returns - the storage path of the verifier contract file. */ export const getVerifierContractStorageFilePath = ( circuitPrefix: string, completeVerifierContractFilename: string ): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${completeVerifierContractFilename}` /** * Get transcript file path tied to a particular circuit of a ceremony in the storage. * @notice each R1CS file in the storage must be stored in the following path: `circuits//`. * nb. This is a rule that must be satisfied. This is NOT an optional convention. * @param circuitPrefix - the prefix of the circuit. * @param completeTranscriptFilename - the complete transcript filename (name + ext). * @returns - the storage path of the transcript file. */ export const getTranscriptStorageFilePath = (circuitPrefix: string, completeTranscriptFilename: string): string => `${commonTerms.collections.circuits.name}/${circuitPrefix}/${commonTerms.foldersAndPathsTerms.transcripts}/${completeTranscriptFilename}`