{"version":3,"file":"quilts.mjs","names":[],"sources":["../../src/utils/quilts.ts"],"sourcesContent":["// Copyright (c) Mysten Labs, Inc.\n// SPDX-License-Identifier: Apache-2.0\n\nimport { bcs } from '@mysten/bcs';\nimport type { EncodingType } from '../types.js';\nimport type { QuiltPatchV1 } from './bcs.js';\nimport { QuiltIndexV1, QuiltPatchBlobHeader, QuiltPatchId, QuiltPatchTags } from './bcs.js';\nimport {\n\tfromUrlSafeBase64,\n\tgetSourceSymbols,\n\tMAX_SYMBOL_SIZE_BY_ENCODING_TYPE,\n\tREQUIRED_ALIGNMENT_BY_ENCODING_TYPE,\n\turlSafeBase64,\n} from './index.js';\n\nexport const QUILT_INDEX_SIZE_BYTES_LENGTH = 4;\nexport const QUILT_VERSION_BYTES_LENGTH = 1;\nexport const QUILT_INDEX_PREFIX_SIZE = QUILT_VERSION_BYTES_LENGTH + QUILT_INDEX_SIZE_BYTES_LENGTH;\nexport const QUILT_PATCH_BLOB_HEADER_SIZE = 1 + 4 + 1; // bcs length of QuiltPatchBlobHeader\n\nexport const BLOB_IDENTIFIER_SIZE_BYTES_LENGTH = 2;\nexport const TAGS_SIZE_BYTES_LENGTH = 2;\nexport const MAX_BLOB_IDENTIFIER_BYTES_LENGTH = (1 << (8 * BLOB_IDENTIFIER_SIZE_BYTES_LENGTH)) - 1;\nexport const MAX_NUM_SLIVERS_FOR_QUILT_INDEX = 10;\n\nexport const HAS_TAGS_FLAG = 1 << 0;\n\n/**\n * Finds the minimum symbol size needed to store blobs in a fixed number of columns.\n * Each blob must be stored in consecutive columns exclusively.\n *\n * A binary search is used to find the minimum symbol size:\n * 1. Compute the upper and lower bounds for the symbol size.\n * 2. Check if the all the blobs can be fit into the quilt with the current symbol size.\n * 3. Adjust the bounds based on the result and repeat until the symbol size is found.\n *\n * @param blobsSizes - Slice of blob lengths, including the index size as the first element.\n *   Note that the len of the blob_size should be between 1 and n_columns.\n * @param nColumns - Number of columns available.\n * @param nRows - Number of rows available.\n * @param maxNumColumnsForQuiltIndex - The maximum number of columns that can be used to\n *   store the quilt index.\n * @param encodingType - The encoding type to use.\n *\n * @returns The minimum symbol size needed.\n **/\nexport function computeSymbolSize(\n\tblobsSizes: number[],\n\tnColumns: number,\n\tnRows: number,\n\tmaxNumColumnsForQuiltIndex: number,\n\tencodingType: EncodingType = 'RS2',\n): number {\n\tif (blobsSizes.length > nColumns) {\n\t\tthrow new Error('Too many blobs, the number of blobs must be less than the number of columns');\n\t}\n\n\tif (blobsSizes.length === 0) {\n\t\tthrow new Error('No blobs provided');\n\t}\n\n\tlet minVal = Math.max(\n\t\tblobsSizes.reduce((acc, size) => acc + size, 0) / (nColumns * nRows),\n\t\tblobsSizes[0] / (nRows * maxNumColumnsForQuiltIndex),\n\t\tMath.ceil(QUILT_INDEX_PREFIX_SIZE / nRows),\n\t);\n\n\tlet maxVal = Math.ceil((Math.max(...blobsSizes) / (nColumns / blobsSizes.length)) * nRows);\n\n\twhile (minVal < maxVal) {\n\t\tconst mid = Math.floor((minVal + maxVal) / 2);\n\t\tif (canBlobsFitIntoMatrix(blobsSizes, nColumns, mid * nRows)) {\n\t\t\tmaxVal = mid;\n\t\t} else {\n\t\t\tminVal = mid + 1;\n\t\t}\n\t}\n\tconst symbolSize =\n\t\tMath.ceil(minVal / REQUIRED_ALIGNMENT_BY_ENCODING_TYPE[encodingType]) *\n\t\tREQUIRED_ALIGNMENT_BY_ENCODING_TYPE[encodingType];\n\n\tif (!canBlobsFitIntoMatrix(blobsSizes, nColumns, symbolSize * nRows)) {\n\t\tthrow new Error('Quilt oversize');\n\t}\n\n\tif (symbolSize > MAX_SYMBOL_SIZE_BY_ENCODING_TYPE[encodingType]) {\n\t\tthrow new Error(\n\t\t\t`Quilt oversize: the resulting symbol size ${symbolSize} is larger than the maximum symbol size ${MAX_SYMBOL_SIZE_BY_ENCODING_TYPE[encodingType]}; remove some blobs`,\n\t\t);\n\t}\n\n\treturn symbolSize;\n}\n\nfunction canBlobsFitIntoMatrix(\n\tblobsSizes: number[],\n\tnColumns: number,\n\tcolumnSize: number,\n): boolean {\n\treturn blobsSizes.reduce((acc, size) => acc + Math.ceil(size / columnSize), 0) <= nColumns;\n}\n\nexport function parseQuiltPatchId(id: string) {\n\treturn QuiltPatchId.parse(fromUrlSafeBase64(id));\n}\n\nexport function encodeQuiltPatchId(id: typeof QuiltPatchId.$inferInput) {\n\treturn urlSafeBase64(QuiltPatchId.serialize(id).toBytes());\n}\n\nexport function parseWalrusId(id: string) {\n\tconst bytes = fromUrlSafeBase64(id);\n\n\tif (bytes.length === 32) {\n\t\treturn {\n\t\t\tkind: 'blob' as const,\n\t\t\tid,\n\t\t};\n\t}\n\n\treturn {\n\t\tkind: 'quiltPatch' as const,\n\t\tid: parseQuiltPatchId(id),\n\t};\n}\n\nexport interface EncodeQuiltOptions {\n\tblobs: {\n\t\tcontents: Uint8Array;\n\t\tidentifier: string;\n\t\ttags?: Record<string, string>;\n\t}[];\n\tnumShards: number;\n\tencodingType?: EncodingType;\n}\n\nexport function encodeQuilt({ blobs, numShards, encodingType }: EncodeQuiltOptions) {\n\tconst { primarySymbols: nRows, secondarySymbols: nCols } = getSourceSymbols(\n\t\tnumShards,\n\t\tencodingType,\n\t);\n\n\tconst sortedBlobs = [...blobs].sort((a, b) => (a.identifier < b.identifier ? -1 : 1));\n\tconst identifiers = new Set<string>();\n\tconst index = {\n\t\tpatches: [] as (typeof QuiltPatchV1.$inferInput & { startIndex: number })[],\n\t};\n\tconst tags = sortedBlobs.map((blob) =>\n\t\tblob.tags && Object.keys(blob.tags).length > 0\n\t\t\t? QuiltPatchTags.serialize(blob.tags).toBytes()\n\t\t\t: null,\n\t);\n\n\tfor (const blob of sortedBlobs) {\n\t\tif (identifiers.has(blob.identifier)) {\n\t\t\tthrow new Error(`Duplicate blob identifier: ${blob.identifier}`);\n\t\t}\n\t\tidentifiers.add(blob.identifier);\n\t\tindex.patches.push({\n\t\t\tstartIndex: 0,\n\t\t\tendIndex: 0,\n\t\t\tidentifier: blob.identifier,\n\t\t\ttags: blob.tags ?? {},\n\t\t});\n\t}\n\n\tconst indexSize = QUILT_INDEX_PREFIX_SIZE + QuiltIndexV1.serialize(index).toBytes().length;\n\tconst blobMetadata = sortedBlobs.map((blob, i) => {\n\t\tconst identifierBytes = bcs.string().serialize(blob.identifier).toBytes();\n\t\tlet metadataSize =\n\t\t\tQUILT_PATCH_BLOB_HEADER_SIZE + BLOB_IDENTIFIER_SIZE_BYTES_LENGTH + identifierBytes.length;\n\n\t\tlet mask = 0;\n\t\tlet offset = 0;\n\n\t\tif (tags[i]) {\n\t\t\tmetadataSize += TAGS_SIZE_BYTES_LENGTH + tags[i].length;\n\t\t\tmask |= HAS_TAGS_FLAG << 0;\n\t\t}\n\n\t\tconst metadata = new Uint8Array(metadataSize);\n\t\tconst metadataView = new DataView(metadata.buffer);\n\n\t\tconst header = QuiltPatchBlobHeader.serialize({\n\t\t\tversion: 1,\n\t\t\tlength: metadataSize - QUILT_PATCH_BLOB_HEADER_SIZE + blob.contents.length,\n\t\t\tmask,\n\t\t}).toBytes();\n\n\t\tmetadata.set(header, offset);\n\t\toffset += header.length;\n\n\t\tmetadataView.setUint16(offset, identifierBytes.length, true);\n\t\toffset += BLOB_IDENTIFIER_SIZE_BYTES_LENGTH;\n\t\tmetadata.set(identifierBytes, offset);\n\t\toffset += identifierBytes.length;\n\n\t\tif (tags[i]) {\n\t\t\tmetadataView.setUint16(offset, tags[i].length, true);\n\t\t\toffset += TAGS_SIZE_BYTES_LENGTH;\n\t\t\tmetadata.set(tags[i], offset);\n\t\t\toffset += tags[i].length;\n\t\t}\n\n\t\treturn metadata;\n\t});\n\n\tconst blobSizes = [\n\t\tindexSize,\n\t\t...sortedBlobs.map((blob, i) => {\n\t\t\tif (blob.identifier.length > MAX_BLOB_IDENTIFIER_BYTES_LENGTH) {\n\t\t\t\tthrow new Error(`Blob identifier too long: ${blob.identifier}`);\n\t\t\t}\n\n\t\t\treturn blobMetadata[i].length + blob.contents.length;\n\t\t}),\n\t];\n\n\tconst symbolSize = computeSymbolSize(\n\t\tblobSizes,\n\t\tnCols,\n\t\tnRows,\n\t\tMAX_NUM_SLIVERS_FOR_QUILT_INDEX,\n\t\tencodingType,\n\t);\n\n\tconst rowSize = symbolSize * nCols;\n\tconst columnSize = symbolSize * nRows;\n\tconst indexColumnsNeeded = Math.ceil(indexSize / columnSize);\n\n\tif (indexColumnsNeeded > MAX_NUM_SLIVERS_FOR_QUILT_INDEX) {\n\t\tthrow new Error('Index too large');\n\t}\n\n\tconst quilt = new Uint8Array(rowSize * nRows);\n\tlet currentColumn = indexColumnsNeeded;\n\n\tfor (let i = 0; i < sortedBlobs.length; i++) {\n\t\tconst blob = sortedBlobs[i];\n\t\tindex.patches[i].startIndex = currentColumn;\n\t\tcurrentColumn += writeBlobToQuilt(\n\t\t\tquilt,\n\t\t\tblob.contents,\n\t\t\trowSize,\n\t\t\tcolumnSize,\n\t\t\tsymbolSize,\n\t\t\tcurrentColumn,\n\t\t\tblobMetadata[i],\n\t\t);\n\t\tindex.patches[i].endIndex = currentColumn;\n\t}\n\n\tconst indexBytes = QuiltIndexV1.serialize(index).toBytes();\n\tconst quiltIndex = new Uint8Array(QUILT_INDEX_PREFIX_SIZE + indexBytes.length);\n\tconst view = new DataView(quiltIndex.buffer);\n\tview.setUint8(0, 1);\n\tview.setUint32(1, indexBytes.length, true);\n\tquiltIndex.set(indexBytes, QUILT_INDEX_PREFIX_SIZE);\n\n\twriteBlobToQuilt(quilt, quiltIndex, rowSize, columnSize, symbolSize, 0);\n\n\treturn { quilt, index };\n}\n\nfunction writeBlobToQuilt(\n\tquilt: Uint8Array,\n\tblob: Uint8Array,\n\trowSize: number,\n\tcolumnSize: number,\n\tsymbolSize: number,\n\tstartColumn: number,\n\tprefix?: Uint8Array,\n) {\n\tconst nRows = columnSize / symbolSize;\n\tlet bytesWritten = 0;\n\n\tif (rowSize % symbolSize !== 0) {\n\t\tthrow new Error('Row size must be divisible by symbol size');\n\t}\n\n\tif (columnSize % symbolSize !== 0) {\n\t\tthrow new Error('Column size must be divisible by symbol size');\n\t}\n\n\tif (prefix) {\n\t\twriteBytes(prefix);\n\t}\n\n\twriteBytes(blob);\n\n\treturn Math.ceil(bytesWritten / columnSize);\n\n\tfunction writeBytes(bytes: Uint8Array) {\n\t\tconst offset = bytesWritten;\n\t\tconst symbolsToSkip = Math.floor(offset / symbolSize);\n\t\tlet remainingOffset = offset % symbolSize;\n\t\tlet currentCol = startColumn + Math.floor(symbolsToSkip / nRows);\n\t\tlet currentRow = symbolsToSkip % nRows;\n\n\t\tlet index = 0;\n\t\twhile (index < bytes.length) {\n\t\t\tconst baseIndex = currentRow * rowSize + currentCol * symbolSize;\n\t\t\tconst startIndex = baseIndex + remainingOffset;\n\t\t\tconst len = Math.min(symbolSize - remainingOffset, bytes.length - index);\n\n\t\t\tfor (let i = 0; i < len; i++) {\n\t\t\t\tquilt[startIndex + i] = bytes[index + i];\n\t\t\t}\n\t\t\tindex += len;\n\t\t\tremainingOffset = 0;\n\t\t\tcurrentRow = (currentRow + 1) % nRows;\n\t\t\tif (currentRow === 0) {\n\t\t\t\tcurrentCol++;\n\t\t\t}\n\t\t}\n\n\t\tbytesWritten += bytes.length;\n\t}\n}\n"],"mappings":";;;;;AAeA,MAAa,gCAAgC;AAC7C,MAAa,6BAA6B;AAC1C,MAAa,0BAA0B,6BAA6B;AACpE,MAAa,+BAA+B;AAE5C,MAAa,oCAAoC;AACjD,MAAa,yBAAyB;AACtC,MAAa,oCAAoC,KAAM,IAAI,qCAAsC;AACjG,MAAa,kCAAkC;AAE/C,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;AAqB7B,SAAgB,kBACf,YACA,UACA,OACA,4BACA,eAA6B,OACpB;AACT,KAAI,WAAW,SAAS,SACvB,OAAM,IAAI,MAAM,8EAA8E;AAG/F,KAAI,WAAW,WAAW,EACzB,OAAM,IAAI,MAAM,oBAAoB;CAGrC,IAAI,SAAS,KAAK,IACjB,WAAW,QAAQ,KAAK,SAAS,MAAM,MAAM,EAAE,IAAI,WAAW,QAC9D,WAAW,MAAM,QAAQ,6BACzB,KAAK,KAAK,0BAA0B,MAAM,CAC1C;CAED,IAAI,SAAS,KAAK,KAAM,KAAK,IAAI,GAAG,WAAW,IAAI,WAAW,WAAW,UAAW,MAAM;AAE1F,QAAO,SAAS,QAAQ;EACvB,MAAM,MAAM,KAAK,OAAO,SAAS,UAAU,EAAE;AAC7C,MAAI,sBAAsB,YAAY,UAAU,MAAM,MAAM,CAC3D,UAAS;MAET,UAAS,MAAM;;CAGjB,MAAM,aACL,KAAK,KAAK,SAAS,oCAAoC,cAAc,GACrE,oCAAoC;AAErC,KAAI,CAAC,sBAAsB,YAAY,UAAU,aAAa,MAAM,CACnE,OAAM,IAAI,MAAM,iBAAiB;AAGlC,KAAI,aAAa,iCAAiC,cACjD,OAAM,IAAI,MACT,6CAA6C,WAAW,0CAA0C,iCAAiC,cAAc,qBACjJ;AAGF,QAAO;;AAGR,SAAS,sBACR,YACA,UACA,YACU;AACV,QAAO,WAAW,QAAQ,KAAK,SAAS,MAAM,KAAK,KAAK,OAAO,WAAW,EAAE,EAAE,IAAI;;AAGnF,SAAgB,kBAAkB,IAAY;AAC7C,QAAO,aAAa,MAAM,kBAAkB,GAAG,CAAC;;AAGjD,SAAgB,mBAAmB,IAAqC;AACvE,QAAO,cAAc,aAAa,UAAU,GAAG,CAAC,SAAS,CAAC;;AAG3D,SAAgB,cAAc,IAAY;AAGzC,KAFc,kBAAkB,GAAG,CAEzB,WAAW,GACpB,QAAO;EACN,MAAM;EACN;EACA;AAGF,QAAO;EACN,MAAM;EACN,IAAI,kBAAkB,GAAG;EACzB;;AAaF,SAAgB,YAAY,EAAE,OAAO,WAAW,gBAAoC;CACnF,MAAM,EAAE,gBAAgB,OAAO,kBAAkB,UAAU,iBAC1D,WACA,aACA;CAED,MAAM,cAAc,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,MAAO,EAAE,aAAa,EAAE,aAAa,KAAK,EAAG;CACrF,MAAM,8BAAc,IAAI,KAAa;CACrC,MAAM,QAAQ,EACb,SAAS,EAAE,EACX;CACD,MAAM,OAAO,YAAY,KAAK,SAC7B,KAAK,QAAQ,OAAO,KAAK,KAAK,KAAK,CAAC,SAAS,IAC1C,eAAe,UAAU,KAAK,KAAK,CAAC,SAAS,GAC7C,KACH;AAED,MAAK,MAAM,QAAQ,aAAa;AAC/B,MAAI,YAAY,IAAI,KAAK,WAAW,CACnC,OAAM,IAAI,MAAM,8BAA8B,KAAK,aAAa;AAEjE,cAAY,IAAI,KAAK,WAAW;AAChC,QAAM,QAAQ,KAAK;GAClB,YAAY;GACZ,UAAU;GACV,YAAY,KAAK;GACjB,MAAM,KAAK,QAAQ,EAAE;GACrB,CAAC;;CAGH,MAAM,YAAY,0BAA0B,aAAa,UAAU,MAAM,CAAC,SAAS,CAAC;CACpF,MAAM,eAAe,YAAY,KAAK,MAAM,MAAM;EACjD,MAAM,kBAAkB,IAAI,QAAQ,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;EACzE,IAAI,eACH,+BAA+B,oCAAoC,gBAAgB;EAEpF,IAAI,OAAO;EACX,IAAI,SAAS;AAEb,MAAI,KAAK,IAAI;AACZ,mBAAgB,yBAAyB,KAAK,GAAG;AACjD,WAAQ,iBAAiB;;EAG1B,MAAM,WAAW,IAAI,WAAW,aAAa;EAC7C,MAAM,eAAe,IAAI,SAAS,SAAS,OAAO;EAElD,MAAM,SAAS,qBAAqB,UAAU;GAC7C,SAAS;GACT,QAAQ,eAAe,+BAA+B,KAAK,SAAS;GACpE;GACA,CAAC,CAAC,SAAS;AAEZ,WAAS,IAAI,QAAQ,OAAO;AAC5B,YAAU,OAAO;AAEjB,eAAa,UAAU,QAAQ,gBAAgB,QAAQ,KAAK;AAC5D,YAAU;AACV,WAAS,IAAI,iBAAiB,OAAO;AACrC,YAAU,gBAAgB;AAE1B,MAAI,KAAK,IAAI;AACZ,gBAAa,UAAU,QAAQ,KAAK,GAAG,QAAQ,KAAK;AACpD,aAAU;AACV,YAAS,IAAI,KAAK,IAAI,OAAO;AAC7B,aAAU,KAAK,GAAG;;AAGnB,SAAO;GACN;CAaF,MAAM,aAAa,kBAXD,CACjB,WACA,GAAG,YAAY,KAAK,MAAM,MAAM;AAC/B,MAAI,KAAK,WAAW,SAAS,iCAC5B,OAAM,IAAI,MAAM,6BAA6B,KAAK,aAAa;AAGhE,SAAO,aAAa,GAAG,SAAS,KAAK,SAAS;GAC7C,CACF,EAIA,OACA,OACA,iCACA,aACA;CAED,MAAM,UAAU,aAAa;CAC7B,MAAM,aAAa,aAAa;CAChC,MAAM,qBAAqB,KAAK,KAAK,YAAY,WAAW;AAE5D,KAAI,qBAAqB,gCACxB,OAAM,IAAI,MAAM,kBAAkB;CAGnC,MAAM,QAAQ,IAAI,WAAW,UAAU,MAAM;CAC7C,IAAI,gBAAgB;AAEpB,MAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;EAC5C,MAAM,OAAO,YAAY;AACzB,QAAM,QAAQ,GAAG,aAAa;AAC9B,mBAAiB,iBAChB,OACA,KAAK,UACL,SACA,YACA,YACA,eACA,aAAa,GACb;AACD,QAAM,QAAQ,GAAG,WAAW;;CAG7B,MAAM,aAAa,aAAa,UAAU,MAAM,CAAC,SAAS;CAC1D,MAAM,aAAa,IAAI,WAAW,0BAA0B,WAAW,OAAO;CAC9E,MAAM,OAAO,IAAI,SAAS,WAAW,OAAO;AAC5C,MAAK,SAAS,GAAG,EAAE;AACnB,MAAK,UAAU,GAAG,WAAW,QAAQ,KAAK;AAC1C,YAAW,IAAI,YAAY,wBAAwB;AAEnD,kBAAiB,OAAO,YAAY,SAAS,YAAY,YAAY,EAAE;AAEvE,QAAO;EAAE;EAAO;EAAO;;AAGxB,SAAS,iBACR,OACA,MACA,SACA,YACA,YACA,aACA,QACC;CACD,MAAM,QAAQ,aAAa;CAC3B,IAAI,eAAe;AAEnB,KAAI,UAAU,eAAe,EAC5B,OAAM,IAAI,MAAM,4CAA4C;AAG7D,KAAI,aAAa,eAAe,EAC/B,OAAM,IAAI,MAAM,+CAA+C;AAGhE,KAAI,OACH,YAAW,OAAO;AAGnB,YAAW,KAAK;AAEhB,QAAO,KAAK,KAAK,eAAe,WAAW;CAE3C,SAAS,WAAW,OAAmB;EACtC,MAAM,SAAS;EACf,MAAM,gBAAgB,KAAK,MAAM,SAAS,WAAW;EACrD,IAAI,kBAAkB,SAAS;EAC/B,IAAI,aAAa,cAAc,KAAK,MAAM,gBAAgB,MAAM;EAChE,IAAI,aAAa,gBAAgB;EAEjC,IAAI,QAAQ;AACZ,SAAO,QAAQ,MAAM,QAAQ;GAE5B,MAAM,aADY,aAAa,UAAU,aAAa,aACvB;GAC/B,MAAM,MAAM,KAAK,IAAI,aAAa,iBAAiB,MAAM,SAAS,MAAM;AAExE,QAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IACxB,OAAM,aAAa,KAAK,MAAM,QAAQ;AAEvC,YAAS;AACT,qBAAkB;AAClB,iBAAc,aAAa,KAAK;AAChC,OAAI,eAAe,EAClB;;AAIF,kBAAgB,MAAM"}