const encoder = new TextEncoder(); const decoder = new TextDecoder('utf-8', { fatal: true }); // EOD marker for container parsing const EOD = Symbol('EOD'); const BASE64_CHARS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_$'; const BASE64_LOOKUP = new Uint8Array(128).fill(255); // Use 255 as invalid marker; for (let i = 0; i < BASE64_CHARS.length; ++i) BASE64_LOOKUP[BASE64_CHARS.charCodeAt(i)] = i; const USE_COLORS = typeof process !== 'undefined' && process?.stdout?.isTTY; const COLORS = USE_COLORS ? ['\x1b[32m', '\x1b[33m', '\x1b[34m', '\x1b[35m'] : ['']; // green, yellow, blue, magenta const RESET_COLOR = USE_COLORS ? '\x1b[0m' : ''; const ERROR_COLOR = USE_COLORS ? '\x1b[31m' : ''; // red const COLLECTION_BOUNDARIES = {'array': 5, 'object': 6, 'map': 7, 'set': 8, 'end': 9}; let toStringTermCount = 0; let useExtendedLogging = typeof process !== 'undefined' ? !!process.env?.DATAPACK_EXTENDED_LOGGING : false; /** * A byte buffer for efficient reading and writing of primitive values and bit sequences. * * The DataPack class provides methods for serializing and deserializing various data types * including numbers, strings, bit sequences, and other primitive values to/from byte buffers. * It supports both reading and writing operations with automatic buffer management. */ export default class DataPack { private buffer: Uint8Array; private dataView?: DataView; public readPos: number = 0; public writePos: number = 0; static EOD = EOD; /** * Backward compatibility: Access to the internal buffer */ get _buffer(): Uint8Array { return this.buffer; } /** * Create a new DataPack instance. * @param data Optional initial data as Uint8Array or buffer size as number. */ constructor(data: Uint8Array | number = 1900) { if (data instanceof Uint8Array) { this.buffer = data; this.writePos = data.length; } else { this.buffer = new Uint8Array(data); } } /** * Helper function to write a multi-byte integer with length prefix * @param value The value to write * @param headerType The type bits (0-7) for the header * @param invertBytes Whether to invert bytes (for negative numbers) * @param invertByteCount Whether to invert the byte count (for type 0) */ private writeMultiByteNumber(value: number, headerType: number, invertBytes: boolean = false, invertByteCount: boolean = false): void { let byteCount = 0; let temp = value; while (temp > 0) { byteCount++; temp = Math.floor(temp / 256); } const encodedByteCount = invertByteCount ? ((~byteCount) & 0x1F) : byteCount; this.buffer[this.writePos++] = (headerType << 5) | encodedByteCount; let max = 1; for (let j = 0; j < byteCount; j++) max *= 256; for (let i = 0; i < byteCount; i++) { max = Math.floor(max / 256); let byte = Math.floor(value / max) % 256; if (invertBytes) byte ^= 0xFF; this.buffer[this.writePos++] = byte; } } /** * Helper function to read a multi-byte integer with length prefix * @param byteCount Number of bytes to read * @param invertBytes Whether to invert bytes (for negative numbers) * @returns The read value */ private readMultiByteNumber(byteCount: number, invertBytes: boolean = false): number { if (this.readPos + byteCount > this.writePos) this.notEnoughData('number'); let value = 0; let multiplier = 1; for (let i = byteCount - 1; i >= 0; i--) { let byte = this.buffer[this.readPos + i]; if (invertBytes) byte ^= 0xFF; value += byte * multiplier; multiplier *= 256; } this.readPos += byteCount; return value; } private notEnoughData(type: string): never { throw new Error(`Not enough data to read a ${type} at position ${this.readPos} in\n${this.toString(true)}`); } /** * Each data item starts with a single byte. The high 3 bits indicate the type. * The low 5 bits indicate a size or a subtype, depending on the type. * * 0: negative integer (byte count encoded as bitwise NOT in lower bits) * 1: small integer 0..31 (encoded in lower bits) * 2: small integer 32...63 (encoded in lower bits) * 3: integer (byte count encoded in lower bits) * 4: * 0: float64 (8 bytes follow) * 1: undefined * 2: null * 3: true * 4: false * 5: array start (followed by a items until EOD) * 6: object start (followed by key+value pairs until EOD) * 7: map start (followed by key+value pairs until EOD) * 8: set start (followed by items until EOD) * 9: EOD * 10: identifier (6 byte positive int follows, represented as a base64 string of length 8) * 11: null-terminated string * 12: Date/Time (varint with whole seconds since epoch follows) * 13: custom type, followed by name string and data value * 5: short string (length in lower bits, 0-31) * 6: string (byte count of length in lower bits) * 7: blob (byte count of length in lower bits) */ write(data: any): DataPack { // Pessimistic capacity allocation - covers most cases this.ensureCapacity(33); // 1 byte header + up to 32 bytes for large integers/length encoding if (typeof data === 'number') { if (Number.isInteger(data) && data <= Number.MAX_SAFE_INTEGER && data >= Number.MIN_SAFE_INTEGER) { if (data >= 0) { if (data < 32) { // Type 1: small positive integer 0...31 this.buffer[this.writePos++] = (1 << 5) | data; } else if (data < 64) { // Type 2: small positive integer 32...63 this.buffer[this.writePos++] = (2 << 5) | (data-32); } else { // Type 3: positive integer (byte count in lower bits) this.writeMultiByteNumber(data-64, 3); } } else { // Type 0: negative integer (byte count as bitwise NOT in lower bits) this.writeMultiByteNumber(-data, 0, true, true); } } else { // Type 4, subtype 0: float64 this.buffer[this.writePos++] = (4 << 5) | 0; this.dataView ||= new DataView(this.buffer.buffer, this.buffer.byteOffset, this.buffer.byteLength); this.dataView.setFloat64(this.writePos, data); this.writePos += 8; } } else if (data === undefined) { this.buffer[this.writePos++] = (4 << 5) | 1; } else if (data === null) { this.buffer[this.writePos++] = (4 << 5) | 2; } else if (data === true) { this.buffer[this.writePos++] = (4 << 5) | 3; } else if (data === false) { this.buffer[this.writePos++] = (4 << 5) | 4; } else if (typeof data === 'string') { const encoded = encoder.encode(data); if (encoded.length < 32) { // Type 5: short string (length in lower bits, 0-31) this.buffer[this.writePos++] = (5 << 5) | encoded.length; } else { // Type 6: string (byte count of length in lower bits) this.writeMultiByteNumber(encoded.length, 6); } this.ensureCapacity(encoded.length); this.buffer.set(encoded, this.writePos); this.writePos += encoded.length; } else if (data instanceof Uint8Array) { // Type 7: blob (byte count of length in lower bits) this.writeMultiByteNumber(data.length, 7); this.ensureCapacity(data.length); this.buffer.set(data, this.writePos); this.writePos += data.length; } else if (Array.isArray(data) || (typeof data.length === 'number' && typeof data[Symbol.iterator] === 'function')) { // Type 4, subtype 5: array start this.buffer[this.writePos++] = (4 << 5) | 5; for (const item of data) { this.write(item); } this.buffer[this.writePos++] = (4 << 5) | 9; // EOD } else if (data instanceof Map) { // Type 4, subtype 7: map start this.buffer[this.writePos++] = (4 << 5) | 7; for (const [key, value] of data) { this.write(key); this.write(value); } this.buffer[this.writePos++] = (4 << 5) | 9; // EOD } else if (data instanceof Set) { // Type 4, subtype 8: set start this.buffer[this.writePos++] = (4 << 5) | 8; for (const item of data) { this.write(item); } this.buffer[this.writePos++] = (4 << 5) | 9; // EOD } else if (data instanceof Date) { // Type 4, subtype 12: Date/Time this.buffer[this.writePos++] = (4 << 5) | 12; // Write a varint -- whole seconds should be plenty of resolution this.write(Math.floor(data.getTime()/1000)); } else if (typeof data === 'object') { if (data instanceof CustomData) { this.writeCustom(data.name, data.data); } else if (typeof data.toDataPack === 'function') { data.toDataPack(this); } else { this.writeAsObject(data); } } else { throw new Error(`Unsupported data type: ${typeof data}`); } return this; } read(customConverters?: {[name: string]: ((data: any) => any)} | undefined): any { if (this.readPos > this.writePos) { throw new Error('Not enough data'); } const header = this.buffer[this.readPos++]; const type = (header >> 5) & 0x07; const subtype = header & 0x1F; switch (type) { case 0: { // Negative integer (byte count as bitwise NOT in lower bits) const byteCount = (~subtype) & 0x1F; return -this.readMultiByteNumber(byteCount, true); } case 1: { // Small positive integer 0...31 return subtype; } case 2: { // Small positive integer 32..63 return subtype + 32; } case 3: { // Positive integer (byte count in lower bits) return this.readMultiByteNumber(subtype) + 64; } case 4: { // Floats, special values and collections switch (subtype) { case 0: { // float64 if (this.readPos + 8 > this.writePos) this.notEnoughData('float64'); this.dataView ||= new DataView(this.buffer.buffer, this.buffer.byteOffset, this.buffer.byteLength); const result = this.dataView.getFloat64(this.readPos); this.readPos += 8; return result; } case 1: return undefined; case 2: return null; case 3: return true; case 4: return false; case 5: { // Array start const result = []; while (true) { const nextValue = this.read(customConverters); if (nextValue === EOD) break; result.push(nextValue); } return result; } case 6: { // Plain object or custom class instance let result: any = {}; while (true) { const key = this.read(customConverters); if (key === EOD) break; const value = this.read(customConverters); result[key] = value; } return result; } case 7: { // Map start const result = new Map(); while (true) { const key = this.read(customConverters); if (key === EOD) break; const value = this.read(customConverters); result.set(key, value); } return result; } case 8: { // Set start const result = new Set(); while (true) { const value = this.read(customConverters); if (value === EOD) break; result.add(value); } return result; } case 9: return EOD; case 10: { // Identifier (6 byte positive int follows, represented as a base64 string of length 8) --this.readPos; // readIdentifier() will read the header byte again return this.readIdentifier(); } case 11: { // Null-terminated string const start = this.readPos; let end = start; while (true) { if (end >= this.writePos) this.notEnoughData('null-terminated string'); if (this.buffer[end] === 0) break; end++; } this.readPos = end + 1; // Skip the null terminator return decoder.decode(this.buffer.subarray(start, end)); } case 12: { // Date/Time (varint with whole seconds since epoch follows) const seconds = this.readPositiveInt(); return new Date(seconds * 1000); } case 13: { // Custom type, followed by name string and data value const name = this.readString(); const data = this.read(); return (customConverters && name in customConverters) ? customConverters[name](data) : new CustomData(name, data); } default: throw new Error(`Unknown type 4 subtype: ${subtype}`); } } case 5: { // Short string (length in lower bits, 0-31) const length = subtype; if (this.readPos + length > this.writePos) this.notEnoughData('short string'); const bytes = this.buffer.subarray(this.readPos, this.readPos + length); this.readPos += length; return decoder.decode(bytes); } case 6: { // String (byte count of length in lower bits) const length = this.readMultiByteNumber(subtype); if (this.readPos + length > this.writePos) this.notEnoughData('string'); const bytes = this.buffer.subarray(this.readPos, this.readPos + length); this.readPos += length; return decoder.decode(bytes); } case 7: { // Blob (byte count of length in lower bits) const length = this.readMultiByteNumber(subtype); if (this.readPos + length > this.writePos) this.notEnoughData('blob'); // This makes an actual copy of the underlying buffer data const result = this.buffer.slice(this.readPos, this.readPos + length); this.readPos += length; return new Uint8Array(result); // Return a copy } default: throw new Error(`Unknown type: ${type}`); } } /** * Ensure the buffer has capacity for additional bytes. * @param bytesNeeded Number of additional bytes needed. */ private ensureCapacity(bytesNeeded: number): void { const needed = this.writePos + bytesNeeded; if (needed <= this.buffer.length) return; // Grow by 1.5x or the needed amount, whichever is larger const newCapacity = Math.max(needed, Math.floor(this.buffer.length * 1.5)); const newBuffer = new Uint8Array(newCapacity); newBuffer.set(this.buffer.subarray(0, this.writePos)); this.buffer = newBuffer; delete this.dataView; } readNumber(): number { const result = this.read(); if (typeof result !== 'number') { throw new Error('Expected number but got ' + typeof result); } return result; } readDate(): Date { const result = this.read(); if (!(result instanceof Date)) { throw new Error('Expected Date but got ' + typeof result); } return result; } readPositiveInt(limit?: number): number { const result = this.read(); if (typeof result !== 'number' || !Number.isInteger(result) || result < 0 || (limit !== undefined && result >= limit)) { throw new Error(`Expected positive integer < ${limit} but got ${result}`); } return result; } readBoolean(): boolean { const result = this.read(); if (typeof result !== 'boolean') { throw new Error('Expected boolean but got ' + typeof result); } return result; } readUint8Array(): Uint8Array { const result = this.read(); if (!(result instanceof Uint8Array)) { throw new Error('Expected Uint8Array but got ' + typeof result); } return result; } readString(): string { const result = this.read(); if (typeof result !== 'string') { throw new Error('Expected string but got ' + typeof result); } return result; } writeIdentifier(id: string | number): DataPack { let num: number = 0; if (typeof id === 'number') { num = id; } else { if (id.length !== 8) { throw new Error(`Identifier must be exactly 8 characters, got ${id.length}`); } // Convert base64 string to 48-bit number let value; for (let i = 0; i < 8; i++) { const char = id.charCodeAt(i); if (char > 127 || (value = BASE64_LOOKUP[char]) === 255) { throw new Error(`Invalid base64 character: ${id[i]}`); } num = num * 64 + value; } } // Write type 4, subtype 10 header this.ensureCapacity(7); // 1 byte header + 6 bytes for the number this.buffer[this.writePos++] = (4 << 5) | 10; // Write the 6-byte number in big-endian format for (let i = 5; i >= 0; i--) { this.buffer[this.writePos + i] = num % 256; num = Math.floor(num / 256); } this.writePos += 6; return this; } readIdentifier(): string { let num = this.readIdentifierNumber(); // Convert 48-bit number back to 8-character base64 string let id = ''; for (let i = 0; i < 8; i++) { id = BASE64_CHARS[num % 64] + id; num = Math.floor(num / 64); } return id; } readIdentifierNumber(): number { // Read the 6-byte number in big-endian format if (this.readPos + 7 > this.writePos) this.notEnoughData('identifier'); const header = this.buffer[this.readPos++]; if (header !== ((4 << 5) | 10)) { throw new Error('Invalid identifier header'); } let num = 0; for (let i = 0; i < 6; i++) { num = (num * 256) + this.buffer[this.readPos++]; } return num; } /** * Like writeString but writes without a length prefix and with a null terminator, for ordered storage. * Can be read with {@link read} or {@link readString} just like any other string. * @param str The string to write. May not contain null characters. */ writeOrderedString(str: string): DataPack { const utf8Bytes = new TextEncoder().encode(str); if (utf8Bytes.includes(0)) { throw new Error('String contains null character'); } this.ensureCapacity(utf8Bytes.length + 2); this.buffer[this.writePos++] = (4 << 5) | 11; // Null-terminated string this.buffer.set(utf8Bytes, this.writePos); this.writePos += utf8Bytes.length; this.buffer[this.writePos++] = 0; // Null terminator return this; } toUint8Array(copyBuffer: boolean = true, startPos: number = 0, endPos: number = this.writePos): Uint8Array { return copyBuffer ? this.buffer.slice(startPos, endPos) : this.buffer.subarray(startPos, endPos); } toBuffer(): ArrayBuffer { return this.buffer.buffer.slice(0, this.writePos) as ArrayBuffer; } clone(copyBuffer: boolean, readPos: number = 0, writePos: number = this.writePos): DataPack { if (copyBuffer) { return new DataPack(this.buffer.slice(readPos, writePos)); } else { const pack = new DataPack(this.buffer); pack.readPos = readPos; pack.writePos = writePos; return pack; } } writeCustom(name: string, data: any) { this.buffer[this.writePos++] = (4 << 5) | 13; this.write(name); this.write(data); } writeAsObject(obj: Record) { // Type 4, subtype 6: object start this.buffer[this.writePos++] = (4 << 5) | 6; for (const key of Object.keys(obj)) { this.write(key); this.write(obj[key]); } this.buffer[this.writePos++] = (4 << 5) | 9; // EOD return this; } /** * Write a collection (array, set, object, or map) using a callback to add items/fields. * @param type 'array' | 'set' | 'object' | 'map' * @param bodyFunc Callback function to add items/fields. It accepts a function to add values or key-value pairs (depending on the collection type). * @returns The DataPack instance for chaining. * @example * // Writing an array * pack.writeCollection('array', add => { * add(1); * add(2); * add(3); * }); * * // Writing an object * pack.writeCollection('object', (add) => { * add('key1', 'value1'); * add('key2', 42); * }); */ writeCollection(type: 'array' | 'set', bodyFunc: (addField: (value: any) => void) => void): DataPack; writeCollection(type: 'object', bodyFunc: (addField: (field: number|string, value: any) => void) => void): DataPack; writeCollection(type: 'map', bodyFunc: (addField: (field: any, value: any) => void) => void): DataPack; writeCollection(type: 'array' | 'set' | 'object' | 'map', bodyFunc: (addField: (a: any, b?: any) => void) => void): DataPack { this.writeCollectionBoundary(type); bodyFunc((type === 'array' || type === 'set') ? (value: any) => { this.write(value); } : (type === 'object') ? (name, value) => { this.writeObjectKey(name); this.write(value); } : (name, value) => { // map this.write(name); this.write(value); }); return this.writeCollectionBoundary('end'); } /** * Write a collection boundary marker (start or end) for arrays, sets, objects, or maps. * This is a low-level method for advanced use cases. Use `writeCollection` (or just `write`) * if possible. */ writeCollectionBoundary(marker: 'array' | 'set' | 'object' | 'map' | 'end'): DataPack { let subType = COLLECTION_BOUNDARIES[marker]; if (!subType) throw new Error(`Invalid collection type: ${marker}`); this.buffer[this.writePos++] = (4 << 5) | subType; return this; } /** * Writes either a string or a number, converting strings to numbers if they represent javascript-safe integers. * This is useful for writing object keys, which are always strings but may be more compactly represented as numbers. */ writeObjectKey(key: number | string): void { if (typeof key === 'string') { const num = parseInt(key, 10); if (num.toString() == key) key = num; } else if (typeof key !== 'number') { key = '' + key; } this.write(key); } /** * Read and consume a collection boundary marker, validating it matches the expected type. */ readCollectionBoundary(expected: 'array' | 'set' | 'object' | 'map' | 'end'): void { if (this.readPos >= this.writePos) this.notEnoughData('collection boundary'); const byte = this.buffer[this.readPos++]; const expectedByte = (4 << 5) | COLLECTION_BOUNDARIES[expected]; if (byte !== expectedByte) throw new Error(`Expected ${expected} boundary but got byte ${byte}`); } /** * Increment the last byte of the buffer. If it was already 255 set it to 0 and * increment the previous byte, and so on. If all bytes were 255, return undefined. * This is useful for creating an exclusive end key for range scans. * This may result in a DataPack instance that cannot be parsed (or represented by * {@link toString}). */ increment(): DataPack | undefined { for(let byte=this.writePos-1; byte >= 0; byte--) { if (this.buffer[byte] === 255) { // Byte is all 1s, set to 0 and continue this.buffer[byte] = 0; } else { this.buffer[byte]++; return this; } } return undefined; // All bytes were 255 (and are now 0) } toString(extended: boolean | undefined = undefined, startPos: number = 0, endPos: number = this.writePos): string { if (extended === undefined) extended = useExtendedLogging; let oldReadPos = this.readPos; this.readPos = startPos; let lastPos = 0; let vals = ''; let hexs = ''; const orgTermCount = toStringTermCount; const resetColor = orgTermCount > 0 ? COLORS[(orgTermCount-1) % COLORS.length] : RESET_COLOR; try { while(this.readPos < endPos) { const color = COLORS[toStringTermCount++ % COLORS.length]; vals += color + toText(this.read()) + ' '; if (extended) { hexs += color; while(lastPos < this.readPos) { hexs += this.buffer[lastPos].toString(16).padStart(2, '0') + ' '; lastPos++; } } } } catch(e) { if (!extended) { this.readPos = oldReadPos; toStringTermCount = orgTermCount; return this.toString(true, startPos, endPos); } vals += ERROR_COLOR + "ERROR "; hexs += ERROR_COLOR; while(lastPos < this.writePos) { hexs += this.buffer[lastPos].toString(16).padStart(2, '0') + ' '; lastPos++; } } this.readPos = oldReadPos; if (!orgTermCount) toStringTermCount = 0; if (extended) { return "DataPack{" + hexs + resetColor + "→ " + vals.trimEnd() + resetColor + "}"; } else { return "DataPack{" + vals.trimEnd() + resetColor + "}"; } } [Symbol.for('nodejs.util.inspect.custom')](): string { return this.toString(); } readAvailable(): boolean { return this.readPos < this.writePos; } static generateIdentifier(): string { // Combine a timestamp with randomness, to create locality of reference as well as a high chance of uniqueness. // Bits 9...48 are the date in ms (wrapping about every 34 years), providing some `locality of reference // Bit 0...14 are random bits (partly overlapping with the date, adding up to 62ms of jitter) let num = Math.floor(+new Date() * (1<<8) + Math.random() * (1<<14)); let id = ''; for(let i = 0; i < 8; i++) { id = BASE64_CHARS[num & 0x3f] + id; num = Math.floor(num / 64); } return id; } static createBuffer(...args: any) { const pack = new DataPack(); for (const arg of args) { pack.write(arg); } return pack.toBuffer(); } static createUint8Array(...args: any) { const pack = new DataPack(); for (const arg of args) { pack.write(arg); } return pack.toUint8Array(false); } } function toText(v: any): string { if (v===undefined) return 'undefined'; if (typeof v === 'object' && v !== null) { if (v instanceof Uint8Array) return new DataPack(v).toString(); if (v instanceof Array) return '[' + v.map(vv => toText(vv)).join(', ') + ']'; if (v instanceof Map) return 'Map{' + Array.from(v.entries()).map(([k, val]) => `${toText(k)}=>${toText(val)}`).join(', ') + '}'; if (v instanceof Set) return 'Set{' + Array.from(v.values()).map(vv => toText(vv)).join(', ') + '}'; if (typeof v === 'object' && v !== null) return '{' + Object.entries(v).map(([k, val]) => `${k}:${toText(val)}`).join(', ') + '}'; } return JSON.stringify(v); } class CustomData { constructor(public name: string, public data: any) {} } export default interface DataPack { CustomData: typeof CustomData; } DataPack.prototype.CustomData = CustomData;