import {Type} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {BUCKET_LENGTH} from "./const.js"; import {KeyValue} from "./controller/index.js"; import {Db, DbBatch, DbReqOpts, FilterOptions} from "./controller/interface.js"; import {encodeKey} from "./util.js"; type Id = Uint8Array | string | number | bigint; /** * Repository is a high level kv storage * This abstract repository is designed in a way to store items with different prefixed * Specially when those prefixed data is not available in the object to be stored * * By default, SSZ-encoded values, */ export abstract class PrefixedRepository { private readonly dbReqOpts: DbReqOpts; /** Inclusive range for the minimum key for the bucket */ private readonly minKey: Uint8Array; /** Exclusive range for the maximum key for the bucket */ private readonly maxKey: Uint8Array; protected constructor( protected config: ChainForkConfig, protected db: Db, protected bucket: number, protected type: Type, private readonly bucketId: string ) { this.dbReqOpts = {bucketId: this.bucketId}; this.minKey = encodeKey(bucket, Buffer.alloc(0)); this.maxKey = encodeKey(bucket + 1, Buffer.alloc(0)); } abstract encodeKeyRaw(prefix: P, id: I): Uint8Array; abstract decodeKeyRaw(raw: Uint8Array): {prefix: P; id: I}; /** * Max key is inclusive * */ abstract getMaxKeyRaw(prefix: P): Uint8Array; /** * Min key is inclusive * */ abstract getMinKeyRaw(prefix: P): Uint8Array; protected encodeValue(value: T): Uint8Array { return this.type.serialize(value); } protected decodeValue(data: Uint8Array): T { return this.type.deserialize(data); } protected wrapKey(raw: Uint8Array): Uint8Array { return encodeKey(this.bucket, raw); } protected unwrapKey(key: Uint8Array): Uint8Array { return key.slice(BUCKET_LENGTH); } // The Id can be inferred from the value getId(value: T): I { return this.type.hashTreeRoot(value) as I; } async get(prefix: P, id: I): Promise { const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); const v = await this.db.get(key, this.dbReqOpts); return v ? this.decodeValue(v) : null; } async getMany(prefix: P, ids: I[]): Promise<(T | undefined)[]> { const keys = []; for (const id of ids) { keys.push(this.wrapKey(this.encodeKeyRaw(prefix, id))); } const values = await this.db.getMany(keys, this.dbReqOpts); const result = []; for (const value of values) { result.push(value ? this.decodeValue(value) : undefined); } return result; } async getManyBinary(prefix: P, ids: I[]): Promise<(Uint8Array | undefined)[]> { const keys = []; for (const id of ids) { keys.push(this.wrapKey(this.encodeKeyRaw(prefix, id))); } return await this.db.getMany(keys, this.dbReqOpts); } async getBinary(prefix: P, id: I): Promise { const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); return await this.db.get(key, this.dbReqOpts); } async put(prefix: P, item: T): Promise { const id = this.getId(item); const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); await this.db.put(key, this.encodeValue(item), this.dbReqOpts); } async putMany(prefix: P, items: T[]): Promise { const batch: KeyValue[] = []; for (const item of items) { const id = this.getId(item); const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); batch.push({key, value: this.encodeValue(item)}); } await this.db.batchPut(batch, this.dbReqOpts); } async putBinary(prefix: P, id: I, bytes: Uint8Array): Promise { const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); await this.db.put(key, bytes, this.dbReqOpts); } async putManyBinary(prefix: P, items: KeyValue[]): Promise { const batch: KeyValue[] = []; for (const {key, value} of items) { batch.push({key: this.wrapKey(this.encodeKeyRaw(prefix, key)), value: value}); } await this.db.batchPut(batch, this.dbReqOpts); } async delete(prefix: P, id: I): Promise { const key = this.wrapKey(this.encodeKeyRaw(prefix, id)); await this.db.delete(key, this.dbReqOpts); } async deleteMany(prefix: P | P[]): Promise { const keys: Uint8Array[][] = []; for (const p of Array.isArray(prefix) ? prefix : [prefix]) { const prefixedKeys = await this.db.keys({ gte: this.wrapKey(this.getMinKeyRaw(p)), lte: this.wrapKey(this.getMaxKeyRaw(p)), bucketId: this.bucketId, }); keys.push(prefixedKeys); } await this.db.batchDelete(keys.flat(), this.dbReqOpts); } async batch(prefix: P, batch: DbBatch): Promise { const batchWithKeys = []; for (const b of batch) { if (b.type === "del") { batchWithKeys.push({type: b.type, key: this.wrapKey(this.encodeKeyRaw(prefix, b.key))}); } else { batchWithKeys.push({ type: b.type, key: this.wrapKey(this.encodeKeyRaw(prefix, b.key)), value: this.encodeValue(b.value), }); } } await this.db.batch(batchWithKeys, this.dbReqOpts); } async batchBinary(prefix: P, batch: DbBatch): Promise { const batchWithKeys = []; for (const b of batch) { batchWithKeys.push({...b, key: this.wrapKey(this.encodeKeyRaw(prefix, b.key))}); } await this.db.batch(batchWithKeys, this.dbReqOpts); } async values(prefix: P | P[]): Promise { const result: T[] = []; for (const p of Array.isArray(prefix) ? prefix : [prefix]) { for await (const vb of this.db.valuesStream({ gte: this.wrapKey(this.getMinKeyRaw(p)), lte: this.wrapKey(this.getMaxKeyRaw(p)), bucketId: this.bucketId, })) { result.push(this.decodeValue(vb)); } } return result; } async valuesBinary(prefix: P | P[]): Promise { const result: Uint8Array[] = []; for (const p of Array.isArray(prefix) ? prefix : [prefix]) { for await (const vb of this.db.valuesStream({ gte: this.wrapKey(this.getMinKeyRaw(p)), lte: this.wrapKey(this.getMaxKeyRaw(p)), bucketId: this.bucketId, })) { result.push(vb); } } return result; } async *valuesStream(prefix: P | P[]): AsyncIterable { for (const p of Array.isArray(prefix) ? prefix : [prefix]) { for await (const vb of this.db.valuesStream({ gte: this.wrapKey(this.getMinKeyRaw(p)), lte: this.wrapKey(this.getMaxKeyRaw(p)), bucketId: this.bucketId, })) { yield this.decodeValue(vb); } } } async *valuesStreamBinary(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: Uint8Array}> { for (const p of Array.isArray(prefix) ? prefix : [prefix]) { for await (const {key, value} of this.db.entriesStream({ gte: this.wrapKey(this.getMinKeyRaw(p)), lte: this.wrapKey(this.getMaxKeyRaw(p)), bucketId: this.bucketId, })) { const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key)); yield { prefix, id, value, }; } } } async *entriesStream(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: T}> { for (const v of Array.isArray(prefix) ? prefix : [prefix]) { for await (const {key, value} of this.db.entriesStream({ gte: this.wrapKey(this.getMinKeyRaw(v)), lte: this.wrapKey(this.getMaxKeyRaw(v)), bucketId: this.bucketId, })) { const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key)); yield { prefix, id, value: this.decodeValue(value), }; } } } async *entriesStreamBinary(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: Uint8Array}> { for (const v of Array.isArray(prefix) ? prefix : [prefix]) { for await (const {key, value} of this.db.entriesStream({ gte: this.wrapKey(this.getMinKeyRaw(v)), lte: this.wrapKey(this.getMaxKeyRaw(v)), bucketId: this.bucketId, })) { const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key)); yield { prefix, id: id, value, }; } } } async keys(opts?: FilterOptions<{prefix: P; id: I}>): Promise<{prefix: P; id: I}[]> { const optsBuff: FilterOptions = { bucketId: this.bucketId, }; if (opts?.gte !== undefined) { optsBuff.gte = this.wrapKey(this.encodeKeyRaw(opts.gte.prefix, opts.gte.id)); } else if (opts?.gt !== undefined) { optsBuff.gt = this.wrapKey(this.encodeKeyRaw(opts.gt.prefix, opts.gt.id)); } else { optsBuff.gte = this.minKey; } if (opts?.lte !== undefined) { optsBuff.lte = this.wrapKey(this.encodeKeyRaw(opts.lte.prefix, opts.lte.id)); } else if (opts?.lt !== undefined) { optsBuff.lt = this.wrapKey(this.encodeKeyRaw(opts.lt.prefix, opts.lt.id)); } else { optsBuff.lt = this.maxKey; } if (opts?.reverse !== undefined) optsBuff.reverse = opts.reverse; if (opts?.limit !== undefined) optsBuff.limit = opts.limit; const data = await this.db.keys(optsBuff); return (data ?? []).map((data) => this.decodeKeyRaw(this.unwrapKey(data))); } }