import { CollectionQuery, QueryResultCardinality } from '@triplit/db'; import { EntityStore, KVStoreOrTransaction } from './types.js'; import { DBSchema } from './db.js'; export declare class EntityStoreQueryEngine { private storage; private store; executionStack: { collectionName: string; data: any; }[]; schema: DBSchema | undefined; private queryViews; constructor(storage: KVStoreOrTransaction, store: EntityStore, schema: DBSchema | undefined); fetch(query: CollectionQuery): Promise; private loadQuery; private loadIncludes; loadSubquery(entity: { collectionName: string; data: any; }, relation: { subquery: CollectionQuery; cardinality: QueryResultCardinality; }): Promise; loadVariable(variable: string): Promise; /** * Load non inclusion data as needed, should clean this data after use */ loadEntityDataAtPath(entity: { collectionName: string; data: any; }, dataPath: string | Iterable): Promise; } /** * Delta format: * { * [collection]: { * [id]: { * [attr]: [value] * } * } * } * * The entity store is essentially some "before" state * The addition of the write buffer also represents a "before" state for the transaction (optimally we have MVCC guarantees here) * * Given the write buffer should be "committed" data, our indexing should probably reflect that * - Leave it to the entity store implementation to handle write buffer * * Fulfillment: * { * collectionName: boolean, * where: boolean[], * after: boolean, * order: boolean, // need to do full ordering * limit: boolean * } * - merge fulfillments with logical AND (can ignore if no candidates) * * Overlays: * - Write buffer * - (Outbox buffer) - write buffer could be pass through * - Transaction * * ALGORITHMS * simple * - look up possible candidates in entity store via indexes * - consider all write buffered and transaction data as candidates * - Perform filtering on loaded candidates * * - If we have no overlay data, we can avoid in memory checks ... perform some of the steps of fetch in the index lookup * - Basically in read heavy cases, fetch becomes very efficient * - The entity store basically has two potential candidate selection paths (smart and dumb) * * index all stores (evaluting -> is there a qualifying change) * - lookup possible candidates in entity store via indexes * - merge write buffer entity ids that are not in candidates and pass simple some filter * - merge transaction entity ids that are not in candidates and pass simple some filter * * merge all entities (MVCC) */ /** * Writeup: * Our best scenario is nothing is a read only transaction and an empty write buffer (or empty after candidate eval - so only impacted for related data) * Optimally we're hitting this in read heavy apps * Optimally for write heavy apps (ie write buffer might fill up) its still relatively fast */ /** * class FetchExecutor { * constructor(entityStore: EntityStore, writeBuffer: writeBuffer, tx: Transaction) { * ... * } * * fetch(query: CollectionQuery): Iterable { * const writeBufferEvaluation = this.writeBuffer.getCandidates(this, query); * const txEvaluation = this.tx.getCandidates(this, query); * const storeEvaluation = this.store.getCandidates(this, query); * const fulfillment = mergeFulfillments(writeBufferEvaluation.fulfillment, txEvaluation.fulfillment, storeEvaluation.fulfillment); * const candidates = mergeCandidates(writeBufferEvaluation.candidates, txEvaluation.candidates, storeEvaluation.candidates); * * for(const candidate of candidates) { * const entity = addOverlays(loadEntity(candidate)); * if(!fulfillment.collectionName) // check collectionName * // filters * // order * // limit * } * } * } */ /** * KV Store * interact with an index stored in the KV store * * Memory * Use iterators and filter down candidates * * SQLite * Perform a query on the SQLite database with some filters / orders applied, based on indexes * Can also load entities into memory at this point (maybe) * - think through how that can remain memory efficient * - iterators would be helpful here * * IndexedDB * Create indexes on specified attrs probably? * * Open question? * - should you specify indexed attributes? * - probably, then later we can figure out the right way to do it automatically * - You should get write performance benefits with less indexing * - How does this work with MVCC, what are our isolation guarantees with the delta format? * - A write buffer that's a log of changes and locking buffer flushes when txs are open makes sense to me * - A transaction probably calculates a single delta from the log of changes * - This creates a bit of garbage for systems with lots of transactions * - How would this work with server syncing? */ /** * Buffer fetch * - what am i actually fetching here? * - !=? * - not really do-able ... or you couldnt guarantee the fulfillment of it * - order and limit merging * - */