/** * FlatSQL-WASM engine record store (loop D.1) — THE SDNNode store. * * ONE store: the same FlatSQL-WASM engine that runs inside sdn-server hosts * the browser node's records. Layout mirrors the server * (sdn-server/internal/storage): * * - Record envelopes are indexed in a plain SQL control table named * `sdn_record_index` created through engine DDL (server: flatsql.go) — * schema_name / cid / source_timestamp columns keep the server's naming. * Payload bytes are NOT stored in SQLite rows (server keeps them in * append-only stream files); here they live in an in-memory envelope map * whose durable substrate is the snapshot journal below. * - SDS FlatBuffer records are mirrored into per-standard engine databases * (src/local-flatsql.ts): per-provider `registerSource` shadow tables * (`OMM@`) + unified views whose `_source` column carries the * full shadow-table name — identical to engine_records.go. Records * stored through the node surface default to the server's `local` * source partition. * - Durability: the engine is in-memory; boot = replay. The envelope * journal (same size-prefixed codec the pre-D.1 FlatSQLStorage snapshots * used, so existing snapshots keep loading) persists through a pluggable * SnapshotPersistence — IndexedDB by default when a persistenceKey is * given, or Helia/memory when injected. Per-standard record streams * persist per (standard, source) via the engine store. * * The public surface satisfies both `NodeRecordStorage` (SDNNode) and the * `ModuleHostRecordStore` contract (module-host-adapters.ts), so module * hosting drops in unchanged. */ import type { EngineEpochQueryProfilesConfig, EngineEpochQueryRequest } from './epoch-query-sql'; import type { FlatSqlPublishedShard, FlatSqlSyncChunk, FlatSqlSyncManifest, FlatSqlSyncManifestSegment } from './flatsql-sync'; import type { SchemaName } from './schemas'; import type { StoredRecord, QueryFilter } from './storage'; import { type LocalFlatSqlPersistenceStore, type LocalFlatSqlPinLedgerEntry, type LocalFlatSqlPinLedgerQuery, type LocalFlatSqlSchema, type LocalFlatSqlStore } from './local-flatsql'; /** Positional query parameter accepted by the engine (flatsql/wasm QueryParam). */ export type EngineQueryParam = null | boolean | number | string | Uint8Array; export interface SnapshotPersistence { load(): Promise; save(bytes: Uint8Array): Promise; } /** In-memory persistence (tests / ephemeral nodes). */ export declare class MemorySnapshotPersistence implements SnapshotPersistence { private snapshot; load(): Promise; save(bytes: Uint8Array): Promise; } /** Minimal Helia surface the persistence needs (helia + @helia/unixfs). */ export interface HeliaLike { addBytes(bytes: Uint8Array): Promise<{ toString(): string; }>; catBytes(cid: string): Promise; } /** * Helia-backed persistence: snapshot bytes become a unixfs block; the root * CID is remembered in a Storage-like ref store (localStorage in browsers). */ export declare class HeliaSnapshotPersistence implements SnapshotPersistence { private readonly helia; private readonly refStore; private readonly refKey; constructor(helia: HeliaLike, refStore: Pick, refKey?: string); rootCid(): string | null; load(): Promise; save(bytes: Uint8Array): Promise; } /** * SnapshotPersistence over a LocalFlatSqlPersistenceStore key — the default * journal backend (IndexedDB `sdn-local-flatsql`/`datastores`) when the * store opens with a persistenceKey and no explicit persistence. */ export declare class PersistenceStoreSnapshotPersistence implements SnapshotPersistence { private readonly store; private readonly key; constructor(store: LocalFlatSqlPersistenceStore, key: string); load(): Promise; save(bytes: Uint8Array): Promise; } /** Engine database surface the record store drives (subset of flatsql/wasm). */ interface ControlDatabase { query(sql: string, params?: Array): { columns: string[]; rows: unknown[][]; }; destroy(): void; } export interface FlatSQLEngineRecordStoreOptions { /** Envelope journal backend. Defaults to IndexedDB when persistenceKey is set. */ persistence?: SnapshotPersistence; /** Persist the journal after every mutation (default true when a journal exists). */ flushOnWrite?: boolean; hashHex?: (data: Uint8Array) => Promise; nowMs?: () => number; /** * SDS standards mirrored into per-standard engine databases (server * layout). THIS OPTION DECIDES DURABILITY: with no schemas there is no * per-standard disk-backed lane at all and everything falls to the * whole-blob envelope journal (see `envelopeJournalOnly`). */ schemas?: LocalFlatSqlSchema[]; /** Key namespacing persisted engine state (streams, journal) in IndexedDB. */ persistenceKey?: string | null; /** * Acknowledge whole-blob envelope-journal persistence: REQUIRED when a * `persistenceKey` is given without `schemas`, ignored otherwise. * * A `persistenceKey` looks like a request for durable storage, and until * 2.0.18 a schemas-less key silently downgraded to ONE `:record-envelopes` * blob rewritten in full on every flush and re-ingested in full at boot — * measured at 10,544,950 B for a live catalogue cache. That downgrade is * now refused rather than silent: pass `schemas` for the disk-backed * per-source lane, or pass this flag to say the journal is what you meant. */ envelopeJournalOnly?: boolean; desktopPersistenceBaseUrl?: string | null; fetch?: ((input: string | URL | Request, init?: RequestInit) => Promise) | null; /** Injectable key/value persistence backend (tests / embedders). */ persistenceStore?: LocalFlatSqlPersistenceStore | null; /** Source partition for records stored through the node surface (default `local`). */ defaultSource?: string; /** * Per-standard default query profiles (loop D.2) — retrieval-module * config shape, keyed by schema name (`OMM.fbs`) or standard id (`OMM`). */ queryProfiles?: EngineEpochQueryProfilesConfig | null; /** Clock for defaulted query epochs (tests). */ nowSeconds?: (() => number) | null; } /** * A standard registered on this store: the routing key (`OMM`), the 4-byte * FlatBuffer file identifier payloads are ROUTED BY, and the engine table its * frames land in. */ interface RegisteredStandard { standardId: string; fileId: string; tableName: string; } interface EngineRecordStoreContext { controlDb: ControlDatabase; standards: LocalFlatSqlStore | null; standardIds: Map; options: FlatSQLEngineRecordStoreOptions; /** Compact durable index for datasync-fed envelope rows (loop D.4). */ syncEnvelopePersistence?: { store: LocalFlatSqlPersistenceStore; key: string; } | null; } /** Provenance fallbacks for sync-chunk ingest (per-record refs win). */ export interface EngineSyncChunkIngestOptions { /** Explicit source partition override (wins over all provenance). */ source?: string | null; /** Fallback provider when a record ref carries no provider tag. */ providerId?: string | null; /** Fallback source name when a record ref carries no source tag. */ sourceName?: string | null; /** Persist engine streams + envelope index after ingest (default true). */ persist?: boolean; } export interface EngineSyncIngestResult { standardId: string; /** Frames delivered by the peer (before dedupe). */ totalRecords: number; /** New engine rows materialized (dedupe makes replay 0). */ ingestedRecords: number; /** Envelope index rows written for the first time. */ indexedEnvelopes: number; /** Source partitions touched (server layout: `registerSource` per provider). */ sources: string[]; /** True when the pin ledger proved this delivery was already materialized. */ replayed: boolean; } export interface EnginePublishedShardIngestOptions { /** Explicit source partition override (wins over header/manifest provenance). */ source?: string | null; /** libp2p peer the shard was fetched from (pin-ledger provenance). */ providerPeerId?: string | null; providerPublicKey?: string | null; /** Manifest the shard was announced in (provenance fallback: provider/source/batch/head). */ manifest?: Pick | null; /** Manifest segment for this shard (feed head / chunk-hash fallback). */ segment?: Pick | null; /** Write per-record envelope index rows (cid = sha256, the server computeCID). Default true. */ indexEnvelopes?: boolean; /** Persist engine streams + pin ledger + envelope index after ingest (default true). */ persist?: boolean; } /** * Admit options for `storeStream` — the shard-shaped admit point. The * standard is NOT one of them: it is routed from the frames' file identifier. */ export interface EngineStreamStoreOptions { /** Source partition the shard lands in (default: the store's defaultSource). */ source?: string | null; /** * ASSERTION only — the standard the caller believes this shard is. Routing * is by file identifier; a mismatch is refused, never overridden. */ standardId?: string | null; /** Publication content id; keys the frames so re-admitting the shard is a no-op. */ shardCid?: string | null; /** Expected sha256 of the stream bytes, verified before ingest when given. */ sha256?: string | null; /** Provenance rows for the pin ledger (provider / source / batch attribution). */ pinLedgerEntries?: LocalFlatSqlPinLedgerEntry[]; /** Flush the lane after ingest (default true). */ persist?: boolean; /** * Write a per-FRAME envelope index row. DEFAULT FALSE: a shard's durable * substrate is the lane, so envelope rows would be a second durable copy of * bytes the lane already holds (measured: 21,104,976 B for one 10.5 MB * shard). Opt in only when the frames must also be reachable through the * record envelope surface (`get`/`query`/`count`). */ indexEnvelopes?: boolean; } export interface EngineStreamStoreResult { /** Standard the frames' file identifier routed to. */ standardId: string; source: string; /** Frames in the delivered stream. */ frames: number; /** Frames newly materialized in the lane (replay makes this 0). */ ingested: number; /** Envelope index rows written (0 unless `indexEnvelopes`). */ indexedEnvelopes: number; bytes: number; /** True when the ingested-keys ledger proved this shard was already resident. */ replayed: boolean; } export interface EngineStreamReadOptions { /** * Read ONE source partition in ingest order — required for byte-identical * round trips. Omitted, the read spans every partition (a union). */ source?: string | null; } export declare class FlatSQLEngineRecordStore { private readonly controlDb; private readonly standards; private readonly standardIds; private readonly byCid; /** Datasync-fed envelope index rows (schema|cid → row), payloads live in the engine partitions. */ private readonly syncEnvelopes; private readonly syncEnvelopePersistence; private readonly persistence?; private readonly flushOnWrite; private readonly hashHex; private readonly nowMs; private readonly defaultSource; private closed; protected constructor(context: EngineRecordStoreContext); static open(this: typeof FlatSQLEngineRecordStore, options?: FlatSQLEngineRecordStoreOptions): Promise; /** Per-standard engine store (per-source shadow tables + unified views). */ get standardsStore(): LocalFlatSqlStore | null; /** * PRIMARY public query API (loop D.2): run an engine-native epoch profile * (`nearest` — the default — / `as_of` / `forward`; the server's retrieval * profiles, SQL byte-identical to sdn-server engine_records.go, params * bound in the same positional order) over the unified per-standard view * and return the ALIGNED size-prefixed FlatBuffer frame stream — the wire * format, byte-identical to the Go host for the same store contents. * * Defaults resolve request > per-standard `queryProfiles` config > * compiled fallback (`nearest`, epoch = now, limit 50000) — the same * precedence as the retrieval module. */ queryEpochRawStream(standardId: string, request?: EngineEpochQueryRequest | null): Uint8Array; /** * Decoded-record convenience over `queryEpochRawStream`: an iterator of * per-record FlatBuffer frames that are zero-copy subarray VIEWS into the * aligned stream. */ queryEpochFrames(standardId: string, request?: EngineEpochQueryRequest | null): Generator; /** * Ingest an aligned size-prefixed FlatBuffer record stream straight into * the per-standard engine database (loop D.3: HTTP bulk-stream bodies feed * through verbatim — no base64, no JSON re-encode, no per-record * round-trips). Delegates to the engine store's stream ingest (dedupe via * the ingested-keys ledger; per-(standard,source) persistence). */ ingestFlatBufferStream(standardId: string, streamBytes: Uint8Array, options?: Parameters[2]): Promise; /** * Admit an aligned size-prefixed SDS shard into the disk-backed * per-source standards lane. The standard is resolved from the FRAMES' * FlatBuffer file identifier — header-only routing, never a caller-supplied * name (`options.standardId` is an ASSERTION, refused on mismatch). * * Requires `schemas` (there is no lane without them) and refuses a payload * that is not a size-prefixed stream of a registered standard. */ storeStream(streamBytes: Uint8Array, options?: EngineStreamStoreOptions): Promise; /** * Read a standard's lane back as ONE aligned size-prefixed stream — the * paired read for `storeStream`, and the wire format itself. * * With a `source` this reads that per-source shadow table in `_rowid` * order, which is exactly the order the frames were ingested in, so a shard * admitted through `storeStream` comes back BYTE-IDENTICAL (verified at * 32,141 frames / 10,544,168 B, across teardown + reopen). Without a * `source` it reads the unified view across every partition, which is a * union — use a source when byte identity is what you need. * * Returns an empty stream when the standard has no such partition yet. */ readStream(standardId: string, options?: EngineStreamReadOptions): Uint8Array; /** Standards registered on this store (routing key, file id, engine table). */ registeredStandards(): Array<{ standardId: string; fileId: string; tableName: string; }>; /** Resolve a size-prefixed stream to its standard by FILE IDENTIFIER alone. */ private routeStreamByFileId; /** * `store()` is the RECORD admit point. A multi-frame SDS shard handed to it * used to be accepted, enveloped whole, and mirrored through a one-record * ingest that produced zero queryable rows while doubling storage. The * detection fix alone would make that silent (envelope only, no mirror); * refusing makes it loud and names the surface that is actually wanted. * * Only genuine shards are refused: the payload must tile the buffer exactly * into 2+ frames that ALL carry a registered standard's file identifier. */ private refuseMultiFrameStream; private requireRegisteredStandard; /** * Datasync-fed ingest (loop D.4): materialize a flatsql-sync `read_chunk` * response into the engine store with its TRUE provenance. Every record * ref's provider/source/batch tags route the record into its per-provider * source partition (`registerSource` shadow tables — the server layout), * NOT into the node's default `local` partition; the envelope index * (`sdn_record_index` mirror) gains a row per ref (cid / peer / * source_timestamp), with the payload bytes living in the engine * partition exactly like the server keeps them in stream files. * * Idempotent under sync re-delivery: dedupe keys are the D.1 * ingested-keys ledger keys (`schema|cid|provider|source|batch|timestamp` * — identical to the webUI sync worker's `flatSqlRecordKeys`), and * envelope rows upsert by (schema, cid). */ ingestSyncChunk(chunk: FlatSqlSyncChunk, options?: EngineSyncChunkIngestOptions): Promise; /** * Datasync-fed ingest (loop D.4): materialize a published FlatSQL shard * (`read_published_shard` response, or a manifest segment fetched over * IPFS) into the engine store. The shard's provider/source/batch * provenance routes into a per-provider engine source partition, the pin * ledger records the shard exactly like the webUI sync worker does * (role `shard`, verified, byteHash/head/rowCount), and — by default — * every record frame is indexed as an envelope row whose cid is the * sha256 of the frame (byte-identical to the server's computeCID). * * Idempotent under re-delivery: a verified pin-ledger entry for the same * (cid, standard, provider/source/batch) short-circuits to a no-op, and * the ingested-keys ledger (`shard:` keys) catches replays even when * the ledger entry is missing. */ ingestPublishedShard(shard: FlatSqlPublishedShard, options?: EnginePublishedShardIngestOptions): Promise; /** Pin-ledger provenance surface (provider/source/batch attribution queries). */ listPinLedgerEntries(query?: LocalFlatSqlPinLedgerQuery): Promise; recordPinLedgerEntries(entries: LocalFlatSqlPinLedgerEntry[]): Promise; private ingestAnonymousSyncStream; /** * Upsert a datasync-fed envelope index row. Server mirror: SQLite holds * index/provenance rows only — the payload stays in the engine partition * (the server's append-only stream files). Returns true on first insert. */ private upsertSyncEnvelope; private replaySyncEnvelopes; private persistSyncEnvelopes; /** * Generic aligned-raw-stream query (server mirror of * FlatSQLStore.QueryRawStream): read-only SQL whose result cells are all * BLOBs, run verbatim with positional params against the standard's * engine database. */ queryRawFlatBufferStream(standardId: string, sql: string, params?: EngineQueryParam[]): Uint8Array; private requireStandards; private replayJournal; private insertEnvelope; /** * Mirror SDS FlatBuffer payloads into the per-standard engine database * (server behavior: engine vtab is a query cache over the durable * envelope; per-record failures are logged and skipped). Idempotent across * journal replays via the engine store's ingested-keys ledger. */ private mirrorIntoStandard; store(schema: SchemaName | string, data: Uint8Array, peerId: string, signature: Uint8Array): Promise; get(schema: SchemaName | string, cid: string): Promise; query(schema: SchemaName | string, filter?: QueryFilter): Promise; /** Raw SQL over the engine's control table(s) (schema/cid provenance). */ sql(query: string): { columns: string[]; rows: unknown[][]; rowCount: number; }; delete(cid: string): Promise; count(schema?: SchemaName | string): Promise; listRecords(): StoredRecord[]; flush(): Promise; close(): Promise; } /** * True when the payload is ONE FlatBuffer RECORD carrying the given 4-byte * file identifier, either bare (identifier at bytes 4-7) or size-prefixed * (identifier at bytes 8-11). * * The size-prefixed arm requires the u32 length prefix to account for the * WHOLE buffer — the server's rule verbatim (sdn-server/internal/storage/ * engine_records.go:137 `engineRecordPayload`: `uint32(data[:4])+4 == * len(data) && SizePrefixedOMMBufferHasIdentifier(data)`). Without that * length equality an aligned size-prefixed STREAM of N frames matches as a * single record — its first frame's u32 prefix puts the file identifier at * bytes 8-11 — and a 10.5 MB shard gets handed to a one-record ingest, * which is the false positive measured in 2.0.17 (0 rows, doubled * IndexedDB, +342 ms). sdn-js was MORE permissive than the Go host it * mirrors; this closes that divergence. Multi-frame streams are classified * by `flatBufferStreamMatchesFileId`. */ export declare function flatBufferMatchesFileId(data: Uint8Array, fileId: string): boolean; /** * True when the payload is an ALIGNED SIZE-PREFIXED STREAM whose frames all * carry the given file identifier — the SDS shard shape (the wire format, * and what `queryRawFlatBufferStream` returns). * * Strict by construction: the frame lengths must tile the buffer exactly and * EVERY frame must carry the identifier, so opaque record bytes can never be * mistaken for a shard. A one-frame stream satisfies this and also satisfies * `flatBufferMatchesFileId` — that overlap is deliberate: a single record is * legitimately admissible through either lane. */ export declare function flatBufferStreamMatchesFileId(data: Uint8Array, fileId: string): boolean; /** * Frame count of an aligned size-prefixed stream whose frames ALL carry * `fileId`; 0 when the buffer is not such a stream. Never throws — this is a * classifier, run on untrusted bytes. */ export declare function countFlatBufferStreamFramesWithFileId(data: Uint8Array, fileId: string): number; export {}; //# sourceMappingURL=engine-record-store.d.ts.map