/// /// import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig, RemoteConfig, SourceConfig, SyncActivity } from "../IArchives"; import { StoreSync } from "./storeSync"; import { StoreConfig } from "./storeConfig"; export declare const WINDOW_END_FLUSH_MARGIN: number; export declare const HISTORY_MIN_BYTES: number; /** The multiple of a store's live bytes its deletion history may grow to. Async so it can later become dynamic and user-configurable; for now it is a constant. */ export declare function getHistoryFactor(): Promise; /** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see TransactionFile). */ type IndexValue = { size: number; sourcesListIndex: number; }; /** One file we hold, as everything outside the index sees it. */ export type IndexEntry = IndexValue & { writeTime: number; changedAt: number; }; export type BlobSourceSpec = { identity: string; url: string; validWindows: [number, number][]; route?: [number, number]; noFullSync?: boolean; intermediate?: string; sourceConfig?: SourceConfig; create: () => IArchives; applyConfig?: (source: IArchives) => void; }; export declare class BlobStore { folder: string; /** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */ storeName: string; private config?; stopped: { stop: boolean; }; syncStarted: boolean; /** Its sources, in config order: slot 0 is always its own disk folder, the rest are the peers it synchronizes with. Filled by updateSources, which is also how they change. The store OWNS them - writes pick among them, reads resolve holders through them - and StoreSync only scans whatever is in here at the time. */ sources: ArchivesSource[]; private discardedUploads; private nextDiscardedUpload; private sourcesList; private slotSourcesListIndexes; private slotRegistrations; private index; /** Keeping the index in agreement with the sources: scanning, pulling, pushing, and the maintenance that follows from holding an index (disk-limit eviction, tombstone expiry). It reads and writes this store's index and sources - it does not own them. */ sync: StoreSync; constructor(folder: string, /** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */ storeName: string, config?: { /** Whether a config entry is THIS SERVER (same account, same bucket, our own address). Injected because a store knows nothing about servers - it only needs to tell its own entries apart from its peers'. Absent means nothing is us, which is what a bare store (no server around it) wants. */ isSelf?: ((source: SourceConfig) => boolean) | undefined; /** Builds one of its sources. Injected for the same reason, and because the delay it is created with is this store's policy. */ createSource?: ((config: { sourceConfig?: SourceConfig; writeDelay: number; }) => IArchives) | undefined; /** Hands a running source a changed config, so an endpoint we already talk to is never rebuilt just because a flag moved. */ applySource?: ((source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay: number) => void) | undefined; onIndexChanged?: ((key: string) => void) | undefined; /** Called every time this store applies a routing config to itself (startup, an operator's write, a peer's copy arriving) - the store is the one that knows when a config landed, and the server arms window-boundary scans from it. */ onRoutingApplied?: ((routing: RemoteConfig) => void) | undefined; /** Asks the client whose request created this store what routing config it intended for our name. Only used when init finds NO configuration in our folder: a store only ever exists because a config names it, so the requester has that config - asking for it lazily is the same information as passing the config on every call, without the per-call kilobytes. */ requestRoutingConfig?: (() => Promise) | undefined; onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined; /** A synchronization transfer: "sync get" is bytes pulled off a source (the backblaze download bill), "sync set" is bytes pushed to one. Injected because sync traffic never passes through the API controller, so nothing else can count it. */ onSyncTransfer?: ((operation: "sync get" | "sync set", path: string, bytes: number) => void) | undefined; resolveSourceUrl?: ((url: string) => IArchives) | undefined; } | undefined); /** This store's folder, unwrapped: the same bytes slot 0 serves, but reached without its write delay. Used for the two things that cannot go through a buffered source - reading our own routing config before we have any sources, and streaming a large upload that must not sit in memory. */ private ownDisk; /** What this store is configured to be. It owns this: the routing config is a file IN the store, so the store reads it, applies it to itself, and re-applies it whenever the file changes - by our own write, or by a peer's copy arriving through synchronization. */ storeConfig: StoreConfig; private appliedRoutingVersion; private appliedRouting; init: { (): Promise; reset(): void; set(newValue: Promise): void; }; /** * Re-reads the routing config out of this store and applies it to itself. Called at startup and * whenever that file changes here - which is the ONE mechanism: a config written by an operator * and a config pulled off a peer are the same event, a write of that path into this store. * * A store with no routing config configures itself as its own disk, valid always, for the whole * key space. That is a complete, working store - it just has nobody to synchronize with - and it * is what lets a store exist before it has ever heard of a configuration. */ applyRoutingConfig(): Promise; private readRoutingConfig; /** The version of the routing config this store is running, so a copy found on a peer is only taken when it is genuinely newer. -1 means it has none. */ routingVersion(): number; private routingApplies; reapplyRoutingConfig(): void; private planSources; dispose(): Promise; get2(config: { path: string; range?: { start: number; end: number; }; internal?: boolean; includeTombstones?: boolean; includeMarked?: boolean; }): Promise<{ data: Buffer; writeTime: number; size: number; } | undefined>; set(config: { path: string; data: Buffer; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean; undelete?: boolean; }): Promise; del(config: { path: string; lastModified?: number; internal?: boolean; }): Promise; /** A node-side move: the bytes never travel through the client. Deliberately just get2 + set + del rather than a disk rename, so the destination write passes EVERY rule a set passes (windows, routes, immutability, only-take-latest, index, fan-out to peers) and the deletion propagates as a normal tombstone - a rename would bypass all of it. The set stamps fresh, so the moved file beats any tombstone at its new path. */ move(config: { fromPath: string; toPath: string; }): Promise; getInfo(config: { path: string; includeTombstones?: boolean; }): Promise<{ writeTime: number; size: number; } | undefined>; findInfo(config: FindConfig & { prefix: string; }): Promise; getChangesAfter2(config: ChangesAfterConfig): Promise; getSyncStatus(): Promise; /** The index's totals plus any in-progress background synchronization. */ getSyncProgress(): { index: { fileCount: number; byteCount: number; }; marked: { fileCount: number; byteCount: number; oldestDeleteTime?: number; }; sources: { debugName: string; fileCount: number; byteCount: number; }[]; readerDiskLimit?: number; syncing: SyncActivity[]; }; /** getSyncProgress's totals, but loading the index first, so they are never the zeroes of a store nothing has touched yet. */ computeIndexTotals(): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number; }[]; }>; private namedIndexTotals; /** * The store's sources, as the current routing config says they should be. This is the ONLY way * they are ever set: the first call populates an empty store, every later one applies a change to * the running one. Windows, routes and flags move in place, genuinely new endpoints are added and * start scanning, and endpoints that are gone go dead (their scans stop, their index entries * drop). * * A store is never rebuilt for a config change. Its name decides its folder and its identity, and * a config change cannot change either - so there is nothing a change can do to a store except * this. */ updateSources(specs: BlobSourceSpec[]): void; /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */ rescanBase(): Promise; /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes). */ boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number]; }): Promise; startLargeUpload(config?: { path?: string; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean; }): Promise; appendLargeUpload(config: { id: string; data: Buffer; offset?: number; }): Promise; finishLargeUpload(config: { id: string; path: string; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean; }): Promise; cancelLargeUpload(config: { id: string; }): Promise; /** Bytes of read cache the disk may hold; see CommonConfig.readerDiskLimit (StoreSync enforces it). Read from the config in effect, so raising or removing the limit takes effect on the next eviction pass. */ get readerDiskLimit(): number | undefined; /** The write time a new write has to beat, or 0 when we have never heard of the key. Counts DELETIONS too: a write older than the deletion that removed it must not bring it back. The index is authoritative even for a write still buffered in a delayed source, since the entry is recorded when the write is accepted rather than when it reaches storage. */ currentWriteTime(key: string): number; private isLive; registerSlot(slot: number): Promise; /** The persistent sourcesListIndex of a slot, or undefined when the slot never got that far (a source removed before its registration resolved). */ slotSourcesListIndex(slot: number): number | undefined; sourcesListIndexOfSlot(slot: number): number; slotForSourcesListIndex(sourcesListIndex: number): number | undefined; getEntryHolder(entry: IndexEntry): Promise; private loadIndex; /** A file we hold. A deleted one is not one: it is a tombstone, and only getDeletedEntry knows about it. */ getIndexEntry(key: string): IndexEntry | undefined; /** When a key was deleted, if it was. A deletion is an absence with a time attached - that time is what makes it propagate and what expires it. */ getDeletedEntry(key: string): { writeTime: number; changedAt: number; } | undefined; /** Every file we hold, for the passes that walk them all (listings, scans, reconciliation, eviction). Deletions are not in here - see deletedEntries. Live: deleting entries while iterating is expected here, and safe. */ indexEntries(): IterableIterator<[string, IndexEntry]>; /** Every deletion we know of. A much smaller walk than the files, which is what makes expiring them cheap. */ deletedEntries(): IterableIterator<[string, { writeTime: number; changedAt: number; }]>; /** A file MARKED for deletion: its kept index value plus when it was deleted. Undefined when the key is live, never existed, or its history was already dropped. */ getMarkedEntry(key: string): (IndexEntry & { deleteTime: number; }) | undefined; /** Every file marked for deletion - the deletion history, walked by retention and by includeMarked listings. */ markedEntries(): IterableIterator<[string, IndexEntry & { deleteTime: number; }]>; /** The deletion history's totals: how many marked files, their bytes, and the delete time of the OLDEST one - which is how far back the history reaches. */ markedTotals(): { fileCount: number; byteCount: number; oldestDeleteTime?: number; }; /** Physically removes a marked file's bytes from our disk and drops its kept value, leaving a plain tombstone that ages out normally - retention calling time on the oldest history. */ dropMarkedHistory(key: string): Promise; /** See SetConfig.undelete: flips a marked deletion back to live (fresh write time, so the restore outranks the deletion everywhere it propagated) - the bytes never left the disk, so reads just work again. Internal restores are a peer's propagation and tolerate having nothing to restore (this node may never have held the file); a caller's restore throws instead. */ private undeleteKey; /** How many files we hold, deletions excluded. */ indexSize(): number; /** Totals over the files we hold, broken down by the slot holding each (entries can name a source that is no longer configured, which counts towards the total but no slot). */ indexTotals(): { fileCount: number; byteCount: number; slots: { fileCount: number; byteCount: number; }[]; }; /** Records a file, as of its write time. Returns false, having changed nothing, when we already know something at least as new - the index cannot be made to go backwards, whichever path the write came in by. */ setIndexEntry(key: string, entry: { writeTime: number; size: number; sourcesListIndex: number; }): boolean; /** Records a DELETION, as of its time: the key stops existing here, and the tombstone is what makes that fact propagate and reconcile like any other write. Same ordering rule as setIndexEntry. */ setIndexDeleted(key: string, writeTime: number): boolean; /** Forgets a key entirely, tombstone included. NOT a deletion: it says nothing happened to the file, only that we no longer know anything about it - for an entry whose holder turned out not to have it, and for a tombstone old enough that everyone has heard. */ purgeIndexEntry(key: string): void; /** Counts a synchronization transfer in the server's access statistics (see getStore's wiring): "sync get" for bytes pulled off a source, "sync set" for bytes pushed to one. */ noteSyncTransfer(operation: "sync get" | "sync set", path: string, bytes: number): void; /** * Every write, however it is stamped, has to be one we are actually meant to hold - because the * alternative is not a smaller problem, it is a silent one. A write that lands on a store that * does not serve its route (or on a server that is not in the bucket's config at all) goes into a * folder nothing scans and no peer reconciles: it succeeds, and then it is gone. The markers make * the client re-read the routing config and retry, which is exactly the right outcome when the * reason it aimed here is that its config was stale. */ private assertWriteTarget; /** Exactly why this store has no configuration entries - which of the three possible reasons it is, with the values that decided it, because "not configured" alone is undiagnosable. */ private unconfiguredDetail; /** * Whether a routing config may be written here. Two rules, and this is the one place either is * applied - a config only ever enters the system through a write, so a config that got in is a * config that passed, and reading one back never judges it again. * * The config has to be valid as a whole (see assertValidRemoteConfig), and it has to outrank what * we are running: the same version means the same config, so re-writing it is harmless, but a * lower one is an older config arriving late and must never undo a newer one. */ private assertRoutingConfigWritable; private assertFreshWriteTarget; private assertMutable; private assertInternalWriteAccepted; /** Internal (store-to-store) read: never goes to OTHER sources - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form - but the INDEX still gates, because it is the source of truth: a marked deletion keeps its bytes on disk as history (see writeToSources), so the disk alone would happily serve a DELETED file as live. Index says live -> the disk provides the bytes (past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes). Index says deleted -> the tombstone is the answer, never the disk. No window or route checks. */ private getInternal2; /** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Only-take-latest still applies here. */ private setInternal; private cacheRead; private setOrDelete; /** The instant every delayed write must be on its source: the end of our own write window that contains now, minus the flush margin (so the next window's source finds the data on handoff). The LATEST end among covering windows - overlapping windows hand off at the last one. No window contains now (an inert store, or a moment between our windows) -> 0, i.e. nothing may be delayed at all. */ writeFlushDeadline(): number; private getWritableSources; private writeToSources; private getDiskSource; /** Writes everything still held by a delayed source (see ArchivesDelayed). force also writes what isn't due yet - shutdown cannot leave writes in memory. */ private flushDelayedWrites; } export {};