import knex, { Knex } from "knex"; import config from "../../knexfile.js"; import stringify from "json-stable-stringify"; import XXH from "xxhashjs"; import { ERPObjType } from "../../types/erp-types.js"; import { CacheMetrics } from "./index.js"; import { CoreConfiguration } from "../data-sync-service/configuration-manager.js"; import { logger } from "../reporting-service/index.js"; import { EntityTransformer } from "../mm-api-service/types/entity-transformer.js"; import { IToRESTApiObject } from "../mm-api-service/index.js"; type HashedCacheManagerOptions = { ttl?: number; tableName?: string; /** Injectable Knex handle (tests); defaults to the SDK's local SQLite. */ db?: Knex; }; export class HashedCacheManager { private static TABLE_NAME = "sdk_cache"; private db: Knex; private ownsDb: boolean; private options: HashedCacheManagerOptions; private static readonly SEED = 0xabcd; // Arbitrary seed for hashing private isDestroyed: boolean = false; private metrics: CacheMetrics = { recordCounts: {}, }; constructor(options?: HashedCacheManagerOptions) { this.options = { ttl: options?.ttl || CoreConfiguration.inst().cacheTTL, tableName: options?.tableName || HashedCacheManager.TABLE_NAME, }; if (options?.db) { this.db = options.db; this.ownsDb = false; } else { this.db = knex({ ...config.local, pool: { min: 0, max: 10, }, }); this.ownsDb = true; } } /** * Checks if the cache manager is still valid * @throws Error if the cache manager has been destroyed */ private checkValid(): void { if (this.isDestroyed) { throw new Error("Cache manager has been destroyed"); } } /** * Generates a stable hash of a record using JSON stringify + xxhash */ public static hashRecord(record: object): string { try { const serialized = stringify(record); if (!serialized) { throw new Error("Failed to serialize record for hashing"); } const hash = XXH.h64(serialized, HashedCacheManager.SEED).toString(16); return hash; } catch (error) { if (error instanceof Error && error.message.includes("circular")) { throw new Error("Failed to serialize record for hashing"); } throw error; } } /** * Derives a stable per-entity identity for a record, or null when it cannot be * computed (e.g. required key fields are missing). A null identity means the * record is never deduped and never cached — we must not silently drop a write * for a record we cannot identify. */ private static entityKeyFor( type: ERPObjType, record: object ): string | null { try { return EntityTransformer.extractPrimaryKey( type, record as IToRESTApiObject ); } catch (error) { // Identity couldn't be computed (e.g. a missing required key field): the // record won't be deduped or cached. Log at debug so it's diagnosable // without adding noise or changing behavior. Avoid logging the record itself. logger.debug("Cache: skipping dedupe; could not derive entity key", { type, error: error instanceof Error ? error.message : String(error), }); return null; } } /** * Gets the stored content hash for an entity, keyed by its primary key. * @returns the stored row's content hash, or null if the entity is uncached */ private async getRecord( type: ERPObjType, entityKey: string ): Promise<{ content_hash: string } | null> { this.checkValid(); return this.db(this.options.tableName) .select("content_hash") .where({ type, entity_key: entityKey }) .first(); } /** * Stores (upserts) an entity's last-sent content hash, keyed by its primary key. * @returns true if the stored content changed (new entity or different content) */ public async store(type: ERPObjType, record: object): Promise { if (!this.isDestroyed && record) { try { const entityKey = HashedCacheManager.entityKeyFor(type, record); // No identity -> cannot key the row; skip caching so it is always re-sent. if (entityKey === null) { return false; } const contentHash = HashedCacheManager.hashRecord(record); const existing = await this.getRecord(type, entityKey); const changed = !existing || existing.content_hash !== contentHash; // Overwrite the entity's remembered content hash. Unlike the old // insert-only set-membership model, this is an upsert so a value that // returns to a previously-seen state is no longer treated as "seen". await this.db(this.options.tableName) .insert({ type, entity_key: entityKey, content_hash: contentHash, updated_at: Date.now(), }) .onConflict(["type", "entity_key"]) .merge(["content_hash", "updated_at"]); return changed; } catch (error) { logger.error("Error storing record:", error); throw error; } } return false; } /** * Checks if a record has changed since the last content we sent for that entity. * @param type The type of record * @param record The record to check * @returns true if the record is new, its content differs from the last stored * content for that entity, or its identity cannot be computed */ async hasChanged(type: ERPObjType, record: object): Promise { this.checkValid(); const entityKey = HashedCacheManager.entityKeyFor(type, record); // Unidentifiable record: never dedupe (avoid silently dropping the write). if (entityKey === null) { return true; } const contentHash = HashedCacheManager.hashRecord(record); const existing = await this.getRecord(type, entityKey); return !existing || existing.content_hash !== contentHash; } /** * Checks if a record has changed and stores it if it has * @param type The type of record * @param record The record to check and store * @returns true if the record was changed or is new */ async upsert(type: ERPObjType, record: object): Promise { this.checkValid(); const hasChanged = await this.hasChanged(type, record); if (hasChanged) { await this.store(type, record as Record); } return hasChanged; } /** * Removes expired records based on TTL */ async removeExpiredObjects(): Promise { this.checkValid(); const ttl = this.options.ttl; if (!ttl) { logger.info("Cache cleanup: TTL disabled; skipping removeExpiredObjects", { tableName: this.options.tableName, ttl, }); return; } const ttlMilliseconds = ttl * 1000; // `updated_at` is stored as epoch ms (see store()), so compare directly. const cutoffEpochMs = Date.now() - ttlMilliseconds; const deletedCount = await this.db(this.options.tableName) .where("updated_at", "<", cutoffEpochMs) .del(); logger.info("Cache cleanup: removed expired objects", { tableName: this.options.tableName, deletedCount, }); } /** * Gets the entity keys of all cached records of a specific type */ async getRecordsByType(type: ERPObjType): Promise { this.checkValid(); const records = await this.db(this.options.tableName) .select("entity_key") .where({ type }); return records.map((record) => record.entity_key); } /** * Removes all records of a specific type */ async removeRecordsByType(type: ERPObjType): Promise { this.checkValid(); await this.db(this.options.tableName).where({ type }).del(); } /** * Removes a specific record */ public async removeRecord(type: ERPObjType, record: object): Promise { if (!this.isDestroyed) { try { const entityKey = HashedCacheManager.entityKeyFor(type, record); if (entityKey === null) { return; // unidentifiable record was never cached } await this.db(this.options.tableName) .where({ type, entity_key: entityKey }) .del(); } catch (error) { logger.error("Error removing record:", error); throw error; } } } /** * Clears all records from the cache */ async clear(): Promise { this.checkValid(); await this.db(this.options.tableName).del(); } /** * Cleans up database connection and marks the cache manager as destroyed */ async destroy(): Promise { if (!this.isDestroyed) { if (this.ownsDb) { await this.db.destroy(); } this.isDestroyed = true; } } /** * Gets the current cache metrics * @returns The current cache metrics */ async getMetrics(): Promise { this.checkValid(); // Get counts for each type const counts = (await this.db(this.options.tableName) .select("type") .count("* as count") .groupBy("type")) as Array<{ type: string; count: string }>; // Update metrics this.metrics.recordCounts = counts.reduce( (acc, row) => { acc[row.type] = parseInt(row.count, 10); return acc; }, {} as Record ); return this.metrics; } }