import { mkdir, realpath, unlink } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' import { NamzuError } from '../../types/errors/index.js' import type { MemoryId } from '../../types/ids/index.js' import type { CreateMemoryParams, MemoryContent, MemoryIndexEntry, MemoryRecord, MemorySearchParams, MemorySearchResult, MemoryStore, UpdateMemoryParams, } from '../../types/memory/index.js' import { assertMemoryStatus, isMemoryType } from '../../types/memory/index.js' import { generateMemoryId, isEntityId } from '../../utils/id.js' import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js' import { type Logger, resolveLogger } from '../../utils/logger.js' import { DiskRecordStore } from '../kv/record-store.js' import { SCHEMA_VERSION_KEY, defineSchema } from '../schema.js' import { InMemoryMemoryIndex, searchMemoryEntries } from './index.js' import { MemoryNameConflictError, assertOptionalMemoryFields, isMemoryName, nameHolder, withOptionalFields, } from './naming.js' import { DEFAULT_MEMORY_LOCK_TIMEOUT_MS, acquireMemoryOperationLock, validateMemoryLockTimeout, } from './operation-lock.js' /** * This store's on-disk format, versioned as a unit — which is how a * migration would actually be written and shipped, and it keeps every call * site free of schema plumbing. * * Bump `current` and add the migration for the step you are leaving when * the shape changes. */ const SCHEMA = defineSchema({ kind: 'memory-store', current: 1, migrations: {} }) interface StorageLocation { readonly indexPath: string readonly contentDir: string } function invalidIndex( reason: string, details: { entryIndex?: number; field?: string } = {}, ): never { throw new NamzuError({ code: 'storage_error', message: `Memory index is invalid${ details.entryIndex === undefined ? '' : ` at entry ${details.entryIndex}` }: ${reason}. Refusing to treat unreadable durable memory as empty.`, details, retryable: false, }) } function invalidContent(id: string, reason: string, details: { field?: string } = {}): never { throw new NamzuError({ code: 'storage_error', message: `Memory content for ${id} is invalid: ${reason}. Refusing to treat indexed durable memory as absent or valid.`, details: { id, ...details }, retryable: false, }) } /** A typed ID is not automatically a safe filesystem segment. */ function assertStorageMemoryId( value: unknown, refuse: (reason: string) => never, ): asserts value is MemoryId { if (!isEntityId(value, 'memory')) { refuse('id must be a UUID; prefixed memory IDs are not supported') } } /** * Validate the domain shape before anything is installed in the live index. * * `DiskRecordStore` owns the version envelope and JSON parsing; it cannot * know that this particular array contains memory entries. Keeping this * check here avoids turning the generic primitive into a schema registry, * while ensuring valid JSON with a wrong field type cannot be accepted and * later written back beside a new record. */ function assertMemoryIndexEntries(value: unknown): asserts value is readonly MemoryIndexEntry[] { if (!Array.isArray(value)) invalidIndex('the top-level value must be an array') const ids = new Set() const names = new Set() for (const [entryIndex, candidate] of value.entries()) { if (candidate === null || typeof candidate !== 'object' || Array.isArray(candidate)) { invalidIndex('each entry must be an object', { entryIndex }) } const entry = candidate as Record assertStorageMemoryId(entry.id, (reason) => invalidIndex(reason, { entryIndex, field: 'id', }), ) if (ids.has(entry.id)) { invalidIndex('ids must be unique', { entryIndex, field: 'id' }) } ids.add(entry.id) for (const field of ['title', 'summary'] as const) { if (typeof entry[field] !== 'string') { invalidIndex(`${field} must be a string`, { entryIndex, field }) } } if (!Array.isArray(entry.tags) || !entry.tags.every((tag) => typeof tag === 'string')) { invalidIndex('tags must be an array of strings', { entryIndex, field: 'tags' }) } if (entry.status !== 'active' && entry.status !== 'archived') { invalidIndex('status must be active or archived', { entryIndex, field: 'status' }) } for (const field of ['createdAt', 'updatedAt'] as const) { if (typeof entry[field] !== 'number' || !Number.isFinite(entry[field])) { invalidIndex(`${field} must be a finite number`, { entryIndex, field }) } } // Optional since typed memory; absent on every index written before it. if (entry.name !== undefined) { if (!isMemoryName(entry.name)) { invalidIndex('name must be a kebab-case memory name', { entryIndex, field: 'name' }) } if (names.has(entry.name)) invalidIndex('names must be unique', { entryIndex, field: 'name' }) names.add(entry.name) } if (entry.description !== undefined && typeof entry.description !== 'string') { invalidIndex('description must be a string', { entryIndex, field: 'description' }) } if (entry.type !== undefined && !isMemoryType(entry.type)) { invalidIndex('type must be user, feedback, project or reference', { entryIndex, field: 'type', }) } } } function assertMemoryContent(value: unknown, expectedId: MemoryId): asserts value is MemoryContent { if (value === null || typeof value !== 'object' || Array.isArray(value)) { invalidContent(expectedId, 'the record must be an object') } const content = value as Record assertStorageMemoryId(content.id, (reason) => invalidContent(expectedId, reason, { field: 'id' })) if (content.id !== expectedId) { invalidContent(expectedId, 'record id does not match its index owner', { field: 'id' }) } if (typeof content.content !== 'string') { invalidContent(expectedId, 'content must be a string', { field: 'content' }) } if (content.format !== 'text' && content.format !== 'markdown' && content.format !== 'json') { invalidContent(expectedId, 'format must be text, markdown, or json', { field: 'format' }) } if ( content.metadata !== undefined && (content.metadata === null || typeof content.metadata !== 'object' || Array.isArray(content.metadata)) ) { invalidContent(expectedId, 'metadata must be an object when present', { field: 'metadata' }) } } export interface DiskMemoryStoreConfig { baseDir: string /** Exact memory directory; defaults to `baseDir/memory`. Hosts can partition by workspace. */ directory?: string logger?: Logger /** Maximum wait for another process's operation; default 10 seconds. Never breaks stale locks. */ lockTimeoutMs?: number } export class DiskMemoryStore implements MemoryStore { private readonly baseDir: string private readonly log: Logger private readonly lockTimeoutMs: number private readonly index = new InMemoryMemoryIndex() private location?: StorageLocation // Two instances of one primitive rather than two copies of its body. // Typed separately because the index file holds an ARRAY and a content // file holds one record, and a single `DiskRecordStore` would // have put the cast back at every call site. private readonly records = new DiskRecordStore(SCHEMA) private readonly indexRecords = new DiskRecordStore(SCHEMA) constructor(config: DiskMemoryStoreConfig) { this.baseDir = config.directory === undefined ? join(config.baseDir, 'memory') : resolve(config.directory) this.log = resolveLogger(config.logger).child({ [SCOPE_ATTRIBUTE]: 'store/memory/disk' }) this.lockTimeoutMs = config.lockTimeoutMs ?? DEFAULT_MEMORY_LOCK_TIMEOUT_MS validateMemoryLockTimeout(this.lockTimeoutMs) } private get contentDir(): string { return join(this.baseDir, 'content') } private async storageLocation(): Promise { if (this.location) return this.location await mkdir(this.contentDir, { recursive: true }) const [canonicalBaseDir, canonicalContentDir] = await Promise.all([ realpath(this.baseDir), realpath(this.contentDir), ]) if (dirname(canonicalContentDir) !== canonicalBaseDir) { invalidIndex('the content directory resolves outside the memory store') } this.location = { indexPath: join(canonicalBaseDir, 'index.json'), contentDir: canonicalContentDir, } return this.location } private contentPath(location: StorageLocation, id: MemoryId): string { assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' })) const path = resolve(location.contentDir, `${id}.json`) if (dirname(path) !== location.contentDir) { invalidContent(id, 'resolved path escapes the content directory', { field: 'id' }) } return path } private async reloadIndex(location: StorageLocation): Promise { try { const entries = await this.indexRecords.read(location.indexPath) if (entries === null) { this.index.rebuild([]) return } assertMemoryIndexEntries(entries) this.index.rebuild([...entries]) this.log.info('Memory index loaded', { 'namzu.store.count': entries.length }) } catch (err) { this.log.error('Failed to read memory index — refusing to start empty', { 'exception.message': String(err), }) throw err } } private async withAuthoritativeIndex( operation: (location: StorageLocation) => Promise, ): Promise { const location = await this.storageLocation() const release = await acquireMemoryOperationLock( join(dirname(location.indexPath), 'operation.lock'), this.lockTimeoutMs, ) try { await this.reloadIndex(location) return await operation(location) } finally { await release() } } private async readContent(location: StorageLocation, id: MemoryId): Promise { const content = await this.records.read(this.contentPath(location, id)) if (content === null) invalidContent(id, 'the indexed content record is missing') assertMemoryContent(content, id) // The version stamp is the file's, not the record's: leaving it on // handed every reader a `schemaVersion` field MemoryContent does not have. const { [SCHEMA_VERSION_KEY]: _stamp, ...record } = content as MemoryContent & { [SCHEMA_VERSION_KEY]?: unknown } return record } private async writeIndex( location: StorageLocation, entries: readonly MemoryIndexEntry[], ): Promise { await this.indexRecords.write(location.indexPath, entries) } async create( params: CreateMemoryParams, ): Promise<{ entry: MemoryIndexEntry; content: MemoryContent }> { assertOptionalMemoryFields(params) return this.withAuthoritativeIndex(async (location) => { if (params.name !== undefined) { const holder = nameHolder(this.index.allEntries(), params.name) if (holder) throw new MemoryNameConflictError(params.name, holder.id) } const id = generateMemoryId() const now = Date.now() const entry: MemoryIndexEntry = withOptionalFields( { id, title: params.title, summary: params.summary, tags: params.tags ? [...params.tags] : [], status: 'active', createdAt: now, updatedAt: now, }, params, ) const memoryContent: MemoryContent = { id, content: params.content, format: params.format ?? 'text', metadata: params.metadata ? { ...params.metadata } : undefined, } const contentPath = this.contentPath(location, id) const entries = [...this.index.allEntries(), entry] await this.records.write(contentPath, memoryContent) try { await this.writeIndex(location, entries) } catch (error) { await unlink(contentPath).catch(() => undefined) throw error } this.index.rebuild(entries) this.log.info('Memory created', { 'namzu.memory.id': id, 'namzu.store.title': params.title, }) return { entry, content: memoryContent } }) } async get(id: MemoryId): Promise { return this.withAuthoritativeIndex(async (location) => { assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' })) if (!this.index.getEntry(id)) return undefined return this.readContent(location, id) }) } async getRecord(id: MemoryId): Promise { return this.withAuthoritativeIndex(async (location) => { assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' })) const entry = this.index.getEntry(id) if (!entry) return undefined return structuredClone({ entry, content: await this.readContent(location, id) }) }) } async update(id: MemoryId, updates: UpdateMemoryParams): Promise { if (updates.status !== undefined) assertMemoryStatus(updates.status) assertOptionalMemoryFields(updates) return this.withAuthoritativeIndex(async (location) => { assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' })) const existing = this.index.getEntry(id) if (!existing) return undefined if (updates.name !== undefined) { const holder = nameHolder(this.index.allEntries(), updates.name, id) if (holder) throw new MemoryNameConflictError(updates.name, holder.id) } const existingContent = await this.readContent(location, id) const updated: MemoryIndexEntry = withOptionalFields( { ...existing, title: updates.title ?? existing.title, summary: updates.summary ?? existing.summary, tags: updates.tags ? [...updates.tags] : existing.tags, status: updates.status ?? existing.status, updatedAt: Date.now(), }, updates, ) const updatesContent = updates.content !== undefined || updates.format !== undefined || updates.metadata !== undefined if (updatesContent) { const updatedContent: MemoryContent = { ...existingContent, content: updates.content ?? existingContent.content, format: updates.format ?? existingContent.format, metadata: updates.metadata !== undefined ? { ...updates.metadata } : existingContent.metadata, } await this.records.write(this.contentPath(location, id), updatedContent) } const entries = this.index.allEntries().map((entry) => (entry.id === id ? updated : entry)) await this.writeIndex(location, entries) this.index.rebuild([...entries]) this.log.info('Memory updated', { 'namzu.memory.id': id }) return updated }) } async delete(id: MemoryId): Promise { return this.withAuthoritativeIndex(async (location) => { assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' })) if (!this.index.getEntry(id)) return false try { await unlink(this.contentPath(location, id)) } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error } const entries = this.index.allEntries().filter((entry) => entry.id !== id) await this.writeIndex(location, entries) this.index.rebuild([...entries]) this.log.info('Memory deleted', { 'namzu.memory.id': id }) return true }) } async list(params?: MemorySearchParams): Promise { return this.withAuthoritativeIndex(async (location) => { if (!params?.query?.trim() && !params?.requiredIdentifiers?.length) return this.index.search(params ?? {}) // Search the current bodies inside the same operation as the index // snapshot. No retained cache can hide another process's update. const candidates = this.index.search({ ...params, query: undefined, limit: undefined, requiredIdentifiers: undefined, }) const contents = new Map() for (const entry of candidates.entries) { contents.set(entry.id, (await this.readContent(location, entry.id)).content) } return searchMemoryEntries(candidates.entries, params, (id) => contents.get(id) ?? '') }) } getIndex(): InMemoryMemoryIndex { return this.index } }