import * as plugins from './plugins.js'; import { FlexDomainHeadModel, FlexHeadModel, FlexMigrationModel, FlexPrivateCandidateModel, FlexPrivateChunkModel, FlexWriterLeaseModel, type IFlexDomainHeadDocument, type IFlexHeadDocument, type IFlexMigrationDocument, type IFlexPrivateCandidateDocument, type IFlexPrivateCandidateManifest, type IFlexPrivateChunkDocument, type IFlexWriterLeaseDocument, type TFlexDomainKind, assertFlexDomainHeadDocument, assertFlexHeadDocument, assertFlexMigrationDocument, assertFlexPrivateCandidateDocument, assertFlexPrivateChunkDocument, assertFlexWriterLeaseDocument, flexDomainHeadId, flexMigrationId, flexPrivateChunkBytes, flexWriterLeaseId, initFlexPrivateModels, registerFlexPrivateModels, } from './classes.flexmodels.private.js'; import { FlexPublicCandidateModel, FlexPublicHeadModel, FlexPublicMessageRecordModel, FlexPublicSessionRecordModel, FlexProjectionMessageRecordModel, FlexProjectionSourceModel, type IFlexProjectionMessageDescriptor, type IFlexProjectionMessageRecordDocument, type IFlexProjectionSourceDocument, type IFlexProjectionSourceManifest, type IFlexPublicCandidateDocument, type IFlexPublicCandidateManifest, type IFlexPublicHeadDocument, type IFlexPublicMessageRecordDocument, type IFlexPublicProjectionSource, type IFlexPublicSessionRecordDocument, type IFlexRecordManifest, assertFlexPublicCandidateDocument, assertFlexPublicHeadDocument, assertFlexPublicMessageRecordDocument, assertFlexPublicSessionRecordDocument, assertFlexProjectionMessageRecordDocument, assertFlexProjectionSourceDocument, flexHeadId, flexPublicHeadId, flexPublicMessageBytesLimit, flexPublicMessageLimit, flexPublicSessionLimit, flexPublicSingleMessageBytesLimit, initFlexPublicModels, registerFlexPublicModels, } from './classes.flexmodels.public.js'; type TFlexLegacySnapshot = plugins.flexharnessMigration.IFlexLegacyHarnessSnapshot; type TFlexScopeSnapshot = plugins.flexharness.IFlexScopeSnapshot; type TFlexProjectionSnapshot = plugins.flexharness.IFlexProjectionSnapshot; type TFlexPermissionSnapshot = plugins.flexharness.IFlexPermissionSnapshot; type TFlexMessage = plugins.flexharness.IFlexMessage; type TFlexSession = plugins.flexharness.IFlexSession; type TAgentEventSnapshot = plugins.flexAgent.IAgentEventSnapshotV2; type TAgentEventArchive = plugins.flexAgent.IAgentEventArchiveV2; type TToolJobSnapshot = plugins.flexTools.IToolJobSnapshot; type TStoredDomainHead = plugins.smartdata.TStoredDocument; type TStoredLegacyHead = plugins.smartdata.TStoredDocument; type TStoredMigration = plugins.smartdata.TStoredDocument; type TStoredPublicHead = plugins.smartdata.TStoredDocument; type TStoredLease = plugins.smartdata.TStoredDocument; type TFlexObjectId = TStoredDomainHead['_id']; export interface IFlexProjectionOrderMigrationPage { sessionIds: string[]; nextCursor?: TFlexObjectId; } export interface IFlexStoreOptions { database: plugins.smartdata.SmartdataDb; controllerId: string; projectManagementHost: IFlexProjectManagementHost; onFatalError?: (errorArg: Error) => void; } export interface IFlexProjectManagementHost { load( storageKeyArg: string, sessionIdArg: string, sessionContextArg: Readonly, ): Promise; save( storageKeyArg: string, sessionIdArg: string, snapshotArg: plugins.flexharness.IFlexProjectManagementSnapshot, expectedRevisionArg: number, writeContextArg: plugins.flexharness.IFlexProjectManagementWriteContext, ): Promise; tombstoneSession( storageKeyArg: string, sessionIdArg: string, tombstoneArg: plugins.flexharness.IFlexProjectManagementTombstone, expectedRevisionArg: number, sessionContextArg: Readonly, ): Promise; purgeNamespace(storageKeyArg: string): Promise; } const cloneProjectManagementSessionContext = ( valueArg: Readonly | undefined, ): Readonly => { if ( !valueArg || typeof valueArg.sessionGenerationId !== 'string' || valueArg.sessionGenerationId.trim().length === 0 || Buffer.byteLength(valueArg.sessionGenerationId, 'utf8') > plugins.flexharness.FLEX_SESSION_GENERATION_ID_MAX_BYTES || !Number.isSafeInteger(valueArg.sessionGenerationSequence) || valueArg.sessionGenerationSequence < 1 ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex project-management operation requires an authoritative session context.', ); } const subagent = valueArg.subagent === undefined ? undefined : Object.freeze(structuredClone(valueArg.subagent)); return Object.freeze({ sessionGenerationId: valueArg.sessionGenerationId, sessionGenerationSequence: valueArg.sessionGenerationSequence, ...(subagent === undefined ? {} : { subagent }), }); }; export interface IFlexCommittedScopeSource { head: IFlexDomainHeadDocument; snapshot: TFlexScopeSnapshot; } export interface IFlexCommittedProjectionSource { head: IFlexDomainHeadDocument; snapshot: TFlexProjectionSnapshot; } export interface IFlexPublicProjectionBuild { candidate: IFlexPublicCandidateDocument; sessions: IFlexPublicSessionRecordDocument[]; messages: IFlexPublicMessageRecordDocument[]; } interface IFlexProjectionSourceBuild { manifest: IFlexProjectionSourceManifest; source: IFlexProjectionSourceDocument; messages: IFlexProjectionMessageRecordDocument[]; } interface IFlexOrderedMessage { messageIndex: number; message: TFlexMessage; } export interface IFlexProjectionSourceLimits { messageLimit: number; messageBytesLimit: number; singleMessageBytesLimit: number; } interface IFlexCandidatePreparation { candidateId: string; incarnationId: string; revision: number; createdAt: string; } interface IFlexDomainIdentity { storageKey: string; domain: TFlexDomainKind; sessionId?: string; archiveId?: string; } interface IFlexLoadedDomain { head: TStoredDomainHead; snapshot: TSnapshot; } interface IFlexPrivateDomainSource { identity: IFlexDomainIdentity; incarnationId: string; } interface IFlexLeaseTransitionImage extends IFlexWriterLeaseDocument { _id: TFlexObjectId; _smartdataRevision?: string; } interface IFlexUncertainLeaseTransition { preimage: IFlexLeaseTransitionImage; postimage: IFlexLeaseTransitionImage; } interface IFlexHeldLease { storageKey: string; ownerToken: string; epoch: number; expiresAt: number; lost: boolean; releasing: boolean; uncertainTransition?: IFlexUncertainLeaseTransition; transitionTask?: Promise; heartbeatAbortController?: AbortController; heartbeatTask?: Promise; } interface IFlexHeadMutationOptions { lease: IFlexHeldLease; collection: typeof FlexDomainHeadModel.collection.mongoDbCollection; current: ({ _id: unknown; _smartdataRevision?: string } & THead) | null; next: THead | null; selector: Record; readCurrent: ( optionsArg?: IFlexReadOptions, ) => Promise<({ _id: unknown; _smartdataRevision?: string } & THead) | null>; project: (valueArg: THead) => THead; targetName: string; } interface IFlexReadOptions { signal?: AbortSignal; timeoutMS?: number; } interface IFlexRawMaintenanceDocument { id: string; controllerId: string; scopeId: string; candidateId: string; createdAt: string; } export const flexOrphanGraceMs = 60 * 60 * 1000; const flexMaintenancePageLimit = 128; const flexOrphanMaintenanceIntervalMs = 15 * 60 * 1000; const flexLeaseDurationMs = 30_000; const flexLeaseHeartbeatMs = 10_000; const flexLeaseHeartbeatOperationTimeoutMs = 5_000; const flexLeaseTransitionTimeoutMs = 20_000; const flexMaintenanceOperationTimeoutMs = 10_000; const sha256 = (valueArg: Uint8Array | string): string => plugins.crypto.createHash('sha256').update(valueArg).digest('hex'); const serializedBytes = (valueArg: unknown): Buffer => Buffer.from(JSON.stringify(valueArg), 'utf8'); const canonicalJson = (valueArg: unknown): string => JSON.stringify(valueArg, (_key, value) => { if (!value || typeof value !== 'object' || Array.isArray(value)) return value; return Object.fromEntries(Object.keys(value).sort().map((key) => [key, value[key]])); }); const randomCandidateId = (): string => plugins.crypto.randomBytes(16).toString('base64url'); const randomOwnerToken = (): string => plugins.crypto.randomBytes(32).toString('base64url'); const privateCandidateDocumentId = (candidateIdArg: string): string => `flex-private-candidate:${candidateIdArg}`; const privateChunkDocumentId = (candidateIdArg: string, indexArg: number): string => `flex-private-chunk:${candidateIdArg}:${indexArg}`; const publicCandidateDocumentId = (candidateIdArg: string): string => `flex-public-candidate:${candidateIdArg}`; const publicSessionDocumentId = ( candidateIdArg: string, sessionIdArg: string, ): string => `flex-public-session:${candidateIdArg}:${sha256(sessionIdArg).slice(0, 32)}`; const publicMessageDocumentId = ( candidateIdArg: string, sessionIdArg: string, messageIdArg: string, ): string => `flex-public-message:${candidateIdArg}:${sha256(`${sessionIdArg}\0${messageIdArg}`).slice(0, 32)}`; const projectionSourceDocumentId = (candidateIdArg: string): string => `flex-projection-source:${candidateIdArg}`; const projectionMessageDocumentId = (candidateIdArg: string, indexArg: number): string => `flex-projection-message:${candidateIdArg}:${String(indexArg).padStart(16, '0')}`; const providerKey = (storageKeyArg: string, sessionIdArg: string): string => JSON.stringify([storageKeyArg, sessionIdArg]); const compareNewestSession = (leftArg: TFlexSession, rightArg: TFlexSession): number => rightArg.updatedAt.localeCompare(leftArg.updatedAt) || rightArg.createdAt.localeCompare(leftArg.createdAt) || leftArg.sessionId.localeCompare(rightArg.sessionId); const compareNewestOrderedMessage = ( leftArg: IFlexOrderedMessage, rightArg: IFlexOrderedMessage, ): number => { if (leftArg.message.sessionId === rightArg.message.sessionId) { return rightArg.messageIndex - leftArg.messageIndex; } return rightArg.message.createdAt.localeCompare(leftArg.message.createdAt) || leftArg.message.sessionId.localeCompare(rightArg.message.sessionId) || leftArg.message.messageId.localeCompare(rightArg.message.messageId) || rightArg.messageIndex - leftArg.messageIndex; }; const compareNewestProjectionDescriptor = ( leftArg: IFlexProjectionMessageDescriptor, rightArg: IFlexProjectionMessageDescriptor, ): number => leftArg.sessionId === rightArg.sessionId ? (rightArg.messageIndex ?? rightArg.index) - (leftArg.messageIndex ?? leftArg.index) : rightArg.messageCreatedAt.localeCompare(leftArg.messageCreatedAt) || leftArg.sessionId.localeCompare(rightArg.sessionId) || leftArg.messageId.localeCompare(rightArg.messageId) || (rightArg.messageIndex ?? rightArg.index) - (leftArg.messageIndex ?? leftArg.index); const retainNewest = ( valuesArg: Iterable, limitArg: number, compareArg: (leftArg: TValue, rightArg: TValue) => number, ): TValue[] => { const retained: TValue[] = []; for (const value of valuesArg) { let low = 0; let high = retained.length; while (low < high) { const middle = (low + high) >>> 1; if (compareArg(value, retained[middle]!) < 0) high = middle; else low = middle + 1; } if (low >= limitArg) continue; retained.splice(low, 0, value); if (retained.length > limitArg) retained.pop(); } return retained; }; const retainPublicSessions = (sessionsArg: TFlexSession[]): TFlexSession[] => { const roots = retainNewest( sessionsArg.filter((session) => session.parentSessionId === undefined), flexPublicSessionLimit, compareNewestSession, ); const children = retainNewest( sessionsArg.filter((session) => session.parentSessionId !== undefined), flexPublicSessionLimit - roots.length, compareNewestSession, ); return [...roots, ...children].sort(compareNewestSession); }; const flexTaskPlaceholderBytesLimit = 384 * 1024; const taskOutputOmittedNotice = '[Task output omitted because its public projection exceeded 480 KiB.]'; const placeholderMessage = (messageArg: TFlexMessage): TFlexMessage => { const linkedTaskParts = messageArg.parts.flatMap((part) => ( part.type === 'tool' && part.childSessionId !== undefined ? [{ partId: part.partId, type: 'tool' as const, toolCallId: part.toolCallId, toolName: part.toolName, status: part.status, input: {}, output: taskOutputOmittedNotice, childSessionId: part.childSessionId, ...(part.model === undefined ? {} : { model: part.model }), }] : [] )); if (linkedTaskParts.length === 0) { return { ...messageArg, parts: [{ partId: messageArg.parts[0]?.partId ?? messageArg.messageId, type: 'text', text: '[Message content omitted because its public projection exceeded 480 KiB.]', }], }; } const retainedTaskParts: typeof linkedTaskParts = []; for (const part of linkedTaskParts) { const candidate = { ...messageArg, parts: [...retainedTaskParts, part] }; if (serializedBytes(candidate).byteLength > flexTaskPlaceholderBytesLimit) break; retainedTaskParts.push(part); } if (retainedTaskParts.length < linkedTaskParts.length) { retainedTaskParts.at(-1)!.output = `${taskOutputOmittedNotice} ${ linkedTaskParts.length - retainedTaskParts.length } additional task link(s) omitted.`; } return { ...messageArg, parts: retainedTaskParts }; }; const recordManifest = (documentArg: { id: string }): IFlexRecordManifest => { const bytes = serializedBytes(documentArg); return { id: documentArg.id, sha256: sha256(bytes), bytes: bytes.byteLength }; }; const publicCandidateManifestBytes = ( manifestArg: Omit, ): Buffer => serializedBytes(manifestArg); const projectionSourceManifestBytes = ( manifestArg: Omit, ): Buffer => Buffer.from(canonicalJson(manifestArg), 'utf8'); const projectionMessagePayload = ( messageArg: TFlexMessage, previousArg?: IFlexProjectionMessageDescriptor, ): { message: TFlexMessage; previous?: IFlexProjectionMessageDescriptor } => ({ message: messageArg, ...(previousArg === undefined ? {} : { previous: previousArg }), }); const projectionMessagePayloadBytes = ( messageArg: TFlexMessage, previousArg?: IFlexProjectionMessageDescriptor, ): Buffer => Buffer.from(canonicalJson(projectionMessagePayload(messageArg, previousArg)), 'utf8'); const projectionMessageBodyBytes = (messageArg: TFlexMessage): number => Buffer.byteLength(canonicalJson(messageArg), 'utf8'); const cloneJson = (valueArg: TValue, pathArg = '$'): TValue => { const seen = new WeakSet(); const visit = (value: unknown, path: string): unknown => { if (value === null || typeof value === 'string' || typeof value === 'boolean') return value; if (typeof value === 'number') { if (!Number.isFinite(value)) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${path} contains a non-finite number.`, ); } return value; } if (typeof value === 'undefined') return undefined; if (typeof value !== 'object') { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${path} contains ${typeof value}, which is not JSON-safe.`, ); } if (seen.has(value)) { throw new plugins.flexharness.FlexHarnessStoreFormatError(`${path} contains a cycle.`); } const prototype = Object.getPrototypeOf(value); if (!Array.isArray(value) && prototype !== Object.prototype && prototype !== null) { throw new plugins.flexharness.FlexHarnessStoreFormatError(`${path} is not a plain object.`); } if (Object.getOwnPropertySymbols(value).length > 0) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${path} contains a symbol-keyed property.`, ); } seen.add(value); if (Array.isArray(value)) { const result = value.map((entry, index) => { if (!(index in value) || entry === undefined) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${path}[${index}] is not JSON-safe.`, ); } return visit(entry, `${path}[${index}]`); }); seen.delete(value); return result; } const result: Record = {}; for (const key of Object.getOwnPropertyNames(value)) { const descriptor = Object.getOwnPropertyDescriptor(value, key)!; if (!descriptor.enumerable || !('value' in descriptor)) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${path}.${key} is not a plain JSON property.`, ); } const entry = visit(descriptor.value, `${path}.${key}`); if (entry !== undefined) result[key] = entry; } seen.delete(value); return result; }; return visit(valueArg, pathArg) as TValue; }; const assertIdentifier = (valueArg: string, nameArg: string): void => { if ( typeof valueArg !== 'string' || valueArg.trim().length === 0 || Buffer.byteLength(valueArg, 'utf8') > 512 ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `${nameArg} must be a non-empty string of at most 512 bytes.`, ); } }; const assertExpectedNextRevision = ( snapshotRevisionArg: number, expectedRevisionArg: number, ): void => { if (!Number.isSafeInteger(expectedRevisionArg) || expectedRevisionArg < 0) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'expectedRevision must be a non-negative integer.', ); } if (snapshotRevisionArg !== expectedRevisionArg + 1) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `Snapshot revision ${snapshotRevisionArg} must equal expected revision ${expectedRevisionArg} plus one.`, ); } }; const smartRevisionSelector = (valueArg: { _smartdataRevision?: string }): unknown => valueArg._smartdataRevision === undefined ? { $exists: false } : valueArg._smartdataRevision; const privateChunkManifestsEqual = ( leftArg: IFlexPrivateCandidateManifest['chunks'], rightArg: IFlexPrivateCandidateManifest['chunks'], ): boolean => leftArg.length === rightArg.length && leftArg.every((left, index) => { const right = rightArg[index]; return right !== undefined && left.id === right.id && left.index === right.index && left.sha256 === right.sha256 && left.bytes === right.bytes; }); const publicManifestsEqual = ( leftArg: IFlexPublicCandidateManifest, rightArg: IFlexPublicCandidateManifest, ): boolean => canonicalJson(leftArg) === canonicalJson(rightArg); const domainHeadBody = (valueArg: IFlexDomainHeadDocument): IFlexDomainHeadDocument => ({ id: valueArg.id, controllerId: valueArg.controllerId, storageKey: valueArg.storageKey, domain: valueArg.domain, incarnationId: valueArg.incarnationId, revision: valueArg.revision, currentPrivateCandidateId: valueArg.currentPrivateCandidateId, privateCandidate: valueArg.privateCandidate, ...(valueArg.projectionSource !== undefined ? { projectionSource: valueArg.projectionSource } : {}), ...(valueArg.sessionId !== undefined ? { sessionId: valueArg.sessionId } : {}), ...(valueArg.archiveId !== undefined ? { archiveId: valueArg.archiveId } : {}), }); const publicHeadBody = (valueArg: IFlexPublicHeadDocument): IFlexPublicHeadDocument => ({ id: valueArg.id, controllerId: valueArg.controllerId, storageKey: valueArg.storageKey, revision: valueArg.revision, currentPublicCandidateId: valueArg.currentPublicCandidateId, publicCandidates: valueArg.publicCandidates, }); const leaseBody = (valueArg: IFlexWriterLeaseDocument): IFlexWriterLeaseDocument => ({ id: valueArg.id, controllerId: valueArg.controllerId, storageKey: valueArg.storageKey, ownerToken: valueArg.ownerToken, epoch: valueArg.epoch, state: valueArg.state, expiresAt: valueArg.expiresAt, updatedAt: valueArg.updatedAt, }); const leaseTransitionImage = ( valueArg: IFlexWriterLeaseDocument & { _id: TFlexObjectId; _smartdataRevision?: string }, ): IFlexLeaseTransitionImage => ({ _id: valueArg._id, ...leaseBody(valueArg), ...(valueArg._smartdataRevision !== undefined ? { _smartdataRevision: valueArg._smartdataRevision } : {}), }); const leaseTransitionImagesEqual = ( leftArg: IFlexWriterLeaseDocument & { _id: TFlexObjectId; _smartdataRevision?: string }, rightArg: IFlexLeaseTransitionImage, ): boolean => String(leftArg._id) === String(rightArg._id) && canonicalJson(leaseBody(leftArg)) === canonicalJson(leaseBody(rightArg)) && leftArg._smartdataRevision === rightArg._smartdataRevision; const migrationBody = (valueArg: IFlexMigrationDocument): IFlexMigrationDocument => ({ id: valueArg.id, controllerId: valueArg.controllerId, storageKey: valueArg.storageKey, migration: valueArg.migration, legacyRevision: valueArg.legacyRevision, legacyCandidateId: valueArg.legacyCandidateId, completedAt: valueArg.completedAt, }); const projectPrivateChunk = (modelArg: FlexPrivateChunkModel): IFlexPrivateChunkDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, revision: modelArg.revision, index: modelArg.index, sha256: modelArg.sha256, bytes: modelArg.bytes, data: modelArg.data, createdAt: modelArg.createdAt, ...(modelArg.domain !== undefined ? { domain: modelArg.domain } : {}), ...(modelArg.incarnationId !== undefined ? { incarnationId: modelArg.incarnationId } : {}), ...(modelArg.sessionId !== undefined ? { sessionId: modelArg.sessionId } : {}), ...(modelArg.archiveId !== undefined ? { archiveId: modelArg.archiveId } : {}), }); const projectPrivateCandidate = ( modelArg: FlexPrivateCandidateModel, ): IFlexPrivateCandidateDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, revision: modelArg.revision, sha256: modelArg.sha256, bytes: modelArg.bytes, chunks: modelArg.chunks, createdAt: modelArg.createdAt, ...(modelArg.domain !== undefined ? { domain: modelArg.domain } : {}), ...(modelArg.incarnationId !== undefined ? { incarnationId: modelArg.incarnationId } : {}), ...(modelArg.sessionId !== undefined ? { sessionId: modelArg.sessionId } : {}), ...(modelArg.archiveId !== undefined ? { archiveId: modelArg.archiveId } : {}), }); const projectPublicCandidate = ( modelArg: FlexPublicCandidateModel, ): IFlexPublicCandidateDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, revision: modelArg.revision, sha256: modelArg.sha256, bytes: modelArg.bytes, sessionRecords: modelArg.sessionRecords, messageRecords: modelArg.messageRecords, sessionsTruncated: modelArg.sessionsTruncated, messagesTruncated: modelArg.messagesTruncated, messageBytesTruncated: modelArg.messageBytesTruncated, visibleDigest: modelArg.visibleDigest, scopeSource: modelArg.scopeSource, projectionSources: modelArg.projectionSources, createdAt: modelArg.createdAt, }); const projectPublicSession = ( modelArg: FlexPublicSessionRecordModel, ): IFlexPublicSessionRecordDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, revision: modelArg.revision, session: modelArg.session, createdAt: modelArg.createdAt, }); const projectPublicMessage = ( modelArg: FlexPublicMessageRecordModel, ): IFlexPublicMessageRecordDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, revision: modelArg.revision, messageIndex: modelArg.messageIndex, message: modelArg.message, createdAt: modelArg.createdAt, }); const projectProjectionSource = ( modelArg: FlexProjectionSourceModel, ): IFlexProjectionSourceDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, sessionId: modelArg.sessionId, candidateId: modelArg.candidateId, incarnationId: modelArg.incarnationId, revision: modelArg.revision, ...(modelArg.orderVersion === undefined ? {} : { orderVersion: modelArg.orderVersion }), sha256: modelArg.sha256, bytes: modelArg.bytes, messageCount: modelArg.messageCount, messagesTruncated: modelArg.messagesTruncated, visibleDigest: modelArg.visibleDigest, ...(modelArg.latestMessage === undefined ? {} : { latestMessage: modelArg.latestMessage }), createdAt: modelArg.createdAt, }); const projectProjectionMessage = ( modelArg: FlexProjectionMessageRecordModel, ): IFlexProjectionMessageRecordDocument => ({ id: modelArg.id, controllerId: modelArg.controllerId, scopeId: modelArg.scopeId, candidateId: modelArg.candidateId, sessionId: modelArg.sessionId, incarnationId: modelArg.incarnationId, revision: modelArg.revision, index: modelArg.index, ...(modelArg.messageIndex === undefined ? {} : { messageIndex: modelArg.messageIndex }), sha256: modelArg.sha256, bytes: modelArg.bytes, messageCreatedAt: modelArg.messageCreatedAt, messageId: modelArg.messageId, message: modelArg.message, ...(modelArg.previous === undefined ? {} : { previous: modelArg.previous }), createdAt: modelArg.createdAt, }); const insertOrdinary = async >( modelConstructorArg: { new (): TModel; insert(documentArg: TModel): Promise }, documentArg: object, ): Promise => { const model = Object.assign(new modelConstructorArg(), documentArg); await modelConstructorArg.insert(model); }; const inBatches = async ( valuesArg: TValue[], operationArg: (valueArg: TValue) => Promise, ): Promise => { for (let index = 0; index < valuesArg.length; index += 32) { await Promise.all(valuesArg.slice(index, index + 32).map(operationArg)); } }; const createPrivateProjection = ( controllerIdArg: string, storageKeyArg: string, candidateIdArg: string, revisionArg: number, snapshotArg: unknown, createdAtArg: string, sourceArg?: IFlexPrivateDomainSource, ): { candidate: IFlexPrivateCandidateDocument; chunks: IFlexPrivateChunkDocument[] } => { const complete = serializedBytes(snapshotArg); const chunks: IFlexPrivateChunkDocument[] = []; for (let offset = 0, index = 0; offset < complete.byteLength || index === 0; index++) { const chunk = complete.subarray(offset, Math.min(offset + flexPrivateChunkBytes, complete.byteLength)); const id = privateChunkDocumentId(candidateIdArg, index); chunks.push({ id, controllerId: controllerIdArg, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: revisionArg, index, sha256: sha256(chunk), bytes: chunk.byteLength, data: chunk.toString('base64'), createdAt: createdAtArg, ...(sourceArg ? { domain: sourceArg.identity.domain, incarnationId: sourceArg.incarnationId, ...(sourceArg.identity.sessionId !== undefined ? { sessionId: sourceArg.identity.sessionId } : {}), ...(sourceArg.identity.archiveId !== undefined ? { archiveId: sourceArg.identity.archiveId } : {}), } : {}), }); offset += chunk.byteLength; if (complete.byteLength === 0) break; } const candidate: IFlexPrivateCandidateDocument = { id: privateCandidateDocumentId(candidateIdArg), controllerId: controllerIdArg, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: revisionArg, sha256: sha256(complete), bytes: complete.byteLength, chunks: chunks.map(({ id, index, sha256: digest, bytes }) => ({ id, index, sha256: digest, bytes, })), createdAt: createdAtArg, ...(sourceArg ? { domain: sourceArg.identity.domain, incarnationId: sourceArg.incarnationId, ...(sourceArg.identity.sessionId !== undefined ? { sessionId: sourceArg.identity.sessionId } : {}), ...(sourceArg.identity.archiveId !== undefined ? { archiveId: sourceArg.identity.archiveId } : {}), } : {}), }; assertFlexPrivateCandidateDocument(candidate); for (const chunk of chunks) assertFlexPrivateChunkDocument(chunk); return { candidate, chunks }; }; export const createProjectionSource = ( controllerIdArg: string, storageKeyArg: string, sessionIdArg: string, snapshotArg: TFlexProjectionSnapshot, preparationArg: IFlexCandidatePreparation, limitsArg: IFlexProjectionSourceLimits = { messageLimit: flexPublicMessageLimit, messageBytesLimit: flexPublicMessageBytesLimit, singleMessageBytesLimit: flexPublicSingleMessageBytesLimit, }, ): IFlexProjectionSourceBuild => { const retainedMessages = snapshotArg.messages.slice(-limitsArg.messageLimit); const buildChain = ( orderedMessagesArg: TFlexMessage[], normalizeOversizedArg: boolean, messageIndexOffsetArg: number, ): { messages: IFlexProjectionMessageRecordDocument[]; replaced: boolean } => { const messages: IFlexProjectionMessageRecordDocument[] = []; let previous: IFlexProjectionMessageDescriptor | undefined; let replaced = false; for (const [index, sourceMessage] of orderedMessagesArg.entries()) { let message = sourceMessage; let payload = projectionMessagePayloadBytes(message, previous); if (payload.byteLength > limitsArg.singleMessageBytesLimit) { if (!normalizeOversizedArg) { throw new Error('Flex projection source normalization invariant failed.'); } const placeholder = placeholderMessage(sourceMessage); const placeholderPayload = projectionMessagePayloadBytes(placeholder, previous); if ( projectionMessageBodyBytes(placeholder) >= projectionMessageBodyBytes(sourceMessage) || placeholderPayload.byteLength > limitsArg.singleMessageBytesLimit ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source message metadata exceeds its projection limit.', ); } message = placeholder; payload = placeholderPayload; replaced = true; } const descriptor: IFlexProjectionMessageDescriptor = { id: projectionMessageDocumentId(preparationArg.candidateId, index), index, messageIndex: messageIndexOffsetArg + index, sha256: sha256(payload), bytes: payload.byteLength, messageCreatedAt: message.createdAt, sessionId: sessionIdArg, messageId: message.messageId, }; const record: IFlexProjectionMessageRecordDocument = { ...descriptor, controllerId: controllerIdArg, scopeId: storageKeyArg, candidateId: preparationArg.candidateId, incarnationId: preparationArg.incarnationId, revision: preparationArg.revision, message, ...(previous === undefined ? {} : { previous }), createdAt: preparationArg.createdAt, }; assertFlexProjectionMessageRecordDocument(record); messages.push(record); previous = descriptor; } return { messages, replaced }; }; const normalized = buildChain( retainedMessages, true, snapshotArg.messages.length - retainedMessages.length, ); const selectedNewest: TFlexMessage[] = []; let selectedMessageBytes = 0; for (let index = normalized.messages.length - 1; index >= 0; index--) { const message = normalized.messages[index]!.message; const bodyBytes = projectionMessageBodyBytes(message); if (selectedMessageBytes + bodyBytes > limitsArg.messageBytesLimit) break; selectedMessageBytes += bodyBytes; selectedNewest.push(message); } const finalChain = buildChain( selectedNewest.reverse(), false, snapshotArg.messages.length - selectedNewest.length, ); const messages = finalChain.messages; const previous = messages.at(-1); const latestMessage: IFlexProjectionMessageDescriptor | undefined = previous ? { id: previous.id, index: previous.index, messageIndex: previous.messageIndex, sha256: previous.sha256, bytes: previous.bytes, messageCreatedAt: previous.messageCreatedAt, sessionId: previous.sessionId, messageId: previous.messageId, } : undefined; const messagesTruncated = retainedMessages.length < snapshotArg.messages.length || messages.length < retainedMessages.length || normalized.replaced; const visibleHash = plugins.crypto.createHash('sha256'); visibleHash.update('order:1', 'utf8'); visibleHash.update(messagesTruncated ? '1' : '0', 'utf8'); for (const message of messages) { visibleHash.update('\0', 'utf8'); visibleHash.update(String(message.messageIndex), 'utf8'); visibleHash.update('\0', 'utf8'); visibleHash.update(sha256(Buffer.from(canonicalJson(message.message), 'utf8')), 'utf8'); } const unhashedManifest: Omit = { candidateId: preparationArg.candidateId, orderVersion: 1, messageCount: messages.length, messagesTruncated, visibleDigest: visibleHash.digest('hex'), ...(latestMessage === undefined ? {} : { latestMessage }), }; const manifestBytes = projectionSourceManifestBytes(unhashedManifest); const manifest: IFlexProjectionSourceManifest = { ...unhashedManifest, sha256: sha256(manifestBytes), bytes: manifestBytes.byteLength, }; const source: IFlexProjectionSourceDocument = { id: projectionSourceDocumentId(preparationArg.candidateId), controllerId: controllerIdArg, scopeId: storageKeyArg, sessionId: sessionIdArg, incarnationId: preparationArg.incarnationId, revision: preparationArg.revision, ...manifest, createdAt: preparationArg.createdAt, }; assertFlexProjectionSourceDocument(source); return { manifest, source, messages }; }; export const createFlexPublicProjection = ( controllerIdArg: string, storageKeyArg: string, candidateIdArg: string, publicationRevisionArg: number, scopeSourceArg: IFlexCommittedScopeSource, projectionSourcesArg: IFlexCommittedProjectionSource[], createdAtArg = new Date().toISOString(), ): IFlexPublicProjectionBuild => { const retainedSessions = retainPublicSessions(scopeSourceArg.snapshot.sessions); const retainedSessionIds = new Set(retainedSessions.map((session) => session.sessionId)); const projectionBySession = new Map(projectionSourcesArg.map((source) => [ source.head.sessionId!, source, ])); const sessions = retainedSessions.map((session) => ({ id: publicSessionDocumentId(candidateIdArg, session.sessionId), controllerId: controllerIdArg, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: publicationRevisionArg, session: cloneJson(session), createdAt: createdAtArg, })); const scopeSessionIds = new Set( scopeSourceArg.snapshot.sessions.map((session) => session.sessionId), ); const originalMessageCount = projectionSourcesArg.reduce((total, source) => ( source.head.sessionId && scopeSessionIds.has(source.head.sessionId) ? total + source.snapshot.messages.length : total ), 0); const sourceEntries = projectionSourcesArg.flatMap((source) => { if (!source.head.sessionId || !retainedSessionIds.has(source.head.sessionId)) return []; const sourceTail = source.snapshot.messages.slice(-flexPublicMessageLimit); return sourceTail.length === 0 ? [] : [{ messages: sourceTail, sourceIndex: sourceTail.length - 1, messageIndex: source.snapshot.messages.length - 1, }]; }); const allMessages: IFlexOrderedMessage[] = []; while (sourceEntries.length > 0 && allMessages.length < flexPublicMessageLimit) { let newestIndex = 0; for (let index = 1; index < sourceEntries.length; index++) { const candidate = sourceEntries[index]!; const newest = sourceEntries[newestIndex]!; if (compareNewestOrderedMessage( { messageIndex: candidate.messageIndex, message: candidate.messages[candidate.sourceIndex]! }, { messageIndex: newest.messageIndex, message: newest.messages[newest.sourceIndex]! }, ) < 0) newestIndex = index; } const newest = sourceEntries[newestIndex]!; allMessages.push({ messageIndex: newest.messageIndex, message: newest.messages[newest.sourceIndex]!, }); newest.sourceIndex -= 1; newest.messageIndex -= 1; if (newest.sourceIndex < 0) sourceEntries.splice(newestIndex, 1); } const messages: IFlexPublicMessageRecordDocument[] = []; let messageBytes = 0; let messageBytesTruncated = false; let oversizedMessagesReplaced = false; for (const sourceEntry of allMessages) { if (messages.length >= flexPublicMessageLimit) break; const createRecord = (messageArg: TFlexMessage): IFlexPublicMessageRecordDocument => ({ id: publicMessageDocumentId(candidateIdArg, messageArg.sessionId, messageArg.messageId), controllerId: controllerIdArg, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: publicationRevisionArg, messageIndex: sourceEntry.messageIndex, message: cloneJson(messageArg), createdAt: createdAtArg, }); let record = createRecord(sourceEntry.message); let recordBytes = serializedBytes(record).byteLength; if (recordBytes > flexPublicSingleMessageBytesLimit) { record = createRecord(placeholderMessage(sourceEntry.message)); recordBytes = serializedBytes(record).byteLength; oversizedMessagesReplaced = true; if (recordBytes > flexPublicSingleMessageBytesLimit) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex public message metadata exceeds its projection limit.', ); } } if (messageBytes + recordBytes > flexPublicMessageBytesLimit) { messageBytesTruncated = true; break; } messageBytes += recordBytes; messages.push(record); } const sessionsTruncated = retainedSessions.length < scopeSourceArg.snapshot.sessions.length; const messagesTruncated = sessionsTruncated || messages.length < originalMessageCount || oversizedMessagesReplaced; const sessionRecords = sessions.map(recordManifest); if (new Set(messages.map((record) => ( `${record.message.sessionId}\0${record.messageIndex}` ))).size !== messages.length) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex public projection contains duplicate message indexes.', ); } const messageRecords = messages.map(recordManifest); const visibleDigest = sha256(serializedBytes({ sessions: sessions.map((record) => record.session), messages: messages.map((record) => ({ messageIndex: record.messageIndex, message: record.message, })), sessionsTruncated, messagesTruncated, messageBytesTruncated, })); const projectionSources: IFlexPublicProjectionSource[] = retainedSessions.flatMap((session) => { const source = projectionBySession.get(session.sessionId); return source ? [{ sessionId: session.sessionId, incarnationId: source.head.incarnationId, revision: source.head.revision, }] : []; }); const unhashedManifest: Omit = { candidateId: candidateIdArg, revision: publicationRevisionArg, sessionRecords, messageRecords, sessionsTruncated, messagesTruncated, messageBytesTruncated, visibleDigest, scopeSource: { incarnationId: scopeSourceArg.head.incarnationId, revision: scopeSourceArg.head.revision, }, projectionSources, }; const candidateBytes = publicCandidateManifestBytes(unhashedManifest); const candidate: IFlexPublicCandidateDocument = { id: publicCandidateDocumentId(candidateIdArg), controllerId: controllerIdArg, scopeId: storageKeyArg, ...unhashedManifest, sha256: sha256(candidateBytes), bytes: candidateBytes.byteLength, createdAt: createdAtArg, }; assertFlexPublicCandidateDocument(candidate); for (const session of sessions) assertFlexPublicSessionRecordDocument(session); for (const message of messages) assertFlexPublicMessageRecordDocument(message); return { candidate, sessions, messages }; }; class FlexLeaseLostError extends Error { constructor(storageKeyArg: string) { super(`Flex writer lease for storage key "${storageKeyArg}" is not owned or has expired.`); this.name = 'FlexLeaseLostError'; } } class FlexHeadCasError extends Error { constructor(public readonly actualRevision: number) { super('Flex head compare-and-swap did not match.'); this.name = 'FlexHeadCasError'; } } const waitForFlexStoreDeadline = async ( promiseArg: Promise, timeoutMsArg: number, messageArg: string, ): Promise => new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error(messageArg)), timeoutMsArg); timer.unref?.(); void promiseArg.then(resolve, reject).finally(() => clearTimeout(timer)); }); const waitForFlexStoreRetry = async ( timeoutMsArg: number, abortSignalArg?: AbortSignal, ): Promise => new Promise((resolve, reject) => { const timer = setTimeout(finish, timeoutMsArg); timer.unref?.(); const abort = () => finish(abortSignalArg!.reason, true); function finish(errorArg?: unknown, rejectedArg = false): void { clearTimeout(timer); abortSignalArg?.removeEventListener('abort', abort); if (rejectedArg) reject(errorArg); else resolve(); } if (abortSignalArg?.aborted) abort(); else abortSignalArg?.addEventListener('abort', abort, { once: true }); }); export class FlexStore implements plugins.flexharness.IFlexHarnessStores { public readonly scopes: plugins.flexharness.IFlexScopeStore; public readonly projections: plugins.flexharness.IFlexProjectionStore; public readonly permissions: plugins.flexharness.IFlexPermissionStore; public readonly projectManagement: plugins.flexharness.IFlexProjectManagementStore; public readonly agentEvents: plugins.flexharness.IFlexAgentEventStoreProvider; public readonly jobs: plugins.flexharness.IFlexToolJobStoreProvider; private readonly database: plugins.smartdata.SmartdataDb; private readonly controllerId: string; private readonly onFatalError?: (errorArg: Error) => void; private readonly manager: { db: plugins.smartdata.SmartdataDb }; private initialized = false; private initPromise?: Promise; private closed = false; private closeComplete = false; private closePromise?: Promise; private closeDrainTimeoutMs = flexMaintenanceOperationTimeoutMs; private maintenanceOpen = true; private readonly maintenanceAbortController = new AbortController(); private orphanMaintenanceTask?: Promise; private orphanMaintenanceTimer?: ReturnType; private heartbeatsOpen = true; private leaseHeartbeatTimer?: ReturnType; private leaseTransitionTimeoutMs = flexLeaseTransitionTimeoutMs; private fatalError?: Error; private readonly leaseTransitionAbortControllers = new Set(); private readonly heldLeases = new Map(); private readonly storageQueues = new Map>(); private readonly activeCandidateIds = new Set(); private readonly agentEventStores = new Map(); private readonly jobStores = new Map(); constructor(optionsArg: IFlexStoreOptions) { assertIdentifier(optionsArg.controllerId, 'controllerId'); if ( !optionsArg.projectManagementHost || typeof optionsArg.projectManagementHost.load !== 'function' || typeof optionsArg.projectManagementHost.save !== 'function' || typeof optionsArg.projectManagementHost.tombstoneSession !== 'function' || typeof optionsArg.projectManagementHost.purgeNamespace !== 'function' ) throw new Error('FlexStore requires a durable project-management host.'); this.database = optionsArg.database; this.controllerId = optionsArg.controllerId; this.onFatalError = optionsArg.onFatalError; this.manager = { db: this.database }; registerFlexPrivateModels(this.manager); registerFlexPublicModels(this.manager); this.scopes = { load: (storageKey) => this.loadScope(storageKey), save: (storageKey, snapshot, expectedRevision) => this.saveScope(storageKey, snapshot, expectedRevision), }; this.projections = { load: (storageKey, sessionId) => this.loadProjection(storageKey, sessionId), save: (storageKey, sessionId, snapshot, expectedRevision) => this.saveProjection(storageKey, sessionId, snapshot, expectedRevision), deleteSession: (storageKey, sessionId) => this.deleteProjection(storageKey, sessionId), }; this.permissions = { load: (storageKey, sessionId) => this.loadPermission(storageKey, sessionId), save: (storageKey, sessionId, snapshot, expectedRevision) => this.savePermission(storageKey, sessionId, snapshot, expectedRevision), deleteSession: (storageKey, sessionId) => this.deletePermission(storageKey, sessionId), }; this.projectManagement = { load: (storageKey, sessionId, sessionContext) => { const clonedContext = cloneProjectManagementSessionContext(sessionContext); return optionsArg.projectManagementHost.load( storageKey, sessionId, clonedContext, ); }, save: (storageKey, sessionId, snapshot, expectedRevision, writeContext) => optionsArg.projectManagementHost.save( storageKey, sessionId, snapshot, expectedRevision, writeContext, ), tombstoneSession: ( storageKey, sessionId, tombstone, expectedRevision, sessionContext, ) => optionsArg.projectManagementHost.tombstoneSession( storageKey, sessionId, tombstone, expectedRevision, cloneProjectManagementSessionContext(sessionContext), ), purgeNamespace: (storageKey) => optionsArg.projectManagementHost.purgeNamespace(storageKey), }; this.agentEvents = { getStore: (storageKey, sessionId) => this.getAgentEventStore(storageKey, sessionId), releaseSession: (storageKey, sessionId) => { this.agentEventStores.delete(providerKey(storageKey, sessionId)); }, deleteSession: (storageKey, sessionId) => this.deleteAgentEventSession(storageKey, sessionId), }; this.jobs = { getStore: (storageKey, sessionId) => this.getJobStore(storageKey, sessionId), releaseSession: (storageKey, sessionId) => { this.jobStores.delete(providerKey(storageKey, sessionId)); }, deleteSession: (storageKey, sessionId) => this.deleteJobSession(storageKey, sessionId), }; } public async init(): Promise { this.throwIfFatal(); if (this.closed) throw new Error('The Flex store is closed.'); if (this.initialized) return; if (this.initPromise) return this.initPromise; const initPromise = (async () => { await initFlexPrivateModels(); await initFlexPublicModels(); this.initialized = true; this.queueOrphanMaintenance(); })(); this.initPromise = initPromise; try { await initPromise; } finally { if (this.initPromise === initPromise) this.initPromise = undefined; } } public getLegacyMigrationStores(): plugins.flexharness.IFlexHarnessStores { return { scopes: { ...this.scopes, save: (storageKey, snapshot, expectedRevision) => this.saveScope(storageKey, snapshot, expectedRevision, false), }, projections: { ...this.projections, save: (storageKey, sessionId, snapshot, expectedRevision) => this.saveProjection(storageKey, sessionId, snapshot, expectedRevision, false), }, permissions: this.permissions, projectManagement: this.projectManagement, agentEvents: this.agentEvents, jobs: this.jobs, }; } public async loadLegacy(storageKeyArg: string): Promise { return this.runStorage(storageKeyArg, async () => { const head = await this.readLegacyHead(storageKeyArg); if (!head) return undefined; const manifest = head.privateCandidates.find( (candidate) => candidate.candidateId === head.currentPrivateCandidateId, ); if (!manifest || manifest.revision !== head.revision) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex legacy private head manifest is invalid.', ); } return this.loadPrivateCandidate( storageKeyArg, manifest, async (value) => value as TFlexLegacySnapshot, ); }); } public async hasCompletedLegacyMigration(storageKeyArg: string): Promise { return this.runStorage(storageKeyArg, async () => { const marker = await this.readMigration(storageKeyArg); if (!marker) return false; const legacyHead = await this.readLegacyHead(storageKeyArg); if ( !legacyHead || marker.legacyRevision !== legacyHead.revision || marker.legacyCandidateId !== legacyHead.currentPrivateCandidateId ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Completed Flex migration does not match its retained legacy source.', ); } return true; }); } public async completeLegacyMigration(storageKeyArg: string): Promise { await this.runStorage(storageKeyArg, async (lease) => { const legacyHead = await this.readLegacyHead(storageKeyArg); if (!legacyHead) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Cannot complete Flex migration without its retained legacy source.', ); } const current = await this.readMigration(storageKeyArg); if (current) { if ( current.legacyRevision !== legacyHead.revision || current.legacyCandidateId !== legacyHead.currentPrivateCandidateId ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Completed Flex migration does not match its retained legacy source.', ); } return; } const next: IFlexMigrationDocument = { id: flexMigrationId(this.controllerId, storageKeyArg), controllerId: this.controllerId, storageKey: storageKeyArg, migration: 'flexharness-v3', legacyRevision: legacyHead.revision, legacyCandidateId: legacyHead.currentPrivateCandidateId, completedAt: new Date().toISOString(), }; assertFlexMigrationDocument(next); await this.commitHeadMutation({ lease, collection: FlexMigrationModel.collection.mongoDbCollection, current: null, next, selector: { id: next.id }, readCurrent: (options) => this.readMigration(storageKeyArg, options), project: migrationBody, targetName: `migration:${next.id}`, }); }); } public async ensurePublicProjection(storageKeyArg: string): Promise { await this.runStorage(storageKeyArg, async (lease) => this.publishPublicProjection( storageKeyArg, lease, )); } public async listProjectionOrderMigrationPage( storageKeyArg: string, cursorArg?: TFlexObjectId, ): Promise { return this.runStorage(storageKeyArg, async () => { const heads = await this.readDomainHeadPage({ controllerId: this.controllerId, storageKey: storageKeyArg, domain: 'projection', }, cursorArg); const sessionIds: string[] = []; for (const head of heads) { if (!head.sessionId) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection head is missing its session identity.', ); } this.assertDomainHeadIdentity(head, { storageKey: storageKeyArg, domain: 'projection', sessionId: head.sessionId, }); if (head.projectionSource?.orderVersion !== 1) sessionIds.push(head.sessionId); } const last = heads.at(-1); return { sessionIds, ...(last && heads.length === flexMaintenancePageLimit ? { nextCursor: last._id } : {}), }; }); } public async rewriteProjectionSourceOrder( storageKeyArg: string, sessionIdArg: string, ): Promise { return this.runStorage(storageKeyArg, async (lease) => { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'projection', sessionId: sessionIdArg, }; const current = await this.readDomainHead(identity); if (!current || current.projectionSource?.orderVersion === 1) return false; const snapshot = await this.loadPrivateCandidate( storageKeyArg, current.privateCandidate, (value) => this.validateProjectionSnapshot(value, sessionIdArg), undefined, { identity, incarnationId: current.incarnationId }, ); const candidateId = randomCandidateId(); const build = createProjectionSource( this.controllerId, storageKeyArg, sessionIdArg, snapshot, { candidateId, incarnationId: current.incarnationId, revision: current.revision, createdAt: new Date().toISOString(), }, ); const next: IFlexDomainHeadDocument = { ...domainHeadBody(current), projectionSource: build.manifest, }; assertFlexDomainHeadDocument(next); this.activeCandidateIds.add(candidateId); let committed = false; try { await this.writeAndVerifyProjectionSource(build); await this.commitDomainHead(lease, current, next, identity); committed = true; if (current.projectionSource) { await this.cleanupProjectionSourceByCandidate( current.projectionSource.candidateId, ).catch(() => undefined); } } finally { this.activeCandidateIds.delete(candidateId); if (!committed) this.queueOrphanMaintenance(); } return true; }); } public async purgeScope(storageKeyArg: string): Promise { await this.runStorage(storageKeyArg, async (lease) => { this.evictProviderScope(storageKeyArg); let lastDomainHeadId: TFlexObjectId | undefined; while (true) { const domainHeads = await this.readDomainHeadPage({ controllerId: this.controllerId, storageKey: storageKeyArg, }, lastDomainHeadId); for (const head of domainHeads) { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: head.domain, ...(head.sessionId !== undefined ? { sessionId: head.sessionId } : {}), ...(head.archiveId !== undefined ? { archiveId: head.archiveId } : {}), }; this.assertDomainHeadIdentity(head, identity); await this.deleteDomainHead(lease, head, identity); } const last = domainHeads.at(-1); if (!last || domainHeads.length < flexMaintenancePageLimit) break; lastDomainHeadId = last._id; } const publicHead = await this.readPublicHead(storageKeyArg); if (publicHead) await this.deletePublicHead(lease, publicHead, storageKeyArg); const migration = await this.readMigration(storageKeyArg); if (migration) { await this.commitHeadMutation({ lease, collection: FlexMigrationModel.collection.mongoDbCollection, current: migration, next: null, selector: { _id: migration._id, id: migration.id, controllerId: this.controllerId, storageKey: storageKeyArg, _smartdataRevision: smartRevisionSelector(migration), }, readCurrent: (options) => this.readMigration(storageKeyArg, options), project: migrationBody, targetName: `migration:${migration.id}`, }); } const legacyRaw = await FlexHeadModel.collection.mongoDbCollection.findOne({ id: flexHeadId(this.controllerId, storageKeyArg), controllerId: this.controllerId, storageKey: storageKeyArg, }); if (legacyRaw) { await this.commitHeadMutation({ lease, collection: FlexHeadModel.collection.mongoDbCollection, current: legacyRaw as unknown as { _id: unknown; _smartdataRevision?: string } & IFlexHeadDocument, next: null, selector: { _id: legacyRaw._id, id: flexHeadId(this.controllerId, storageKeyArg), controllerId: this.controllerId, storageKey: storageKeyArg, _smartdataRevision: smartRevisionSelector( legacyRaw as { _smartdataRevision?: string }, ), }, readCurrent: async (options) => await FlexHeadModel.collection.mongoDbCollection.findOne({ id: flexHeadId(this.controllerId, storageKeyArg), }, options) as unknown as ({ _id: unknown; _smartdataRevision?: string } & IFlexHeadDocument) | null, project: (value) => value, targetName: `legacy:${storageKeyArg}`, }); } await this.deleteScopeRecords(lease, storageKeyArg); const remaining = await Promise.all([ FlexDomainHeadModel.exact.count({ controllerId: this.controllerId, storageKey: storageKeyArg }), FlexPublicHeadModel.exact.count({ controllerId: this.controllerId, storageKey: storageKeyArg }), FlexMigrationModel.exact.count({ controllerId: this.controllerId, storageKey: storageKeyArg }), FlexHeadModel.collection.mongoDbCollection.countDocuments({ controllerId: this.controllerId, storageKey: storageKeyArg, }), ]); if (remaining.some((count) => count !== 0)) { throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', 'Flex scope purge could not confirm all head deletions.', ); } await this.releaseHeldLease(lease); }); } public async reconcileOrphans( nowArg = Date.now(), abortSignalArg?: AbortSignal, ): Promise { await this.init(); abortSignalArg?.throwIfAborted(); const cutoff = new Date(nowArg - flexOrphanGraceMs).toISOString(); await this.reconcileCandidateOrphans('private', cutoff, abortSignalArg); await this.reconcileCandidateOrphans('public', cutoff, abortSignalArg); await this.reconcileCandidateOrphans('projectionSource', cutoff, abortSignalArg); } public async close(): Promise { if (this.closeComplete) return; if (this.closePromise) return this.closePromise; const closePromise = (async () => { const errors: unknown[] = []; this.closed = true; for (const controller of this.leaseTransitionAbortControllers) { controller.abort(new Error('The Flex store is closing.')); } this.maintenanceOpen = false; this.maintenanceAbortController.abort(new Error('The Flex store is closing.')); if (this.orphanMaintenanceTimer) { clearTimeout(this.orphanMaintenanceTimer); this.orphanMaintenanceTimer = undefined; } if (this.orphanMaintenanceTask) { try { await waitForFlexStoreDeadline( this.orphanMaintenanceTask, this.closeDrainTimeoutMs, 'Flex orphan maintenance did not stop before the close deadline.', ); } catch (errorArg) { errors.push(errorArg); } } try { await waitForFlexStoreDeadline( Promise.allSettled([...this.storageQueues.values()]), this.closeDrainTimeoutMs, 'Flex storage operations did not drain before the close deadline.', ); } catch (errorArg) { errors.push(errorArg); for (const controller of this.leaseTransitionAbortControllers) { controller.abort(errorArg); } } this.heartbeatsOpen = false; if (this.leaseHeartbeatTimer) { clearInterval(this.leaseHeartbeatTimer); this.leaseHeartbeatTimer = undefined; } this.agentEventStores.clear(); this.jobStores.clear(); const releaseResults = await Promise.allSettled( [...this.heldLeases.values()].map((lease) => this.releaseHeldLease( lease, this.closeDrainTimeoutMs, )), ); errors.push(...releaseResults .filter((result): result is PromiseRejectedResult => result.status === 'rejected') .map((result) => result.reason)); if (errors.length > 0) throw new AggregateError(errors, 'Flex store close failed.'); this.closeComplete = true; })(); this.closePromise = closePromise; try { await closePromise; } finally { if (!this.closeComplete && this.closePromise === closePromise) this.closePromise = undefined; } } private async loadScope(storageKeyArg: string): Promise { return this.runStorage(storageKeyArg, async () => ( await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'scope' }, (value) => this.validateScopeSnapshot(value), ) )?.snapshot); } private async saveScope( storageKeyArg: string, snapshotArg: TFlexScopeSnapshot, expectedRevisionArg: number, publishPublicProjectionArg = true, ): Promise { assertExpectedNextRevision(snapshotArg.revision, expectedRevisionArg); const snapshot = await this.validateScopeSnapshot(snapshotArg); await this.runStorage(storageKeyArg, async (lease) => { await this.saveDomain( lease, { storageKey: storageKeyArg, domain: 'scope' }, snapshot, expectedRevisionArg, (actual) => new plugins.flexharness.FlexHarnessStoreConflictError( storageKeyArg, expectedRevisionArg, actual, ), ); if (!publishPublicProjectionArg) return; try { await this.publishPublicProjection(storageKeyArg, lease); } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreCommitUncertainError( flexPublicHeadId(this.controllerId, storageKeyArg), 'write', { cause: errorArg }, ); } }); } private async loadProjection( storageKeyArg: string, sessionIdArg: string, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); return this.runStorage(storageKeyArg, async () => ( await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'projection', sessionId: sessionIdArg }, (value) => this.validateProjectionSnapshot(value, sessionIdArg), ) )?.snapshot); } private async saveProjection( storageKeyArg: string, sessionIdArg: string, snapshotArg: TFlexProjectionSnapshot, expectedRevisionArg: number, publishPublicProjectionArg = true, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); assertExpectedNextRevision(snapshotArg.revision, expectedRevisionArg); const snapshot = await this.validateProjectionSnapshot(snapshotArg, sessionIdArg); await this.runStorage(storageKeyArg, async (lease) => { let visibleSourceChanged = true; await this.saveDomain( lease, { storageKey: storageKeyArg, domain: 'projection', sessionId: sessionIdArg }, snapshot, expectedRevisionArg, (actual) => new plugins.flexharness.FlexHarnessStoreConflictError( providerKey(storageKeyArg, sessionIdArg), expectedRevisionArg, actual, ), undefined, undefined, async (preparation, current) => { const source = createProjectionSource( this.controllerId, storageKeyArg, sessionIdArg, snapshot, preparation, ); if ( current?.projectionSource?.orderVersion === 1 && current.projectionSource.visibleDigest === source.manifest.visibleDigest ) { await this.loadProjectionSource(current); visibleSourceChanged = false; return current.projectionSource; } await this.writeAndVerifyProjectionSource(source); return source.manifest; }, ); if (!visibleSourceChanged || !publishPublicProjectionArg) return; try { await this.publishPublicProjection(storageKeyArg, lease); } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreCommitUncertainError( flexPublicHeadId(this.controllerId, storageKeyArg), 'write', { cause: errorArg }, ); } }); } private async deleteProjection(storageKeyArg: string, sessionIdArg: string): Promise { assertIdentifier(sessionIdArg, 'sessionId'); await this.runStorage(storageKeyArg, async (lease) => { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'projection', sessionId: sessionIdArg, }; const head = await this.readDomainHead(identity); if (head) await this.deleteDomainHead(lease, head, identity); try { await this.publishPublicProjection(storageKeyArg, lease); } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreCommitUncertainError( flexPublicHeadId(this.controllerId, storageKeyArg), 'write', { cause: errorArg }, ); } }); } private async loadPermission( storageKeyArg: string, sessionIdArg: string, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); return this.runStorage(storageKeyArg, async () => ( await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'permission', sessionId: sessionIdArg }, (value) => this.validatePermissionSnapshot(value), ) )?.snapshot); } private async savePermission( storageKeyArg: string, sessionIdArg: string, snapshotArg: TFlexPermissionSnapshot, expectedRevisionArg: number, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); assertExpectedNextRevision(snapshotArg.revision, expectedRevisionArg); const snapshot = await this.validatePermissionSnapshot(snapshotArg); await this.runStorage(storageKeyArg, async (lease) => { await this.saveDomain( lease, { storageKey: storageKeyArg, domain: 'permission', sessionId: sessionIdArg }, snapshot, expectedRevisionArg, (actual) => new plugins.flexharness.FlexHarnessStoreConflictError( providerKey(storageKeyArg, sessionIdArg), expectedRevisionArg, actual, ), ); }); } private async deletePermission(storageKeyArg: string, sessionIdArg: string): Promise { assertIdentifier(sessionIdArg, 'sessionId'); await this.runStorage(storageKeyArg, async (lease) => { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'permission', sessionId: sessionIdArg, }; const head = await this.readDomainHead(identity); if (head) await this.deleteDomainHead(lease, head, identity); }); } private async getAgentEventStore( storageKeyArg: string, sessionIdArg: string, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); return this.runStorage(storageKeyArg, async () => { const key = providerKey(storageKeyArg, sessionIdArg); const existing = this.agentEventStores.get(key); if (existing) return existing; const wrapper: plugins.flexAgent.IAgentEventStoreV2 = { eventSchemaVersion: 2, load: (sessionId) => this.loadAgentEventSnapshot(storageKeyArg, sessionIdArg, sessionId), save: (sessionId, events, expectedRevision) => this.saveAgentEventSnapshot( storageKeyArg, sessionIdArg, sessionId, events, expectedRevision, ), archive: (archive) => this.saveAgentEventArchive(storageKeyArg, sessionIdArg, archive), loadArchive: (sessionId, archiveId) => this.loadAgentEventArchive( storageKeyArg, sessionIdArg, sessionId, archiveId, ), }; this.agentEventStores.set(key, wrapper); return wrapper; }); } private assertBoundSession(boundSessionIdArg: string, requestedSessionIdArg: string): void { if (boundSessionIdArg !== requestedSessionIdArg) { throw new plugins.flexharness.FlexHarnessStoreFormatError( `Store is bound to session "${boundSessionIdArg}", not "${requestedSessionIdArg}".`, ); } } private async loadAgentEventSnapshot( storageKeyArg: string, boundSessionIdArg: string, sessionIdArg: string, ): Promise { this.assertBoundSession(boundSessionIdArg, sessionIdArg); return this.runStorage(storageKeyArg, async () => ( await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'agentEvents', sessionId: sessionIdArg }, (value) => this.validateAgentEventSnapshot(value, sessionIdArg), ) )?.snapshot); } private async saveAgentEventSnapshot( storageKeyArg: string, boundSessionIdArg: string, sessionIdArg: string, eventsArg: readonly plugins.flexAgent.TAgentEvent[], expectedRevisionArg: number, ): Promise { this.assertBoundSession(boundSessionIdArg, sessionIdArg); assertExpectedNextRevision(expectedRevisionArg + 1, expectedRevisionArg); const snapshot = this.validateAgentEventSnapshot({ schemaVersion: 2, sessionId: sessionIdArg, revision: expectedRevisionArg + 1, updatedAt: eventsArg.reduce((latest, event) => Math.max(latest, event.timestamp), 0), events: cloneJson([...eventsArg], '$events'), }, sessionIdArg); await this.runStorage(storageKeyArg, async (lease) => { await this.saveDomain( lease, { storageKey: storageKeyArg, domain: 'agentEvents', sessionId: sessionIdArg }, snapshot, expectedRevisionArg, (actual) => new plugins.flexAgent.AgentEventStoreConflictError( sessionIdArg, expectedRevisionArg, actual, ), ); }); return snapshot.revision; } private async saveAgentEventArchive( storageKeyArg: string, boundSessionIdArg: string, archiveArg: TAgentEventArchive, ): Promise { this.assertBoundSession(boundSessionIdArg, archiveArg.sessionId); assertIdentifier(archiveArg.archiveId, 'archiveId'); const archive = this.validateAgentEventArchive( cloneJson(archiveArg, '$archive'), boundSessionIdArg, archiveArg.archiveId, ); await this.runStorage(storageKeyArg, async (lease) => { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'agentArchive', sessionId: boundSessionIdArg, archiveId: archive.archiveId, }; const current = await this.loadDomainDirect( identity, (value) => this.validateAgentEventArchive(value, boundSessionIdArg, archive.archiveId), ); if (current) { if (canonicalJson(current.snapshot) !== canonicalJson(archive)) { throw new Error(`Archive "${archive.archiveId}" already exists with different content.`); } return; } try { await this.saveDomain( lease, identity, archive, 0, () => new Error(`Archive "${archive.archiveId}" already exists.`), 1, ); } catch (errorArg) { const reconciled = await this.loadDomainDirect( identity, (value) => this.validateAgentEventArchive(value, boundSessionIdArg, archive.archiveId), ); if (reconciled && canonicalJson(reconciled.snapshot) === canonicalJson(archive)) return; throw errorArg; } }); } private async loadAgentEventArchive( storageKeyArg: string, boundSessionIdArg: string, sessionIdArg: string, archiveIdArg: string, ): Promise { this.assertBoundSession(boundSessionIdArg, sessionIdArg); assertIdentifier(archiveIdArg, 'archiveId'); return this.runStorage(storageKeyArg, async () => ( await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'agentArchive', sessionId: sessionIdArg, archiveId: archiveIdArg, }, (value) => this.validateAgentEventArchive(value, sessionIdArg, archiveIdArg), ) )?.snapshot); } private async deleteAgentEventSession(storageKeyArg: string, sessionIdArg: string): Promise { assertIdentifier(sessionIdArg, 'sessionId'); await this.runStorage(storageKeyArg, async (lease) => { let lastId: TFlexObjectId | undefined; while (true) { const heads = await this.readDomainHeadPage({ controllerId: this.controllerId, storageKey: storageKeyArg, sessionId: sessionIdArg, domain: { $in: ['agentEvents', 'agentArchive'] }, }, lastId); for (const head of heads) { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: head.domain, sessionId: sessionIdArg, ...(head.archiveId !== undefined ? { archiveId: head.archiveId } : {}), }; this.assertDomainHeadIdentity(head, identity); await this.deleteDomainHead(lease, head, identity); } const last = heads.at(-1); if (!last || heads.length < flexMaintenancePageLimit) break; lastId = last._id; } this.agentEventStores.delete(providerKey(storageKeyArg, sessionIdArg)); }); } private async getJobStore( storageKeyArg: string, sessionIdArg: string, ): Promise { assertIdentifier(sessionIdArg, 'sessionId'); return this.runStorage(storageKeyArg, async () => { const key = providerKey(storageKeyArg, sessionIdArg); const existing = this.jobStores.get(key); if (existing) return existing; const wrapper: plugins.flexTools.IToolJobStore = { load: (abortSignal) => this.loadJobSnapshot(storageKeyArg, sessionIdArg, abortSignal), save: (jobs, expectedRevision, abortSignal) => this.saveJobSnapshot( storageKeyArg, sessionIdArg, jobs, expectedRevision, abortSignal, ), }; this.jobStores.set(key, wrapper); return wrapper; }); } private async loadJobSnapshot( storageKeyArg: string, sessionIdArg: string, abortSignalArg?: AbortSignal, ): Promise { abortSignalArg?.throwIfAborted(); const result = await this.runStorage(storageKeyArg, async () => { abortSignalArg?.throwIfAborted(); const loaded = await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'jobs', sessionId: sessionIdArg }, (value) => this.validateToolJobSnapshot(value), abortSignalArg, ); abortSignalArg?.throwIfAborted(); return loaded?.snapshot; }); abortSignalArg?.throwIfAborted(); return result; } private async saveJobSnapshot( storageKeyArg: string, sessionIdArg: string, jobsArg: readonly plugins.flexTools.IToolJobState[], expectedRevisionArg: number, abortSignalArg?: AbortSignal, ): Promise { abortSignalArg?.throwIfAborted(); assertExpectedNextRevision(expectedRevisionArg + 1, expectedRevisionArg); const snapshot = this.validateToolJobSnapshot({ schemaVersion: 1, revision: expectedRevisionArg + 1, updatedAt: Date.now(), jobs: cloneJson([...jobsArg], '$jobs'), }); await this.runStorage(storageKeyArg, async (lease) => { abortSignalArg?.throwIfAborted(); await this.saveDomain( lease, { storageKey: storageKeyArg, domain: 'jobs', sessionId: sessionIdArg }, snapshot, expectedRevisionArg, (actual) => new plugins.flexTools.ToolJobStoreConflictError( expectedRevisionArg, actual, ), snapshot.revision, abortSignalArg, ); }); return snapshot.revision; } private async deleteJobSession(storageKeyArg: string, sessionIdArg: string): Promise { assertIdentifier(sessionIdArg, 'sessionId'); await this.runStorage(storageKeyArg, async (lease) => { const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'jobs', sessionId: sessionIdArg, }; const head = await this.readDomainHead(identity); if (head) await this.deleteDomainHead(lease, head, identity); this.jobStores.delete(providerKey(storageKeyArg, sessionIdArg)); }); } private async validateScopeSnapshot(valueArg: unknown): Promise { const value = valueArg as TFlexScopeSnapshot; if (!Number.isSafeInteger(value?.revision) || value.revision < 0) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid scope snapshot revision.'); } const stores = new plugins.flexharness.InMemoryFlexHarnessStores(); await stores.scopes.save('__validation__', { ...value, revision: 1 }, 0); return cloneJson(value); } private async validateProjectionSnapshot( valueArg: unknown, sessionIdArg: string, ): Promise { const value = valueArg as TFlexProjectionSnapshot; if (!Number.isSafeInteger(value?.revision) || value.revision < 0) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid projection snapshot revision.'); } const stores = new plugins.flexharness.InMemoryFlexHarnessStores(); await stores.projections.save( '__validation__', sessionIdArg, { ...value, revision: 1 } as plugins.flexharness.IFlexProjectionSnapshotCurrent, 0, ); return cloneJson(value); } private async validatePermissionSnapshot(valueArg: unknown): Promise { const value = valueArg as TFlexPermissionSnapshot; if (!Number.isSafeInteger(value?.revision) || value.revision < 0) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid permission snapshot revision.'); } const stores = new plugins.flexharness.InMemoryFlexHarnessStores(); await stores.permissions.save('__validation__', '__validation__', { ...value, revision: 1 }, 0); return cloneJson(value); } private validateAgentEventSnapshot(valueArg: unknown, sessionIdArg: string): TAgentEventSnapshot { try { return cloneJson( plugins.flexAgent.validateAgentEventSnapshotV2(valueArg, sessionIdArg), ); } catch (errorArg) { if (errorArg instanceof plugins.flexharness.FlexHarnessStoreFormatError) throw errorArg; throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Invalid schema-2 Agent event snapshot.', { cause: errorArg }, ); } } private validateAgentEventArchive( valueArg: unknown, sessionIdArg: string, archiveIdArg: string, ): TAgentEventArchive { try { return cloneJson( plugins.flexAgent.validateAgentEventArchiveV2(valueArg, sessionIdArg, archiveIdArg), ); } catch (errorArg) { if (errorArg instanceof plugins.flexharness.FlexHarnessStoreFormatError) throw errorArg; throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Invalid schema-2 Agent event archive.', { cause: errorArg }, ); } } private validateToolJobSnapshot(valueArg: unknown): TToolJobSnapshot { const value = cloneJson(valueArg) as Partial; if ( !value || value.schemaVersion !== 1 || !Number.isSafeInteger(value.revision) || (value.revision ?? -1) < 1 || !Number.isSafeInteger(value.updatedAt) || (value.updatedAt ?? -1) < 0 || !Array.isArray(value.jobs) || Object.keys(value).some((key) => !['schemaVersion', 'revision', 'updatedAt', 'jobs'].includes(key)) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job snapshot.'); } const executionIds = new Set(); for (const job of value.jobs) { if ( !job || typeof job !== 'object' || Array.isArray(job) || typeof job.executionId !== 'string' || job.executionId.length === 0 || executionIds.has(job.executionId) || typeof job.type !== 'string' || job.type.length === 0 || !['running', 'finished', 'failed', 'aborted'].includes(job.state) || Object.keys(job).some((key) => ![ 'executionId', 'type', 'state', 'request', 'exitCode', 'stdout', 'stderr', 'signal', 'startedAt', 'updatedAt', 'finishedAt', ].includes(key)) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job state.'); } executionIds.add(job.executionId); if ( (job.exitCode !== undefined && job.exitCode !== null && !Number.isSafeInteger(job.exitCode)) || (job.stdout !== undefined && typeof job.stdout !== 'string') || (job.stderr !== undefined && typeof job.stderr !== 'string') || (job.signal !== undefined && typeof job.signal !== 'string') || [job.startedAt, job.updatedAt, job.finishedAt].some((time) => ( time !== undefined && (!Number.isSafeInteger(time) || time < 0) )) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job state values.'); } if (job.request !== undefined) { const request = job.request; if ( !request || typeof request !== 'object' || Array.isArray(request) || typeof request.type !== 'string' || request.type.length === 0 || Object.keys(request).some((key) => ![ 'type', 'command', 'cwd', 'timeoutMs', 'metadata', ].includes(key)) || (request.cwd !== undefined && typeof request.cwd !== 'string') || (request.timeoutMs !== undefined && ( !Number.isSafeInteger(request.timeoutMs) || request.timeoutMs < 0 )) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job request.'); } if (request.command !== undefined && ( !request.command || typeof request.command !== 'object' || Array.isArray(request.command) || typeof request.command.executable !== 'string' || request.command.executable.length === 0 || !Array.isArray(request.command.args) || !request.command.args.every((argument) => typeof argument === 'string') || Object.keys(request.command).some((key) => !['executable', 'args'].includes(key)) )) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job command.'); } if (request.metadata !== undefined && ( !request.metadata || typeof request.metadata !== 'object' || Array.isArray(request.metadata) )) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid tool job metadata.'); } } } return value as TToolJobSnapshot; } private async saveDomain( leaseArg: IFlexHeldLease, identityArg: IFlexDomainIdentity, snapshotArg: TSnapshot, expectedRevisionArg: number, conflictArg: (actualRevisionArg: number) => Error, revisionArg = (snapshotArg as { revision?: number }).revision, abortSignalArg?: AbortSignal, prepareCandidateArg?: ( preparationArg: IFlexCandidatePreparation, currentArg: TStoredDomainHead | null, ) => Promise, ): Promise { if (!Number.isSafeInteger(revisionArg) || Number(revisionArg) < 1) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid domain revision.'); } const revision = Number(revisionArg); const current = await this.readDomainHead(identityArg); const actualRevision = current?.revision ?? 0; if (actualRevision !== expectedRevisionArg) throw conflictArg(actualRevision); abortSignalArg?.throwIfAborted(); const candidateId = randomCandidateId(); const createdAt = new Date().toISOString(); const incarnationId = current?.incarnationId ?? randomCandidateId(); const projection = createPrivateProjection( this.controllerId, identityArg.storageKey, candidateId, revision, snapshotArg, createdAt, { identity: identityArg, incarnationId }, ); const manifest: IFlexPrivateCandidateManifest = { candidateId, revision, sha256: projection.candidate.sha256, bytes: projection.candidate.bytes, chunks: projection.candidate.chunks, }; this.activeCandidateIds.add(candidateId); let committed = false; try { await this.writeAndVerifyPrivate(projection.candidate, projection.chunks, abortSignalArg); abortSignalArg?.throwIfAborted(); const projectionSource = await prepareCandidateArg?.({ candidateId, incarnationId, revision, createdAt, }, current); abortSignalArg?.throwIfAborted(); const next: IFlexDomainHeadDocument = { id: flexDomainHeadId( this.controllerId, identityArg.storageKey, identityArg.domain, identityArg.sessionId, identityArg.archiveId, ), controllerId: this.controllerId, storageKey: identityArg.storageKey, domain: identityArg.domain, incarnationId, revision, currentPrivateCandidateId: candidateId, privateCandidate: manifest, ...(projectionSource === undefined ? {} : { projectionSource }), ...(identityArg.sessionId !== undefined ? { sessionId: identityArg.sessionId } : {}), ...(identityArg.archiveId !== undefined ? { archiveId: identityArg.archiveId } : {}), }; assertFlexDomainHeadDocument(next); this.assertDomainHeadIdentity(next, identityArg); try { await this.commitDomainHead(leaseArg, current, next, identityArg); } catch (errorArg) { if (errorArg instanceof FlexHeadCasError) throw conflictArg(errorArg.actualRevision); throw errorArg; } committed = true; if (current) { await this.cleanupPrivateByCandidate(current.privateCandidate).catch(() => undefined); if ( current.domain === 'projection' && current.projectionSource?.candidateId !== projectionSource?.candidateId ) { await this.cleanupProjectionSourceByCandidate( current.projectionSource!.candidateId, ).catch(() => undefined); } } return await this.readDomainHead(identityArg) ?? (() => { throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', 'Committed Flex domain head is missing on readback.', ); })(); } finally { if (!committed) this.queueOrphanMaintenance(); this.activeCandidateIds.delete(candidateId); } } private async loadDomainDirect( identityArg: IFlexDomainIdentity, validateArg: (valueArg: unknown) => TSnapshot | Promise, abortSignalArg?: AbortSignal, ): Promise | undefined> { abortSignalArg?.throwIfAborted(); const head = await this.readDomainHead(identityArg); if (!head) return undefined; const snapshot = await this.loadPrivateCandidate( identityArg.storageKey, head.privateCandidate, validateArg, abortSignalArg, { identity: identityArg, incarnationId: head.incarnationId }, ); abortSignalArg?.throwIfAborted(); const snapshotRevision = (snapshot as { revision?: unknown }).revision; if ( identityArg.domain !== 'agentArchive' && snapshotRevision !== head.revision ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex domain snapshot revision does not match its head.', ); } return { head, snapshot }; } private async readLegacyHead( storageKeyArg: string, optionsArg?: IFlexReadOptions, ): Promise { const expectedId = flexHeadId(this.controllerId, storageKeyArg); const raw = await FlexHeadModel.collection.mongoDbCollection.findOne( { id: expectedId }, optionsArg, ); const head = raw as unknown as TStoredLegacyHead | null; if (!head) return null; const body: IFlexHeadDocument = { id: head.id, controllerId: head.controllerId, storageKey: head.storageKey, revision: head.revision, currentPrivateCandidateId: head.currentPrivateCandidateId, currentPublicCandidateId: head.currentPublicCandidateId, privateCandidates: head.privateCandidates, publicCandidates: head.publicCandidates, }; try { assertFlexHeadDocument(body); } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex legacy head is invalid.', { cause: errorArg }, ); } if ( body.id !== expectedId || body.controllerId !== this.controllerId || body.storageKey !== storageKeyArg ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex legacy head identity mismatch.', ); } return head; } private async readMigration( storageKeyArg: string, optionsArg?: IFlexReadOptions, ): Promise { const expectedId = flexMigrationId(this.controllerId, storageKeyArg); const marker = await FlexMigrationModel.collection.mongoDbCollection.findOne( { id: expectedId }, optionsArg, ) as unknown as TStoredMigration | null; if (!marker) return null; const body = migrationBody(marker); try { assertFlexMigrationDocument(body); } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex migration marker is invalid.', { cause: errorArg }, ); } if ( body.id !== expectedId || body.controllerId !== this.controllerId || body.storageKey !== storageKeyArg ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex migration marker identity mismatch.', ); } return marker; } private async readDomainHead( identityArg: IFlexDomainIdentity, optionsArg?: IFlexReadOptions, ): Promise { const id = flexDomainHeadId( this.controllerId, identityArg.storageKey, identityArg.domain, identityArg.sessionId, identityArg.archiveId, ); const head = await FlexDomainHeadModel.collection.mongoDbCollection.findOne( { id }, optionsArg, ) as unknown as TStoredDomainHead | null; if (head) this.assertDomainHeadIdentity(head, identityArg); return head; } private async readDomainHeadPage( filterArg: Record, lastIdArg?: TFlexObjectId, ): Promise { const cursor = FlexDomainHeadModel.collection.mongoDbCollection.find({ ...filterArg, ...(lastIdArg ? { _id: { $gt: lastIdArg } } : {}), }).sort({ _id: 1 }).limit(flexMaintenancePageLimit); try { return await cursor.toArray() as unknown as TStoredDomainHead[]; } finally { await cursor.close(); } } private assertDomainHeadIdentity( headArg: IFlexDomainHeadDocument, identityArg: IFlexDomainIdentity, ): void { const body = domainHeadBody(headArg); assertFlexDomainHeadDocument(body); if ( body.id !== flexDomainHeadId( this.controllerId, identityArg.storageKey, identityArg.domain, identityArg.sessionId, identityArg.archiveId, ) || body.controllerId !== this.controllerId || body.storageKey !== identityArg.storageKey || body.domain !== identityArg.domain || body.sessionId !== identityArg.sessionId || body.archiveId !== identityArg.archiveId ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex domain head identity mismatch.', ); } } private async commitDomainHead( leaseArg: IFlexHeldLease, currentArg: TStoredDomainHead | null, nextArg: IFlexDomainHeadDocument, identityArg: IFlexDomainIdentity, ): Promise { await this.commitHeadMutation({ lease: leaseArg, collection: FlexDomainHeadModel.collection.mongoDbCollection, current: currentArg, next: nextArg, selector: currentArg ? { _id: currentArg._id, id: currentArg.id, controllerId: this.controllerId, storageKey: identityArg.storageKey, domain: identityArg.domain, revision: currentArg.revision, _smartdataRevision: smartRevisionSelector(currentArg), } : { id: nextArg.id }, readCurrent: (options) => this.readDomainHead(identityArg, options), project: domainHeadBody, targetName: `domain:${nextArg.id}`, }); } private async deleteDomainHead( leaseArg: IFlexHeldLease, headArg: TStoredDomainHead, identityArg: IFlexDomainIdentity, ): Promise { await this.commitHeadMutation({ lease: leaseArg, collection: FlexDomainHeadModel.collection.mongoDbCollection, current: headArg, next: null, selector: { _id: headArg._id, id: headArg.id, controllerId: this.controllerId, storageKey: identityArg.storageKey, domain: identityArg.domain, revision: headArg.revision, _smartdataRevision: smartRevisionSelector(headArg), }, readCurrent: (options) => this.readDomainHead(identityArg, options), project: domainHeadBody, targetName: `domain:${headArg.id}`, }); await this.cleanupPrivateByCandidate(headArg.privateCandidate).catch(() => undefined); if (headArg.domain === 'projection') { await this.cleanupProjectionSourceByCandidate( headArg.projectionSource!.candidateId, ).catch(() => undefined); } } private async loadPrivateCandidate( storageKeyArg: string, manifestArg: IFlexPrivateCandidateManifest, validateArg: (valueArg: unknown) => TSnapshot | Promise, abortSignalArg?: AbortSignal, sourceArg?: IFlexPrivateDomainSource, ): Promise { abortSignalArg?.throwIfAborted(); const candidateModel = await FlexPrivateCandidateModel.getInstance({ id: privateCandidateDocumentId(manifestArg.candidateId), controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }); if (!candidateModel) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Flex private candidate is missing.'); } const candidate = projectPrivateCandidate(candidateModel); assertFlexPrivateCandidateDocument(candidate); if ( candidate.id !== privateCandidateDocumentId(manifestArg.candidateId) || candidate.controllerId !== this.controllerId || candidate.scopeId !== storageKeyArg || candidate.candidateId !== manifestArg.candidateId || !privateChunkManifestsEqual(candidate.chunks, manifestArg.chunks) || candidate.sha256 !== manifestArg.sha256 || candidate.bytes !== manifestArg.bytes || candidate.revision !== manifestArg.revision || (sourceArg && ( candidate.domain !== sourceArg.identity.domain || candidate.incarnationId !== sourceArg.incarnationId || candidate.sessionId !== sourceArg.identity.sessionId || candidate.archiveId !== sourceArg.identity.archiveId )) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Flex private candidate mismatch.'); } const chunkModels = await FlexPrivateChunkModel.getInstances({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }); if (chunkModels.length !== manifestArg.chunks.length) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex private candidate has an unexpected chunk set.', ); } const chunksById = new Map(chunkModels.map((model) => [model.id, model])); const chunks: Buffer[] = []; for (const chunkManifest of manifestArg.chunks) { abortSignalArg?.throwIfAborted(); const model = chunksById.get(chunkManifest.id); if (!model) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Flex private chunk is missing.'); } const document = projectPrivateChunk(model); assertFlexPrivateChunkDocument(document); const bytes = Buffer.from(document.data, 'base64'); if ( document.id !== privateChunkDocumentId(manifestArg.candidateId, chunkManifest.index) || bytes.toString('base64') !== document.data || document.controllerId !== this.controllerId || document.scopeId !== storageKeyArg || document.candidateId !== manifestArg.candidateId || document.revision !== manifestArg.revision || document.createdAt !== candidate.createdAt || (sourceArg && ( document.domain !== sourceArg.identity.domain || document.incarnationId !== sourceArg.incarnationId || document.sessionId !== sourceArg.identity.sessionId || document.archiveId !== sourceArg.identity.archiveId )) || document.index !== chunkManifest.index || document.sha256 !== chunkManifest.sha256 || document.bytes !== chunkManifest.bytes || bytes.byteLength !== chunkManifest.bytes || sha256(bytes) !== chunkManifest.sha256 ) { bytes.fill(0); throw new plugins.flexharness.FlexHarnessStoreFormatError('Flex private chunk mismatch.'); } chunks.push(bytes); } const complete = Buffer.concat(chunks, manifestArg.bytes); for (const chunk of chunks) chunk.fill(0); try { if (complete.byteLength !== manifestArg.bytes || sha256(complete) !== manifestArg.sha256) { throw new plugins.flexharness.FlexHarnessStoreFormatError('Flex private snapshot hash mismatch.'); } let parsed: unknown; try { parsed = JSON.parse(complete.toString('utf8')) as unknown; } catch (errorArg) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex private snapshot is not JSON.', { cause: errorArg }, ); } abortSignalArg?.throwIfAborted(); try { return await validateArg(parsed); } catch (errorArg) { if (errorArg instanceof plugins.flexharness.FlexHarnessStoreFormatError) throw errorArg; throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex private snapshot has an invalid schema.', { cause: errorArg }, ); } } finally { complete.fill(0); } } private async writeAndVerifyPrivate( candidateArg: IFlexPrivateCandidateDocument, chunksArg: IFlexPrivateChunkDocument[], abortSignalArg?: AbortSignal, ): Promise { await insertOrdinary(FlexPrivateCandidateModel, candidateArg); abortSignalArg?.throwIfAborted(); await inBatches(chunksArg, async (chunk) => { abortSignalArg?.throwIfAborted(); await insertOrdinary(FlexPrivateChunkModel, chunk); abortSignalArg?.throwIfAborted(); }); abortSignalArg?.throwIfAborted(); const storedCandidate = await FlexPrivateCandidateModel.getInstance({ id: candidateArg.id }); if (!storedCandidate) throw new Error('Flex private candidate verification failed.'); const projectedCandidate = projectPrivateCandidate(storedCandidate); assertFlexPrivateCandidateDocument(projectedCandidate); if (canonicalJson(projectedCandidate) !== canonicalJson(candidateArg)) { throw new Error('Flex private candidate verification failed.'); } const storedChunks = await FlexPrivateChunkModel.getInstances({ controllerId: candidateArg.controllerId, scopeId: candidateArg.scopeId, candidateId: candidateArg.candidateId, }); if (storedChunks.length !== chunksArg.length) { throw new Error('Flex private chunk verification failed.'); } const expectedById = new Map(chunksArg.map((chunk) => [chunk.id, chunk])); for (const stored of storedChunks) { abortSignalArg?.throwIfAborted(); const projected = projectPrivateChunk(stored); assertFlexPrivateChunkDocument(projected); const expected = expectedById.get(projected.id); if (!expected || canonicalJson(projected) !== canonicalJson(expected)) { throw new Error('Flex private chunk verification failed.'); } } } private async writeAndVerifyProjectionSource( buildArg: IFlexProjectionSourceBuild, ): Promise { await insertOrdinary(FlexProjectionSourceModel, buildArg.source); await inBatches( buildArg.messages, (message) => insertOrdinary(FlexProjectionMessageRecordModel, message), ); const storedSource = await FlexProjectionSourceModel.getInstance({ id: buildArg.source.id }); if (!storedSource) throw new Error('Flex projection source verification failed.'); const source = projectProjectionSource(storedSource); assertFlexProjectionSourceDocument(source); if (canonicalJson(source) !== canonicalJson(buildArg.source)) { throw new Error('Flex projection source verification failed.'); } await inBatches(buildArg.messages, async (expected) => { const stored = await FlexProjectionMessageRecordModel.getInstance({ id: expected.id }); if (!stored) throw new Error('Flex projection message verification failed.'); const message = projectProjectionMessage(stored); assertFlexProjectionMessageRecordDocument(message); if (canonicalJson(message) !== canonicalJson(expected)) { throw new Error('Flex projection message verification failed.'); } }); } private async loadProjectionSource( headArg: TStoredDomainHead, ): Promise { if (headArg.domain !== 'projection' || !headArg.sessionId || !headArg.projectionSource) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection head does not contain a projection source.', ); } const model = await FlexProjectionSourceModel.getInstance({ id: projectionSourceDocumentId(headArg.projectionSource.candidateId), }); if (!model) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source is missing.', ); } const source = projectProjectionSource(model); assertFlexProjectionSourceDocument(source); const unhashedManifest: Omit = { candidateId: source.candidateId, ...(source.orderVersion === undefined ? {} : { orderVersion: source.orderVersion }), messageCount: source.messageCount, messagesTruncated: source.messagesTruncated, visibleDigest: source.visibleDigest, ...(source.latestMessage === undefined ? {} : { latestMessage: source.latestMessage }), }; const bytes = projectionSourceManifestBytes(unhashedManifest); const manifest: IFlexProjectionSourceManifest = { ...unhashedManifest, sha256: sha256(bytes), bytes: bytes.byteLength, }; if ( source.id !== projectionSourceDocumentId(headArg.projectionSource.candidateId) || source.controllerId !== this.controllerId || source.scopeId !== headArg.storageKey || source.sessionId !== headArg.sessionId || source.candidateId !== headArg.projectionSource.candidateId || source.incarnationId !== headArg.incarnationId || source.revision > headArg.revision || canonicalJson(manifest) !== canonicalJson(headArg.projectionSource) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source does not match its domain head.', ); } return source; } private async loadProjectionMessageRecord( sourceArg: IFlexProjectionSourceDocument, descriptorArg: IFlexProjectionMessageDescriptor, ): Promise { const model = await FlexProjectionMessageRecordModel.getInstance({ id: descriptorArg.id }); if (!model) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection message record is missing.', ); } const record = projectProjectionMessage(model); assertFlexProjectionMessageRecordDocument(record); const descriptor: IFlexProjectionMessageDescriptor = { id: record.id, index: record.index, ...(record.messageIndex === undefined ? {} : { messageIndex: record.messageIndex }), sha256: record.sha256, bytes: record.bytes, messageCreatedAt: record.messageCreatedAt, sessionId: record.sessionId, messageId: record.messageId, }; const payload = projectionMessagePayloadBytes(record.message, record.previous); if ( record.id !== projectionMessageDocumentId(sourceArg.candidateId, record.index) || record.controllerId !== this.controllerId || record.scopeId !== sourceArg.scopeId || record.candidateId !== sourceArg.candidateId || record.sessionId !== sourceArg.sessionId || record.incarnationId !== sourceArg.incarnationId || record.revision !== sourceArg.revision || canonicalJson(descriptor) !== canonicalJson(descriptorArg) || payload.byteLength !== record.bytes || sha256(payload) !== record.sha256 || (record.index === 0 ? record.previous !== undefined : !record.previous || record.previous.index !== record.index - 1 || record.previous.id !== projectionMessageDocumentId( sourceArg.candidateId, record.index - 1, ) ) ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection message chain is invalid.', ); } return record; } private async buildPublicProjectionFromSources( storageKeyArg: string, candidateIdArg: string, publicationRevisionArg: number, scopeArg: IFlexLoadedDomain, sourcesArg: IFlexProjectionSourceDocument[], createdAtArg = new Date().toISOString(), ): Promise { const retainedSessions = retainPublicSessions(scopeArg.snapshot.sessions); const retainedSessionIds = new Set(retainedSessions.map((session) => session.sessionId)); const sessions: IFlexPublicSessionRecordDocument[] = retainedSessions.map((session) => ({ id: publicSessionDocumentId(candidateIdArg, session.sessionId), controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: publicationRevisionArg, session: cloneJson(session), createdAt: createdAtArg, })); const sourceBySession = new Map(sourcesArg.map((source) => [source.sessionId, source])); if (sourceBySession.size !== sourcesArg.length) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source set contains duplicate sessions.', ); } const sources = retainedSessions.flatMap((session) => { const source = sourceBySession.get(session.sessionId); return source && retainedSessionIds.has(source.sessionId) ? [source] : []; }); const entries = sources.flatMap((source) => source.latestMessage ? [{ source, descriptor: source.latestMessage }] : []); const messages: IFlexPublicMessageRecordDocument[] = []; let messageBytes = 0; let messageBytesTruncated = false; let oversizedMessagesReplaced = false; while (entries.length > 0 && messages.length < flexPublicMessageLimit) { let newestIndex = 0; for (let index = 1; index < entries.length; index++) { if (compareNewestProjectionDescriptor( entries[index]!.descriptor, entries[newestIndex]!.descriptor, ) < 0) newestIndex = index; } const entry = entries[newestIndex]!; const sourceRecord = await this.loadProjectionMessageRecord(entry.source, entry.descriptor); if (sourceRecord.messageIndex === undefined) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source message order is unavailable.', ); } const messageIndex = sourceRecord.messageIndex; const createRecord = (messageArg: TFlexMessage): IFlexPublicMessageRecordDocument => ({ id: publicMessageDocumentId(candidateIdArg, messageArg.sessionId, messageArg.messageId), controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: candidateIdArg, revision: publicationRevisionArg, messageIndex, message: cloneJson(messageArg), createdAt: createdAtArg, }); let publicRecord = createRecord(sourceRecord.message); let publicRecordBytes = serializedBytes(publicRecord).byteLength; if (publicRecordBytes > flexPublicSingleMessageBytesLimit) { publicRecord = createRecord(placeholderMessage(sourceRecord.message)); publicRecordBytes = serializedBytes(publicRecord).byteLength; oversizedMessagesReplaced = true; if (publicRecordBytes > flexPublicSingleMessageBytesLimit) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex public message metadata exceeds its projection limit.', ); } } if (messageBytes + publicRecordBytes > flexPublicMessageBytesLimit) { messageBytesTruncated = true; break; } messageBytes += publicRecordBytes; messages.push(publicRecord); if (sourceRecord.previous) { entries[newestIndex] = { source: entry.source, descriptor: sourceRecord.previous }; } else { entries.splice(newestIndex, 1); } } const sessionsTruncated = retainedSessions.length < scopeArg.snapshot.sessions.length; let availableMessageCount = 0; for (const source of sources) { availableMessageCount = Math.min( flexPublicMessageLimit + 1, availableMessageCount + source.messageCount, ); } const messagesTruncated = sessionsTruncated || sources.some((source) => source.messagesTruncated) || messages.length < availableMessageCount || oversizedMessagesReplaced; const sessionRecords = sessions.map(recordManifest); if (new Set(messages.map((record) => ( `${record.message.sessionId}\0${record.messageIndex}` ))).size !== messages.length) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex public projection contains duplicate message indexes.', ); } const messageRecords = messages.map(recordManifest); const visibleDigest = sha256(serializedBytes({ sessions: sessions.map((record) => record.session), messages: messages.map((record) => ({ messageIndex: record.messageIndex, message: record.message, })), sessionsTruncated, messagesTruncated, messageBytesTruncated, })); const projectionSources: IFlexPublicProjectionSource[] = sources.map((source) => ({ sessionId: source.sessionId, incarnationId: source.incarnationId, revision: source.revision, })); const unhashedManifest: Omit = { candidateId: candidateIdArg, revision: publicationRevisionArg, sessionRecords, messageRecords, sessionsTruncated, messagesTruncated, messageBytesTruncated, visibleDigest, scopeSource: { incarnationId: scopeArg.head.incarnationId, revision: scopeArg.head.revision, }, projectionSources, }; const candidateBytes = publicCandidateManifestBytes(unhashedManifest); const candidate: IFlexPublicCandidateDocument = { id: publicCandidateDocumentId(candidateIdArg), controllerId: this.controllerId, scopeId: storageKeyArg, ...unhashedManifest, sha256: sha256(candidateBytes), bytes: candidateBytes.byteLength, createdAt: createdAtArg, }; assertFlexPublicCandidateDocument(candidate); for (const session of sessions) assertFlexPublicSessionRecordDocument(session); for (const message of messages) assertFlexPublicMessageRecordDocument(message); return { candidate, sessions, messages }; } private async publishPublicProjection( storageKeyArg: string, leaseArg: IFlexHeldLease, ): Promise { const scope = await this.loadDomainDirect( { storageKey: storageKeyArg, domain: 'scope' }, (value) => this.validateScopeSnapshot(value), ); if (!scope) return; const retainedSessionIds = retainNewest( scope.snapshot.sessions, flexPublicSessionLimit, compareNewestSession, ) .map((session) => session.sessionId); const projectionHeads: TStoredDomainHead[] = []; if (retainedSessionIds.length > 0) { const cursor = FlexDomainHeadModel.collection.mongoDbCollection.find({ controllerId: this.controllerId, storageKey: storageKeyArg, domain: 'projection', sessionId: { $in: retainedSessionIds }, }).limit(retainedSessionIds.length + 1); try { projectionHeads.push(...await cursor.toArray() as unknown as TStoredDomainHead[]); } finally { await cursor.close(); } if (projectionHeads.length > retainedSessionIds.length) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection head set exceeds the retained session set.', ); } } const sources: IFlexProjectionSourceDocument[] = []; for (let index = 0; index < projectionHeads.length; index += 32) { const batch = projectionHeads.slice(index, index + 32); sources.push(...await Promise.all(batch.map(async (head) => { if (!head.sessionId) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection head is missing its session identity.', ); } const identity: IFlexDomainIdentity = { storageKey: storageKeyArg, domain: 'projection', sessionId: head.sessionId, }; this.assertDomainHeadIdentity(head, identity); const source = await this.loadProjectionSource(head); if (source.orderVersion !== 1) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex projection source ordering migration is required before publication.', ); } return source; }))); } const currentHead = await this.readPublicHead(storageKeyArg); const publicationRevision = (currentHead?.revision ?? 0) + 1; const candidateId = randomCandidateId(); const build = await this.buildPublicProjectionFromSources( storageKeyArg, candidateId, publicationRevision, scope, sources, ); const currentManifest = currentHead?.publicCandidates[0]; if ( currentManifest?.visibleDigest === build.candidate.visibleDigest && await this.verifyPublicCandidate(storageKeyArg, currentManifest).catch(() => false) ) return; const manifest: IFlexPublicCandidateManifest = { candidateId: build.candidate.candidateId, revision: build.candidate.revision, sha256: build.candidate.sha256, bytes: build.candidate.bytes, sessionRecords: build.candidate.sessionRecords, messageRecords: build.candidate.messageRecords, sessionsTruncated: build.candidate.sessionsTruncated, messagesTruncated: build.candidate.messagesTruncated, messageBytesTruncated: build.candidate.messageBytesTruncated, visibleDigest: build.candidate.visibleDigest, scopeSource: build.candidate.scopeSource, projectionSources: build.candidate.projectionSources, }; const nextHead: IFlexPublicHeadDocument = { id: flexPublicHeadId(this.controllerId, storageKeyArg), controllerId: this.controllerId, storageKey: storageKeyArg, revision: publicationRevision, currentPublicCandidateId: candidateId, publicCandidates: [ manifest, ...(currentHead?.publicCandidates ?? []).slice(0, 3), ], }; assertFlexPublicHeadDocument(nextHead); this.activeCandidateIds.add(candidateId); let committed = false; try { await this.writeAndVerifyPublic(build); await this.commitPublicHead(leaseArg, currentHead, nextHead, storageKeyArg); committed = true; if (currentHead) { const retained = new Set(nextHead.publicCandidates.map((entry) => entry.candidateId)); for (const evicted of currentHead.publicCandidates) { if (!retained.has(evicted.candidateId)) { await this.cleanupPublicByCandidate(evicted, storageKeyArg).catch(() => undefined); } } } } finally { this.activeCandidateIds.delete(candidateId); if (!committed) this.queueOrphanMaintenance(); } } private async readPublicHead( storageKeyArg: string, optionsArg?: IFlexReadOptions, ): Promise { const id = flexPublicHeadId(this.controllerId, storageKeyArg); const rawHead = await this.database.mongoDb.collection('flex_public_heads').findOne( { id }, optionsArg, ); const head = rawHead as unknown as TStoredPublicHead | null; if (head) { const body = publicHeadBody(head); assertFlexPublicHeadDocument(body); if ( body.id !== id || body.controllerId !== this.controllerId || body.storageKey !== storageKeyArg ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex public head identity mismatch.', ); } } return head; } private async commitPublicHead( leaseArg: IFlexHeldLease, currentArg: TStoredPublicHead | null, nextArg: IFlexPublicHeadDocument, storageKeyArg: string, ): Promise { await this.commitHeadMutation({ lease: leaseArg, collection: this.database.mongoDb.collection('flex_public_heads'), current: currentArg, next: nextArg, selector: currentArg ? { _id: currentArg._id, id: currentArg.id, controllerId: this.controllerId, storageKey: storageKeyArg, revision: currentArg.revision, _smartdataRevision: smartRevisionSelector(currentArg), } : { id: nextArg.id }, readCurrent: (options) => this.readPublicHead(storageKeyArg, options), project: publicHeadBody, targetName: `public:${nextArg.id}`, }); } private async deletePublicHead( leaseArg: IFlexHeldLease, headArg: TStoredPublicHead, storageKeyArg: string, ): Promise { await this.commitHeadMutation({ lease: leaseArg, collection: this.database.mongoDb.collection('flex_public_heads'), current: headArg, next: null, selector: { _id: headArg._id, id: headArg.id, controllerId: this.controllerId, storageKey: storageKeyArg, revision: headArg.revision, _smartdataRevision: smartRevisionSelector(headArg), }, readCurrent: (options) => this.readPublicHead(storageKeyArg, options), project: publicHeadBody, targetName: `public:${headArg.id}`, }); } private async writeAndVerifyPublic(buildArg: IFlexPublicProjectionBuild): Promise { await insertOrdinary(FlexPublicCandidateModel, buildArg.candidate); await inBatches(buildArg.sessions, (session) => insertOrdinary(FlexPublicSessionRecordModel, session)); await inBatches(buildArg.messages, (message) => insertOrdinary(FlexPublicMessageRecordModel, message)); const manifest: IFlexPublicCandidateManifest = { candidateId: buildArg.candidate.candidateId, revision: buildArg.candidate.revision, sha256: buildArg.candidate.sha256, bytes: buildArg.candidate.bytes, sessionRecords: buildArg.candidate.sessionRecords, messageRecords: buildArg.candidate.messageRecords, sessionsTruncated: buildArg.candidate.sessionsTruncated, messagesTruncated: buildArg.candidate.messagesTruncated, messageBytesTruncated: buildArg.candidate.messageBytesTruncated, visibleDigest: buildArg.candidate.visibleDigest, scopeSource: buildArg.candidate.scopeSource, projectionSources: buildArg.candidate.projectionSources, }; if (!await this.verifyPublicCandidate(buildArg.candidate.scopeId, manifest)) { throw new Error('Flex public candidate verification failed.'); } } private async verifyPublicCandidate( storageKeyArg: string, manifestArg: IFlexPublicCandidateManifest, ): Promise { const model = await FlexPublicCandidateModel.getInstance({ id: publicCandidateDocumentId(manifestArg.candidateId), controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }); if (!model) return false; const candidate = projectPublicCandidate(model); assertFlexPublicCandidateDocument(candidate); const projectedManifest: IFlexPublicCandidateManifest = { candidateId: candidate.candidateId, revision: candidate.revision, sha256: candidate.sha256, bytes: candidate.bytes, sessionRecords: candidate.sessionRecords, messageRecords: candidate.messageRecords, sessionsTruncated: candidate.sessionsTruncated, messagesTruncated: candidate.messagesTruncated, messageBytesTruncated: candidate.messageBytesTruncated, visibleDigest: candidate.visibleDigest, scopeSource: candidate.scopeSource, projectionSources: candidate.projectionSources, }; const unhashed = { candidateId: candidate.candidateId, revision: candidate.revision, sessionRecords: candidate.sessionRecords, messageRecords: candidate.messageRecords, sessionsTruncated: candidate.sessionsTruncated, messagesTruncated: candidate.messagesTruncated, messageBytesTruncated: candidate.messageBytesTruncated, visibleDigest: candidate.visibleDigest, scopeSource: candidate.scopeSource, projectionSources: candidate.projectionSources, }; const bytes = publicCandidateManifestBytes(unhashed); if ( candidate.id !== publicCandidateDocumentId(manifestArg.candidateId) || candidate.controllerId !== this.controllerId || candidate.scopeId !== storageKeyArg || !publicManifestsEqual(projectedManifest, manifestArg) || candidate.bytes !== bytes.byteLength || candidate.sha256 !== sha256(bytes) ) return false; const [sessionModels, messageModels] = await Promise.all([ FlexPublicSessionRecordModel.getInstances({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }), FlexPublicMessageRecordModel.getInstances({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }), ]); if ( sessionModels.length !== manifestArg.sessionRecords.length || messageModels.length !== manifestArg.messageRecords.length ) return false; const manifests = new Map([ ...manifestArg.sessionRecords, ...manifestArg.messageRecords, ].map((manifest) => [manifest.id, manifest])); for (const sessionModel of sessionModels) { const document = projectPublicSession(sessionModel); assertFlexPublicSessionRecordDocument(document); if ( document.controllerId !== this.controllerId || document.scopeId !== storageKeyArg || document.candidateId !== manifestArg.candidateId || document.revision !== manifestArg.revision || canonicalJson(recordManifest(document)) !== canonicalJson(manifests.get(document.id)) ) return false; } for (const messageModel of messageModels) { const document = projectPublicMessage(messageModel); assertFlexPublicMessageRecordDocument(document); if ( document.controllerId !== this.controllerId || document.scopeId !== storageKeyArg || document.candidateId !== manifestArg.candidateId || document.revision !== manifestArg.revision || canonicalJson(recordManifest(document)) !== canonicalJson(manifests.get(document.id)) ) return false; } return true; } private async commitHeadMutation( optionsArg: IFlexHeadMutationOptions, ): Promise { return this.runLeaseTransition( optionsArg.lease, (abortSignal, timeoutMs) => this.commitHeadMutationWithLease( optionsArg, abortSignal, timeoutMs, ), ); } private async commitHeadMutationWithLease( optionsArg: IFlexHeadMutationOptions, abortSignalArg: AbortSignal, timeoutMsArg: number, ): Promise { const deadline = Date.now() + timeoutMsArg; const session = this.database.startSession(); let callbackError: unknown; try { await session.withTransaction(async () => { abortSignalArg.throwIfAborted(); const now = Date.now(); const leasePostimage = await FlexWriterLeaseModel.collection.mongoDbCollection.findOneAndUpdate( { id: flexWriterLeaseId(this.controllerId, optionsArg.lease.storageKey), controllerId: this.controllerId, storageKey: optionsArg.lease.storageKey, ownerToken: optionsArg.lease.ownerToken, epoch: optionsArg.lease.epoch, state: 'active', expiresAt: { $gt: now }, }, { $set: { expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), _smartdataRevision: plugins.crypto.randomUUID(), }, }, { session, returnDocument: 'after', includeResultMetadata: false, }, ); abortSignalArg.throwIfAborted(); if (!leasePostimage) throw new FlexLeaseLostError(optionsArg.lease.storageKey); if (!optionsArg.current && optionsArg.next) { try { await optionsArg.collection.insertOne({ ...optionsArg.next, _smartdataRevision: plugins.crypto.randomUUID(), }, { session }); abortSignalArg.throwIfAborted(); } catch (errorArg) { callbackError = errorArg; throw errorArg; } return; } if (optionsArg.current && optionsArg.next) { const postimage = await optionsArg.collection.findOneAndReplace( optionsArg.selector, { ...optionsArg.next, _smartdataRevision: plugins.crypto.randomUUID() }, { session, upsert: false, returnDocument: 'after', includeResultMetadata: false, }, ); abortSignalArg.throwIfAborted(); if (!postimage) { throw new FlexHeadCasError( (optionsArg.current as { revision?: number }).revision ?? 0, ); } return; } if (optionsArg.current && !optionsArg.next) { const deletion = await optionsArg.collection.deleteOne(optionsArg.selector, { session, }); abortSignalArg.throwIfAborted(); if (deletion.deletedCount !== 1) { throw new FlexHeadCasError( (optionsArg.current as { revision?: number }).revision ?? 0, ); } } }, { timeoutMS: Math.max(1, deadline - Date.now()), maxCommitTimeMS: Math.max(1, deadline - Date.now()), }); this.throwIfFatal(); await this.verifyHeldLease( optionsArg.lease, false, abortSignalArg, false, Math.max(1, deadline - Date.now()), ); return; } catch (errorArg) { this.throwIfFatal(); const operationError = callbackError ?? errorArg; const reconciliationOptions = { signal: abortSignalArg, timeoutMS: Math.max(1, deadline - Date.now()), }; let leaseCurrent: TStoredLease | null; let targetCurrent: ({ _id: unknown; _smartdataRevision?: string } & THead) | null; try { [leaseCurrent, targetCurrent] = await Promise.all([ FlexWriterLeaseModel.collection.mongoDbCollection.findOne({ id: flexWriterLeaseId(this.controllerId, optionsArg.lease.storageKey), }, reconciliationOptions) as unknown as Promise, optionsArg.readCurrent(reconciliationOptions), ]); } catch (readbackError) { throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', `Flex ${optionsArg.targetName} mutation failed and readback was unavailable.`, { cause: new AggregateError([operationError, readbackError]) }, ); } const uncertainTransition = optionsArg.lease.uncertainTransition; if ( leaseCurrent && uncertainTransition && leaseTransitionImagesEqual(leaseCurrent, uncertainTransition.postimage) ) { optionsArg.lease.ownerToken = leaseCurrent.ownerToken; optionsArg.lease.epoch = leaseCurrent.epoch; optionsArg.lease.expiresAt = leaseCurrent.expiresAt; optionsArg.lease.uncertainTransition = undefined; } const sameLease = !!leaseCurrent && leaseCurrent.controllerId === this.controllerId && leaseCurrent.storageKey === optionsArg.lease.storageKey && leaseCurrent.ownerToken === optionsArg.lease.ownerToken && leaseCurrent.epoch === optionsArg.lease.epoch && leaseCurrent.state === 'active' && leaseCurrent.expiresAt > Date.now(); const requestedCommitted = optionsArg.next ? !!targetCurrent && canonicalJson(optionsArg.project(targetCurrent)) === canonicalJson(optionsArg.project(optionsArg.next)) : targetCurrent === null; if (sameLease && requestedCommitted) { optionsArg.lease.expiresAt = leaseCurrent!.expiresAt; return; } if (!sameLease) { if ( uncertainTransition && leaseCurrent && leaseTransitionImagesEqual(leaseCurrent, uncertainTransition.preimage) ) throw operationError; throw this.markLeaseLost(optionsArg.lease); } if (errorArg instanceof FlexHeadCasError) { throw new FlexHeadCasError((targetCurrent as { revision?: number } | null)?.revision ?? 0); } if (targetCurrent && optionsArg.current) { const stillCurrent = canonicalJson(optionsArg.project(targetCurrent)) === canonicalJson(optionsArg.project(optionsArg.current)); if (stillCurrent) throw operationError; } if (!targetCurrent && !optionsArg.current) throw operationError; throw new FlexHeadCasError((targetCurrent as { revision?: number } | null)?.revision ?? 0); } finally { await session.endSession(); } } private async deleteScopeRecords( leaseArg: IFlexHeldLease, storageKeyArg: string, ): Promise { const collections = [ FlexPrivateChunkModel.collection.mongoDbCollection, FlexPrivateCandidateModel.collection.mongoDbCollection, FlexPublicSessionRecordModel.collection.mongoDbCollection, FlexPublicMessageRecordModel.collection.mongoDbCollection, FlexPublicCandidateModel.collection.mongoDbCollection, FlexProjectionMessageRecordModel.collection.mongoDbCollection, FlexProjectionSourceModel.collection.mongoDbCollection, ]; for (const collection of collections) { let lastId: TFlexObjectId | undefined; while (true) { const cursor = collection.find({ controllerId: this.controllerId, scopeId: storageKeyArg, ...(lastId ? { _id: { $gt: lastId } } : {}), }, { projection: { _id: 1 } }).sort({ _id: 1 }).limit(flexMaintenancePageLimit); let documents: Array<{ _id: TFlexObjectId }>; try { documents = await cursor.toArray() as Array<{ _id: TFlexObjectId }>; } finally { await cursor.close(); } if (documents.length === 0) break; const documentIds = documents.map((document) => document._id); await this.deleteScopeRecordBatch(leaseArg, storageKeyArg, collection, documentIds); const last = documents.at(-1); if (!last || documents.length < flexMaintenancePageLimit) break; lastId = last._id; } } } private async deleteScopeRecordBatch( leaseArg: IFlexHeldLease, storageKeyArg: string, collectionArg: typeof FlexPrivateChunkModel.collection.mongoDbCollection, documentIdsArg: TFlexObjectId[], ): Promise { return this.runLeaseTransition( leaseArg, (abortSignal, timeoutMs) => this.deleteScopeRecordBatchWithLease( leaseArg, storageKeyArg, collectionArg, documentIdsArg, abortSignal, timeoutMs, ), ); } private async deleteScopeRecordBatchWithLease( leaseArg: IFlexHeldLease, storageKeyArg: string, collectionArg: typeof FlexPrivateChunkModel.collection.mongoDbCollection, documentIdsArg: TFlexObjectId[], abortSignalArg: AbortSignal, timeoutMsArg: number, ): Promise { const deadline = Date.now() + timeoutMsArg; const session = this.database.startSession(); try { await session.withTransaction(async () => { abortSignalArg.throwIfAborted(); const now = Date.now(); const leasePostimage = await FlexWriterLeaseModel.collection.mongoDbCollection.findOneAndUpdate( { id: flexWriterLeaseId(this.controllerId, storageKeyArg), controllerId: this.controllerId, storageKey: storageKeyArg, ownerToken: leaseArg.ownerToken, epoch: leaseArg.epoch, state: 'active', expiresAt: { $gt: now }, }, { $set: { expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), _smartdataRevision: plugins.crypto.randomUUID(), }, }, { session, returnDocument: 'after', includeResultMetadata: false, }, ); abortSignalArg.throwIfAborted(); if (!leasePostimage) throw new FlexLeaseLostError(storageKeyArg); await collectionArg.deleteMany({ controllerId: this.controllerId, scopeId: storageKeyArg, _id: { $in: documentIdsArg }, }, { session }); abortSignalArg.throwIfAborted(); }, { timeoutMS: Math.max(1, deadline - Date.now()), maxCommitTimeMS: Math.max(1, deadline - Date.now()), }); this.throwIfFatal(); await this.verifyHeldLease( leaseArg, false, abortSignalArg, false, Math.max(1, deadline - Date.now()), ); } catch (errorArg) { this.throwIfFatal(); const [currentLease, remaining] = await Promise.all([ FlexWriterLeaseModel.collection.mongoDbCollection.findOne({ id: flexWriterLeaseId(this.controllerId, storageKeyArg), }, { signal: abortSignalArg, timeoutMS: Math.max(1, deadline - Date.now()), }) as unknown as Promise, collectionArg.countDocuments({ _id: { $in: documentIdsArg } }, { signal: abortSignalArg, timeoutMS: Math.max(1, deadline - Date.now()), }), ]); const uncertainTransition = leaseArg.uncertainTransition; if ( currentLease && uncertainTransition && leaseTransitionImagesEqual(currentLease, uncertainTransition.postimage) ) { leaseArg.ownerToken = currentLease.ownerToken; leaseArg.epoch = currentLease.epoch; leaseArg.expiresAt = currentLease.expiresAt; leaseArg.uncertainTransition = undefined; } const sameLease = !!currentLease && currentLease.controllerId === this.controllerId && currentLease.storageKey === storageKeyArg && currentLease.ownerToken === leaseArg.ownerToken && currentLease.epoch === leaseArg.epoch && currentLease.state === 'active' && currentLease.expiresAt > Date.now(); if (sameLease && remaining === 0) { leaseArg.expiresAt = currentLease.expiresAt; return; } if (!sameLease) { if ( uncertainTransition && currentLease && leaseTransitionImagesEqual(currentLease, uncertainTransition.preimage) ) throw errorArg; throw this.markLeaseLost(leaseArg); } throw errorArg; } finally { await session.endSession(); } } private async runStorage( storageKeyArg: string, operationArg: (leaseArg: IFlexHeldLease) => Promise, ): Promise { assertIdentifier(storageKeyArg, 'storageKey'); this.throwIfFatal(); await this.init(); if (this.closed) throw new Error('The Flex store is closed.'); const prior = this.storageQueues.get(storageKeyArg) ?? Promise.resolve(); const operation = prior.catch(() => undefined).then(async () => { this.throwIfFatal(); if (this.closed) throw new Error('The Flex store is closed.'); const lease = await this.ensureLease(storageKeyArg); if (this.closed) { await this.releaseHeldLease(lease, this.closeDrainTimeoutMs); throw new Error('The Flex store is closed.'); } return operationArg(lease); }); const settled = operation.then(() => undefined, () => undefined); this.storageQueues.set(storageKeyArg, settled); void settled.then(() => { if (this.storageQueues.get(storageKeyArg) === settled) { this.storageQueues.delete(storageKeyArg); } }); return operation; } private async ensureLease(storageKeyArg: string): Promise { this.throwIfFatal(); if (this.closed) throw new Error('The Flex store is closed.'); const held = this.heldLeases.get(storageKeyArg); if (held) { if (held.lost || held.releasing) throw new FlexLeaseLostError(storageKeyArg); await this.runLeaseTransition(held, (abortSignal, timeoutMs) => this.verifyHeldLease( held, true, abortSignal, false, timeoutMs, )); return held; } const id = flexWriterLeaseId(this.controllerId, storageKeyArg); const ownerToken = randomOwnerToken(); while (true) { if (this.closed) throw new Error('The Flex store is closed.'); const now = Date.now(); const current = await FlexWriterLeaseModel.exact.findStoredOne({ id }); if (!current) { const document: IFlexWriterLeaseDocument = { id, controllerId: this.controllerId, storageKey: storageKeyArg, ownerToken, epoch: 1, state: 'active', expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), }; assertFlexWriterLeaseDocument(document); const insertion = await FlexWriterLeaseModel.exact.insert(document); if (insertion.status === 'conflict') continue; const lease: IFlexHeldLease = { storageKey: storageKeyArg, ownerToken, epoch: 1, expiresAt: document.expiresAt, lost: false, releasing: false, }; this.heldLeases.set(storageKeyArg, lease); if (this.closed) { await this.releaseHeldLease(lease, this.closeDrainTimeoutMs); throw new Error('The Flex store is closed.'); } this.startLeaseHeartbeat(); return lease; } const body = leaseBody(current); assertFlexWriterLeaseDocument(body); if ( body.id !== id || body.controllerId !== this.controllerId || body.storageKey !== storageKeyArg ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex writer lease identity mismatch.', ); } if (body.state === 'active' && body.expiresAt > now) { throw new Error(`Storage key "${storageKeyArg}" has an active foreign Flex writer lease.`); } const transition = await FlexWriterLeaseModel.exact.transition({ current, change: (model) => { model.ownerToken = ownerToken; model.epoch = current.epoch + 1; model.state = 'active'; model.expiresAt = now + flexLeaseDurationMs; model.updatedAt = new Date(now).toISOString(); }, }); if (transition.status === 'concurrent_change') continue; const lease: IFlexHeldLease = { storageKey: storageKeyArg, ownerToken, epoch: current.epoch + 1, expiresAt: now + flexLeaseDurationMs, lost: false, releasing: false, }; this.heldLeases.set(storageKeyArg, lease); if (this.closed) { await this.releaseHeldLease(lease, this.closeDrainTimeoutMs); throw new Error('The Flex store is closed.'); } this.startLeaseHeartbeat(); return lease; } } private async verifyHeldLease( leaseArg: IFlexHeldLease, renewIfNeededArg: boolean, abortSignalArg?: AbortSignal, forceRenewalArg = false, timeoutMsArg = flexMaintenanceOperationTimeoutMs, ): Promise { if (leaseArg.lost) throw new FlexLeaseLostError(leaseArg.storageKey); this.throwIfFatal(); const id = flexWriterLeaseId(this.controllerId, leaseArg.storageKey); const collection = FlexWriterLeaseModel.collection.mongoDbCollection; const deadline = Date.now() + timeoutMsArg; const operationOptions = () => ({ timeoutMS: Math.max(1, deadline - Date.now()), ...(abortSignalArg ? { signal: abortSignalArg } : {}), }); while (true) { abortSignalArg?.throwIfAborted(); this.throwIfFatal(); if (Date.now() >= deadline) { throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', `Flex writer lease transition for "${leaseArg.storageKey}" exceeded its deadline.`, ); } const uncertain = leaseArg.uncertainTransition; if (uncertain) { let current: TStoredLease | null; try { current = await collection.findOne({ id }, operationOptions()) as unknown as TStoredLease | null; } catch (readbackError) { if (uncertain.postimage.expiresAt <= Date.now()) { throw this.markLeaseLost(leaseArg); } throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', `Flex writer lease transition for "${leaseArg.storageKey}" remains uncertain.`, { cause: readbackError }, ); } this.throwIfFatal(); if (uncertain.postimage.expiresAt <= Date.now()) { throw this.markLeaseLost(leaseArg); } if (current && leaseTransitionImagesEqual(current, uncertain.postimage)) { leaseArg.ownerToken = current.ownerToken; leaseArg.epoch = current.epoch; leaseArg.expiresAt = current.expiresAt; leaseArg.uncertainTransition = undefined; continue; } if (!current || !leaseTransitionImagesEqual(current, uncertain.preimage)) { throw this.markLeaseLost(leaseArg); } let transition: TStoredLease | null; try { transition = await collection.findOneAndReplace({ _id: uncertain.preimage._id, ...leaseBody(uncertain.preimage), _smartdataRevision: smartRevisionSelector(uncertain.preimage), }, { ...leaseBody(uncertain.postimage), _smartdataRevision: uncertain.postimage._smartdataRevision, }, { returnDocument: 'after', includeResultMetadata: false, ...operationOptions(), }) as unknown as TStoredLease | null; } catch { await waitForFlexStoreRetry(10, abortSignalArg); continue; } this.throwIfFatal(); if (transition && leaseTransitionImagesEqual(transition, uncertain.postimage)) { leaseArg.ownerToken = transition.ownerToken; leaseArg.epoch = transition.epoch; leaseArg.expiresAt = transition.expiresAt; leaseArg.uncertainTransition = undefined; continue; } if (transition && !leaseTransitionImagesEqual(transition, uncertain.preimage)) { throw this.markLeaseLost(leaseArg); } await waitForFlexStoreRetry(10, abortSignalArg); continue; } let current: TStoredLease | null; try { current = await collection.findOne( { id }, operationOptions(), ) as unknown as TStoredLease | null; } catch (readError) { if (leaseArg.expiresAt <= Date.now()) throw this.markLeaseLost(leaseArg); throw readError; } this.throwIfFatal(); if ( !current || current.controllerId !== this.controllerId || current.storageKey !== leaseArg.storageKey || current.ownerToken !== leaseArg.ownerToken || current.epoch !== leaseArg.epoch || current.state !== 'active' ) { throw this.markLeaseLost(leaseArg); } leaseArg.expiresAt = current.expiresAt; const now = Date.now(); const expired = current.expiresAt <= now; if ( !expired && ( !renewIfNeededArg || (!forceRenewalArg && current.expiresAt - now > flexLeaseHeartbeatMs) ) ) return; const previousOwnerToken = leaseArg.ownerToken; const previousEpoch = leaseArg.epoch; const nextOwnerToken = expired ? randomOwnerToken() : previousOwnerToken; const nextEpoch = expired ? previousEpoch + 1 : previousEpoch; const nextRevision = plugins.crypto.randomUUID(); const next: IFlexWriterLeaseDocument = { ...leaseBody(current), ownerToken: nextOwnerToken, epoch: nextEpoch, expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), }; assertFlexWriterLeaseDocument(next); const uncertainTransition: IFlexUncertainLeaseTransition = { preimage: leaseTransitionImage(current), postimage: leaseTransitionImage({ _id: current._id, ...next, _smartdataRevision: nextRevision, }), }; leaseArg.uncertainTransition = uncertainTransition; let transition: TStoredLease | null; try { transition = await collection.findOneAndReplace({ _id: uncertainTransition.preimage._id, ...leaseBody(uncertainTransition.preimage), _smartdataRevision: smartRevisionSelector(uncertainTransition.preimage), }, { ...leaseBody(uncertainTransition.postimage), _smartdataRevision: uncertainTransition.postimage._smartdataRevision, }, { returnDocument: 'after', includeResultMetadata: false, ...operationOptions(), }) as unknown as TStoredLease | null; } catch { continue; } this.throwIfFatal(); if (transition && leaseTransitionImagesEqual(transition, uncertainTransition.postimage)) { leaseArg.ownerToken = nextOwnerToken; leaseArg.epoch = nextEpoch; leaseArg.expiresAt = transition.expiresAt; leaseArg.uncertainTransition = undefined; if (transition.expiresAt > Date.now()) return; await waitForFlexStoreRetry(10, abortSignalArg); continue; } if (transition && !leaseTransitionImagesEqual(transition, uncertainTransition.preimage)) { throw this.markLeaseLost(leaseArg); } await waitForFlexStoreRetry(10, abortSignalArg); } } private async runLeaseTransition( leaseArg: IFlexHeldLease, operationArg: (abortSignalArg: AbortSignal, timeoutMsArg: number) => Promise, ): Promise { this.throwIfFatal(); const deadline = Date.now() + this.leaseTransitionTimeoutMs; const abortController = new AbortController(); this.leaseTransitionAbortControllers.add(abortController); const prior = leaseArg.transitionTask ?? Promise.resolve(); const operation = prior.catch(() => undefined).then(async () => { this.throwIfFatal(); if (leaseArg.lost) throw new FlexLeaseLostError(leaseArg.storageKey); abortController.signal.throwIfAborted(); if (leaseArg.uncertainTransition) { await this.verifyHeldLease( leaseArg, false, abortController.signal, false, Math.max(1, deadline - Date.now()), ); } abortController.signal.throwIfAborted(); return operationArg( abortController.signal, Math.max(1, deadline - Date.now()), ); }); let operationSettled = false; const settled = operation.then( () => { operationSettled = true; }, () => { operationSettled = true; }, ); leaseArg.transitionTask = settled; void settled.then(() => { this.leaseTransitionAbortControllers.delete(abortController); if (leaseArg.transitionTask === settled) leaseArg.transitionTask = undefined; }); try { await waitForFlexStoreDeadline( prior, Math.max(1, deadline - Date.now()), `Flex writer lease transition queue for "${leaseArg.storageKey}" stalled.`, ); } catch (errorArg) { abortController.abort(errorArg); throw this.failStore(errorArg, leaseArg); } try { return await waitForFlexStoreDeadline( operation, Math.max(1, deadline - Date.now()), `Flex writer lease transition for "${leaseArg.storageKey}" stalled.`, ); } catch (errorArg) { if (!operationSettled) { abortController.abort(errorArg); throw this.failStore(errorArg, leaseArg); } throw errorArg; } } private throwIfFatal(): void { if (this.fatalError) throw this.fatalError; } private markLeaseLost(leaseArg: IFlexHeldLease): Error { return this.failStore(new FlexLeaseLostError(leaseArg.storageKey), leaseArg); } private failStore(errorArg: unknown, leaseArg?: IFlexHeldLease): Error { const error = errorArg instanceof Error ? errorArg : new Error(String(errorArg)); if (leaseArg) leaseArg.lost = true; if (this.fatalError) return this.fatalError; this.fatalError = error; this.heartbeatsOpen = false; this.maintenanceOpen = false; this.maintenanceAbortController.abort(error); if (this.orphanMaintenanceTimer) { clearTimeout(this.orphanMaintenanceTimer); this.orphanMaintenanceTimer = undefined; } if (this.leaseHeartbeatTimer) { clearInterval(this.leaseHeartbeatTimer); this.leaseHeartbeatTimer = undefined; } for (const controller of this.leaseTransitionAbortControllers) controller.abort(error); for (const lease of this.heldLeases.values()) { lease.lost = true; lease.heartbeatAbortController?.abort(error); } try { this.onFatalError?.(error); } catch { // Store fencing is independent from health-listener behavior. } return error; } private startLeaseHeartbeat(): void { if (this.leaseHeartbeatTimer || !this.heartbeatsOpen) return; this.leaseHeartbeatTimer = setInterval( () => this.heartbeatHeldLeases(), flexLeaseHeartbeatMs, ); this.leaseHeartbeatTimer.unref?.(); } private heartbeatHeldLeases(): void { for (const lease of this.heldLeases.values()) { if (lease.lost || lease.releasing || lease.heartbeatTask) continue; const abortController = new AbortController(); const task = this.runLeaseTransition(lease, (transitionSignal, timeoutMs) => ( this.verifyHeldLease( lease, true, AbortSignal.any([abortController.signal, transitionSignal]), true, Math.min(timeoutMs, flexLeaseHeartbeatOperationTimeoutMs), ) )).catch((errorArg) => { if (errorArg instanceof FlexLeaseLostError) { if (!lease.releasing) lease.lost = true; return; } if (!lease.releasing && this.heartbeatsOpen && !abortController.signal.aborted) { console.error( `Flex writer lease heartbeat for "${lease.storageKey}" failed; retrying on the next interval.`, errorArg, ); } }).finally(() => { if (lease.heartbeatTask === task) lease.heartbeatTask = undefined; if (lease.heartbeatAbortController === abortController) { lease.heartbeatAbortController = undefined; } }); lease.heartbeatAbortController = abortController; lease.heartbeatTask = task; } } private async releaseHeldLease( leaseArg: IFlexHeldLease, timeoutMsArg = flexMaintenanceOperationTimeoutMs, ): Promise { const deadline = Date.now() + timeoutMsArg; leaseArg.releasing = true; try { const heartbeatTask = leaseArg.heartbeatTask; const heartbeatAbortController = leaseArg.heartbeatAbortController; heartbeatAbortController?.abort(new Error('The Flex writer lease is being released.')); if (heartbeatTask) { await waitForFlexStoreDeadline( heartbeatTask, Math.max(1, deadline - Date.now()), `Flex writer lease heartbeat for "${leaseArg.storageKey}" did not stop before the release deadline.`, ); if (leaseArg.heartbeatTask === heartbeatTask) leaseArg.heartbeatTask = undefined; if (leaseArg.heartbeatAbortController === heartbeatAbortController) { leaseArg.heartbeatAbortController = undefined; } } const transitionTask = leaseArg.transitionTask; if (transitionTask) { try { await waitForFlexStoreDeadline( transitionTask, Math.max(1, deadline - Date.now()), `Flex writer lease transition for "${leaseArg.storageKey}" did not stop before the release deadline.`, ); } catch { leaseArg.lost = true; } } if (leaseArg.lost || this.fatalError) { this.forgetHeldLease(leaseArg); if (this.closed) this.leaseTransitionAbortControllers.clear(); return; } const remainingMs = deadline - Date.now(); if (remainingMs <= 0) { throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', `Flex writer lease release for "${leaseArg.storageKey}" exceeded its deadline.`, ); } await this.releaseLease(leaseArg, remainingMs); this.forgetHeldLease(leaseArg); } catch (errorArg) { leaseArg.releasing = false; if (!this.heldLeases.has(leaseArg.storageKey)) { this.heldLeases.set(leaseArg.storageKey, leaseArg); } throw errorArg; } } private forgetHeldLease(leaseArg: IFlexHeldLease): void { if (this.heldLeases.get(leaseArg.storageKey) === leaseArg) { this.heldLeases.delete(leaseArg.storageKey); } if (this.heldLeases.size === 0 && this.leaseHeartbeatTimer) { clearInterval(this.leaseHeartbeatTimer); this.leaseHeartbeatTimer = undefined; } } private async releaseLease( leaseArg: IFlexHeldLease, timeoutMsArg = flexMaintenanceOperationTimeoutMs, ): Promise { const id = flexWriterLeaseId(this.controllerId, leaseArg.storageKey); const collection = FlexWriterLeaseModel.collection.mongoDbCollection; const deadline = Date.now() + timeoutMsArg; while (Date.now() < deadline) { const remainingMs = Math.max(1, deadline - Date.now()); const current = await collection.findOne( { id }, { timeoutMS: remainingMs }, ) as unknown as TStoredLease | null; if ( !current || current.ownerToken !== leaseArg.ownerToken || current.epoch !== leaseArg.epoch || current.state === 'released' ) return; const now = Date.now(); const next: IFlexWriterLeaseDocument = { ...leaseBody(current), state: 'released', expiresAt: now, updatedAt: new Date(now).toISOString(), }; assertFlexWriterLeaseDocument(next); const transition = await collection.findOneAndReplace({ _id: current._id, id, ownerToken: leaseArg.ownerToken, epoch: leaseArg.epoch, state: current.state, _smartdataRevision: smartRevisionSelector(current), }, { ...next, _smartdataRevision: plugins.crypto.randomUUID(), }, { returnDocument: 'after', includeResultMetadata: false, timeoutMS: Math.max(1, deadline - Date.now()), }); if (transition) return; } throw new plugins.smartdata.SmartdataExactPersistenceError( 'ambiguous_write', `Flex writer lease release for "${leaseArg.storageKey}" exceeded its deadline.`, ); } private async cleanupPrivateByCandidate( manifestArg: { candidateId: string }, timeoutMsArg?: number, ): Promise { const options = timeoutMsArg === undefined ? {} : { timeoutMS: timeoutMsArg }; await FlexPrivateChunkModel.collection.mongoDbCollection.deleteMany({ controllerId: this.controllerId, candidateId: manifestArg.candidateId, }, options); await FlexPrivateCandidateModel.collection.mongoDbCollection.deleteMany({ controllerId: this.controllerId, candidateId: manifestArg.candidateId, }, options); } private async cleanupProjectionSourceByCandidate( candidateIdArg: string, timeoutMsArg?: number, ): Promise { const filter = { controllerId: this.controllerId, candidateId: candidateIdArg }; const options = timeoutMsArg === undefined ? {} : { timeoutMS: timeoutMsArg }; await FlexProjectionMessageRecordModel.collection.mongoDbCollection.deleteMany(filter, options); await FlexProjectionSourceModel.collection.mongoDbCollection.deleteMany(filter, options); } private async cleanupPublicByCandidate( manifestArg: { candidateId: string }, storageKeyArg: string, timeoutMsArg?: number, ): Promise { await Promise.all([ FlexPublicSessionRecordModel.collection.mongoDbCollection.deleteMany({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }, timeoutMsArg === undefined ? {} : { timeoutMS: timeoutMsArg }), FlexPublicMessageRecordModel.collection.mongoDbCollection.deleteMany({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }, timeoutMsArg === undefined ? {} : { timeoutMS: timeoutMsArg }), ]); await FlexPublicCandidateModel.collection.mongoDbCollection.deleteMany({ controllerId: this.controllerId, scopeId: storageKeyArg, candidateId: manifestArg.candidateId, }, timeoutMsArg === undefined ? {} : { timeoutMS: timeoutMsArg }); } private async ensureMaintenanceLease( storageKeyArg: string, abortSignalArg: AbortSignal, ): Promise<{ lease: IFlexHeldLease; acquired: boolean } | undefined> { abortSignalArg.throwIfAborted(); const id = flexWriterLeaseId(this.controllerId, storageKeyArg); const collection = FlexWriterLeaseModel.collection.mongoDbCollection; const readLease = async (withAbortSignalArg = true): Promise => { const raw = await collection.findOne({ id }, { ...(withAbortSignalArg ? { signal: abortSignalArg } : {}), timeoutMS: flexMaintenanceOperationTimeoutMs, }) as unknown as TStoredLease | null; if (!raw) return null; const body = leaseBody(raw); assertFlexWriterLeaseDocument(body); if ( body.id !== id || body.controllerId !== this.controllerId || body.storageKey !== storageKeyArg ) { throw new plugins.flexharness.FlexHarnessStoreFormatError( 'Flex writer lease identity mismatch.', ); } return raw; }; const held = this.heldLeases.get(storageKeyArg); if (held) { if (held.lost || held.releasing) return undefined; try { await this.runLeaseTransition( held, (transitionSignal, timeoutMs) => this.verifyHeldLease( held, false, AbortSignal.any([abortSignalArg, transitionSignal]), false, Math.min(timeoutMs, flexMaintenanceOperationTimeoutMs), ), ); } catch (errorArg) { if (abortSignalArg.aborted) throw abortSignalArg.reason; return undefined; } return { lease: held, acquired: false }; } const ownerToken = randomOwnerToken(); const now = Date.now(); let current = await readLease(); if (!current) { const document: IFlexWriterLeaseDocument = { id, controllerId: this.controllerId, storageKey: storageKeyArg, ownerToken, epoch: 1, state: 'active', expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), }; assertFlexWriterLeaseDocument(document); try { await collection.insertOne({ ...document, _smartdataRevision: plugins.crypto.randomUUID(), }, { timeoutMS: flexMaintenanceOperationTimeoutMs }); current = await readLease(false); } catch (errorArg) { current = await readLease(false); if (!current) throw errorArg; } if ( !current || current.ownerToken !== ownerToken || current.epoch !== 1 || current.state !== 'active' ) return undefined; } else { if (current.state === 'active' && current.expiresAt > now) return undefined; const next: IFlexWriterLeaseDocument = { ...leaseBody(current), ownerToken, epoch: current.epoch + 1, state: 'active', expiresAt: now + flexLeaseDurationMs, updatedAt: new Date(now).toISOString(), }; assertFlexWriterLeaseDocument(next); const postimage = await collection.findOneAndReplace({ _id: current._id, id, ownerToken: current.ownerToken, epoch: current.epoch, state: current.state, _smartdataRevision: smartRevisionSelector(current), }, { ...next, _smartdataRevision: plugins.crypto.randomUUID(), }, { returnDocument: 'after', includeResultMetadata: false, timeoutMS: flexMaintenanceOperationTimeoutMs, }); if (!postimage) return undefined; current = postimage as unknown as TStoredLease; } const lease: IFlexHeldLease = { storageKey: storageKeyArg, ownerToken, epoch: current.epoch, expiresAt: current.expiresAt, lost: false, releasing: false, }; this.heldLeases.set(storageKeyArg, lease); this.startLeaseHeartbeat(); return { lease, acquired: true }; } private async runIdleMaintenanceStorage( storageKeyArg: string, abortSignalArg: AbortSignal, operationArg: () => Promise, ): Promise { abortSignalArg.throwIfAborted(); if (this.storageQueues.has(storageKeyArg)) return false; const operation = (async () => { const leaseResult = await this.ensureMaintenanceLease(storageKeyArg, abortSignalArg); if (!leaseResult) return false; try { abortSignalArg.throwIfAborted(); await operationArg(); return true; } finally { if (leaseResult.acquired) { await this.releaseHeldLease(leaseResult.lease, flexMaintenanceOperationTimeoutMs); } } })(); const settled = operation.then(() => undefined, () => undefined); this.storageQueues.set(storageKeyArg, settled); void settled.then(() => { if (this.storageQueues.get(storageKeyArg) === settled) { this.storageQueues.delete(storageKeyArg); } }); return operation; } private async reconcileCandidateOrphans( kindArg: 'private' | 'public' | 'projectionSource', cutoffArg: string, abortSignalArg?: AbortSignal, ): Promise { const model = kindArg === 'private' ? FlexPrivateCandidateModel : kindArg === 'public' ? FlexPublicCandidateModel : FlexProjectionSourceModel; let lastPosition: { createdAt: string; id: TFlexObjectId } | undefined; while (true) { abortSignalArg?.throwIfAborted(); const cursor = model.collection.mongoDbCollection.find({ controllerId: this.controllerId, ...(lastPosition ? { $or: [{ createdAt: { $gt: lastPosition.createdAt, $lte: cutoffArg }, }, { createdAt: lastPosition.createdAt, _id: { $gt: lastPosition.id }, }], } : { createdAt: { $lte: cutoffArg } }), }, abortSignalArg ? { signal: abortSignalArg } : {}) .sort({ createdAt: 1, _id: 1 }) .limit(flexMaintenancePageLimit); let documents: Array<{ _id: TFlexObjectId } & IFlexRawMaintenanceDocument>; try { documents = await cursor.toArray() as Array<{ _id: TFlexObjectId; } & IFlexRawMaintenanceDocument>; } finally { await cursor.close(); } for (const document of documents) { abortSignalArg?.throwIfAborted(); if (this.activeCandidateIds.has(document.candidateId)) continue; try { await this.runIdleMaintenanceStorage( document.scopeId, abortSignalArg ?? this.maintenanceAbortController.signal, async () => { if (await this.hasCandidateHeadReference( kindArg, document.candidateId, abortSignalArg, )) return; abortSignalArg?.throwIfAborted(); if (kindArg === 'private') { await this.cleanupPrivateByCandidate( { candidateId: document.candidateId }, flexMaintenanceOperationTimeoutMs, ); } else if (kindArg === 'public') { await this.cleanupPublicByCandidate( document, document.scopeId, flexMaintenanceOperationTimeoutMs, ); } else { await this.cleanupProjectionSourceByCandidate( document.candidateId, flexMaintenanceOperationTimeoutMs, ); } }, ); } catch (errorArg) { if (abortSignalArg?.aborted) throw abortSignalArg.reason; } } const last = documents.at(-1); if (!last || documents.length < flexMaintenancePageLimit) break; lastPosition = { createdAt: last.createdAt, id: last._id }; } } private async hasCandidateHeadReference( kindArg: 'private' | 'public' | 'projectionSource', candidateIdArg: string, abortSignalArg?: AbortSignal, ): Promise { if (kindArg === 'private') { const [legacy, domain] = await Promise.all([ FlexHeadModel.collection.mongoDbCollection.findOne({ controllerId: this.controllerId, currentPrivateCandidateId: candidateIdArg, }, { projection: { _id: 1 }, ...(abortSignalArg ? { signal: abortSignalArg } : {}) }), FlexDomainHeadModel.collection.mongoDbCollection.findOne({ controllerId: this.controllerId, currentPrivateCandidateId: candidateIdArg, }, { projection: { _id: 1 }, ...(abortSignalArg ? { signal: abortSignalArg } : {}) }), ]); return !!legacy || !!domain; } if (kindArg === 'projectionSource') { return !!await FlexDomainHeadModel.collection.mongoDbCollection.findOne({ controllerId: this.controllerId, domain: 'projection', 'projectionSource.candidateId': candidateIdArg, }, { projection: { _id: 1 }, ...(abortSignalArg ? { signal: abortSignalArg } : {}) }); } const [legacy, current] = await Promise.all([ FlexHeadModel.collection.mongoDbCollection.findOne({ controllerId: this.controllerId, 'publicCandidates.candidateId': candidateIdArg, }, { projection: { _id: 1 }, ...(abortSignalArg ? { signal: abortSignalArg } : {}) }), FlexPublicHeadModel.collection.mongoDbCollection.findOne({ controllerId: this.controllerId, 'publicCandidates.candidateId': candidateIdArg, }, { projection: { _id: 1 }, ...(abortSignalArg ? { signal: abortSignalArg } : {}) }), ]); return !!legacy || !!current; } private evictProviderScope(storageKeyArg: string): void { for (const key of this.agentEventStores.keys()) { if ((JSON.parse(key) as [string, string])[0] === storageKeyArg) this.agentEventStores.delete(key); } for (const key of this.jobStores.keys()) { if ((JSON.parse(key) as [string, string])[0] === storageKeyArg) this.jobStores.delete(key); } } private queueOrphanMaintenance(): void { if (!this.maintenanceOpen) return; if (this.orphanMaintenanceTask) return; const task = Promise.resolve().then(async () => { await this.reconcileOrphans( Date.now(), this.maintenanceAbortController.signal, ).catch(() => undefined); }).finally(() => { if (this.orphanMaintenanceTask === task) this.orphanMaintenanceTask = undefined; this.scheduleOrphanMaintenance(); }); this.orphanMaintenanceTask = task; } private scheduleOrphanMaintenance(): void { if (!this.maintenanceOpen || this.orphanMaintenanceTimer) return; this.orphanMaintenanceTimer = setTimeout(() => { this.orphanMaintenanceTimer = undefined; if (!this.maintenanceOpen) return; this.queueOrphanMaintenance(); }, flexOrphanMaintenanceIntervalMs); this.orphanMaintenanceTimer.unref?.(); } }