// mind/mind.ts — perceive, deposit, recall, think, express. // // Memory is a content-addressed node graph (see store.ts). Learning is // DEPOSITION: perceive a stream into a tree and intern every node, so equal — // and, by resonance, similar — subtrees collapse to one shared node. A fact is // an EDGE between node ids; recall traverses edges; thinking completes the // query's OWN tree, node by node, to a fixed point. No whole, no weights. // // Architecture: 4 primitives × 2 patterns = all inference. // Implementation split across src/mind/*.ts — this file assembles the Mind class. import { cosine, makeKeyring, rng, setVecConfig, Vec } from "../vec.js"; import { bindSeat, fold, Sema, Space } from "../sema.js"; import { Alphabet } from "../alphabet.js"; import { bytesToTree, contentFoldIncremental, Grid, gridToTree, hilbertBytes, reachThreshold, stackGrids, } from "../geometry.js"; import type { ContentFold } from "../geometry.js"; import { BoundedMap, type Store } from "../store.js"; import { SQliteStore } from "../store-sqlite.js"; import { type MindConfig, resolveConfig } from "../config.js"; import { type Canon, canonHash, textCanon, textEdgeTrim } from "../canon.js"; import { type CandidateSpan, coverSequence, lightestDerivation, } from "../derive/src/index.js"; import { bytesEqual, concat2, concatBytes, indexOf } from "../bytes.js"; import { type ComputedResult, type DerivationItem, type DerivationStep, GraphSearch, type Leaf, type Seg, type Site, } from "./graph-search.js"; import { Alu } from "../alu/src/index.js"; import type { ComputedSpan, ExtensionHost } from "../extension.js"; export type { ComputedSpan, ExtensionHost }; import { decodeText, type InspectRationale, Rationale, type RationaleItem, } from "./rationale.js"; export type { InspectRationale, RationaleItem, RationaleStep, } from "./rationale.js"; // Public types re-exported export type Input = string | Uint8Array | Grid | Grid[]; export interface Response { v: Vec | null; bytes: Uint8Array; /** How the answer was grounded (see {@link Provenance}). `"recall-echo"` * marks the last-resort fallback that returned the nearest stored form's * own bytes verbatim — an echo, NOT a grounded fact. Absent when there is * no answer. */ provenance?: import("./pipeline.js").Provenance; } /** Serializable state of a conversation — can be saved and restored across * sessions. The Mind never interprets the bytes; it only tracks their * cumulative lengths so the caller can reconstruct turn boundaries later * without inspecting content. */ export interface ConversationState { /** The accumulated context bytes — raw concatenation of every turn's * bytes in order. No separator is inserted; the boundary offsets * ({@link boundaries}) tell the caller where each turn ends. */ context: Uint8Array; /** Cumulative byte length after each completed turn. Sorted, strictly * increasing, each {@code < context.length}. The first turn's length * is `boundaries[0]`; the second turn starts at that offset, and so * on. Empty for a single-turn or new conversation. */ boundaries: number[]; /** Byte spans occupied by replies produced by this Mind. Unlike boundary * parity, this remains exact when a turn receives an empty reply. Optional * so states saved before the field existed remain restorable. */ answeredSpans?: Array<[number, number]>; } /** An active conversation handle. Opaque — interact through the Mind's * conversation methods ({@link Mind.beginConversation}, * {@link Mind.respondTurn}, {@link Mind.endConversation}). */ export interface Conversation { readonly id: number; } /** Internal per-conversation state. * * The {@link pyramid} IS the conversation — the accumulated internal * processing state. {@link bytes} is the raw accumulated input, kept * in sync with the pyramid for O(1) concatenation. * * Memos persist across turns so the inference pipeline does not * re-process the prefix. Content-keyed (latin1) — each turn's fresh * {@code Uint8Array} would never hit an object-identity key. * * {@link resolvedSubtrees} caches foldTree resolutions at the Sema-node * level. When the pyramid reuses prefix subtrees (identical objects), * foldTree recovers their ids without touching the store. A walk that * passes no `visit` callback can stop at a cached subtree outright and is * O(suffix); a walk that DOES pass one — recognition and attention both do — * still descends in full and spends O(context), banking the elided store * probes rather than an elided traversal. That asymmetry is deliberate and * load-bearing: see foldTree in primitives.ts. */ interface ConversationData { tree: Sema; bytes: Uint8Array; boundaries: number[]; /** The plain fold's reusable segment state (see {@link ContentFold}). A * grown context reuses every content segment it already folded and folds * only the new turn — O(turn) instead of O(context) — and, because the * reused segments are the SAME Sema objects, `resolvedSubtrees` (keyed by * node identity) hits across turns, so recognition recovers the prefix's * ids without re-probing the store for any of them. It still WALKS the * prefix — it must, or it would emit fewer sites on a warm cache than a * cold one (see foldTree) — so the saving is in probes, not in traversal. * Undefined until the first grow. * * No turn boundaries are involved: reuse comes from content cuts being * stable under append, and imposing boundaries would only change the tree * away from what the deposit path folded. `boundaries` beside this field * is API metadata, not an input to the fold. */ content?: ContentFold; answeredSpans: Array<[number, number]>; perceiveMemo: Map; recogniseMemo: Map; climbMemo: Map>; resolvedSubtrees: WeakMap; } // Mind module imports import type { AttentionRead, MindContext, Recognition } from "./types.js"; import { changedNodes, liftAnswer, spliceAll } from "./types.js"; import { foldTree, gistOf, inputBytes, latin1Key, perceive as perceiveImpl, perceiveKey, read, resolve as resolveImpl, } from "./primitives.js"; import { chooseNext, edgeAncestors as edgeAncestorsFn, invalidateStructuralCaches, leadsSomewhere, } from "./traverse.js"; import { invalidateJunctionCache } from "./junction.js"; import { follow } from "./match.js"; import { recognise, segment } from "./recognition.js"; import { meaningOf } from "./resonance.js"; import { climbAttention as climbAttentionFn, naturalBreak as naturalBreakFn, } from "./attention.js"; export type { AnchorRejectionReason, ClimbConsensusData, ConsensusAnchorTrace, ConsensusReachTrace, ConsensusRegionTrace, CrossRegionTier, JunctionVoteTrace, RegionOutcome, } from "./attention.js"; export type { AncestorReach, SaturationReason, SaturationStop, } from "./types.js"; import { aluToMechanism, defaultMechanisms, think } from "./pipeline.js"; import { articulate } from "./articulation.js"; import { ingest } from "./learning.js"; import { rItem } from "./trace.js"; // The work meter is exported from src/index.ts (via src/meter.ts) — the one // definition; the Mind only consumes it. import { type CostReport, Meter } from "../meter.js"; // ── MindOptions ─────────────────────────────────────────────────────────── export interface MindOptions { seed?: number; recallQueryK?: number; haloQueryK?: number; normalizeEpsilon?: number; cosineEpsilon?: number; geometry?: Partial; alphabet?: Partial; storeConfig?: Partial; store?: Store; /** Additional grounding mechanisms (appended after the built-in defaults). */ mechanisms?: import("./pipeline-mechanism.js").PipelineMechanism[]; /** Factories that receive the {@link ExtensionHost} and return mechanisms. */ mechanismFactories?: (( host: import("../extension.js").ExtensionHost, ) => import("./pipeline-mechanism.js").PipelineMechanism)[]; /** Measure the computational usage of every inference call — see * src/meter.ts. Off by default and free when off (one null check per store * read); on, each `respond`/`respondTurn` leaves a {@link Mind.lastCost} * report behind. Counters are deterministic, so two runs of the same query on * the same store are diffable; the millisecond fields are not. Profiling * NEVER changes an answer — but attaching a RATIONALE does: a traced response * bypasses the ctx memos (memoization.md), so profile without a trace. */ profile?: boolean; /** Content canonicalizer applied to EVERY response (any modality) for * equivalence-class resolution — see src/canon.ts. Text entry points * ({@link Mind.respondText}, {@link Mind.respondTurnText}) inject the * Unicode text canonicalizer automatically when this is unset; pass * `false` to disable canonical resolution everywhere. */ canon?: Canon | false; } // ═══════════════════════════════════════════════════════════════════════════ // THE MIND // ═══════════════════════════════════════════════════════════════════════════ export class Mind implements MindContext { readonly space: Space; readonly alphabet: Alphabet; readonly store: Store; readonly cfg: MindConfig; /** The lightest-derivation engine over the Sema graph. */ readonly search: GraphSearch; /** The grounding mechanisms iterated by {@link think}. */ readonly mechanisms: import("./pipeline-mechanism.js").PipelineMechanism[] = []; /** The live rationale tracer for the inference currently in flight, or null. */ trace: Rationale | null = null; /** The content canonicalizer for the response in flight — see * {@link MindContext.canon}. Injected per response by the modality entry * point; null when the response carries no equivalence. */ canon: Canon | null = null; /** Per-response canonical-resolution memo — see {@link MindContext.canonMemo}. */ canonMemo: Map | null = null; /** The Mind-level canon option: a canonicalizer to use for EVERY response, * `false` to disable canonical resolution, or null to let each entry * point decide (text entry points inject {@link textCanon}). */ private _canonOpt: Canon | false | null = null; /** The work accumulator for the inference call in flight — see * {@link MindContext.meter}. Non-null only between beginResponse and * endResponse, and only when the Mind was constructed with * `{ profile: true }`. */ meter: Meter | null = null; /** Whether {@link MindOptions.profile} was set. */ private _profile = false; /** The computational-usage report of the LAST completed inference call, or * null when profiling is off (or nothing has been asked yet). Overwritten * by every `respond`/`respondTurn`; copy it if you are aggregating. See * {@link import("../meter.js").CostReport} and `sumReports`/`formatReport` * for battery-level aggregation. */ lastCost: CostReport | null = null; /** Memo of the consensus climb — content-keyed. See {@link MindContext.climbMemo}. */ climbMemo: Map> | null = null; _structMemoKey: object = {}; /** Memo of recognise() — content-keyed. See {@link MindContext.recogniseMemo}. */ recogniseMemo: Map | null = null; /** Memo of perceive() — content-keyed. See {@link MindContext.perceiveMemo}. */ perceiveMemo: Map | null = null; /** Subtree-resolution cache. See {@link MindContext._resolvedSubtrees}. */ _resolvedSubtrees: | WeakMap< import("../sema.js").Sema, { id: number; len: number } > | null = null; answeredSpans: ReadonlyArray = []; currentTurnStart = 0; /** The perceived gist of the query currently being answered. Set by `think` * before the graph search runs; `chooseNext` consults it as a gate (a null * guide means no query is in flight, so structural walkers keep plain * first-edge behaviour) and the reverse projection uses it for * reverse-recall disambiguation via `chooseAmong`. */ _edgeGuide: Vec | null = null; /** Per-response memo of {@link chooseNext} picks — ensures every mechanism * of a single response follows the SAME continuation for each ambiguous * context node. */ _edgeChoice: Map = new Map(); /** Previous deposit's seen node ids for incremental change detection. */ _prevSeen: Set | null = null; /** Session cache of node-id → perceived gist for candidate scoring — see * {@link MindContext._gistCache}. 32 MB ≈ 8K gists at D=1024; hub * candidate sets (√N at most) fit comfortably and recur across queries. */ _gistCache: BoundedMap = new BoundedMap( 32_000_000, (v) => v.byteLength, ); // Deposit-path fold-pyramid cache (see MindContext) — ENTRY-count // bounded: a pyramid costs ~KB per content byte (one D-float gist per // interior node), and only the few live conversation chains need to stay // warm, so 8 entries is the honest budget. _depositTrees: BoundedMap = new BoundedMap< string, import("./types.js").DepositCacheEntry >(8); _depositLens: Set = new Set(); _internIds: WeakMap = new WeakMap< import("../sema.js").Sema, number >(); // ── Conversation state ────────────────────────────────────────────────── private _nextConvId = 1; private _conversations = new Map(); // ── GraphSearchHost implementation ───────────────────────────────────── /** Canonical node id of a byte span. Required by GraphSearchHost & MindContext. */ resolve(bytes: Uint8Array): number | null { return resolveImpl(this, bytes); } /** Whether a node leads somewhere — the admission predicate, delegating to * `traverse.ts`'s ONE definition (edge or halo, with its response-scoped * cache). The search holds a bare Store and cannot reach that cache itself, * so it asks through this hook; a bare host keeps its raw-store fallback. */ leadsSomewhere(id: number): boolean { return leadsSomewhere(this, id); } // recogniseSpan wraps recognise recogniseSpan(bytes: Uint8Array): { sites: ReadonlyArray; leaves: ReadonlyArray; splits: ReadonlySet; starts: ReadonlySet; } { const r = recognise(this, bytes); return { sites: r.sites, leaves: r.leaves, splits: r.splits, starts: r.starts, }; } /** Disambiguate among multiple learnt continuations of the same context node. * Required by {@link GraphSearchHost} — the graph search calls this through the * host interface when a recognised form has more than one outgoing edge. * Delegates to the standalone {@link chooseNext} which picks the candidate * with the most distributional evidence (highest `prevOf` count — the * structural manifestation of its halo). When evidence is equal the * first-inserted edge wins. */ chooseNext(node: number): number | undefined { return chooseNext(this, node, this._edgeGuide); } // ── construction ───────────────────────────────────────────────────────── constructor(opts?: MindOptions); constructor(cfg: MindConfig, store: Store, _fromStore: true); constructor( optsOrCfg?: MindOptions | MindConfig, storeArg?: Store, _fromStore?: true, ) { let userMechanisms: import("./pipeline-mechanism.js").PipelineMechanism[] = []; let userFactories: (( host: import("../extension.js").ExtensionHost, ) => import("./pipeline-mechanism.js").PipelineMechanism)[] = []; if (_fromStore !== undefined) { this.cfg = resolveConfig(optsOrCfg as Partial); this.store = storeArg!; } else { const { store: optsStore, mechanisms: userMechs, mechanismFactories: userFacts, canon: optsCanon, profile: optsProfile, ...rest } = (optsOrCfg ?? {}) as MindOptions; this._canonOpt = optsCanon ?? null; this._profile = optsProfile === true; // `explicitSeed` is read BEFORE resolveConfig folds the default in, so // the store can be consulted only when the caller did not choose. const explicitSeed = (rest as Partial).seed; this.cfg = resolveConfig(rest as Partial); this.store = optsStore ?? new SQliteStore({ maxGroup: this.cfg.geometry.maxGroup, }); // THE ARTIFACT'S SEED GOVERNS. `train.seed` is recovered by the store at // open, exactly like `train.D` and `geometry.maxGroup`. The seed feeds // `makeKeyring`, `Space.rand` and the `Alphabet` below, so folding a // query under config.ts's default (42) against a store trained with // another seed (e.g. 7) lands in a DIFFERENT vector space than the one // the artifact's nodes were folded into: recognition and resonance read // the wrong space, and answers silently diverge — pinned by test/97, // where only adoption reproduces the artifact's own answer. An explicit // caller seed still wins — this only replaces the unconfigured default. if (explicitSeed === undefined && this.store.trainSeed !== null) { this.cfg.seed = this.store.trainSeed; } userMechanisms = userMechs ?? []; userFactories = userFacts ?? []; } setVecConfig({ normalizeEpsilon: this.cfg.normalizeEpsilon, cosineEpsilon: this.cfg.cosineEpsilon, }); const seedRand = rng((this.cfg.seed ^ 0x9e3779) >>> 0); const seats = makeKeyring( this.store.D, Math.max(8, this.cfg.geometry.maxGroup), seedRand, ); this.space = { D: this.store.D, seats, rand: rng((this.cfg.seed ^ 0x51f15e) >>> 0), maxGroup: this.cfg.geometry.maxGroup, }; this.alphabet = new Alphabet( this.cfg.seed, this.store.D, this.cfg.alphabet, ); this.search = new GraphSearch( this.store, this.space.maxGroup, this, // MindContext extends GraphSearchHost ); // Build the mechanism list: default grounding + ALU + user mechanisms. for (const m of defaultMechanisms) this.mechanisms.push(m); const host = this.extensionHost(); if (this.cfg.alu.enabled) { const alu = new Alu({ tol: this.cfg.alu.tol, maxIter: this.cfg.alu.maxIter, precision: this.cfg.alu.precision, }, host); this.mechanisms.push(aluToMechanism(alu)); } for (const m of userMechanisms) this.mechanisms.push(m); for (const f of userFactories) this.mechanisms.push(f(host)); } // ── Public API ─────────────────────────────────────────────────────────── /** Exposed for tests: the consensus climb over query sub-regions. */ climbAttention( query: Uint8Array, k: number, mode: import("./types.js").DFMode = "inverse", ): Promise { return climbAttentionFn(this, query, k, mode); } /** Exposed for tests: climb the structural DAG from a node to its * edge-bearing ancestor contexts. */ edgeAncestors( id: number, contextCount: number, ): import("./types.js").AncestorReach { return edgeAncestorsFn(this, id, contextCount); } /** Exposed for tests: find the natural break point in a sorted vote list. */ naturalBreak(votes: number[]): number { return naturalBreakFn(votes); } // ── respond ─────────────────────────────────────────────────────────── /** Perceive input into a content-defined tree. Deterministic — identical * bytes always produce an identical tree. Public for ingest-cache. */ perceive( input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, ): Sema { return perceiveImpl(this, input, leafAt, lookup); } /** Open one response's transient state — the tracer, the per-response * memos, the work meter. The ONE place this state is created, and it * serves BOTH entry points: `respond` takes fresh per-response memos, * `respondTurn` passes its conversation, whose memos persist across turns * (content-keyed, so the previous turn's results are found by this turn's * sub-span calls) and whose `resolvedSubtrees` spares foldTree the store * probes for every prefix subtree — and, for walks that pass no visitor, * the descent as well. respondTurn used to inline its own copy of this * and of {@link endResponse}; the two drifted (a memo added to one was * silently absent from the other), so there is exactly one pair now. */ private beginResponse( inspectRationale?: InspectRationale, canon?: Canon | null, conv?: ConversationData, ): void { this.trace = inspectRationale ? new Rationale(inspectRationale) : null; this.climbMemo = conv ? conv.climbMemo : new Map(); this.recogniseMemo = conv ? conv.recogniseMemo : new Map(); this.perceiveMemo = conv ? conv.perceiveMemo : new Map(); this._resolvedSubtrees = conv ? conv.resolvedSubtrees : null; // Inference is a pure function of cumulative bytes. Conversation // boundaries remain persistence/API metadata and must not select a // different mechanism path than respond() on the identical byte stream. // answeredSpans and currentTurnStart ARE restored from the conversation, // however — they are pure functions of the cumulative byte stream (the // assistant's own prior replies, and where the current user turn starts, // are deterministic given the full transcript). Without them confluence, // cover, the weave, and the consensus climb treat prior assistant turns // as fresh query content — re-deriving them as constraints, voting // anchors, and alignment points. this.answeredSpans = conv ? conv.answeredSpans : []; this.currentTurnStart = conv && conv.boundaries.length > 0 ? conv.boundaries[conv.boundaries.length - 1] : 0; this.canon = canon ?? null; this.canonMemo = canon ? new Map() : null; this._beginMeter(); } /** Open (or leave closed) the response's work accumulator. Separate from * {@link beginResponse} because {@link respondTurn} keeps its own * conversation-scoped lifecycle and must not create fresh per-response * memos — but it DOES meter, through this same pair. */ private _beginMeter(): void { if (!this._profile) return; this.meter = new Meter(); this.store.meter = this.meter; } /** Close the accumulator and publish its report. Detaching from the store * matters: a Mind that shares a store with another Mind must not keep * charging that store's reads to a finished response. */ private _endMeter(queryBytes: number): void { if (this.meter === null) return; this.lastCost = this.meter.report(queryBytes); this.store.meter = null; this.meter = null; } /** The canonicalizer a response should carry: the Mind-level option when * set (or none when explicitly disabled), else the entry point's own * default — text entry points pass {@link textCanon}, binary ones null. */ private _canonFor(entryDefault: Canon | null): Canon | null { if (this._canonOpt === false) return null; return this._canonOpt ?? entryDefault; } /** Close one response's transient state — every per-response field, incl. * the edge guide/choices `think` sets mid-flight, and the meter's report. * * A conversation's memo MAPS were mutated in place, so `data.*` still * points at them and there is nothing to save back. Clearing the Mind's * references is what matters: a concurrently-started `respond()` swaps its * own fresh maps into these pointers, and copying back from them here * would inject a foreign response's memos into the conversation. */ private endResponse(queryBytes: number): void { this._endMeter(queryBytes); this.trace = null; this.climbMemo = null; this.recogniseMemo = null; this.perceiveMemo = null; this._resolvedSubtrees = null; this.answeredSpans = []; this.currentTurnStart = 0; this.canon = null; this.canonMemo = null; this._edgeGuide = null; this._edgeChoice.clear(); } /** Shared response core — the one path from bytes to voiced answer. * `respond` calls this directly; `respondTurn` has its own path * with conversation-persistent memos and incremental perception. */ private async _respondImpl( queryBytes: Uint8Array, inspectRationale?: InspectRationale, traceLabel = "respond", canon: Canon | null = null, ): Promise { this.beginResponse(inspectRationale, canon); try { return await this._groundAndVoice(queryBytes, traceLabel); } finally { this.endResponse(queryBytes.length); } } /** The ONE path from query bytes to a voiced answer: ground (think), then * re-voice in the asker's words (articulate). Both entry points run * exactly this — they differ only in the LIFECYCLE around it (fresh * per-response memos vs. a conversation's persistent ones) and in what * they do with the answer afterwards. It must be called between * {@link beginResponse} and {@link endResponse}. */ private async _groundAndVoice( queryBytes: Uint8Array, traceLabel: string, ): Promise { const top = this.trace?.enter(traceLabel, [rItem(queryBytes, "query")]); const meter = this.meter; const thought = meter ? await meter.time( "think", () => think(this, queryBytes, this.mechanisms), ) : await think(this, queryBytes, this.mechanisms); if (thought === null) { top?.done([], "nothing to perceive or an empty store — no answer"); return { v: null, bytes: new Uint8Array(0) }; } const voiced = meter ? await meter.time( "articulate", () => articulate(this, thought.bytes, queryBytes), ) : await articulate(this, thought.bytes, queryBytes); top?.done( [rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)], "the answer, re-voiced in the asker's words", ); return { v: gistOf(this, voiced), bytes: voiced, provenance: thought.provenance, }; } /** Answer ONE self-contained input. * * A MULTI-TURN context is not that, and this is the wrong entry point for * it. `respond` folds the bytes it is handed with no boundary set, because * nothing in a flat byte string says where one turn ended — only the caller * who assembled it knows, which is the whole reason `boundaries` is a * parameter of {@link perceiveImpl} and never inferred from content. A * conversation deposited through {@link ingest} folds its contexts over * those turn boundaries, so a hand-concatenated transcript passed here * folds differently from the way it was learnt and reaches the trained * context node only by luck (measured on a 7-turn conversation: 5/7 here * against 7/7 through {@link respondTurn}, same bytes). Use * {@link beginConversation} + {@link respondTurn}, or {@link addTurn} to * replay turns the Mind should hear but not answer. */ async respond( input: Input, inspectRationale?: InspectRationale, ): Promise { // A STRING input is text by nature: it carries the text equivalence even // through the generic entry point. Raw bytes / grids carry only the // Mind-level canon option, if any. const canon = this._canonFor(typeof input === "string" ? textCanon : null); // EDGE WHITESPACE IS NOT PART OF THE QUESTION — trim it once, here, so // every mechanism downstream sees the same question regardless of how the // caller spaced it. See canon.ts's textEdgeTrim for why the outer edges of a // whole input are exactly where canon.ts's no-trimming hazard cannot arise. // Gated on the SAME modality test as the canonicalizer above: for bytes and // grids 0x20 is content, and nothing is trimmed. // // Measured on the 15.7M-node store: without this, one leading space took // `Who wrote Romeo and Juliet?` and `What is the chemical symbol for // water?` from answered to silent, because a shift re-seats every fold // boundary — the whole of analyze_training.ts's K2 phase-robustness gap. // The caller's EXACT bytes are tried first and the trim is a RETRY, not a // pre-filter. Trimming up front is asymmetric — it normalises the query but // not the stored forms — so it breaks byte-exact identity for a form trained // WITH edge whitespace: test/04 deposits [" ice ", "cold"] and asks // " ice ", which must keep answering. Retrying preserves that (the raw // query resolves on the first pass) while still reaching the padded case // (the raw query grounds nothing, the trimmed one does). // // COST: nothing on any answering path. The retry needs BOTH silence AND // edge whitespace on the query, the same "only on the already-failed path" // discipline test/44 and the bridge's own trim retry use. The conversation // entry point (respondTurn) is deliberately NOT trimmed — it tracks // turn-boundary offsets into its accumulated context, and shifting the bytes // under those offsets would desync them. const bytes = inputBytes(this, input); const first = await this._respondImpl( bytes, inspectRationale, "respond", canon, ); if (first.bytes.length > 0 || typeof input !== "string") return first; const trimmed = textEdgeTrim(bytes); if (trimmed.length === bytes.length || trimmed.length === 0) return first; return this._respondImpl( trimmed, inspectRationale, "respond", canon, ); } /** Text view of {@link respond}. NUL bytes (0x00) are stripped before * decoding — they are structural padding in text answers. LOSSY for a * binary answer that legitimately contains NULs: use {@link respond} and * read `bytes` directly for binary/grid modalities. * * Injects the TEXT canonicalizer (src/canon.ts) so resolution treats * every character variation of the same text — case, width, whitespace — * as one form, provided the store's canon index is built * ({@link buildCanonIndex}). */ async respondText( input: string, inspectRationale?: InspectRationale, ): Promise { const r = await this.respond(input, inspectRationale); return decodeText(r.bytes); } // ── Conversation API ──────────────────────────────────────────────────── /** Begin a new conversation, optionally restoring from a previously-saved * {@link ConversationState}. The returned handle is required for * {@link respondTurn} and {@link endConversation}. * * Conversations are independent — a Mind can manage several concurrently. * Each tracks the fold pyramid (accumulated internal processing) and * turn-boundary offsets; the geometry never inspects content to guess * where one turn ends and the next begins. */ beginConversation(state?: ConversationState): Conversation { const id = this._nextConvId++; const initBytes = state?.context ?? new Uint8Array(0); // NORMALISE CALLER-SUPPLIED BOUNDARIES. `boundaries` is documented // strictly increasing and every boundary this class produces is (they are // appended as the context grows), but a restored {@link ConversationState} // comes from OUTSIDE — hand-built, migrated, or round-tripped through a // store that did not preserve order. The folds consume boundaries with a // sequential `b > prev` filter, so an out-of-order entry is silently // DROPPED rather than rejected, and the conversation would then fold over // a different cut set than the one the caller believes it restored. // `bytesToTree` used to sort on the way in and absorbed this; the // incremental fold this now calls does not, so the normalisation belongs // here, at the one public door untrusted boundaries come through. const initBoundaries = state?.boundaries ? [...new Set(state.boundaries)] .filter((b) => b > 0 && b < initBytes.length) .sort((a, b) => a - b) : []; const initAnswered = state?.answeredSpans ? state.answeredSpans.map(([start, end]) => [start, end] as [number, number] ) : initBoundaries.flatMap((start, i, cuts) => i % 2 === 0 && i + 1 < cuts.length ? [[start, cuts[i + 1]] as [number, number]] : [] ); // The same incremental fold `_growContext` uses, so a RESTORED // conversation starts with segment state its next turn can reuse — a // resumed conversation is otherwise identical to a live one and must not // pay a full re-fold on every turn for the rest of its life. const restored = contentFoldIncremental( this.space, this.alphabet, initBytes, ); this._conversations.set(id, { tree: restored.tree, content: restored.fold, bytes: initBytes, boundaries: initBoundaries, answeredSpans: initAnswered, perceiveMemo: new Map(), recogniseMemo: new Map(), climbMemo: new Map(), resolvedSubtrees: new WeakMap(), }); return { id }; } /** End a conversation, releasing its internal resources (accumulated * context, boundary offsets, and the fold-pyramid cache). Idempotent. */ endConversation(conv: Conversation): void { this._conversations.delete(conv.id); } /** The current serialisable state of an active conversation. Save this * to resume the conversation later via {@link beginConversation}. */ conversationState(conv: Conversation): ConversationState | null { const data = this._conversations.get(conv.id); if (!data) return null; return { context: data.bytes, boundaries: [...data.boundaries], answeredSpans: data.answeredSpans.map(([start, end]) => [start, end]), }; } /** Append a turn to a conversation's accumulated context WITHOUT * responding — raw byte append plus a boundary offset, never a * separator; the fold pyramid advances by O(turn). * * This is the primitive for turns the Mind should hear but not answer: * replaying a transcript, feeding the OTHER speaker's line in a * prediction harness, or restoring context piecewise. {@link * respondTurn} = addTurn + think + its own reply appended the same way. * * ── ON SEPARATORS: THERE IS NO SEPARATOR QUESTION ──────────────────── * * "Never a separator" above says what this method DOES — it appends the * bytes you give it and records an OFFSET — not that separator bytes are * forbidden, unsupported, or something the engine must be taught about. * Sema is agnostic to them, and reviewers keep mistaking that agnosticism * for a constraint. To be explicit, because the mistake is easy: * * 1. A turn boundary is an OFFSET, held here, in `boundaries`. It is * never a character the geometry scans for. Nothing downstream asks * "what byte separates two turns?" because nothing downstream finds * boundaries by looking at content at all. * 2. A separator in a CORPUS is ordinary content. If a trainer joins * turns with "\n" (example/train_base does), those newlines are * simply bytes inside the stream, folded like every other byte. They * are a property of that corpus, not of this API and not of the fold. * 3. This API can therefore reproduce ANY corpus exactly, with no * convention to agree on: replaying a "\n"-joined corpus means passing * `"\n" + turnText` as the turn. The separator rides along IN the * turn bytes, where it belongs. There is nothing to configure and no * mode to select. * 4. Inference is not exact-match anyway. Recognition works over * sub-spans, canonical equivalence and resonance, so a query that * differs from the trained bytes by punctuation or whitespace still * reaches the trained forms; it degrades, it does not fail closed. * * What follows from 1–4: differing separator bytes between a corpus and a * query is an ordinary CONTENT difference — the same kind as any other * wording difference — and it is measured the same way. It is NOT an * incompatibility between the trainer and this API, and it does NOT * require choosing a project-wide separator convention. A review that * concludes otherwise (this one did, before being corrected) has mistaken * its own harness feeding untrained bytes for an architectural defect. */ addTurn(conv: Conversation, turn: Input): ConversationState { const data = this._conversations.get(conv.id); if (!data) throw new Error(`Conversation ${conv.id} not found`); const turnBytes = inputBytes(this, turn); this._growContext(data, turnBytes); return this.conversationState(conv)!; } /** Grow a conversation's accumulated context by one turn's bytes — raw * append plus a boundary offset, pyramid advanced by O(turn), the grown * context's tree seeded into the conversation's perceive memo. The ONE * place a context grows ({@link addTurn} and {@link respondTurn} both * come through here), so the append semantics cannot drift. */ private _growContext(data: ConversationData, turnBytes: Uint8Array): Sema { const prevLen = data.bytes.length; // An empty turn neither grows the context nor marks a boundary — // boundaries are documented strictly increasing, and a zero-length // "turn" is no turn. Nothing changed, so the existing tree stands. const grow = turnBytes.length > 0; if (!grow) return data.tree; const grown = prevLen > 0 ? concat2(data.bytes, turnBytes) : turnBytes; if (prevLen > 0) data.boundaries.push(prevLen); // THE PLAIN FOLD, INCREMENTALLY. No boundary set is imposed here: the // tree is exactly the tree `perceive(grown)` builds for these bytes, which // is exactly the tree the DEPOSIT path folded when it learnt them. That // agreement is the whole point — it is what lets a cumulative context // resolve to its trained node, and when it was absent the alignment family // went quadratic (measured: 5.2M cells on a 476-byte context, against 0 // when the two sides agree). // // The optimisation is unaffected by dropping the boundaries, because it // never came from them: content cuts are stable under append, so the // incremental fold reuses every segment left of the new turn as the SAME // object (see contentFoldIncremental). That object identity is what // `resolvedSubtrees` — a WeakMap keyed by node identity — needs in order // to hit at all. Measured against the stable-prefix fold it replaces: // ~40 rebuilt nodes per turn either way, flat as the context grows // sevenfold, and ~92% of nodes reused by identity in both. // // `data.boundaries` is still tracked, and is still exact — it is API // metadata (ConversationState, answeredSpans, currentTurnStart), not a // fold instruction. const folded = contentFoldIncremental( this.space, this.alphabet, grown, data.content, ); const tree = folded.tree; data.content = folded.fold; data.tree = tree; data.bytes = grown; // Seeded under the PLAIN content key, and that is now the only key there // is: with no boundary set imposed, this tree IS what `perceive(grown)` // computes, so the memo entry is an ordinary cache hit rather than the // deliberate alias it had to be while the two folds differed. The entry // saves the pipeline re-folding the context it was just handed. data.perceiveMemo.set(perceiveKey(grown), tree); return tree; } /** Process one turn of a conversation. * * `turn` is the raw input for the latest turn — its bytes are appended * to the accumulated context directly (raw concatenation). The Mind * tracks the byte offset where each turn ends; no separator is ever * inserted or inspected. * * Returns the response AND the updated {@link ConversationState} so the * caller can persist it. The conversation handle's internal state is * updated in place — the returned state is a snapshot for storage. * * SINGLE FLIGHT: at most one respondTurn may be in flight per Mind. The * conversation's memo caches are swapped into the Mind-level per-response * pointers for the duration of the turn, so a concurrently-running * respond()/respondTurn() on the SAME Mind would interleave state. * Different Minds (or sequential awaits, as in every test) are safe. */ async respondTurn( conv: Conversation, turn: Input, inspectRationale?: InspectRationale, ): Promise<{ response: Response; state: ConversationState }> { const data = this._conversations.get(conv.id); if (!data) throw new Error(`Conversation ${conv.id} not found`); const turnBytes = inputBytes(this, turn); // Incremental perception — O(turn) instead of O(context). this._growContext(data, turnBytes); const newContext = data.bytes; // The conversation's persistent memos and subtree cache are swapped in // by beginResponse (see there) — the SAME lifecycle respond() uses, so a // memo added in one place can never be missing from the other. A string // turn is text by nature and carries the text equivalence, same as // respond() (see _canonFor). // // No recognise-memo pre-seeding here: that used to be necessary because // the flat/positional fold lost visibility into an earlier turn's own // structure once later bytes shifted its position (foldTree no longer // visited the turn's root node). The STABLE-PREFIX fold (see {@link // ConversationData}) makes every turn's subtree independent of what // follows it by construction, so recognise() finds it correctly on its // own, first-touch, exactly once per turn. this.beginResponse( inspectRationale, this._canonFor(typeof turn === "string" ? textCanon : null), data, ); try { const response = await this._groundAndVoice(newContext, "respondTurn"); // The REPLY joins the accumulated context the same way a turn does // ({@link addTurn}): raw byte append plus a boundary offset — never a // separator. A conversation's context is the full exchange, exactly // the cumulative continuous shape multi-turn training deposits, so a // later turn can refer to what was ANSWERED ("which of those two…"), // not only to what was asked. if (response.bytes.length > 0) { const start = data.bytes.length; this.addTurn(conv, response.bytes); data.answeredSpans.push([start, data.bytes.length]); } return { response, state: this.conversationState(conv)! }; } finally { this.endResponse(newContext.length); } } /** Text view of {@link respondTurn}. See {@link respondText} for the * NUL-stripping caveat. For binary or grid turns use {@link respondTurn} * directly — this is a text-only convenience, like {@link respondText}. */ async respondTurnText( conv: Conversation, turn: string, inspectRationale?: InspectRationale, ): Promise<{ response: string; state: ConversationState }> { const { response, state } = await this.respondTurn( conv, turn, inspectRationale, ); return { response: decodeText(response.bytes), state }; } async embedding(input: Input): Promise { return (await this.respond(input)).v; } /** Kinship note: the vector arm below is a miniature of recall's tier 3 * (resonate → reach gate → read out the nearest form's bytes) — the * read-out direction of the same operation, without recall's grounding * ladder. If either side's acceptance rule changes, revisit the other. */ async express(idOrV: number | Vec): Promise { if (typeof idOrV === "number") return this.store.bytes(idOrV); const [hit] = await this.store.resonate(idOrV, 1); // The same confidence floor recall uses: a vector whose nearest stored // form sits below the reach threshold relates to NOTHING in the store — // returning that form's bytes anyway would fabricate an answer from an // unrelated neighbour. Silence is the honest read-out. if (hit && hit.score >= reachThreshold(this.space.maxGroup)) { return this.store.bytes(hit.id); } return new Uint8Array(0); } // ── Learning ───────────────────────────────────────────────────────────── /** See {@link import("./learning.js").ingest} — `onDeposit`, when given, * reports each ingested item's deposited root node ids * ({@link DepositReport}); purely observational. */ async ingest( input: Input | (Input | [Input, Input])[], second?: Input, onDeposit?: (report: import("./learning.js").DepositReport) => void, /** Witness the DEPOSIT path the way {@link respond}'s callback witnesses * inference — `companyProfile` reports its saturation diagnostics here. * Without it the tracer is never constructed and the emit sites cost * nothing (§ rationale.ts), exactly as on the inference path. */ inspectRationale?: InspectRationale, ): Promise<(Sema & { id: number }) | undefined> { invalidateStructuralCaches(this); invalidateJunctionCache(this); const prevTrace = this.trace; this.trace = inspectRationale ? new Rationale(inspectRationale) : null; try { return await ingest(this, input, second, onDeposit); } finally { this.trace = prevTrace; } } // ── Extension Surface ──────────────────────────────────────────────────── private extensionHost(): ExtensionHost { const mind = this; return { meaningOf: (bytes, anchors) => meaningOf(this, bytes, anchors), continuation: (bytes) => this.groundedContinuation(bytes), segment: (bytes) => segment(this, bytes).map((s) => ({ i: s.start, j: s.end })), get reach() { return mind.space.maxGroup; }, }; } private async groundedContinuation( bytes: Uint8Array, ): Promise { const id = resolveImpl(this, bytes); if (id === null) return null; const grounded = await follow(this, id); if (grounded !== null && !bytesEqual(grounded, bytes)) return grounded; return null; } // ── Content-index repair ─────────────────────────────────────────────── /** Re-index structurally-important nodes whose gists were evicted from the * pending cache before they reached the content index. See {@link * Store.repairContentIndex} for the contract; this method wires the * Mind's perception into the store's repair walk. * * Run this after training or at checkpoints to restore recall reach for * nodes that bridge experiences but were never indexed. A pure interior * node (no edges, no halo) is deliberately skipped — it is scaffolding, * not an experience root or bridge, and regenerating its gist would waste * I/O and index space for no recall benefit. * * @param minParents only repair nodes with ≥ this many structural parents * (default 2 — structural bridges) * @returns number of nodes added to the content index */ async repairContentIndex(minParents = 2): Promise { return this.store.repairContentIndex( async (id) => { const bytes = this.store.bytes(id); if (bytes.length === 0) return null; return gistOf(this, bytes); }, minParents, ); } // ── Canonical-form index ─────────────────────────────────────────────── /** Build (or incrementally refresh) the store's canonical-form index: for * every content-bearing node, record the hash of its CANONICAL key so * resolution can find stored forms across surface variation (case, width, * whitespace — whatever `canon` equates; see src/canon.ts). * * Incremental and idempotent: the last indexed node id is remembered in * store meta (`canon.upto`), so a refresh after further training scans * only the new rows. Run once after training, and again after ingests — * the same operational shape as {@link repairContentIndex}. * * @param canon the canonicalizer to index under — MUST be the same one * queries will carry (text queries carry {@link textCanon} * unless the Mind was constructed with its own) * @returns number of index rows added */ async buildCanonIndex(canon?: Canon): Promise { const c = canon ?? this._canonFor(textCanon); const store = this.store; if (c === null || !store.canonAdd || !store.eachContent) return 0; const from = Number(await store.getMeta("canon.upto") ?? 0); let added = 0; let maxId = from - 1; store.eachContent((id, bytes) => { if (id > maxId) maxId = id; const key = c(bytes); if (key.length === 0) return; // Only index content whose canonical key DIFFERS from its raw bytes — // an already-canonical span is found by the exact lookup (and by the // fallback's own exact probe of the canonical bytes), so indexing it // would only add rows. if (bytesEqual(key, bytes)) return; store.canonAdd!(canonHash(key), id); added++; }, from); await store.setMeta("canon.upto", String(maxId + 1)); store.commit(); return added; } // ── Persistence ────────────────────────────────────────────────────────── async save(): Promise { const meta = new TextEncoder().encode(JSON.stringify(this.cfg)); await this.store.saveSnapshot(meta); return meta; } static async load(snapshot: Uint8Array, store: Store): Promise { const cfg = JSON.parse(new TextDecoder().decode(snapshot)) as MindConfig; return new Mind(cfg, store, true); } static async loadFromStore(store: Store): Promise { const meta = await store.loadSnapshot(); if (!meta) throw new Error("no snapshot in store"); return Mind.load(meta, store); } }