// graph-search.ts — the weighted deduction system that thinking runs on. // // Thinking is ONE thing: a lightest derivation over the Sema graph (derive's // adapted A*LD engine). This file is that one thing, factored out of the mind: it // states the items, axioms, goal, and weighted rules, and returns the chosen // spans of the lightest cover. Every behaviour — covering the query, following // edges to a fixpoint, jumping a concept (halo) link, fusing fragments into a // deeper learned form, splicing a learnt connector between two rewrites, and // re-voicing an answer in the asker's words — is a rule here, not a separate // algorithm. // // It reaches Sema only through the {@link Store} interface and one injected // callback, {@link GraphSearch.resolve} (canonical node id of a byte span). The // async hints the synchronous search cannot gather for itself — concept targets // and connectors — are pre-resolved by the caller (see src/mind/pipeline.ts) and handed in. // So this engine stays decoupled: it knows the graph's shape, never how the // graph was learnt. import { type DeductionSystem, type Derivation, lightestDerivation, type Rule, } from "../derive/src/index.js"; import type { Store } from "../store.js"; import { boundFor } from "../geometry.js"; import { bytesEqual, concat2, concatBytes, indexOf, latin1 } from "../bytes.js"; import type { GraphSearchHost } from "./types.js"; import { ALL } from "./types.js"; /** A recognised form: a span of the query that names a node already in the * store. `payload` is that node id. */ export interface Site { start: number; end: number; payload: number; } /** A perceived leaf of the query tree, carrying the node id it resolves to. */ export interface Leaf { start: number; end: number; bytes: Uint8Array; node: number | null; } /** A COMPUTED span: the result of applying a manual rule (an extension's * operation — e.g. the `alu` extension's arithmetic) to a recognised stretch * of the query. Every * bit of the intelligence — recognising the operator (literally or by * resonance), parsing the operands, evaluating the operation — is done by the * caller (src/mind/pipeline.ts), which hands the search the finished result span; the search * treats it exactly like any other recognised completion (a learned fact that * happens to be computed rather than stored). `node` is the canonical id of * the result bytes when the store already holds them, else undefined. */ export interface ComputedResult { i: number; j: number; bytes: Uint8Array; node?: number; } // ── the deduction system the graph exploration runs on ─────────────────── // // Three item kinds drive one lightest-derivation search over the Sema graph // (see {@link GraphSearch.buildSearch}): // • Cover(p) — the query is covered up to position p; the goal is // Cover(query.length). // • Form(i,j,node,via) — node names the span [i,j); `via` marks a node // reached by walking an edge (vs. a recognised entry). // • Out(i,j,bytes,…) — a span of OUTPUT bytes [i,j); `cover` marks it usable // in a bridge, `rec` that it is a recognised // completion (free to cover), `node` its id when known. export type GItem = | { kind: "cover"; p: number } | { kind: "form"; i: number; j: number; node: number; via: boolean; /** Set when this form was born by RECOMPOSING rewritten parts into a deeper * learned whole (the fuse of ≥2 recognised completions that names an * edge-bearing node). Its onward continuation is charged at MICRO, not * STEP: once parts are recomposed into a learned form, following that form * to its grounded answer is the recomposition completing — so the * consolidated, more-explanatory reading wins over leaving the parts split. * See {@link GraphSearch.fuse} and {@link GraphSearch.formRules}. */ rcmp?: boolean; /** Set on a concept hop's ASKING form: the edge-less form `node` whose halo * target the caller has not granted yet, held at the hop's own cost and * span. Popping it asks for the target; it leads nowhere itself, so it * is never part of a final derivation (see {@link Licence}). */ ask?: boolean; } | { kind: "out"; i: number; j: number; bytes: Uint8Array; cover: boolean; rec: boolean; node?: number; /** Set only for a {@link ComputedResult} axiom (an extension's derived * value, e.g. ALU arithmetic) — as opposed to a genuinely RECOGNISED * learned form. Both set `rec: true` (a computation bridges the cover * exactly like a learned terminal answer), but they mean different * things for `i..j`'s WIDTH: a recognised form's query-span width is * real evidence of how much of the query's meaning it accounts for * (liftAnswer's half-dominance framing decision); a computed span's * width is operand digit-count, uncorrelated with meaning ("1000 - 421" * is wider than "15 * 7" only because the numbers are bigger, not * because subtraction is more "the point" of its query). See * {@link liftAnswer}. */ computed?: boolean; /** Set on the out emitted at a chain's GENUINE FIXPOINT — the one span kind a * recursive re-cover ({@link GraphSearch.recompleteNode}) may deepen. The * re-cover does NOT run here: this marks the span as eligible, and * {@link GraphSearch.deepen} runs it afterwards on the spans the lightest * derivation actually CHOSE. Part of {@link key}, because it decides * whether the span's final bytes may still change. */ fix?: boolean; /** Set on the out DERIVE-THROUGH produced: a produced fact's own contained * entity (the subject the query never named) combined with the query's tail * named a learned key, and that key's continuation is this out. Part of * {@link key} so the derived reading is a distinct chart item from the plain * concatenation of the same bytes. Named for what it does — derive THROUGH * a produced fact — because `join` is already the confluence mechanism's * `Provenance`, a different act at a different layer. */ throughFact?: boolean; }; type OutItem = Extract; /** What {@link GraphSearch.entityProposals} found inside one fact. */ interface EntityProposals { readonly proposed: ReadonlyMap; readonly source: ReadonlyMap; readonly sites: ReadonlyArray; } // The cost ladder is a strict ORDERING, not tuned magic: // • Coverage dominates everything: leaving one query byte unrecognised (PASS) // outweighs any chain of graph steps a covering derivation could take, so // the search always prefers to recognise. // • A direct edge (STEP) is the cheapest move; fusing adjacent fragments is // free (cost 0) — it only re-NAMES a span, it does not advance the cover. // • A concept jump (CONCEPT) is dearer than a direct edge, so a literal // continuation is preferred to a synonym's. // Any constants with this ordering give the same lightest derivations. // // EXPORTED because the ladder is the ONE cost currency of the whole mind: // think's grounding decider (pipeline.ts) weighs every mechanism's candidate // answer in these same units, so a mechanism-level choice and a byte-level // choice are the same kind of decision — a lightest derivation. export const STEP = 1; export const CONCEPT = 10; export const PASS = 1000; /** The cheapest local cost in the ladder: a recognised completion bridging into * the cover. Far below STEP, so connecting two recognised spans never disturbs * the ordering — and, being the minimum per-position cost, it is the per-byte * unit of the admissible search heuristic (see {@link GraphSearch.buildSearch}). */ export const MICRO = 1e-3; /** Append `v` to the list at `mp[k]`, creating the list on first use. */ function pushInto(mp: Map, k: number, v: V): void { const a = mp.get(k); if (a) a.push(v); else mp.set(k, [v]); } /** One chosen span of the cover, left to right. `node` is the graph node the * span resolved to (when known) — for a recognised completion it is the chain's * terminal node, the foothold the bridge walks edges from. */ export interface Seg { i: number; j: number; bytes: Uint8Array; rec: boolean; node?: number; /** See the `computed` field of the "out" {@link GItem} — set only for an * extension's derived value, never a genuinely recognised learned form. */ computed?: boolean; /** See the `fix` field of the "out" {@link GItem} — this span ended a chain at * a genuine fixpoint, so it is the one span kind a recursive re-cover may * still deepen. Consumed by {@link GraphSearch.deepen}. */ fix?: boolean; } /** The BYTE TERM of a derivation's cost: how many bytes its bridge rule charged * at PASS each — read back off the rule that charged them (`bridgeRule`: * `o.rec ? MICRO : PASS * (o.j - o.i)`), so the split exists in ONE place and * the engine's generic accumulator is left alone. * * `derivation.cost - PASS * readBridgedBytes(derivation)` is therefore the * derivation's DISCRETE work — the number a mechanism reports as `moves`, which * the pipeline's one formula then prices together with `PASS * unaccounted`. * The two readings of the byte term coincide by construction: the spans this * sums are exactly the ones the chart could not recognise (`rec === false`), * which are the spans the candidate leaves unaccounted. */ function readBridgedBytes(derivation: Derivation): number { let bytes = 0; let node: Derivation | undefined = derivation; while (node && node.rule) { const o = node.premises[1]?.item; if (o !== undefined && o.kind === "out" && o.rec === false) { bytes += o.j - o.i; } node = node.premises[0]; } return bytes; } /** Read the chosen spans back off a derivation: the goal is a chain of bridge * steps, each whose second premise is the `out` it crossed. Walk the chain to * the axiom and reverse into left-to-right order. */ function readCover(derivation: Derivation): Seg[] { const segs: Seg[] = []; let node: Derivation | undefined = derivation; while (node && node.rule) { const out = node.premises[1].item; if (out.kind === "out") { segs.push({ i: out.i, j: out.j, bytes: out.bytes, rec: out.rec, node: out.node, computed: out.computed, fix: out.fix, }); } node = node.premises[0]; } segs.reverse(); return segs; } /** One rule application inside the cover's lightest derivation — the FINEST * grain of Sema's core reasoning, one node of the adapted A*LD proof tree. `move` * names which deduction rule fired (the reasoning act); `premises` and * `conclusion` are the items it consumed and produced, each rendered as a * positioned, possibly-node-bearing span; `cost` is the rule's local weight * (STEP for a learned edge, CONCEPT for a synonym hop, 0 for a free fuse, …); * `order` is its post-order position so a tracer can replay the proof in the * order it was built (premises before conclusion). */ export interface DerivationStep { order: number; move: DerivationMove; premises: DerivationItem[]; conclusion: DerivationItem; cost: number; /** The `order`s of the steps that produced this one's premises — the EXACT * data-flow edges of the proof tree (a `bridge` names the `ground` whose * `out` it crossed and the earlier `cover` it extended). A premise that is * an axiom (a seed leaf/form/computed result, never a rule conclusion) has no * producer and contributes no edge, so this lists only the derived premises — * the finest dependency structure the inference holds. */ producers: number[]; } /** A premise or conclusion of a {@link DerivationStep}, flattened from a {@link * GItem} into the fields a rationale cares about. */ export interface DerivationItem { kind: "cover" | "form" | "out"; /** `[i, j)` span for a form/out; for a cover item, `[p, p]`. */ span: [number, number]; /** The bytes an `out` carries (the only item kind that holds output bytes). */ bytes?: Uint8Array; /** The graph node a form/out resolves to, when known. */ node?: number; } // CLOSED ON PURPOSE — AND ONLY THIS ONE IS. A derivation move is BRANCHED ON // (`classifyMove`, the rationale's readers, MOVE_NOTE's fallback), so it is a // closed union: adding one without teaching every reader is a compile error, // which is what a closed vocabulary buys. The MECHANISM names in // `rationale.ts` are the opposite case — written and displayed, never // branched on — and they stay free strings that COMPOSE with the nesting // (`["respond", "think", "recognise"]`). That asymmetry is deliberate; do not // "fix" it by uniting the two (see test/126 for the pipeline half of it). /** The reasoning act a derivation rule performs — the human name for which of * {@link GraphSearch}'s rules fired, recovered from the rule's premise/ * conclusion shape (the rules carry no label, so this classifies by structure, * the single place that maps rule geometry to a name). */ export type DerivationMove = | "axiom" // a seed: a perceived leaf, a recognised form, or a computed result | "follow-edge" // form→form via a continuation edge (STEP) — the core "what follows what" | "concept-hop" // form→form via a halo sibling's edge (CONCEPT) — a synonym jump | "voice" // form→out emitting an asker-voice substitute (articulation) | "ground" // form→out: a chain reached its terminal answer | "splice-connector" // out+out→out joined by a learnt connector (the in-search bridge) | "split" // out→out cut at a sub-leaf form boundary | "fuse" // out+out→out: adjacent fragments recomposed toward a learned form | "recompose" // out+out→form: a fused pair that names an edge-bearing node | "derive-through" // out→out: a produced fact's own contained subject + the query's tail names a learned key (derive the answer through the fact) | "bridge" // cover+out→cover: the cover frontier advanced across a span | "pool-vote" // N premises→conclusion, evidence pooled (combine:"sum" — see derive) | "step"; // any other single-premise move (fallback) /** The chart's canonical key — an item's boundary signature (see * {@link DeductionSystem.key}). Module-level because the join license * ({@link GraphSearch.solve}) names facts by the same key the chart does. */ function chartKey(it: GItem): string { if (it.kind === "cover") return "c" + it.p; if (it.kind === "form") { return `f${it.i}.${it.j}.${it.node}.${it.via ? 1 : 0}.${it.rcmp ? 1 : 0}${ it.ask ? "?" : "" }`; } return `o${it.i}.${it.j}.${it.cover ? 1 : 0}.${it.rec ? 1 : 0}.${ it.fix ? 1 : 0 }.${it.throughFact ? 1 : 0}.${it.node ?? -1}.${latin1(it.bytes)}`; } /** The facts a derivation STANDS ON — every node-bearing `out` in its proof * tree, premises before conclusions, each once. */ function factsOf(root: Derivation): OutItem[] { const out: OutItem[] = []; const seen = new Set>(); const walk = (d: Derivation): void => { if (seen.has(d)) return; seen.add(d); for (const p of d.premises) walk(p); if (d.item.kind === "out" && d.item.node !== undefined) out.push(d.item); }; walk(root); return out; } /** The joins granted to the facts a derivation stood on: fact chart key → the * conclusions {@link GraphSearch.deriveThrough} derives through it, computed * once per fact (an empty list is an answer: that fact joins nothing). */ type JoinLicense = Map< string, ReadonlyArray<{ conclusion: GItem; cost: number }> >; /** THE ASYNC PREMISES A COVER USES, RESOLVED ONLY WHERE THE SEARCH REACHES THEM. * * Two of the cover's rules need data the synchronous search cannot fetch — a * connector splice (a bridge between two answers) and a concept hop (an * edge-less form's halo sibling) — and both used to be resolved for EVERY * candidate before the search ran. A rule fires only from popped premises, * and the agenda pops in cost order up to the goal, so most of that work fed * rules the search never reached: measured on the 31.7M-node store, a 262-byte * query recognised whole paid 23 bridges (143,520 junction pops, 7.3 s of its * 7.9 s) for sub-forms whose rewrites cost a STEP each and so never left the * agenda before the one-STEP goal — and then 1.2 s of halo reads for concept * hops priced at CONCEPT, ten times that goal. * * So the caller OFFERS what it can resolve; where the search reaches an offered * key it uses a GRANTED value and ASKS for an ungranted one — a connector when * its splice's premises meet, a concept target when the hop's conclusion, held * as an asking form at the hop's own cost, is popped. A run that asked * nothing is the run every key resolved in advance would have made: the pops * are decided only by the rules fired from popped items, a granted key fires * the very rule it fired before, and an asking form that was never popped fired * none. So its cover is final. A run that asked is PROVISIONAL: the caller * grants the asked keys (the async reads the search cannot run) and covers * again; every round grants at least one offered key, so the rounds end. */ export interface Licence { /** Every key the caller can resolve. */ readonly offered: ReadonlySet; /** The keys resolved so far: their value, or null (nothing learnt). */ readonly granted: ReadonlyMap; /** Filled by the search: the offered keys it reached ungranted. */ readonly asked: Set; } /** Connectors, keyed `L,R` by answer-node ids, valued by the learnt glue. */ export type ConnectorLicence = Licence; /** Concept hops, keyed by an edge-less form's node, valued by the node its * halo sibling's edge leads to. */ export type ConceptLicence = Licence; /** Whether a run asked for anything — its cover is then provisional. */ function asked( concepts: ConceptLicence, connectors: ConnectorLicence | undefined, ): boolean { return concepts.asked.size > 0 || (connectors?.asked.size ?? 0) > 0; } /** Nothing offered: a cover with no async premises (a nested completion, * articulation's revoicing). */ export function noLicence(): Licence { return { offered: new Set(), granted: new Map(), asked: new Set() }; } /** Flatten a {@link GItem} into the {@link DerivationItem} a rationale shows. */ function derivationItem(it: GItem): DerivationItem { if (it.kind === "cover") return { kind: "cover", span: [it.p, it.p] }; if (it.kind === "form") { return { kind: "form", span: [it.i, it.j], node: it.node }; } return { kind: "out", span: [it.i, it.j], bytes: it.bytes, node: it.node }; } /** Name the reasoning act a rule performed, from the shape of what it consumed * and produced — the one mapping from rule geometry to a human move. Mirrors * the rule set in {@link GraphSearch.coverRules}/{@link formRules}/{@link * outRules}/{@link fuse}; kept here beside {@link readCover} so the two readers * of a derivation sit together. */ function classifyMove( premises: ReadonlyArray, conclusion: GItem, articulating: boolean, ): DerivationMove { if (premises.length === 1) { const [p] = premises; if (p.kind === "form" && conclusion.kind === "form") { // form→form is an edge step; CONCEPT cost marks the synonym hop, but the // cost is on the rule, not the item — the caller passes it to refine this. return "follow-edge"; } if (p.kind === "form" && conclusion.kind === "out") { // form→out: grounding to a terminal answer, or — under a substitution map — // the form emitting the asker's own wording (articulation's `voice`). if (!conclusion.rec) return "step"; return articulating ? "voice" : "ground"; } if (p.kind === "out" && conclusion.kind === "out") { // A JOIN derives through a produced fact's own contained subject; a plain // single-premise out→out is the byte-level split. return conclusion.throughFact ? "derive-through" : "split"; } return "step"; } if (premises.length === 2) { const [a, b] = premises; if (a.kind === "cover" || b.kind === "cover") return "bridge"; if (conclusion.kind === "form") return "recompose"; if (conclusion.kind === "out") { // A connector splice concatenates THREE pieces (l + link + r); a plain // fuse concatenates two (concat2). Distinguish by BYTE width: a splice's // conclusion is wider than its two premises summed (the link sits between // them), whereas a fuse's conclusion is exactly their sum. Position can't // tell them apart — in "icefire" the two rewrites are positionally // adjacent yet a connector is still spliced into the bytes. if (a.kind === "out" && b.kind === "out") { const summed = (a.bytes?.length ?? 0) + (b.bytes?.length ?? 0); return (conclusion.bytes?.length ?? 0) > summed ? "splice-connector" : "fuse"; } return "fuse"; } } return "step"; } /** Walk a finished derivation into its rule applications, in post-order * (premises before the conclusion they feed), deduplicating shared * sub-derivations so a node reached by two rules is reported once. This is the * whole proof tree — every STEP, CONCEPT, fuse and bridge the lightest cover * was built from — the finest granularity a rationale can reach. */ function readDerivation( root: Derivation, articulating = false, ): DerivationStep[] { const steps: DerivationStep[] = []; // The `order` assigned to each already-emitted derivation node, so a later // step that consumes it as a premise can name it as a producer — the proof // tree's data-flow edge, preserved exactly. Only RULE-bearing nodes (the // emitted steps) get an entry; an axiom premise (a seed leaf/form/computed // result) has no producer and contributes no edge. `order` therefore stays // contiguous with `steps`, so producers index directly into the emitted list. const orderOf = new Map, number>(); let order = 0; const walk = (d: Derivation): void => { if (orderOf.has(d)) return; // a shared sub-derivation, already emitted for (const p of d.premises) walk(p); if (!d.rule) return; // an axiom: a seed, reported as its consumer's premise const move = classifyMove(d.rule.premises, d.rule.conclusion, articulating); // Refine an edge step to a concept hop by its cost (CONCEPT > STEP). const refined: DerivationMove = move === "follow-edge" && d.rule.cost >= CONCEPT ? "concept-hop" : move; const producers: number[] = []; for (const p of d.premises) { const o = orderOf.get(p); // defined iff p was itself an emitted rule step if (o !== undefined) producers.push(o); } const myOrder = order++; orderOf.set(d, myOrder); steps.push({ order: myOrder, move: refined, premises: d.premises.map((p) => derivationItem(p.item)), conclusion: derivationItem(d.item), cost: d.rule.cost, producers, }); }; walk(root); return steps; } /** The lightest-derivation search over the Sema graph. One instance binds the * store, `maxGroup` (the fusible span ceiling), and the canonical * {@link resolve} callback; {@link cover} then solves one query. */ export class GraphSearch { constructor( private readonly store: Store, private readonly maxGroup: number, /** The host whose capabilities the search consults: resolve (canonical node * id of a byte span), recogniseSpan (content-addressed graph lookup for * recursive completion), and chooseNext (distributional-evidence edge * disambiguation when a recognised form has multiple continuations). */ private readonly host: GraphSearchHost, ) {} /** The hub bound √N (bounded-reads.md) — the ONE fan-out cap, read over the * store this module holds (it never holds a MindContext). */ private hubBound(): number { return boundFor(this.store.edgeSourceCount()); } /** Explore the Sema graph for the lightest cover of the query and return its * chosen spans left-to-right — WITH the derivation's total weight (the g * value of the goal item, in the exported cost ladder), which think's * grounding decider compares against the other mechanisms' candidates — * or null if the query cannot be covered. * * The search runs on the query's tree leaves, not flat bytes — leaf-level * cover axioms, recognised forms as graph entry points — and discovers * cross-leaf forms by fusing adjacent fragments on demand (the only * byte-processing it does, and only where a derivation needs it). When * `substitutions` is given, recognised forms emit substitute bytes directly * (cost 0) — articulation splicing the asker's wording in. * * Any learnt connector between two rewrites is spliced IN by the in-search * connector rule (see {@link outRules}), so the returned spans already carry * it — there is no post-pass. With a {@link ConnectorLicence}, a cover * returned while `connectors.asked` is non-empty is PROVISIONAL: grant the * asked pairs and cover again. */ cover( queryLen: number, sites: ReadonlyArray, concepts: ConceptLicence, leaves: ReadonlyArray, splits: ReadonlySet, substitutions?: ReadonlyMap, connectors?: ConnectorLicence, computedResults?: ReadonlyArray, /** When given, receives each solved span's lightest derivation — the full * adapted A*LD proof tree as classified {@link DerivationStep}s — for the * TOP cover AND every nested completion the sink is threaded into (see * {@link recompleteNode}), so a produced form's own recompositions reach * the rationale instead of stopping at the first layer. Off by default, so * the search pays nothing when no one inspects. */ onDerivation?: (steps: DerivationStep[]) => void, ): { segs: Seg[]; cost: number; moves: number } | null { // Top-level entry: reset the per-call recursion state, then run the one // {@link solve} routine that both the query and any produced composite go // through (completion is cover, recursively — see {@link recompleteNode}). this.recompleteOpen.clear(); this.recompleteMemo = new Map(); this.entityMemo = new Map(); // The top cover's derivation sink is threaded into every nested completion // so the recompositions a produced form needs are reported in the same // trace instead of vanishing after the first layer. this.derivationSink = onDerivation; concepts.asked.clear(); if (connectors) connectors.asked.clear(); const solved = this.solve( queryLen, { sites, leaves, splits }, concepts, substitutions, connectors, computedResults, onDerivation, ); // Deepening runs HERE, once, on the derivation the top cover CHOSE — never // inside the nested solve a completion runs. Nesting the deepening is what // made per-query cost track how densely the corpus interconnects the forms // passed through: every re-cover fanned out into the whole corpus's // continuations instead of following the chain the answer itself licensed. // With deepening only at the top, `recompleteNode` walks the accepted chain // one link at a time (its own memo and stack), so the work is the answer's. // A provisional cover is not deepened: it is about to be re-covered. if (solved === null || asked(concepts, connectors)) return solved; return { segs: this.deepen(solved.segs), cost: solved.cost, moves: solved.moves, }; } /** Build the deduction system for one span and return its lightest cover's * chosen spans — the SINGLE routine the query and every produced composite * run through. `recognition` carries the span's recognised forms; the query * brings its own (with pre-resolved concepts/connectors), a recursive * completion re-recognises the produced bytes (edge/fuse only). * * No depth limit governs nesting — convergence is INTRINSIC, exactly as in the * adapted A*LD chart and {@link completeForward}: a completion only recurses into a * node it has not already entered ({@link recompleteNode}'s cycle guard), and * the node ids are finite, so the recursion must terminate on its own. A * decomposition may therefore run as deep as the graph licenses — three * decomposes, two recomposes, any mix — and stops only when it reaches a node * that leads nowhere new, never at an arbitrary count. */ private solve( spanLen: number, recognition: { sites: ReadonlyArray; leaves: ReadonlyArray; splits: ReadonlySet; }, concepts: ConceptLicence, substitutions?: ReadonlyMap, connectors?: ConnectorLicence, computedResults?: ReadonlyArray, onDerivation?: (steps: DerivationStep[]) => void, ): { segs: Seg[]; cost: number; moves: number } | null { // The query's own bytes, tiled from its perceived leaves. A JOIN reads the // tail a produced fact's contained entity has to combine with, and the // search otherwise only ever sees positions, never the bytes behind them. const queryBytes = new Uint8Array(spanLen); for (const lf of recognition.leaves) { queryBytes.set(lf.bytes.subarray(0, lf.end - lf.start), lf.start); } // The nodes this span's recognition names — read ONCE per solve, on the first // join that asks, instead of re-reading the memo per fact. let queryNodeSet: ReadonlySet | undefined; const queryNodes = (): ReadonlySet => queryNodeSet ??= new Set( (this.host.recogniseSpan?.(queryBytes)?.sites ?? []).map((s) => s.payload ), ); // THE JOIN DERIVES THROUGH THE FACTS THE DERIVATION STANDS ON — never // through every fact the exploration REACHES. How many facts the search // reaches is set by how densely the corpus interconnects the forms on the // way (a dialogue hub's continuations, walked to their fixpoints, are // thousands), and pricing a join from each of them made the work track the // corpus instead of the answer: measured on the 31.7M-node store, a 46-byte // query reached 2,000+ fixpoints in a minute, each paying an entity scan, // past 10 GB. It is the trap {@link recompleteNode} already records, and // the same cure as {@link deepen}: the expensive step runs on what the // lightest derivation CHOSE. // // So the search runs to its lightest derivation, the facts that derivation // stands on are LICENSED — their joins computed once — and it runs again // with those joins as ordinary rules of the same system, priced exactly as // before (the fact's own cost + STEP). A join's conclusion is itself a fact // the next derivation may stand on, so chains extend round by round until // no newly licensed fact joins anything: the depth is the answer's, never a // count. A round is re-run only when a newly licensed fact DOES join, so a // query whose facts join nothing pays one search, as before. // // The joins a covering round licensed are KEPT across a licence's re-covers: // a round that asked for nothing is the very round the next re-cover repeats // (see {@link Licence}), so its joins are the ones that re-cover would grant // again. const joins: JoinLicense = this.joinsKept.get(concepts) ?? new Map(); this.joinsKept.set(concepts, joins); // Search-effort accounting (src/meter.ts): the chart's pops/pushes are // the cover's real cost, and a heuristic that stops being admissible // shows up as a pop count that explodes while the answer stays the same. const meter = this.host.meter; let derivation: Derivation | null; for (;;) { const system = this.buildSearch( spanLen, recognition.sites, concepts, recognition.leaves, recognition.splits, queryBytes, joins, substitutions, connectors, computedResults, ); const stats = meter ? { pops: 0, pushes: 0 } : undefined; derivation = lightestDerivation(system, stats); if (meter && stats) { meter.searches++; meter.searchPops += stats.pops; meter.searchPushes += stats.pushes; } // A round that ASKED for a connector is provisional: return it before any // join is licensed from a derivation the granted connectors may change. if (derivation === null || asked(concepts, connectors)) break; let joined = false; for (const fact of factsOf(derivation)) { const k = chartKey(fact); if (joins.has(k)) continue; if (meter) meter.joinFacts++; const granted = [ ...this.deriveThrough(fact, queryBytes, spanLen, queryNodes), ].map((r) => ({ conclusion: r.conclusion, cost: r.cost })); joins.set(k, granted); if (granted.length > 0) joined = true; } if (!joined) break; } // When covering under a substitution map (articulation), a form→out rule is // the form EMITTING the asker's voice, not grounding to its own answer — so // tell the reader to name those moves `voice` rather than `ground`. if (derivation && onDerivation) { onDerivation(readDerivation(derivation, substitutions !== undefined)); } return derivation ? { segs: readCover(derivation), cost: derivation.cost, moves: derivation.cost - PASS * readBridgedBytes(derivation), } : null; } /** Re-cover the CHOSEN fixpoint spans, in place. * * Completion is still "cover, recursively" — it just runs on the answer * instead of on the exploration. A cover chooses O(segs) spans, so a level * pays O(answer) re-covers however densely the corpus interconnects the forms * the search passed through on the way. That is the bound * {@link recompleteNode}'s contract always claimed and, running per fixpoint * REACHED, never had. * * Deepening cannot change which cover won: the derivation is already final * and every span keeps its i..j and its cost. It only replaces a chosen * span's bytes with the deeper learnt form they rewrite to — what the * recursion was always for. */ private deepen(segs: Seg[]): Seg[] { for (const s of segs) { if (!s.fix || s.node === undefined) continue; const deeper = this.recompleteNode(s.node); if (deeper !== null) s.bytes = deeper; } return segs; } /** The weighted deduction system the graph exploration solves (the four * reductions of adapted A*LD live in {@link lightestDerivation}; this only states the * items, axioms, goal, and rules — see {@link GItem} for the item kinds). * * Forms that span across leaves are discovered BY the rules, which fuse * adjacent fragments toward a known leaf (findLeaf) or branch (findBranch); * a completion fused with its neighbour may spell a deeper learned form the * flat probes can't name, recovered canonically by {@link resolve}. */ private buildSearch( queryLen: number, sites: ReadonlyArray, concepts: ConceptLicence, leaves: ReadonlyArray, splits: ReadonlySet, queryBytes: Uint8Array, joins: JoinLicense, substitutions?: ReadonlyMap, connectors?: ConnectorLicence, computedResults?: ReadonlyArray, ): DeductionSystem { const W = this.maxGroup; // fusible span ceiling (shortest composite bound) // Same corpus-scale hub floor {@link atomIsHub}/{@link atomReach} (traverse.ts) // derive for byte atoms — see {@link hubBound} below for why this module // states the formula itself instead of importing it. const atomsAreHubs = Math.max( 1, Math.ceil((this.store.edgeSourceCount() * W) / 256), ) > this.hubBound(); const nodeBytes = (n: number) => this.store.bytesPrefix(n, ALL); // Content-addressed probes over the store's hash-cons maps — the same keys // training filled. No byte-by-byte trie walk. const findLeafU = (b: Uint8Array) => this.store.findLeaf(b) ?? undefined; const findBranchU = (k: number[]) => this.store.findBranch(k) ?? undefined; // Finalised `out` items, indexed for the binary (bridge, fuse) rules. const coversDone = new Set(); const outsByStart = new Map(); const outsByEnd = new Map(); const outsByNode = new Map(); const coverableByStart = new Map(); // Index the offered connectors by their left and right answer-node, so the // connector rule iterates only this out's FEW offered partners (selective, // and for the N-ary case O(parts) keys) instead of scanning every position // pair — what keeps the in-search bridge bounded when many parts are // recognised at once. const linksByLeft = new Map>(); const linksByRight = new Map>(); if (connectors) { for (const key of connectors.offered) { const comma = key.indexOf(","); const l = Number(key.slice(0, comma)); const r = Number(key.slice(comma + 1)); pushInto(linksByLeft, l, [r, key]); pushInto(linksByRight, r, [l, key]); } } // A SPAN'S CHEAPEST COMPLETION DOMINATES THE REST. Coverage is positional: // every recognised completion of [i, j) advances the cover from i to j at // the same MICRO, so for the goal only the cheapest matters. What else a // completion can do turns on its BYTES — fuse it with a neighbour, splice a // connector, join through it — and those rules fire from the completion the // search would stand on for that span, never from every alternative it // reached: the cure {@link deepen} and the join license already apply, for // the same trap. Measured on the 31.7M-node store, a 46-byte dialogue query // reached 60,530 continuation forms and 57,072 fixpoints for its few spans — // a greeting's thousands of replies, each a fixpoint at the same depth — and // canonically resolved every one of them fused with its neighbours (5.4 s // of an 8.6 s cover) to stand, in the end, on two hops. // // So a form or completion of [i, j) whose cost has reached that of a // completion of [i, j) already yielded is DOMINATED and fires nothing: every // completion it could lead to costs at least as much (local costs are // non-negative and a form's rules keep its span), and a tie goes to the one // yielded first — the evidence-preferred continuation, yielded first by // {@link formRules}. This is also what makes the stop-here offer a real // horizon: a chain deeper than its first hop's stop (STEP + CONCEPT) is // dominated by it, where before the PASS-priced goal let the search buy a // thousand hops to look for one more byte. const spanBest = new Map(); const spanOf = (i: number, j: number) => i * (queryLen + 1) + j; const completes = (it: GItem): it is OutItem => it.kind === "out" && it.rec && it.cover; const dominated = (it: GItem, cost: number): boolean => { if (it.kind === "cover") return false; if (it.kind === "out" && !completes(it)) return false; const best = spanBest.get(spanOf(it.i, it.j)); if (best === undefined || best.cost > cost) { if (completes(it)) spanBest.set(spanOf(it.i, it.j), { cost, item: it }); return false; } return best.item !== it && (it.kind !== "out" || chartKey(best.item) !== chartKey(it)); }; const completing = function* ( rules: Iterable>, costOf: (it: GItem) => number, ): Iterable> { for (const r of rules) { const c = r.conclusion; if (completes(c)) { let cost = r.cost; for (const p of r.premises) cost += costOf(p); const s = spanOf(c.i, c.j); const best = spanBest.get(s); if (best === undefined || cost < best.cost) { spanBest.set(s, { cost, item: c }); } } yield r; } }; const meter = this.host.meter; return { key: chartKey, *axioms() { yield { item: { kind: "cover", p: 0 }, cost: 0 }; // One out per tree leaf — content-defined chunks, far fewer than bytes. // Each carries the node id it resolves to (when known) so it can compose // toward a known branch by findBranch. for (const lf of leaves) { yield { item: { kind: "out", i: lf.start, j: lf.end, bytes: lf.bytes, cover: true, rec: false, node: lf.node ?? undefined, }, cost: 0, }; } for (const s of sites) { yield { item: { kind: "form", i: s.start, j: s.end, node: s.payload, via: false, }, cost: 0, }; } // Computed (extension) results — see {@link ComputedResult}. Each enters as a // RECOGNISED covering completion, exactly like a learned terminal answer: // its bytes are the computed result, it bridges the cover at MICRO (rec), // and it carries the result's canonical node (when the store holds it) so // it can fuse as an operand of an outer form. The STEP base cost marks a // computation as a unit of work — a derived fact, on par with following a // learned edge (STEP), so it decisively beats leaving the span // unrecognised (PASS) but never masquerades as a free perceived leaf. // Because this cost EQUALS a learned edge's, the search would tie a // computation against a colliding recall; the computation-always-wins policy lives // in the caller (src/mind/pipeline.ts think), which masks any recognised site a result // overlaps so the computation is the cover's sole completion there — the // search stays a neutral cost engine with no computation-vs-recall precedence baked // in. When `computedResults` is empty (every non-arithmetic query) this loop // emits nothing, so the search is byte-identical to one with no extension at all. for (const u of computedResults ?? []) { yield { item: { kind: "out", i: u.i, j: u.j, bytes: u.bytes, cover: true, rec: true, node: u.node, computed: true, }, cost: STEP, }; } }, isGoal: (it) => it.kind === "cover" && it.p === queryLen, // Admissible, consistent lower bound on the cost remaining to the goal // cover(queryLen) — this is what makes the adapted A*LD search OUTPUT-SENSITIVE // (work proportional to the answer, not to how densely the corpus enriched // the query's sub-forms). Without it the engine runs as uninformed // Dijkstra and pops every zero-cost fuse before the goal; with it the // agenda is ordered g + h, so the frontier is driven FORWARD toward full // coverage instead of wallowing in low-position fragment fusions. // // The bound: whatever an item is, reaching the goal still requires the // cover frontier to advance to queryLen, and the CHEAPEST any single // covered position can be is the recognised-completion bridge (ε = MICRO, // the minimum local cost in the ladder — every other move costs ≥ STEP). // So the remaining query past the item's right edge is a guaranteed // ≥ ε-per-byte cost. cover(p) still owes [p,queryLen); a form/out at // [i,j) can contribute coverage no further than j, so it still owes // [j,queryLen). Using ε (≤ every real per-byte cost, incl. PASS) keeps it // a true lower bound; counting only the suffix past the right edge keeps it // consistent (each forward rule pays ≥ ε per byte it advances the edge). heuristic: (it) => { const right = it.kind === "cover" ? it.p : it.j; return (queryLen - right) * MICRO; }, rules: (it, costOf) => { if (it.kind === "cover") { return this.coverRules(it, coversDone, coverableByStart); } if (dominated(it, costOf(it))) { if (meter) meter.searchDominated++; return []; } if (it.kind === "form") { return completing( this.formRules(it, concepts, substitutions, nodeBytes), costOf, ); } return completing( this.outRules(it, { W, splits, atomsAreHubs, coversDone, outsByStart, outsByEnd, outsByNode, coverableByStart, findLeafU, findBranchU, linksByLeft, linksByRight, connectors, joins, }), costOf, ); }, }; } /** cover(p): the BRIDGE rule — extend the cover across any coverable out that * begins at p, stepping the frontier from p to that out's end. * * This is what links the rewritten parts of a multi-form answer. An out is * either a recognised completion (`rec`, free) or a literal span the query * carried between known forms — the connective: a space, comma, period, * newline, or any run of bytes that was never recognised. Bridging a literal * costs PASS per byte (so the search still prefers to recognise), but it is * the cheapest — indeed only — way to cross a gap that has no learned form, * and it KEEPS that connective in the cover chain, so the asker's own linking * material reappears when it still coheres ("ice, fire" → "cold, hot", not * "coldhot"). A recognised completion bridges for a tiny ε (1e-3, far below * STEP), so a single connected span beats two separate ones on coherence * without disturbing the cost ladder's ordering. * * Marks p finalised so the symmetric out-side bridge ({@link outRules}) can * fire for outs that arrive after their start position is covered. */ private *coverRules( it: Extract, coversDone: Set, coverableByStart: Map, ): Iterable> { coversDone.add(it.p); for (const o of coverableByStart.get(it.p) ?? []) { yield this.bridgeRule(it, o); } } /** The BRIDGE rule, built once for its two arrival orders (cover-first in * {@link coverRules}, out-first in {@link outRules}): ε for a recognised * completion; PASS per byte for a literal connective — kept in the cover * so the asker's own connector survives where it fits. ONE definition of * the cost expression, so the ladder's application cannot drift between * the two sides. */ private bridgeRule( cover: Extract, o: OutItem, ): Rule { return { premises: [cover, o], conclusion: { kind: "cover", p: o.j }, cost: o.rec ? MICRO : PASS * (o.j - o.i), }; } /** The connector-SPLICE rule for an oriented (l, r) pair, or null when the * pair does not qualify — the ONE body behind {@link outRules}' two * mirror loops (this-as-left over offered right partners, this-as-right * over offered left partners). Fires only when both sides are * recognised, r starts at or after l ends, and the gap between them is * empty or wholly recognised — never across the asker's own literal * separator — and the pair's connector is granted and learnt. A qualifying * pair not yet granted is ASKED for (see {@link ConnectorLicence}). */ private trySplice( l: OutItem, r: OutItem, key: string, licence: ConnectorLicence, outsByEnd: Map, ): Rule | null { if (!l.rec || !r.rec || r.i < l.j) return null; if (!this.gapRecognised(l.j, r.i, outsByEnd)) return null; const link = licence.granted.get(key); if (link === undefined) licence.asked.add(key); if (!link) return null; return { premises: [l, r], conclusion: { kind: "out", i: l.i, j: r.j, bytes: concatBytes([l.bytes, link, r.bytes]), cover: true, rec: true, node: r.node, }, cost: 0, }; } /** form(i,j,node,via): follow the graph out of `node`, or (in articulation) * emit its substitute voice directly. */ private *formRules( it: Extract, concepts: ConceptLicence, substitutions: ReadonlyMap | undefined, nodeBytes: (n: number) => Uint8Array, ): Iterable> { // An asking form reached the agenda's front: the hop it stands for is due. if (it.ask) { concepts.asked.add(it.node); return; } // Articulation: emit voice bytes at the recognised span; the hop/concept/ // emit chain is suppressed — the form contributes only its substitute. if (substitutions) { const voice = substitutions.get(it.node); if (voice !== undefined) { yield { premises: [it], conclusion: { kind: "out", i: it.i, j: it.j, bytes: voice, cover: true, rec: true, node: it.node, }, cost: 0, }; } return; } if (it.via) { // A CHAIN hop (this form was itself reached by following an edge): // read up to the hub bound — the SAME √(max(2,N)) convention {@link // hubBound} in traverse.ts derives, replicated here because // GraphSearch holds only a bare Store, not a MindContext — and fork // across EVERY continuation. A bidirectional pair (SmolSent trains // both directions) puts the back-edge to where we came from ahead of // the forward edge in insertion order; a single first-inserted pick // would walk the search straight into a cycle (the A*LD duplicate-key // guard then dead-ends it) with no way to reach the forward edge. // Forking offers every continuation as its own rule so the one that // genuinely advances (not a duplicate) is still reachable. // // THE OFFER IS THE CORPUS'S OWN STRUCTURE, and the search pays for it. // In the closure law's terms (derivation.ts) a hop is a MOVE — it reaches // structure the derivation has not stood on — so it is admitted on that // ground and priced one STEP; the A*LD search, not a count, decides how // many are worth taking. `hubBound` is the READ cap (invariant 5) and // nothing else bounds the offer: a cap on what a hop may OFFER was tried // and removed as a short-circuit (trap 13). Measured with the hub's // degree below √N: the work is LINEAR in the degree (pushes 262/296/332 at // degrees 35/70/120) and the peak is set by the store, not the fan-out // (test/112 pins the non-quadratic shape). const nx = this.store.nextFirst(it.node, this.hubBound()); // Count what is OFFERED, not what was read: the evidence-preferred // continuation is yielded too, even when it lies outside the cap. if (this.host.meter) this.host.meter.chainOffers += nx.length + 1; if (nx.length) { // The SAME evidence-weighted disambiguation the first hop uses // (below) identifies the most-corroborated continuation. Yielding // it FIRST matters: {@link lightestDerivation}'s chart keeps the // FIRST rule to reach a given cost for an item and discards later // arrivals at an EQUAL cost (`cost < current`, strictly) — so // among same-depth sibling forks that tie in cost, the // evidence-backed edge wins deterministically, never by // exploration-order luck. `preferred` is NOT necessarily an element // of `nx` any more: `nx` is the capped read above, while `chooseNext` // reads its own hub-bounded set (see traverse.ts) — so the evidence // pick may lie outside the cap, and it is still yielded FIRST on // purpose. The cap bounds what the hop EXPLORES; it must never make // the evidence-ranked continuation unreachable, or the derivation the // corpus corroborates would lose to exploration order. Nothing is // reallocated: `nx` is walked with a skip-in-place for the duplicate. const preferred = nx.length > 1 ? this.host.chooseNext?.(it.node) : undefined; const fork = (node: number): Rule => ({ premises: [it], conclusion: { kind: "form", i: it.i, j: it.j, node, via: true }, // A real edge always costs STEP, whether it is the first hop or // the fifth — so a chain's total cost is proportional to its // length and the lightest derivation is always the SHORTEST // successful one. (Charging every hop after the first for // FREE, as an earlier design did, made every stopping point at // any depth tie at the same cost — indistinguishable to the // search, decided by whichever arrived first.) cost: STEP, }); if (preferred !== undefined) yield fork(preferred); for (const c of nx) { if (c !== preferred) yield fork(c); } // Also offer stopping HERE, at CONCEPT above whatever the chain // has already cost — the same "a synonym jump is dearer than a // direct edge" ordering the ladder already uses elsewhere, // repurposed as "giving up early is dearer than one more real // hop". A premature stop at depth D costs D·STEP + CONCEPT, so a // SHORTER premature stop always beats a longer one, and a genuine // fixpoint (below, costing +0) always beats any premature stop at // the same depth — the search only settles here when continuing // genuinely dead-ends (every fork above revisits an already- // explored key) or grows costlier than giving up. // // The stop-here bytes are the node's OWN, as-is — NOT recompleteNode's // recursive re-cover. recompleteNode re-runs the full recognition // + edge/fuse search over the node's bytes, an O(node's own // substructure) cost independent of chain depth; offering it at // EVERY premature stop makes a query's total cost scale with how // many nodes the corpus happens to interconnect here (corpus // density), not with the answer's own hop count — the exact // output-sensitivity the rest of this search is built to keep. // Reserved below for the ONE place it is load-bearing: the true // fixpoint, checked once, at the chain's actual end. yield { premises: [it], conclusion: { kind: "out", i: it.i, j: it.j, bytes: nodeBytes(it.node), cover: true, rec: true, node: it.node, }, cost: CONCEPT, }; return; } // The chain reached a node with no WHOLE-node continuation anywhere // — a genuine fixpoint, not a premature stop. Before emitting it as // terminal, CONTINUE THE EXPLORATION into its own structure: a // composite answer like "p1 p2" leads nowhere as a whole, yet // recognising it surfaces p1, p2 — each of which continues (→ R1, // R2) and recomposes into a deeper learnt form (→ FINAL). {@link // recompleteNode} re-covers the node's bytes through the SAME // recognition + edge/fuse machinery the top query uses (a continued // graph exploration, not a re-perception), and returns the deeper // completion's bytes when it genuinely leads somewhere new. Emit // that; else emit the node itself as the terminal answer. This is // the ONE place the recursive re-cover runs — once, at the chain's // actual end, never per intermediate stop — so its cost tracks the // ANSWER's own structure, not how densely the corpus interconnects // the nodes passed through on the way there. // MARK the fixpoint; do not re-cover it here. Re-covering at this point // pays a full {@link recompleteNode} — a recognition of the node's whole // bytes — for every fixpoint the exploration REACHES, and how many it // reaches is set by how densely the corpus interconnects the forms passed // through. Measured on an 18.9M-node store, a 2-byte query: 12,000 // re-covers inside ONE cover, folding 95,258 distinct spans, ~2.4 GB and // climbing to a V8 fatal. The answer needs a handful. // // {@link deepen} runs it instead on the spans the lightest derivation // CHOSE. The ladder is untouched — this out still costs 0, so a genuine // fixpoint still beats any premature stop at the same depth, exactly as // the ordering above states. yield { premises: [it], conclusion: { kind: "out", i: it.i, j: it.j, bytes: nodeBytes(it.node), cover: true, rec: true, node: it.node, fix: true, }, cost: 0, }; } else { // The FIRST hop out of a recognised site — unchanged from the // original design. LIMIT 2 only senses PLURALITY (does this node // have more than one learnt continuation?); the actual bounded read // and evidence-weighted pick both live inside {@link // Mind.chooseNext} (see traverse.ts), which floors its own hub bound // at √(max(2,N)) regardless of what we read here. const nx = this.store.nextFirst(it.node, 2); if (nx.length) { yield { premises: [it], conclusion: { kind: "form", i: it.i, j: it.j, node: (nx.length > 1 ? this.host.chooseNext?.(it.node) : undefined) ?? nx[0], via: true, rcmp: it.rcmp, }, // A recomposed form's continuation is FREE: the two (or more) parts // were already paid for as their own rewrites, and the single // consolidated span saves one cover-bridge versus leaving them // split — so charging 0 here makes the grounded whole (e.g. // "DE"→F) strictly beat the split ("D","E") by exactly that saved // bridge, deterministically. cost: it.rcmp ? 0 : STEP, }; return; } // Recognised but edge-less: borrow a concept (halo) sibling's edge. No // edge and no concept means the form leads nowhere — it yields no rule, so // a query of only such forms produces no derivation, and think is silent. // An offered form not yet granted yields the hop's ASKING form at the // hop's own cost and span (see {@link Licence}). if (!concepts.offered.has(it.node)) return; const target = concepts.granted.get(it.node); if (target === undefined) { yield { premises: [it], conclusion: { kind: "form", i: it.i, j: it.j, node: it.node, via: true, ask: true, }, cost: CONCEPT, }; } else if (target !== null) { yield { premises: [it], conclusion: { kind: "form", i: it.i, j: it.j, node: target, via: true, }, cost: CONCEPT, }; } } } /** Complete a node that an edge produced but that bears no further whole-edge, * by COVERING ITS OWN BYTES — the very same operation the top query runs. * Completion is not a special pass: a produced composite ("p1 p2") is just * another span to cover, and covering it re-applies recognition, edge-follow, * fusion and recomposition to discover that its parts continue (p1→R1, p2→R2) * and recompose into a deeper learnt form (→ FINAL). This is why a single * edge-target needs no bespoke logic — it routes back through {@link solve}. * * The produced bytes are decomposed by the machinery that owns their shape: * the span's LEAVES and SPLITS drive the split rule, which resolves each half * through findLeaf — so a part that straddles a content-defined cut is still * recovered even though the node's tree children need not align with the * learnt parts (the fold cuts "p1 p2" as "p1 p"|"2"). Recognised SITES are * filtered to the node's own kids: a produced form is completed out of what * it was built from, never by re-recognising arbitrary forms inside it. That * filter is load-bearing — a 37-byte dialogue sentence carries seven hub * openers, and re-covering them chained through the corpus's whole * continuation population: 2.5 GB and OOM for `respond("hi.")`. * * The recovered answer is accepted only when it MOVED and names a LEARNT node * ({@link resolve}) — the graph itself gates against re-expanding a contained * form ("ice is cold" ⊅→ "ice is cold is cold"). An ACCEPTED completion is * then re-covered in turn, so a chain runs as deep as the graph licenses; a * rejected one ends its branch, so no work is spent past it. * * Termination and cost are STRUCTURAL: {@link recompleteOpen} is the chain's * stack (membership is the cycle guard), {@link recompleteMemo} re-covers each * node at most once, only accepted completions recurse, and every level * deepens only the top derivation — so a cover pays for the chain it finds, * not for how densely the corpus interconnects the forms it passes through. * Guard: test/89-completion-recursion.test.mjs. */ private recompleteNode(node: number): Uint8Array | null { if (!this.host.recogniseSpan) return null; const memo = this.recompleteMemo; if (memo.has(node)) return memo.get(node) ?? null; // Re-covering is how a PRODUCED node's bytes enter the search at all: the // cover machinery otherwise only ever sees the QUERY's spans. The recursion // is allowed to nest — a chain IS nested completions — but it is bounded so // the work stays the ANSWER's (bounded-reads.md): the stack below is the // cycle guard, only ACCEPTED completions recurse, and the nested solve // decomposes the form by its own shape instead of re-recognising the // corpus's hub forms inside it. // // `recompleteOpen` IS the stack of the chain being built, so MEMBERSHIP is // the cycle guard: a node already open on this chain cannot re-enter it. // Testing the NODE — not the stack's size — is what lets a completion // recurse as deep as the graph licenses, exactly the intrinsic convergence // {@link solve}'s contract states. Work stays the answer's because the // recursion only ever advances through an ACCEPTED completion (below) and // {@link cover} deepens only the top derivation. if (this.recompleteOpen.has(node)) return null; // A leaf or single-child node has no parts to recompose; skip before the // costly recognition so a plain terminal answer pays nothing. const nrec = this.store.get(node); if (!nrec || nrec.kids === null || nrec.kids.length < 2) { memo.set(node, null); return null; } const bytes = this.store.bytesPrefix(node, ALL); this.recompleteOpen.add(node); try { // Completion is cover, but the produced form is decomposed by its own // shape. The LEAVES and SPLITS are kept whole, so the split rule still // recovers a part that straddles a content-defined cut (the fold cuts // "p1 p2" as "p1 p"|"2", and findLeaf still resolves p1 and p2). The // recognised SITES are filtered to the node's own kids, because // re-recognising arbitrary forms inside the bytes is what let a hub-heavy // utterance explode: a 37-byte dialogue sentence carries seven hub // openers, and re-covering them chained through the corpus's whole // continuation population (2.5 GB and OOM for `"hi."`). No // concepts/connectors either (those need the caller's async // pre-resolution) — the recursion follows edges and fusion, which is what // a deeper rewrite chain is made of. if (this.host.meter) this.host.meter.recompletes++; const rec = this.host.recogniseSpan(bytes); const kids = new Set(nrec.kids); // THE NODE'S OWN KIDS ARE SITES BY STRUCTURE — recognition cannot be the // only source. A produced composite is decomposed by ITS OWN SHAPE, and // at hub scale the recognition of a produced span returns the WHOLE while // deliberately suppressing its atoms (the off-boundary suppression), so // the kid filter below would admit nothing at all and the chain would end // at the intermediate composite. Measured on the chain // `seed → "p q" → (p→r, q→s) → "r s" → "m n"`: below the flip it reaches // "m n" with fuse+recompose, above it stops at "p q" — the trace shows // `recognise("p q") ⇒ form "p q"` alone, no parts. // // Laying the node's kids out over its own bytes restores exactly the // decomposition the node's tree already states; a kid recognition ALREADY // offers is skipped, so below the flip the seed set is byte-identical to // what it was. The filter's guarantee is untouched: nothing beyond the // node's own kids may enter. const recognised = rec.sites.filter((s) => kids.has(s.payload)); // DEDUPED BY SPAN, not by payload. A node whose kids repeat (`"abab"` // folds as ["ab","ab"]) has TWO occurrences of the same node at different // offsets; a payload-keyed set suppressed the structural site for BOTH, so // the second occurrence had no site at all. The set now holds the spans // recognition already covers, and a structural site is added exactly when // nothing covers that PLACE — O(1) lookups, no extra reads. const seenSpans = new Set(recognised.map((s) => `${s.start}:${s.end}`)); const structural: Site[] = []; { let off = 0; for (const k of nrec.kids) { const len = this.store.bytesPrefix(k, ALL).length; if (!seenSpans.has(`${off}:${off + len}`) && len > 0) { structural.push({ start: off, end: Math.min(off + len, bytes.length), payload: k, }); } off += len; } } const solved = this.solve( bytes.length, { sites: [...recognised, ...structural], leaves: rec.leaves, splits: rec.splits, }, noLicence(), undefined, undefined, undefined, this.derivationSink, ); const answer = solved && concatBytes(solved.segs.map((s) => s.bytes)); // ACCEPT, then CONTINUE THE CHAIN — but only along an accepted // completion. A re-cover whose result is not itself a learnt node (the // 70→374-byte concatenations that name nothing) ends its branch here // instead of recursing into work the answer never asked for, and an // accepted one names a node that is re-covered in turn. That is what // keeps a deep chain possible while the work stays proportional to the // chain rather than to the corpus's interconnections. const composed = answer !== null && !bytesEqual(answer, bytes) ? this.host.resolve(answer) : null; if (composed === null) { memo.set(node, null); return null; } const deeper = this.recompleteNode(composed); const out = deeper ?? answer!; memo.set(node, out); return out; } finally { this.recompleteOpen.delete(node); } } /** Per-cover memo of each produced node's completion (so the many terminal * outs of a long query re-cover each distinct node at most once); reset at the * top of {@link cover}. */ private recompleteMemo = new Map(); /** {@link entityProposals}, per fact BYTES — a pure function of them while * the store is read-only (one response), so a fact the chart reaches as * several items (cover/fix variants, nested completions) is scanned once. * Reset at the top of {@link cover}, like {@link recompleteMemo}. */ private entityMemo = new Map(); /** The joins licensed under one cover's {@link Licence}, kept across its * re-covers (see {@link solve}). */ private readonly joinsKept = new WeakMap(); /** The derivation sink of the TOP cover, threaded into every nested * completion so a produced form's own recompositions are reported in the * same trace instead of vanishing after the first layer. Undefined when * nothing is inspecting, so an uninspected response pays nothing. */ private derivationSink?: (steps: DerivationStep[]) => void; /** The chain of nodes currently being re-completed — the recursion STACK. * MEMBERSHIP is the cycle guard ({@link recompleteNode} refuses a node * already open on this chain), which is what lets a completion recurse as * deep as the graph licenses while the work stays the answer's: the * recursion only advances through an ACCEPTED completion, every produced * form is decomposed into its own kids, and the memo re-covers each node at * most once per cover. A Set, not a flag, because it states WHICH node is * open — the invariant a reader needs to check the guard. */ private recompleteOpen = new Set(); /** The ENTITIES a produced fact's bytes contain that lead somewhere — the * candidates a join may travel through. The forms the fact CONTAINS, by the * same recogniser the query went through (so the evidence standard is the * query's), plus — because the recognition of a STORED WHOLE returns the * whole and stops (measured: one site, the fact's own node, for `The * director of Eva is Gustaf Molander.`) — the longest canonically-resolving * form at each offset. A byte atom is never a subject. The scan runs only * for a FORM (≥ W: per letter it measured 20-26 s in test/99), and each probe * is a `canonResolve` whose exact tier decides a miss by its segment probe, * without a fold. The admission predicate is the store's `leadsSomewhere` * (edge or halo); the host lends its memoised form when it can. * * The SOURCE of each proposal travels with it, so a refusal names which path * proposed the candidate. Memoised per fact bytes ({@link entityMemo}). */ private entityProposals(factBytes: Uint8Array): EntityProposals { const memoKey = latin1(factBytes); const hit = this.entityMemo.get(memoKey); if (hit !== undefined) return hit; const leads = (id: number): boolean => this.host.leadsSomewhere !== undefined ? this.host.leadsSomewhere(id) : this.store.leadsSomewhere(id); const sites = this.host.recogniseSpan?.(factBytes)?.sites ?? []; const proposed = new Map(); const source = new Map(); for (const s of sites) { if (s.payload >= 0 && leads(s.payload)) { proposed.set(s.payload, this.store.bytesPrefix(s.payload, ALL)); source.set(s.payload, "recognised site"); } } const W = this.maxGroup; if (this.host.canonResolve !== undefined && factBytes.length >= W) { const canon = this.host.canonResolve.bind(this.host); for (let start = 0; start < factBytes.length; start++) { for (let end = factBytes.length; end - start >= W; end--) { const id = canon(factBytes.subarray(start, end)); if (id === null) continue; if (leads(id)) { proposed.set(id, this.store.bytesPrefix(id, ALL)); source.set(id, "canonical fold"); } break; // the longest form at this offset wins } } } const out = { proposed, source, sites }; this.entityMemo.set(memoKey, out); return out; } /** DERIVE-THROUGH — a RULE this module's DeductionSystem was missing. The * A*LD library is untouched: this is one more `premises → conclusion + cost` * rule in the system {@link buildSearch} hands to {@link lightestDerivation}, * the same kind of rule as `fuse`/`recompose` — not an extension of * `src/derive`, and not a grounding mechanism. It derives the answer THROUGH * a produced fact, without the intermediate key being named in the query. * * A produced fact (`fact.node`) carries the subject the query reached but * never wrote; the query's remaining tail names the relation to follow from * it. The pair IS a learned key — `""` — so the rule asks the * store for that key's continuation and, when it exists, concludes with the * fact reached through it. On the ladder it is one STEP: a direct edge, * exactly as following a literal continuation is. Deterministic and * point-probed (`resolve` + `nextFirst`, no scan), so it adds no read that * grows with the corpus. Asked ONCE per fact a lightest derivation stood on * (the join license, {@link solve}) — never per fact the exploration merely * reached — so the number of facts it prices is the answer's, not the * corpus's. The move is visible in the rationale as its own act * (`classifyMove` reports `derive-through`), distinct from the * byte-concatenating `fuse`/`splice` — and named `derive-through` rather than * `join` so it cannot be read as the confluence mechanism's `Provenance`. */ private *deriveThrough( fact: OutItem, queryBytes: Uint8Array, queryLen: number, queryNodes: () => ReadonlySet, ): Iterable> { if (!this.host.recogniseSpan) return; const tail = queryBytes.subarray(fact.j, queryLen); if (tail.length === 0) return; // Report ONLY the invocations that could have joined: a derivation stands on // node-bearing outs of every width, including the one-byte outs the cover // bridges with — measured, 68 refusals for a single // 3-relation query, all of them letters. A form shorter than one window is // not a fact a join could travel through, so it is not a refusal worth // reporting; W is the same line the rest of the mind draws between a chance // overlap and a form. const reportable = fact.bytes.length >= this.maxGroup; // The entity candidates are the forms the fact's own bytes CONTAIN — the // same recogniser the query went through, so the evidence standard is the // query's. A byte atom is never a subject; the fact's own node is the span // itself, not an entity inside it. The admission predicate has ONE // definition — the store's `leadsSomewhere` (edge or halo); the host LENDS // its memoised form when it can (Mind does, with the response-scoped struct // cache), and a bare host asks the store directly, so the search stays // host-based. const { proposed, source, sites: factSites } = this.entityProposals( fact.bytes, ); // THE QUERY'S OWN SUBJECT, CANONICALLY. The filter used raw bytes and the // store's nodes are canonical, so `Eiffel Tower country` in the query did // not match the deposited `eiffel tower country` — measured, that is the // trap's wrong answer. Read from the recognition the response already // computed (`queryNodes`, once per solve), never from a second offset scan: // a per-offset `canonicalQueryNodes` re-derived what `recognise` had // resolved once, and was the largest single cost the join first added. const leading = [...proposed] .filter(([payload]) => payload !== fact.node && !queryNodes().has(payload) ) .map(([payload, bytes]) => ({ payload, bytes })); // …then prefer the entity the query did NOT name, and the MAXIMAL one. The // join exists to reach the subject the query never wrote, so: // • a candidate the query already contains is the query's OWN subject, and // following its fact answers about that subject instead of the inferred // one (measured: "Gustaf Molander spouse date of death" gave Gustaf's // date of death, not his spouse's); // • a candidate contained in a longer one is not the entity the fact // introduces ("Timur" must not win over "Timur Bekmambetov"). // Byte work over bytes already read, and the pruning REMOVES the // resolve()/nextFirst() probes these candidates would have paid. if (leading.length === 0) { if (this.host.meter) this.host.meter.joinNoEntity++; if (reportable) { // Report WHAT the recognition returned, not just that nothing led: the // count and the first few site texts are the difference between "the // fact was not recognised" and "it was recognised but nothing led". const seen = factSites.slice(0, 3).map((s) => this.store.bytesPrefix(s.payload, ALL) ); this.host.reportSearch?.( "deriveThroughMiss", [fact.bytes, tail, ...seen], `no entity inside the fact leads anywhere — ${factSites.length} site(s) recognised inside it`, ); } } const candidates = leading .filter((c, _i, all) => !all.some((o) => o.bytes.length > c.bytes.length && indexOf(o.bytes, c.bytes, 0) >= 0 ) ); for (const c of candidates) { // THE SHORTEST TAIL PREFIX WHOSE KEY ALSO LEADS SOMEWHERE. // // The conclusion covers only that prefix, so the rest of the tail stays // for the step after — which is what CHAINING is (a whole-tail key // consumed the whole remainder and made every join terminal: measured, // `joinFired=0` on a three-relation query). // // A KEY THAT RESOLVES IS NOT ENOUGH. Measured with a dry run of these // very primitives: for the candidate `Sweden` the first tail prefix that // resolves is `" "` — the key `Sweden ` (a trailing space) — and it leads // NOWHERE (`nextFirst` = 0), so accepting it refused the join while the // key that names the fact (`sweden capital`) sat one prefix further. The // loop therefore asks BOTH questions before accepting, and keeps looking // otherwise. // // EXACT FIRST, THEN CANONICAL: the corpus holds both identities. let key: number | null = null; let used = 0; // The continuation the accepted key leads to, carried out of the loop: // the loop already HAD to read it to accept the key (a key that leads // nowhere is not the relation), so re-reading it after the loop was a // duplicate read and a branch that could never be taken. let next: number | null = null; let keyBytes = c.bytes; // THE CANDIDATE ENDS ARE THE PREFIXES THAT ARE STORED NODES, ASCENDING. // The key is `entity + prefix`, and it names a relation exactly when that // concatenation IS a node — so the ends come from a content-addressed // probe per offset (the host's `contentKeyEnds`, the learning path's own // mechanism: one leaf walk plus one `findBranch` per offset, no `resolve`), // never from the fold's boundaries. A boundary is not a proxy: a stored // member's end is the end of ITS OWN stream, and the fold never cuts at a // stream's end — measured, "stockholm mayor" exists, leads on to the mayor // fact, and its boundary 6 sits in neither the tail's cuts ([4,7]) nor the // concatenation's. Filtering the scan by "is this a node?" cannot change // the winner: a position that is not a node cannot resolve, so skipping it // is invisible; and the order stays SHORTEST FIRST, which is a semantic // law, not an optimisation (test/106, test/108 pin it). // // A host that cannot answer falls back to every prefix: exact and // complete, at a `resolve` per offset. A host that CAN answer is // authoritative even when it answers "none" — if no prefix is a node then // no key exists to resolve, so enumerating would only pay nulls. (A key // reachable through the CANONICAL equivalence alone and ending off every // node end is therefore not tried here; that dimension is unreachable on // this path by construction and is not part of the exact-key law.) const ends = this.host.contentKeyEnds?.(c.bytes, tail); const candidateEnds = function* (): Generator { if (ends !== undefined) { yield* ends; return; } for (let p = 1; p <= tail.length; p++) yield p; }; for (const len of candidateEnds()) { keyBytes = concat2(c.bytes, tail.subarray(0, len)); const k = this.host.resolve(keyBytes) ?? this.host.canonResolve?.(keyBytes) ?? null; if (k === null) continue; const nx = this.store.nextFirst(k, 1); if (nx.length === 0) continue; key = k; used = len; next = nx[0]; break; } if (key === null) { if (this.host.meter) this.host.meter.joinNoKey++; if (reportable) { this.host.reportSearch?.( "deriveThroughMiss", [c.bytes, tail, keyBytes], `no learnt key names this entity and tail together ` + `(candidate #${c.payload}, from the ${ source.get(c.payload) ?? "unknown" } source)`, ); } continue; } if (this.host.meter) this.host.meter.joinFired++; yield { premises: [fact], conclusion: { kind: "out", i: fact.i, j: fact.j + used, bytes: this.store.bytesPrefix(next!, ALL), cover: true, rec: true, node: next!, throughFact: true, }, cost: STEP, }; } } /** out(i,j,bytes,…): index it for the binary rules, then offer splicing a * learnt connector (the in-search bridge), splitting (at a sub-leaf form * boundary), bridging (cover(i) ∧ this → cover(j)), fusing with an adjacent * finalised out, and — for a LICENSED fact — JOINING the entity it contains * with the query's tail ({@link deriveThrough}, granted in {@link solve}). */ private *outRules( it: OutItem, ctx: { W: number; splits: ReadonlySet; atomsAreHubs: boolean; coversDone: Set; outsByStart: Map; outsByEnd: Map; outsByNode: Map; coverableByStart: Map; findLeafU: (b: Uint8Array) => number | undefined; findBranchU: (k: number[]) => number | undefined; linksByLeft: ReadonlyMap>; linksByRight: ReadonlyMap>; connectors?: ConnectorLicence; joins: JoinLicense; }, ): Iterable> { const { splits, coversDone, outsByStart, outsByEnd, coverableByStart } = ctx; const outsByNode = ctx.outsByNode; const byRight = ctx.linksByRight; pushInto(outsByStart, it.i, it); pushInto(outsByEnd, it.j, it); if (it.rec && it.node !== undefined) pushInto(outsByNode, it.node, it); if (it.cover) pushInto(coverableByStart, it.i, it); // ── connector rule (the BRIDGE, in-search) ────────────────────────── // A connector keyed `L,R` carries everything a learnt whole holds BETWEEN // answer L and answer R — for an N-ary whole, that includes the interior // answers (see {@link Mind.resolveGroupConnectors}). Splicing it joins L and // R into one recognised span L+connector+R, priced inside the search. The // rule fires for ADJACENT parts (R begins where L ends) AND across a gap that // is ITSELF wholly recognised (an interior answer absorbed into the whole, // Points 2 & 5) — but NEVER across the asker's own unrecognised separator (a // space, comma), so "ice fire" stays "cold hot", never "cold or hot". // // Cost stays bounded by iterating only the FEW offered connector targets of // this out's node (links are selective and, for the N-ary case, keyed first→ // later — O(parts), not O(parts²)), and matching them against finalised outs // by node id, rather than scanning every position pair. A pair is bridged // only once its two premises have met here ({@link ConnectorLicence}). const byLeft = ctx.linksByLeft; const licence = ctx.connectors; if (licence && it.rec && it.node !== undefined) { // L = this out, R = a later out whose node is an offered target. for (const [rNode, key] of byLeft.get(it.node) ?? []) { for (const r of outsByNode.get(rNode) ?? []) { const rule = this.trySplice(it, r, key, licence, outsByEnd); if (rule) yield rule; } } // R = this out, L = an earlier out whose node has an offered target here. for (const [lNode, key] of byRight.get(it.node) ?? []) { for (const l of outsByNode.get(lNode) ?? []) { const rule = this.trySplice(l, it, key, licence, outsByEnd); if (rule) yield rule; } } } // Split an unrecognised out at a sub-leaf form boundary — demand-driven, // only when a split point sits inside this out's range. Each half carries // its own resolved node id so it can still anchor a fusion. if (it.cover && !it.rec && it.j - it.i > 1) { for (const k of splits) { if (k > it.i && k < it.j) { const lb = it.bytes.subarray(0, k - it.i); const rb = it.bytes.subarray(k - it.i); yield { premises: [it], conclusion: { kind: "out", i: it.i, j: k, bytes: lb, cover: true, rec: false, node: ctx.findLeafU(lb), }, cost: 0, }; yield { premises: [it], conclusion: { kind: "out", i: k, j: it.j, bytes: rb, cover: true, rec: false, node: ctx.findLeafU(rb), }, cost: 0, }; } } } // The BRIDGE, fired from the out side: cover(i) ∧ this → cover(j). Same // rule as coverRules, the other order of arrival — an out finalised after // its start was already covered. Either way the connective (this out, when // unrecognised) is carried into the cover chain so the asker's own link // survives between the rewritten parts. if (it.cover && coversDone.has(it.i)) { yield this.bridgeRule({ kind: "cover", p: it.i }, it); } for (const r of outsByStart.get(it.j) ?? []) yield* this.fuse(it, r, ctx); for (const l of outsByEnd.get(it.i) ?? []) yield* this.fuse(l, it, ctx); // ── DERIVE-THROUGH (one more rule of the DeductionSystem this module builds) // A produced fact may CONTAIN the subject the query never named; the query's // remaining tail then names the relation to follow FROM that subject. The // pair (contained entity, tail) is itself a learned key, and its // continuation is the derived answer — a genuine relational join, distinct // from the confluence mechanism's `join` PROVENANCE — and not the // juxtaposition the cover produces when the intermediate key IS named. // On the ladder and memoised by {@link key} like every rule, but fired only // for a fact the solve has LICENSED (a fact a lightest derivation stood on — // see {@link solve}), from the joins computed for it once. for (const g of ctx.joins.get(chartKey(it)) ?? []) { yield { premises: [it], conclusion: g.conclusion, cost: g.cost }; } } /** Whether the query span [from, to) is wholly covered by RECOGNISED outs — * the test that lets a connector jump across INTERIOR answers (an N-ary whole) * but never across the asker's own unrecognised framing (a space or comma the * asker wrote between parts). Empty span (from === to, the adjacent case) is * trivially recognised. Otherwise step right-to-left: from `to`, find a * recognised out ending there and continue from its start, until reaching * `from`. Greedy-longest is sufficient here — the spans in play are the few * recognised answers of one query, not a general interval cover. */ private gapRecognised( from: number, to: number, outsByEnd: Map, ): boolean { let pos = to; while (pos > from) { let stepped = -1; for (const o of outsByEnd.get(pos) ?? []) { if (o.rec && o.i >= from && o.i < pos) { stepped = Math.min( stepped === -1 ? o.i : stepped, o.i, ); } } if (stepped < 0) return false; // a position not spanned by a recognised out pos = stepped; } return true; } /** Fuse two adjacent finalised outs — the search's own discovery of forms * that cross leaf boundaries. The concatenation may be a known leaf * (findLeaf, when short enough), or — when both sides resolved — their pair a * known branch (findBranch); a completion fused with its neighbour may spell * a deeper learned form, recovered canonically by {@link resolve} (gated on a * completion being present, so it only runs along chains). The fused span * lives on as an intermediate out while it could still grow into a form, and * enters the graph as a form the moment it names a node. */ private *fuse( l: OutItem, r: OutItem, ctx: { W: number; atomsAreHubs: boolean; findLeafU: (b: Uint8Array) => number | undefined; findBranchU: (k: number[]) => number | undefined; }, ): Iterable> { const bytes = concat2(l.bytes, r.bytes); // A PURE leaf-leaf fuse (neither side already a recognised completion) // is opportunistic cross-leaf recovery exactly like recognition.ts's own // canonical chain — findLeaf/findBranch here have no idea WHY these two // leaves are adjacent, only that their concatenation happens to spell a // trained form ("hi" recovered from "W[hi]ch"). At hub scale, where // atoms themselves no longer discriminate, that coincidence is noise. // // A FOLD BOUNDARY IS NOT EVIDENCE. This gate used to exempt any fuse // starting at `starts.has(l.i)` — documented as "a position the query's // OWN fold chose as a boundary (real structural evidence)". The plain // river fold CHOOSES NOTHING: `riverFold` groups fixed-arity // (`for (i = 0; i < complete; i += mg)`), so `starts` is exactly // {0, W, 2W, 3W, …} for every query. Measured on three unrelated // queries: starts = 0,4,8,12,16,20,24,28[,32] in every case — the set // carries ZERO content information, and the exemption therefore fired at // a quarter of all offsets by arithmetic alone. // // What it cost (17.9M-node store, W=4): "In which country is the Eiffel // Tower?" fused the atoms "h"+"i" at offset 4 — trusted only because // 4 ≡ 0 (mod 4) — into the trained form "hi", followed its continuation // edge, and grounded "Hello there. If you are looking for Open // Assistant, look no further." as a FACT, explaining 2 of 37 bytes // (density 0.054, a fifth of the 1/W honesty bar `thinGrounding` // reports and does not enforce). // // So the exemption is removed rather than replaced: there is no cheap // signal here that means what it claimed to mean, and inventing one // would be worse than admitting the absence. Genuine cross-leaf forms // are not lost — recognition.ts's canonical pass already probes EVERY // byte offset for the write side's interned windows and emits them as // sites, which arrive here as `l.rec`/`r.rec` and stay exempt. What // remains excluded at hub scale is precisely the case with no evidence // behind it: two byte atoms that happen to spell something. // // Below hub scale nothing changes (`!atomsAreHubs`): on a small store // coincidence is rare and every chain is real evidence. const trusted = l.rec || r.rec || !ctx.atomsAreHubs; let node = (trusted && bytes.length <= ctx.W) ? ctx.findLeafU(bytes) : undefined; // Whether this pair ACTUALLY forms a 2-child branch — the hard evidence // that the fused bytes are a learned form worth keeping alive. Derived // from the same findBranchU probe that sets `node`; when false, the pair // is structurally unrecognised and an intermediate span that carries no // node cannot contribute to any further fusion (findBranch needs two // nodes, resolve needs a completion, and findLeaf already had its chance). let pairFormsBranch = false; if ( trusted && node === undefined && l.node !== undefined && r.node !== undefined ) { node = ctx.findBranchU([l.node, r.node]); pairFormsBranch = node !== undefined; } if (node === undefined && (l.rec || r.rec)) { // Canonical recovery of a deeper learned form fused from a completion and // its neighbour. const id = this.host.resolve(bytes); if (id !== null) node = id; } // A completed rewrite (rec) must not be absorbed into an unrelated INTERIOR // chunk of a one-shot phrase: that lets the chunk's continuation swallow the // inter-part gap and corrupt the answer ("cold"+" " → "cold " ⊂ "cold or // hot"; "Y"+" " → "Y " ⊂ "X then Y then Z"). A node learnt as a meaningful // unit bears a halo (it took part in an episode); a bare phrase-interior // chunk does not. So when a completion fuses into a node, require that node // to be halo-bearing — a real fused form (a learnt fact context like "4+3") // carries a halo and still passes. if ( node !== undefined && (l.rec || r.rec) && !this.store.learntWhole(node) ) { node = undefined; } // A node-less fused span is kept alive ONLY while it can still grow INTO a // learned form: it's still ≤ W bytes (so a wider fuse might yet name it via // findLeaf), or the pair ACTUALLY forms a branch (so the fused bytes are // a real learned form, even if the halo gate cleared its node above). It is // NOT kept merely because both sides carry a node — that "potential" gate // let every pair of adjacent recognised forms produce an intermediate span // regardless of whether they name a branch together, generating O(N²) chart // items for N abutted forms where only O(N) pairs actually form branches. // The earlier O(2ⁿ) gate (kept alive whenever a side was a completion) is // already superseded by this one — a completion that genuinely deepens names // a node via findBranch or resolve and is yielded as a form regardless. const couldGrow = bytes.length <= ctx.W || pairFormsBranch; if (node === undefined && !couldGrow) return; yield { premises: [l, r], conclusion: { kind: "out", i: l.i, j: r.j, bytes, cover: false, rec: false, node, }, cost: 0, }; if (node !== undefined) { // A RECOMPOSITION: two already-rewritten parts (both rec completions) // fused into a node that itself CONTINUES. Tag the form so its onward // step is charged at MICRO (see formRules) — the graph learned this whole, // so following it to its grounded answer must win over leaving the parts // split. An ordinary cross-leaf fuse (not from rewrites) is not tagged and // keeps the normal STEP cost. const recomposed = l.rec && r.rec && this.store.hasNext(node); yield { premises: [l, r], conclusion: { kind: "form", i: l.i, j: r.j, node, via: false, rcmp: recomposed, }, cost: 0, }; } } }