// attention.ts — Consensus climb / attention pipeline (Section 4 of the mind). // // Every region of the query's perceived tree casts a resonance vote for the // context (learnt fact) it best climbs to. Votes are pooled through the very // deduction engine (lightestDerivation) that GraphSearch covers with — so a // pooled-evidence decision is one weighted rule of the SAME deduction system, // not a hand-rolled tally. The result is one or more independent points of // attention for the rest of the pipeline to follow. import { isChunk } from "../sema.js"; import type { Hit } from "../store.js"; import { type DeductionSystem, lightestDerivation, type PooledConclusion, } from "../derive/src/index.js"; import type { DerivationItem, DerivationStep } from "./graph-search.js"; import type { AItem, AncestorReach, Attention, AttentionRead, DFMode, MindContext, Region, RegionVote, SaturationInfo, SaturationStop, } from "./types.js"; import { composeStructuralGist, consensusFloor, dominates, estimatorNoise, type StructuralPart, } from "../geometry.js"; import { foldTree, gistOf, latin1Key, perceive, read, resolve, } from "./primitives.js"; import { recognise } from "./recognition.js"; import { leafIdRun } from "./canonical.js"; import { atomIsHub, corpusN, edgeAncestors, hubBound, sharedReachMemo, } from "./traverse.js"; import { cachedRead, type Junction, junctionContainersFrom, junctionSeeds, junctionSynonyms, type JunctionSynonymSides, loadJunctionSynonymSides, type SynonymJunction, walkCache, } from "./junction.js"; import type { Vec } from "../vec.js"; import { indexOf } from "../bytes.js"; import type { RationaleItem } from "./rationale.js"; import { rDeriv, rItem, rNode, traceDerivation } from "./trace.js"; // ═══════════════════════════════════════════════════════════════════════════ // climbConsensus / inspectRationale instrumentation. // // Purely additive tracing over the consensus climb: it never changes an // inference result or the human-readable rationale text traceAttention // already produces (see below) — it only exposes, as one structured `data` // payload on the SAME "climbConsensus" step, the machinery that produced // that text: every structural saturation stop, candidate breadth versus // evidence that actually contributed, and the decisions that removed or // accepted evidence (§ objective of the instrumentation spec). // // The mutable collection buffers (the `TraceDraft` below) are allocated ONLY // when `ctx.trace` is set — every call site that would otherwise push onto // one of these arrays or sets is gated by `td?.` / `if (td)`, so a plain // (untraced) climb pays exactly zero allocation for this instrumentation. // ═══════════════════════════════════════════════════════════════════════════ /** How the newly-added graded junction ladder (junction.ts / attention.ts's * {@link CrossRegionTier}) is reported on a `junctionVotes` entry. The * instrumentation spec this implements predates that ladder and only knew * two tiers ("exact" | "synonym"); with the richer `CrossRegionTier` now * the real shape of a junction vote's provenance, `junctionVotes[].tier` * reports it DIRECTLY (the tier as-is: "exact" | "single-synonym" | * "double-synonym" | "structural-resonance") rather than collapsing every * non-exact tier into a lossy "synonym" bucket — the whole point of * exposing `tier` here is to let a debugger tell a halo-sibling * substitution apart from a structural-resonance ANN guess, which the * spec's original two-value type cannot do. */ export type ClimbConsensusJunctionTier = CrossRegionTier; export type RegionOutcome = | "voted" | "no-ann-hit" | "no-structural-reach" | "saturated-abstention" | "nonpositive-df-weight" | "contrastive-margin-rejection"; /** The best DIFFERENT-conclusion rival the contrastive-margin gate found * while scanning an ordinary (approximate) region's ANN hits — spec §1. * Its roots/saturation/contextsReached are already available through * `reaches` (serialiseReaches) for `node`; not duplicated here. */ export interface ConsensusContrastiveRivalTrace { node: number; rank: number; score: number; } export interface ConsensusRegionTrace { index: number; source: "perceived" | "recognised"; span: [number, number]; chunk: boolean; known: boolean; canonicalId?: number; canonicalUsable: boolean; canonicalFailed: boolean; annQueried: boolean; annHitsReturned: number; annHitsExamined: number; selected?: { source: "canonical" | "ann"; node: number; rank?: number; score: number; fallback?: "orphan" | "saturated-tie"; }; reachNode?: number; outcome: RegionOutcome; idf?: number; dfWeight?: number; contrastiveMargin?: number; contrastiveNoiseFloor?: number; contrastiveRival?: ConsensusContrastiveRivalTrace; mutualWeight?: number; voteWeightPerRoot?: number; focusWeightPerRoot?: number; ordinaryVoteProduced: boolean; superseded: boolean; } export interface ConsensusReachTrace { node: number; roots: number[]; contextsReached: number; saturated: boolean; saturation?: SaturationStop; /** Nodes the climb processed — see {@link AncestorReach.visited}. Absent * on payloads recorded before this field existed. */ visited?: number; /** Maximum ascent distance — see {@link AncestorReach.maxDepth}. */ maxDepth?: number; } export type AnchorRejectionReason = | "below-natural-break" | "below-consensus-floor" | "leading-saturation"; export interface ConsensusAnchorTrace { anchor: number; rank: number; pooledVote: number; idfVote: number; candidateBreadth: number; contributingVotes: number; contributingEvidence: number; breadth: number; contributingSpans: Array<[number, number]>; clusters: number; commit: { status: "root" | "overlap" | "rejected"; dominant: boolean; passesNaturalBreak?: boolean; passesConsensusFloor?: boolean; pastLeadingSaturation?: boolean; /** Committed because its margin from the dominant is inside the estimator's * own resolution — the co-dominant band in {@link commitVotes}. Recorded * because commit decisions are kept in the exact shape the gates applied * them: a root admitted this way must never read, in the trace, as one * that cleared the two vote gates. */ tiedWithDominant?: boolean; rejectionReasons: AnchorRejectionReason[]; }; } export interface JunctionVoteTrace { container: number; span: [number, number]; roots: number[]; sourceRegionIndices: number[]; explainedAwayRegionIndices: number[]; absorbed: number; tier?: ClimbConsensusJunctionTier; /** Zero-based index into `crossRegion.probes` — the probe this vote was * produced from (spec §8). */ probe: number; confidence: number; /** "Evidence bytes" — the container-coverage byte count (the existing * `bestCov` variable at the push site). */ evidenceBytes: number; mutualWeight: number; voteWeightPerRoot: number; } /** Whether one DAG/synonym tier attempt was even made for a probe, and how * many candidate containers it returned — spec §2/§3. */ export interface CrossRegionTierAttemptTrace { attempted: boolean; candidatesReturned: number; } /** Aggregate outcome of the container-selection loop for a DAG/synonym tier * that returned at least one container — spec §4. Only aggregate counts * and the final outcome are recorded, never every candidate. */ export interface CrossRegionStructuralTrace { tier: "exact" | "single-synonym" | "double-synonym"; selfEvidenceRejected: number; contradictionRejected: number; passedGuards: number; selectedNode?: number; outcome: | "all-rejected" | "saturated" | "no-roots" | "nonpositive-idf" | "accepted"; } /** One retained structural-resonance variant that actually issued its own * ANN query — spec §5. */ export interface StructuralResonanceVariantTrace { kind: StructuralVariant["kind"]; semanticConfidence: number; leftSiblingId?: number; rightSiblingId?: number; annHitsReturned: number; } /** One merged structural-resonance proposal actually examined via * edgeAncestors — spec §5. Retains node/variant/scores, but NOT * roots/saturation/contextsReached/idf (already in `reaches`). */ export interface StructuralResonanceCandidateTrace { node: number; variant: StructuralVariant["kind"]; leftSiblingId?: number; rightSiblingId?: number; annScore: number; semanticConfidence: number; effectiveScore: number; outcome: | "saturated" | "no-roots" | "nonpositive-idf" | "same-as-endpoint" | "same-as-selected" | "selected" | "contrastive-rival"; } export interface StructuralResonanceTrace { variantBudget: number; variants: StructuralResonanceVariantTrace[]; mergedProposals: number; examined: StructuralResonanceCandidateTrace[]; contrastiveMargin?: number; noiseFloor: number; outcome: | "ineligible" | "empty" | "no-valid-proposal" | "margin-rejected" | "accepted"; ineligibleReasons?: Array< "between-region" | "not-both-strong" | "not-both-known" | "gap-too-large" >; } /** One cross-region pair the ladder actually probed — spec §2. Exactly one * of these is pushed per pair that incremented `probes`. */ export interface CrossRegionProbeTrace { leftRegionIndex: number; rightRegionIndex: number; betweenRegionIndices: number[]; exact: CrossRegionTierAttemptTrace; singleSynonym: CrossRegionTierAttemptTrace; doubleSynonym: CrossRegionTierAttemptTrace; structural?: CrossRegionStructuralTrace; resonance?: StructuralResonanceTrace; outcome: | "accepted" | "structural-rejected" | "resonance-ineligible" | "resonance-rejected"; } export interface ClimbConsensusData { version: 1; cache: { hit: boolean; detailAvailable: boolean; }; config: { annK: number; crossRegionProbeLimit: number; mode: DFMode; corpusN?: number; dimension?: number; hubBound?: number; estimatorNoise?: number; naturalBreak?: number; consensusFloor?: number; }; candidates: { perceived: number; recognised: number; total: number; }; regions?: ConsensusRegionTrace[]; reaches?: ConsensusReachTrace[]; crossRegion?: { eligibleRegions: number; maximalRegions: number; probeLimit: number; probesAttempted: number; junctionVotes: JunctionVoteTrace[]; supersededOrdinaryVotes: number; probes: CrossRegionProbeTrace[]; stopReason: "insufficient-regions" | "probe-limit" | "pairs-exhausted"; }; saturation?: { regionIntervals: Array<{ start: number; end: number }>; hasLeading: boolean; leadingEnd: number; }; pooling?: { inputVotes: number; eligibleVotes: number; saturationMaskedVotes: number; }; anchors?: ConsensusAnchorTrace[]; result: AttentionRead; } /** The mutable collection buffers threaded through one traced consensus * climb — allocated exactly once, in {@link computeAttention}, only when * `ctx.trace` is set. Every field mirrors a `ClimbConsensusData` array/map, * built incrementally as the pipeline runs so commit-time decisions (in * particular) are recorded LIVE, not reconstructed afterward. */ interface TraceDraft { perceivedCount: number; regions: ConsensusRegionTrace[]; crossRegionJunctionVotes: JunctionVoteTrace[]; crossRegionSummary?: { eligibleRegions: number; maximalRegions: number; probeLimit: number; probesAttempted: number; stopReason?: "insufficient-regions" | "probe-limit" | "pairs-exhausted"; }; crossRegionProbes: CrossRegionProbeTrace[]; supersededOrdinaryVotes: number; saturation?: { regionIntervals: Array<{ start: number; end: number }>; hasLeading: boolean; leadingEnd: number; }; pooling?: { inputVotes: number; eligibleVotes: number; saturationMaskedVotes: number; }; anchors: ConsensusAnchorTrace[]; } /** The config/corpus context {@link traceAttention} needs to fill in * `ClimbConsensusData.config` and `.result` at whichever exit fires — * threaded down from {@link computeAttention} rather than re-derived, so * every emission point reports the SAME numbers the real climb used. */ interface ClimbConsensusCfg { k: number; mode: DFMode; perceivedCount: number; totalRegions: number; N?: number; reachMemo?: ReadonlyMap; naturalBreak?: number; consensusFloor?: number; } function newTraceDraft(perceivedCount: number): TraceDraft { return { perceivedCount, regions: [], crossRegionJunctionVotes: [], crossRegionProbes: [], supersededOrdinaryVotes: 0, anchors: [], }; } /** Serialise the shared `reachMemo` into the plain, authoritative saturation * profile (spec §5) — every distinct node any tier's `edgeAncestors` call * climbed from during this response, in insertion (first-consulted) order. */ function serialiseReaches( reachMemo: ReadonlyMap, ): ConsensusReachTrace[] { const out: ConsensusReachTrace[] = []; for (const [node, r] of reachMemo) { out.push({ node, roots: [...r.roots], contextsReached: r.contextsReached, saturated: r.saturated, ...(r.saturation ? { saturation: r.saturation } : {}), ...(r.visited !== undefined ? { visited: r.visited, maxDepth: r.maxDepth } : {}), }); } return out; } // ── Public entry points ─────────────────────────────────────────────────── /** Climb the query's perceived byte regions up the structural DAG via * resonance, pool the evidence, and return only the ROOT points of * attention — those that cleared commitVotes' significance floor. */ export async function climbAttention( ctx: MindContext, query: Uint8Array, k: number, mode: DFMode = "inverse", ): Promise { return (await climbAttentionAll(ctx, query, k, mode)).roots; } /** Full read-out of one consensus climb: both the roots (dominant points of * attention) and the entire ranked list. Cached via ctx.climbMemo, ALWAYS — * see {@link recognise} for why this memo (and recognise()'s own) is never * gated on tracing. The short of it: computeAttention's collectRegions * votes over what walking the query's perceived tree EMITS, and foldTree's * subtree-resolution fast path used to skip that walk on a warm cache, so a * second climb over identical bytes saw less evidence than the first — which * a conversation's shared prefix subtrees guaranteed by the second turn. * foldTree now takes that fast path only when nothing is watching the walk * (see primitives.ts), so the climb is idempotent on its own and this memo * is an accelerator again. It stays unconditional anyway: attaching a trace * must not change which regions attention weighs. * * A cache hit still emits a trace step — abbreviated, since the full * per-sub-region voting detail {@link traceAttention} builds isn't preserved * by the cached read-out — so a traced response is never silently blacked * out for a repeated query. */ export async function climbAttentionAll( ctx: MindContext, query: Uint8Array, k: number, mode: DFMode = "inverse", ): Promise { // Content-keyed memo — works for both single-turn respond() and multi-turn // respondTurn(). if (ctx.climbMemo) { const contentKey = latin1Key(query); const modeKey = `${k}:${mode}`; let byRead = ctx.climbMemo.get(contentKey); if (byRead === undefined) { ctx.climbMemo.set(contentKey, byRead = new Map()); } const hit = byRead.get(modeKey); if (hit !== undefined) { if (ctx.meter) ctx.meter.climbHits++; // Cache-hit exit (spec §9): the abbreviated payload shape — only what // is actually stored in the cached AttentionRead is reported. No // candidate, reach, saturation, pooling or anchor detail is fabricated // (that per-region detail was never retained by the memo). const data: ClimbConsensusData | undefined = ctx.trace ? { version: 1, cache: { hit: true, detailAvailable: false }, config: { annK: k, crossRegionProbeLimit: k, mode }, candidates: { perceived: 0, recognised: 0, total: 0 }, result: hit, } : undefined; ctx.trace?.step( "climbConsensus", [rItem(query, "query")], hit.roots.map((r) => rNode(ctx, r.anchor, "anchor", r.vote)), `(cached) consensus already computed for this query — ` + `${hit.roots.length} point(s) of attention`, undefined, data, ); return hit; } const read = await computeAttention(ctx, query, k, mode); byRead.set(modeKey, read); return read; } return computeAttention(ctx, query, k, mode); } // ── Pipeline ────────────────────────────────────────────────────────────── export async function computeAttention( ctx: MindContext, query: Uint8Array, k: number, mode: DFMode, ): Promise { if (ctx.meter) ctx.meter.climbs++; const regions = collectRegions(ctx, query); const perceivedCount = regions.length; // Recognised sites carry structural evidence that perceived sub-regions // miss: a word crossing a W-boundary is split into chunks whose partial // gists may not resonate distinctively, but the SITE (content-addressed, // exact) names the whole form. Adding sites as climb regions lets the // consensus vote with the full word, at zero cost — recognition is already // memoised per response (ctx.recogniseMemo), and gistOf for short sites is // O(|span|·D). Sites that overlap perceived regions add corroborating // evidence; sites in gaps (like cross-boundary words) fill them. const rec = recognise(ctx, query); for (const s of rec.sites) { regions.push({ v: gistOf(ctx, query.subarray(s.start, s.end)), start: s.start, end: s.end, // NOT a chunk — a precondition, not a judgement about the evidence. // `chunk` admits a region into the saturated-INTERVAL builder (see // crossRegionVotes), which walks regions as a SEQUENCE and merges // neighbouring saturated ones into runs. Its own contract requires the // regions it reads to be DISJOINT and in byte order — true of // leaf-parents, false of sites, which overlap each other and the chunks // ("red", "circle" and "red circle" are all present at once). // // Admitting them was measured both ways and is unprincipled in each // direction: a saturated site EXTENDS a run and masks votes that should // have won (test/37 lost all three — roots became "red " and "hat" // instead of "red circle" and "2"), while a non-saturated one BREAKS a // run and unmasks votes that should have been dropped, which is the only // reason it appeared to fix test/34. Either way the outcome turns on // where an overlapping span happens to fall in the array — the same // positional accident this work exists to remove. chunk: false, known: true, // a recognised site IS a stored form // …and CARRY WHICH ONE. `known: true` claimed exactness while the // identity itself was dropped, leaving the climb to re-derive it from // the gist through the ANN — so which stored node an exact site voted // with turned on approximate rank. Measured on test/34: the site // "square" ([10,16), payload 40) resonated to "quare" (119) instead; // the exact junction tier then found no container holding both "blue" // and 119, fell through to the single-synonym tier, and "blue then // square" attended to "red square" — a context NEITHER attribute // attends to alone. This is the same exact-first economy chunks // already get from canonicalChunkId, and it REMOVES an ANN query // rather than adding one. id: s.payload, }); } // The trace draft (spec §9): allocated ONLY when a trace was requested — // every downstream consumer gates its own writes on `td?` / `if (td)`, so // an untraced climb pays zero allocation for this instrumentation. const td: TraceDraft | undefined = ctx.trace ? newTraceDraft(perceivedCount) : undefined; const cfg0: ClimbConsensusCfg = { k, mode, perceivedCount, totalRegions: regions.length, }; if (regions.length === 0) { traceAttention(ctx, [], [], [], undefined, td, cfg0); return { roots: [], ranked: [] }; } const N = corpusN(ctx); // One climb per distinct anchor for the WHOLE query: regions sharing a // chunk, and canonicalChunkId's prefix probes, all hit this memo instead of // re-reading the anchor's full edge fan-out from the store. The memo is // the SHARED one (traverse.ts) — response-scoped for respond(), // conversation-scoped across turns, and the same map confluence prices // commonality against; it used to be a private per-climb Map, so a // conversation re-climbed its own repeated regions from cold on every // turn. A traced response still gets a fresh one — see sharedReachMemo. const reachMemo = sharedReachMemo(ctx); const rvs = ctx.meter ? await ctx.meter.time( "climb.voteRegions", () => voteRegions(ctx, query, regions, k, mode, N, reachMemo, td), ) : await voteRegions(ctx, query, regions, k, mode, N, reachMemo, td); // ── Cross-region: DIRECT region-to-region interaction ───────────────── // Two regions whose individual climbs land on DIFFERENT contexts leave // their JOINT context — the learnt whole that contains BOTH — with no // vote. crossRegionVotes recovers it by the bridge's content-addressed // junction ascent (see the note above the function). const crossArgs = () => crossRegionVotes(ctx, query, regions, rvs, k, N, reachMemo, td); const cross = ctx.meter ? await ctx.meter.time("climb.crossRegion", crossArgs) : await crossArgs(); // A vote SUPERSEDED by exact joint evidence (its bytes literally live // inside the joint container, yet it climbed elsewhere — grid aliasing) // is dropped, not down-weighted: the joint container explains it away. const allVotes = cross.votes.length > 0 ? [ ...rvs.votes.filter((v) => !cross.superseded.has(v)), ...cross.votes, ] : rvs.votes; // Mark, on the per-region trace, the source region of every superseded // ordinary vote (spec §4's final rule) — an explicit pass over the exact // set crossRegionVotes' explaining-away logic removed, never inferred // from `absorbed`. if (td && cross.superseded.size > 0) { for (const rv of cross.superseded) { const region = td.regions.find( (r) => r.span[0] === rv.start && r.span[1] === rv.end, ); if (region) region.superseded = true; } } // ────────────────────────────────────────────────────────────────────── const cfg: ClimbConsensusCfg = { ...cfg0, N, reachMemo }; if (allVotes.length === 0) { traceAttention(ctx, regions, rvs.voters, [], undefined, td, cfg); return { roots: [], ranked: [] }; } const sat = detectSaturated(ctx, regions, rvs.saturated); if (td) { td.saturation = { regionIntervals: sat.intervals.map((iv) => ({ ...iv })), hasLeading: sat.hasLeading, leadingEnd: sat.leadingEnd, }; } const pooled = poolVotes(ctx, allVotes, sat, N, td); return commitVotes(ctx, pooled, sat, regions, rvs.voters, N, td, cfg); } export function collectRegions(ctx: MindContext, query: Uint8Array): Region[] { const regions: Region[] = []; // A region that DOMINATES the query (covers more than half — the shared // {@link dominates} test liftAnswer uses for a span that swallows its // surroundings) can never itself discriminate between several topics the // query weaves; voting with it only when it is the sole structure (no // narrower region exists) keeps a flat/short query's single point of // attention intact without letting a broad, non-discriminative wrapper // dilute a multi-topic query's vote or masquerade as a genuine second // point of attention. // foldTree (not walkTree): the same post-order walk, but each node also // resolves content-addressed against the store — `known` is what lets the // climb keep exact evidence at full weight while margin-damping the // approximate kind (see voteRegions). One findLeaf/findBranch per tree // node, the same lookups a deposit pays. foldTree(ctx, perceive(ctx, query), 0, (n, start, end, node) => { if (n.kids === null) return; // The dominance filter is about WRAPPERS, not about size. A chunk is the // smallest grouped unit — it wraps no other region — so it can never be the // "broad, non-discriminative wrapper" this rule exists to exclude, however // much of a short query it happens to cover. Testing it by span alone was // safe only while chunks were exactly W bytes: content-defined segments run // up to the keyring's seat count, so on a 15-byte query the 8-byte segment // "is frigi" counted as dominant and was discarded, leaving CAST one point // of attention where it needs two (test/29 D1/D2). Composites are still // filtered exactly as before. if ( isChunk(n) || !dominates(end - start, query.length) || regions.length === 0 ) { regions.push({ v: n.v, start, end, chunk: isChunk(n), known: node !== null, }); } // MEASURED AND REFUTED — subdividing a long segment into W-scale tiles. // A content segment runs from W−1 up to the keyring's seat count (2W) and // folds FLAT, so its only sub-units are single bytes; the grid's regions // were always exactly W. Offering each segment's W-byte tiles as extra // regions (gists only, no stored nodes, anchored on the segment's own // content-defined start so invariance is kept) does restore that finer // grain: on `How is ice like steel?` the climb went from ONE ranked anchor // to three, and test/33's own CAST-candidate spread was recovered. // // It is still wrong, and net worse (measured: test/29 went 9/2 to 7/4). // canonicalWindows governs EXACT identity lookup, which recognition // already probes at every offset — it says nothing about the grain of an // approximate gist, so "the write side's unit scale" was two machineries // conflated. What the tiles actually do is reintroduce a fixed stride // inside the segment, and the extra votes reorder the climb: on // `How is Shakespeare like Leonardo da Vinci?` the short name deposits // outranked the exemplar sentences, claimed their aligned runs first, and // left the sentences CAST needs with no free run at all (C2, C3). A // region must come from the fold, not from a stride over it. }); // ─── FORMS THE QUERY'S OWN CUT SPLIT ──────────────────────────────────── // The walk above enumerates FOLD NODES ONLY, so a stored form the query's // content-defined cut happens to split is not addressable at all — however // discriminative it is. Measured: `request_id=1042` against a 200-record // log, the query's best match, cut as `...uest_id=|10|42 and r`; "1042" // reaches exactly ONE context of 205 (maximal IDF) and cast no vote, while // the scaffolding "=10" — which matches every record 1000–1099 — did. The // climb was voting on the only evidence it could address, and that was the // non-discriminative kind. // // The WRITE path already made these reachable: canonicalWindows interns a // form at both lengths precisely so one straddling a cut resolves from // either side. The read path simply never used the guarantee. So this is // recovered here by lookup — the fold, its invariants and the write path // are untouched. // // Admitting every resolvable window is REFUTED (it is the ascent-sites // failure): on a 5-context corpus a 26-byte query yielded 17 "unique" // windows that were all fragments of ONE word (" pai" "pain" "aint" …). // No per-window threshold separates that from the log case — the two need // the same windows ADMITTED and COLLAPSED at identical per-window IDF. It // is a REDUNDANCY problem, so overlapping admitted windows are COALESCED // into maximal spans: the log query then yields the two disjoint records it // names, and the 17 fragments yield the one span " painted the Mona Lisa". const W = ctx.space.maxGroup; if (query.length > W) { const N = corpusN(ctx); const reachMemo = sharedReachMemo(ctx); // Coalesce while sweeping left to right: a window overlapping (or just // touching) the span under construction extends it. Merging cannot // inflate what the climb pays for this evidence — the merged span votes // as ITSELF, and a longer span is at least as discriminative as its most // discriminative part, i.e. its reach is bounded by the MIN over the // windows that built it. const spans: Array<{ start: number; end: number }> = []; // COVERAGE BY PREFIX MAXIMUM, NOT BY RESCANNING THE REGIONS. // The containment test below is the loop's hot path — it rejects 97% of // windows — and asking it as `regions.some(...)` re-walked every region // at every offset: O(|query| · |regions|). That is quadratic in the // input, and the region count grows with it — measured 1,510 regions on // an 8,195-byte query, i.e. ~12.4M predicate evaluations in ONE call, // against the constant-KB/s law test/14 asserts. // // A region contains the window [o, o+W) exactly when it starts at or // before `o` and ends at or after `o+W`. So the only thing the test // needs from the regions is, per offset, the FARTHEST end among those // starting at or before it — a prefix maximum, built in one pass and // read in O(1). Identical verdict by construction, no behaviour change. const maxEndFrom = new Int32Array(query.length + 1); for (const r of regions) { if (r.start <= query.length && r.end > maxEndFrom[r.start]) { maxEndFrom[r.start] = r.end; } } for (let i = 1; i <= query.length; i++) { if (maxEndFrom[i - 1] > maxEndFrom[i]) maxEndFrom[i] = maxEndFrom[i - 1]; } for (let o = 0; o + W <= query.length; o++) { // A window some fold region wholly contains offers no address the walk // above did not already offer. if (maxEndFrom[o] >= o + W) continue; const ids = leafIdRun(ctx, query, o, o + W); if (ids === null) continue; const wid = ctx.store.findBranch(ids); if (wid === null) continue; const reach = edgeAncestors(ctx, wid, N, reachMemo); // Saturated = the climb ABSTAINED; no roots = it reached nothing that // could corroborate anything. Neither is evidence. if (reach.saturated || reach.roots.length === 0) continue; const last = spans[spans.length - 1]; if (last && o <= last.end) last.end = o + W; else spans.push({ start: o, end: o + W }); } for (const { start, end } of spans) { // The same wrapper filter the fold regions pass through. if (dominates(end - start, query.length) && regions.length > 0) continue; regions.push({ v: gistOf(ctx, query.subarray(start, end)), start, end, // NOT a chunk: `chunk` means "a smallest grouped unit the FOLD // produced", and this span was assembled here. Setting it is // REFUTED — it cost 5 tests (honest silence, fusion direction, both // test/50 probes) where chunk:false costs none. chunk: false, known: true, // EVIDENCE, NOT A POINT OF ATTENTION — see Region.corroborating. corroborating: true, }); } } return regions; } export async function voteRegions( ctx: MindContext, query: Uint8Array, regions: readonly Region[], k: number, mode: DFMode, N: number, reachMemo?: Map, td?: TraceDraft, ): Promise<{ votes: RegionVote[]; saturated: boolean[]; voters: Array<{ id: number; score: number; w: number } | null>; }> { if (ctx.meter) ctx.meter.climbRegions += regions.length; const regionSaturated: boolean[] = new Array(regions.length).fill(false); const regionVotes: RegionVote[] = []; const regionVoter: Array<{ id: number; score: number; w: number } | null> = ctx.trace ? regions.map(() => null) : []; const W = ctx.space.maxGroup; for (let ri = 0; ri < regions.length; ri++) { // `v`/`start`/`end` are rebindable: a long approximate segment may vote // with the sub-span that actually carries its evidence — see below. let { v, start, end } = regions[ri]; const { chunk } = regions[ri]; // BELOW ONE RIVER WINDOW, BYTE IDENTITY IS NOT EVIDENCE. The same // principle identityBar states and recognition's own `emit` already // enforces on sites ("below one river window, byte overlap is chance"), // applied to what the climb calls EXACT. It was unnecessary while the // fold grouped at fixed arity — every chunk was then exactly W bytes — // but content-defined cuts run from W-1 up to the keyring's seat count, // so sub-window segments now exist, and a 3-byte string is interned by // triviality rather than by evidence. // // Such a region is NOT dropped: it still votes on its gist, through the // contrastive-margin gate every approximate region pays. Dropping them // outright was measured and REFUTED — the suite fell 441 -> 406, because // short regions do carry real evidence; what they must not carry is the // EXACT tier's full mutual weight and its exemption from the margin. // // Measured on test/50's junk query: the 3-byte chunk "of " voted exact // at mutual 1.00 with idf 4.22, and an unrelated haiku exemplar's pooled // vote went 1.13 -> 5.94 — past consensusFloor (5.82), making a junk root // TRUSTED and licensing CAST to compare content the query never named. // consensusFloor did not drift; what fed it stopped being evidence. // // A region spanning the WHOLE query is exempt, exactly as the site rule // exempts a whole-query span: it is then not a fragment of something // longer, it is the question ("red" asked on its own — test/34). const subWindow = end - start < W && !(start === 0 && end === query.length); // EXACTNESS IS A PROPERTY OF THE CONTENT, NOT OF THIS QUERY'S GROUPING. // `known` used to mean "these bytes resolve to ONE stored node", which // conflates two different things: whether the store has seen the content, // and whether this query's cut happened to group it the same way the // deposit did. Under fixed-arity folding those coincided; under // content-defined cuts they routinely do not. // // Measured over 42 voting regions (attributes / capitals / artists), // against a graded reading — what fraction of the region's river windows // are content-addressed: // // known=true cov=1.0 90% cov=0 0% 0 { if (end - start < W) return 0; let tot = 0, hit = 0; for (let o = start; o + W <= end; o++) { tot++; if (resolve(ctx, query.subarray(o, o + W)) !== null) hit++; } return tot === 0 ? 0 : hit / tot; }; // A sub-window region pays in full: below one river window its byte // identity is chance, so it has no coverage to claim. const cov = subWindow ? 0 : (regions[ri].known ? 1 : windowCoverage()); const known = cov >= 1 && !subWindow; // Trace-only bookkeeping for this region — allocated only under `td` // (i.e. only when ctx.trace is set); see ConsensusRegionTrace/ // RegionOutcome (spec §4). `examinedIds` tracks distinct ANN hits // whose edgeAncestors reach was actually CONSULTED here (not merely // returned by resonate) — the fallback/margin loops below add to it. const examinedIds = td ? new Set() : undefined; let annQueried = false; let fallbackKind: "orphan" | "saturated-tie" | undefined; const recordRegion = ( outcome: RegionOutcome, extra: Partial = {}, ) => { if (!td) return; td.regions[ri] = { index: ri, source: ri < td.perceivedCount ? "perceived" : "recognised", span: [start, end], chunk, known, canonicalId: canonicalId ?? undefined, canonicalUsable, canonicalFailed, annQueried, annHitsReturned: hits ? hits.length : 0, annHitsExamined: examinedIds ? examinedIds.size : 0, outcome, ordinaryVoteProduced: outcome === "voted", superseded: false, ...extra, }; }; // EXACT-FIRST: a chunk whose canonical anchor is content-addressed needs // no estimator — identity is exact, so its score is 1 BY DEFINITION (the // estimated cosine of a form with itself, minus quantisation noise, and // the caveat atop geometry.ts forbids trusting the estimate over the // exact resolution anyway). The ANN query is deferred behind // `ensureHits` and paid only when actually consulted: the orphan // fallback, the contrastive margin (approximate regions only), or a // region with no usable canonical. On chunk-heavy queries this removes // the resonate() call for most exact regions — the single largest // remaining inference sink — with the anchor choice unchanged (the // canonical branch already ignored hits[0]). let canonicalId = subWindow ? null : (chunk ? canonicalChunkId(ctx, query.subarray(start, end), N, reachMemo) : (regions[ri].id ?? null)); let canonicalUsable = canonicalId !== null && (ctx.store.hasParents(canonicalId) || ctx.store.hasContainers(canonicalId)); let hits: readonly Hit[] | null = null; const ensureHits = async (): Promise => { if (hits === null) { hits = await ctx.store.resonate(v, k); annQueried = true; } return hits; }; // A DILUTED SEGMENT VOTES WITH THE SPAN THAT CARRIES ITS EVIDENCE. // // A content segment runs up to the keyring's seat count and folds FLAT, so // its gist superposes every one of its bytes: an entity inside a longer // segment is averaged together with whatever scaffolding shares the // segment, and the resonance reads the average. Measured on // `How is ice like steel?` against a store holding `Steel is hard`: the // segment `ike stee` resonates to `Ice is c` at 0.297 — the WRONG deposit — // with `Steel ` fourth at 0.123, while the sub-span `stee` resonates to // `Steel ` at 0.627. The evidence is there; the whole-segment read cannot // see it, and `Steel is hard` received no vote at all (test/29 C1). // // Entered only after the EXACT path has already failed — a chunk with a // usable canonical identity has a content-addressed handle on its own bytes // and needs no estimator at all — so this is honest degradation, not extra // work on regions that already resolved. The candidates are the segment's // two EDGE sub-spans at the write side's own unit scale (W) — the scale // `canonicalWindows` interns, and the only one at which a sub-span could // carry a stored identity; a segment of W or less has no interior at all. // Edges because a content cut lands INSIDE a unit, so the remnant it split // sits against the cut: `steel` is cut after `stee`. Offering every // interior offset instead was measured and is worse — it re-anchors // segments on spans no boundary ever separated, and broke three of // test/17's vote-distribution and root-count assertions (428/1 → 424/5). // // Selection is by the SAME quantity the region's vote is weighted by — // score² · idf — never by score alone: the scaffolding window `is i` // resonates at 0.832, far above `stee`, and is worth nothing because its // reach is the whole corpus. Nothing new is being measured here; the // choice the code did not previously make is made with the criterion it // already uses. The region's SPAN narrows with its gist, so breadth, // clusters and cross-region pairing all see where the evidence really sits. if (!canonicalUsable && chunk && !known && end - start > W) { const weigh = (h: Hit): { w: number; id: number } | null => { const r = edgeAncestors(ctx, h.id, N, reachMemo); if (r.saturated || r.roots.length === 0) return null; const idf = Math.log(N / Math.max(1, r.contextsReached)); if (idf <= 0) return null; return { w: h.score * h.score * idf, id: h.id }; }; // The whole-segment candidate reuses the ranking the region needs // anyway, so only the two edge probes are new work. const scoreOf = async ( gist: Vec, ): Promise<{ w: number; id: number } | null> => { const h = await ctx.store.resonate(gist, 1); return h.length === 0 ? null : weigh(h[0]); }; const h0 = await ensureHits(); let best = h0.length > 0 ? weigh(h0[0]) : null; let bestSpan: [number, number, Vec] | null = null; for (const s0 of [start, end - W]) { const sub = gistOf(ctx, query.subarray(s0, s0 + W)); const cand = await scoreOf(sub); if (cand !== null && (best === null || cand.w > best.w)) { best = cand; bestSpan = [s0, s0 + W, sub]; } } if (bestSpan !== null) { [start, end, v] = bestSpan; hits = null; // the whole-segment ranking no longer describes this span canonicalId = canonicalChunkId( ctx, query.subarray(start, end), N, reachMemo, ); canonicalUsable = canonicalId !== null && (ctx.store.hasParents(canonicalId) || ctx.store.hasContainers(canonicalId)); } } const canonicalFailed = chunk && canonicalId === null; let voterId: number; let score: number; let scoreId: number; // the node the score was measured against let selectedSource: "canonical" | "ann"; if (canonicalUsable) { voterId = canonicalId!; score = 1; scoreId = canonicalId!; selectedSource = "canonical"; } else { const h = await ensureHits(); if (h.length === 0) { recordRegion("no-ann-hit"); continue; } voterId = h[0].id; score = h[0].score; scoreId = h[0].id; selectedSource = "ann"; examinedIds?.add(voterId); } let reach = edgeAncestors(ctx, voterId, N, reachMemo); // A region's vote must not die with the TOP hit: `hits[1..k]` were // already fetched, and the top-ranked anchor being a structural orphan // (no edge-bearing ancestors) is an accident of the approximate ranking, // not evidence the region relates to nothing. Walk the remaining hits — // nearest first, climbs memoised — until one climbs. A SATURATED reach // is not an orphan: it is a deliberate abstention, kept as-is. if (reach.roots.length === 0 && !reach.saturated) { for (const h of await ensureHits()) { if (h.id === voterId) continue; const r2 = edgeAncestors(ctx, h.id, N, reachMemo); examinedIds?.add(h.id); if (r2.saturated || r2.roots.length > 0) { ctx.trace?.step( "anchorFallback", [rNode(ctx, voterId, "orphan-anchor", score)], [rNode(ctx, h.id, "anchor", h.score)], "the top-ranked anchor climbs to no context — a lower-ranked hit votes instead", ); reach = r2; voterId = h.id; score = h.score; scoreId = h.id; selectedSource = "ann"; fallbackKind = "orphan"; break; } } } else if (!canonicalUsable && reach.saturated) { // TIE-BAND saturation fallback. A saturated top hit abstains the whole // region (a hub's reach concludes nothing) — but the hub may only CLAIM // that abstention when it is DISTINGUISHABLY the nearest anchor. The // resonance ranking is an estimate: the difference between two scores // against the same query carries √2× the estimator's per-score error, // ≈ 1/√D ({@link estimatorNoise}) — so any hit within that band of the // top is the SAME rank at measurement resolution, and letting the hub // win the tie decides the region by quantisation accident (observed: // a 0.1σ rank inversion flipped a pinned behaviour when the query // estimator sharpened from 4 to 8 bits). Walk the tied hits, nearest // first; the first that climbs somewhere non-saturated votes for the // region. Beyond the band the hub is genuinely nearest and its // abstention stands. A KNOWN (content-addressed) region never enters: // its anchor is exact, not an estimate. const band = estimatorNoise(ctx.store.D); for (const h of await ensureHits()) { if (h.id === voterId) continue; if (h.score < score - band) break; // hits are nearest-first const r2 = edgeAncestors(ctx, h.id, N, reachMemo); examinedIds?.add(h.id); if (!r2.saturated && r2.roots.length > 0) { ctx.trace?.step( "anchorFallback", [rNode(ctx, voterId, "saturated-anchor", score)], [rNode(ctx, h.id, "anchor", h.score)], "the top-ranked anchor is a saturated hub tied within estimator noise — the tied hit votes instead", ); reach = r2; voterId = h.id; score = h.score; scoreId = h.id; selectedSource = "ann"; fallbackKind = "saturated-tie"; break; } } } regionSaturated[ri] = reach.saturated; const selected: ConsensusRegionTrace["selected"] | undefined = !td ? undefined : (() => { const rank: number | undefined = selectedSource === "ann" ? (hits as readonly Hit[] | null)?.findIndex((h: Hit) => h.id === voterId ) : undefined; return { source: selectedSource, node: voterId, score, ...(rank !== undefined ? { rank } : {}), ...(fallbackKind ? { fallback: fallbackKind } : {}), }; })(); if (reach.roots.length === 0) { recordRegion("no-structural-reach", { selected, reachNode: voterId }); continue; } if (reach.saturated) { recordRegion("saturated-abstention", { selected, reachNode: voterId }); continue; } // One IDF per region — dfWeight() and the focus weight used to compute // the same logarithm independently. const idf = Math.log(N / Math.max(1, reach.contextsReached)); const df = Math.log(1 + reach.contextsReached); const wf = mode === "direct" ? df : mode === "combined" ? idf + df : idf; if (wf <= 0) { recordRegion("nonpositive-df-weight", { selected, reachNode: voterId, idf, dfWeight: wf, }); continue; } // CONTRASTIVE-MARGIN GATE — the compensation the linear (byte-proportional) // fold demands, applied to APPROXIMATE evidence only. Under the linear // fold a resonance score reads "fraction of aligned shared bytes", so a // NOVEL span sharing a frame with several stored exemplars scores high // against each of them without being evidence of ANY of them: the shared // scaffolding, not the span's own content, carries the similarity. Such a // frame region resonates ~equally to every framed exemplar, so its top hit // barely beats the best DIFFERENT-conclusion rival (a different climb // root-set) — its discriminative margin, score MINUS that rival, collapses // toward zero. A region votes only when that margin clears the estimator's // own noise floor (1/√D — see {@link estimatorNoise}); below it the margin // is quantisation noise, not evidence. A KNOWN region (content-addressed, // exact) skips the contrast: it IS learnt content, not an approximation. // // The margin GATES; it does NOT scale the weight. A surviving region votes // at its genuine strength (score²·wf) — the SAME scale {@link // consensusFloor} is derived for. Using the margin as a MULTIPLIER // (score·margin) conflated "discriminative" with "strong": a genuinely // discriminative span whose frame-rival happened to score close got a tiny // vote, systematically compressing correct scaffolding-dominated groundings // (reordered / paraphrased queries) below the floor so they grounded // nothing. Gating at the noise floor keeps frame-echo suppression (a frame // region's margin ≈ 0 is gated out) without penalising honest evidence. let contrastiveMargin: number | undefined; let contrastiveRival: ConsensusContrastiveRivalTrace | undefined; if (!known) { let margin = score; const hitsForRival = await ensureHits(); for (let hi = 0; hi < hitsForRival.length; hi++) { const h = hitsForRival[hi]; if (h.id === voterId) continue; const r2 = edgeAncestors(ctx, h.id, N, reachMemo); examinedIds?.add(h.id); if (r2.saturated || r2.roots.length === 0) continue; // concludes nothing if (sameRoots(r2.roots, reach.roots)) continue; // same conclusion margin = score - h.score; // hits are nearest-first: the best rival if (td) { contrastiveRival = { node: h.id, rank: hi, score: h.score }; } break; } contrastiveMargin = margin; // Scaled by what this region does NOT address — see `cov` above. const noiseFloor = estimatorNoise(ctx.store.D) * (1 - cov); if (margin <= noiseFloor) { recordRegion("contrastive-margin-rejection", { selected, reachNode: voterId, idf, dfWeight: wf, contrastiveMargin: margin, contrastiveNoiseFloor: noiseFloor, ...(contrastiveRival ? { contrastiveRival } : {}), }); continue; } } // MUTUAL-EXPLANATION WEIGHT (angle + magnitude). Under the linear fold // cos = shared/(‖r‖·‖h‖) with ‖·‖² = content bytes, so the old score² // was already — implicitly — (shared/len_r)·(shared/len_h): the fraction // of the REGION the hit explains times the fraction of the HIT the // region pins down. Made explicit, each factor is computed from the two // magnitudes (the region's own span; the hit's, read from the store — // contentLen, √bytes being the linear fold's gist norm) and CAPPED at 1: // the estimated cosine can imply more shared content than the smaller // side even holds, and the uncapped square silently credited that // impossible surplus — a small region echoing inside a large context, or // the reverse, voted above its physical evidence. In the uncapped // regime this is exactly score², the scale {@link consensusFloor} is // derived for. (The margin gate above deliberately stays in raw cosine // units: it tests the ESTIMATOR's noise floor, which lives in cosine // space; converting each side by its own hit's magnitude would compare // noise floors of different scales.) const lenR = Math.max(1, end - start); // Cap the magnitude read at lenR·D: past it s/ratio ≤ s/√D — below the // estimator's own noise floor — so the mutual weight is ~0 regardless // and the clamped value yields exactly that; no full walk of a huge hit. const ratio = Math.sqrt( Math.max(1, ctx.store.contentLen(scoreId, lenR * ctx.store.D)) / lenR, ); const mutual = Math.min(1, score * ratio) * Math.min(1, score / ratio); const w = (mutual * wf) / reach.roots.length; const wFocus = (mutual * idf) / reach.roots.length; regionVotes.push({ start, end, canonicalFailed, roots: reach.roots, w, wFocus, // The pool sees VOTES, not regions — carry the region's standing with it. ...(regions[ri].corroborating ? { corroborating: true } : {}), }); if (ctx.trace) { regionVoter[ri] = { id: voterId, score, w: wf }; } recordRegion("voted", { selected, reachNode: voterId, idf, dfWeight: wf, ...(contrastiveMargin !== undefined ? { contrastiveMargin, contrastiveNoiseFloor: estimatorNoise(ctx.store.D), ...(contrastiveRival ? { contrastiveRival } : {}), } : {}), mutualWeight: mutual, voteWeightPerRoot: w, focusWeightPerRoot: wFocus, }); } return { votes: regionVotes, saturated: regionSaturated, voters: regionVoter, }; } /** The consensus vote as EVIDENCE POOLING, not shortest path: each surviving * region is an axiom; it contributes to every root it climbed to (or, for a * terminal answer node, to the contexts that lead to it) by a `combine: * "sum"` rule, so independent regions corroborating the same anchor ADD * rather than compete to be the cheapest route (see {@link Rule.combine} in * derive/src/deduction.ts). Run through the very engine {@link * GraphSearch} covers with — `lightestDerivation` — so a pooled-evidence * decision is, like a followed edge or a spliced connector, one weighted * rule of the SAME deduction system, not a separate hand-rolled tally that * merely logs alongside it. `votesIdf`/`support` are the same two * read-outs {@link commitVotes} always gated on; only how they accumulate * changed. */ export function poolVotes( ctx: MindContext, regionVotes: readonly RegionVote[], sat: SaturationInfo, N: number, td?: TraceDraft, ): { votes: Map; votesIdf: Map; support: Map; /** Per-anchor SCALE-INVARIANT support: Σ RegionVote.absorbed over the * distinct contributing regions — see Attention.breadth. */ regionSupport: Map; /** Per-anchor contributing region spans — see Attention.clusters. */ regionSpans: Map>; /** Per-anchor count of contributing region VOTES (pooled axioms), which is * not the length of `regionSpans`: a joint binding is one vote sitting in * several places. */ regionAxioms: Map; /** Per-anchor LARGEST single-region contribution — see Attention.peak. */ regionPeak: Map; /** Anchors with support from at least one NON-corroborating region. */ anchored: Set; steps: DerivationStep[]; } { const eligible: number[] = []; for (let ri = 0; ri < regionVotes.length; ri++) { const rv = regionVotes[ri]; if ( rv.canonicalFailed && sat.intervals.some((iv) => rv.start >= iv.start && rv.end <= iv.end) ) { continue; } eligible.push(ri); } if (td) { td.pooling = { inputVotes: regionVotes.length, eligibleVotes: eligible.length, saturationMaskedVotes: regionVotes.length - eligible.length, }; } // The one hub bound (traverse.ts) — N here IS corpusN, threaded down from // computeAttention. Read once, not per rule application. const bound = hubBound(ctx); const key = (it: AItem) => it.kind === "region" ? `r${it.ri}` : it.kind === "anchor" ? `a${it.id}` : `x${it.id}`; const pool = new Map>(); const system: DeductionSystem = { key, *axioms() { for (const ri of eligible) { yield { item: { kind: "region", ri }, cost: 0 }; } }, isGoal: () => false, // exhaust every axiom; there is no single goal to stop at // Every region axiom ties at cost 0, so the agenda's pop order among them // is otherwise unspecified; ordering by `ri` here only steers the HEAP // (never added to a stored cost — see relax's use of h) so pooling fires // in exactly the regionVotes array order the original loop used, byte-for- // byte reproducing its accumulation and tie-break order. heuristic: (it) => it.kind === "region" ? it.ri : 0, *rules(it) { if (it.kind !== "region") return; const rv = regionVotes[it.ri]; // The same hub bound the rest of the system uses (edgeAncestors' parent // cutoff, chooseNext's candidate cap): a terminal answer followed by // more than √N contexts is a non-discriminative hub — spreading a // region's vote across its FULL corpus-sized fan-in yields O(corpus) // rule applications per region and near-zero per-target weight anyway. // Cap the redistribution at the first √N contexts (insertion order, // the same convention chooseNext caps by). Hoisted out of the // generator: `rules` is invoked once per popped item, and this used to // re-derive the bound on every one of them. for (const r of rv.roots) { // CAPPED read: only the first hubBound targets are ever credited, so // only they are read — a common continuation's full reverse fan-in // is corpus-sized and is never materialised. const pv = ctx.store.prevFirst(r, bound); const isAnswer = pv.length > 0 && !ctx.store.hasNext(r); const targets = isAnswer ? pv : [r]; for (const t of targets) { yield { premises: [it], conclusion: { kind: "anchor", id: t }, cost: rv.w / targets.length, combine: "sum", }; yield { premises: [it], conclusion: { kind: "anchorFocus", id: t }, cost: rv.wFocus / targets.length, combine: "sum", }; } } }, pool, }; lightestDerivation(system); const votes = new Map(); const votesIdf = new Map(); const support = new Map< number, { start: number; end: number; w: number } >(); const regionSupport = new Map(); const regionSpans = new Map>(); // ONE POOLED AXIOM = ONE REGION VOTE. Counted separately from the spans // below because the two are different quantities: a JOINT binding is a // single vote whose evidence sits in several separate places (RegionVote. // parts), so its span count exceeds its axiom count. Reading the axiom // count off `regionSpans.length` conflated them and broke the accounting // both ways — contributingEvidence (absorbed-weighted, one term per // REGION) could read below it, and it could exceed the query's whole // candidate-region count. const regionAxioms = new Map(); // ANCHORS THE QUERY ITSELF POINTED AT. votesIdf is keyed by anchor node, // but root election has to know something about the REGIONS underneath it: // whether at least one of them is a structure the query wove, rather than a // form its cut split and collectRegions recovered (Region.corroborating). // An anchor standing on corroborating evidence ALONE is a real, well-priced // vote — it just is not a point of attention the query made, so it must not // enter the distribution the root cut is read from, nor the breadth ratio. // // REFUTED: barring such anchors from ROOT CANDIDACY outright. It defeats // the purpose — in the log case the CORRECT record (request_id=1042, one // context of 205) is addressable ONLY through the form the cut split, so // rejecting it handed the answer back to the near-miss 1050 (vote 5.60 -> // 3.81). Grounding follows where the evidence points; what a corroborating // region must not do is make the query look like it wove one more topic // than it did. const anchored = new Set(); // The LARGEST single region's contribution to this anchor's pooled vote. // The pool is a SUM (deliberately — see the pooling note above), so it says // how much evidence there is in total, never whether any ONE place in the // query carries evidence on its own. Consumers that hold an anchor to // consensusFloor(N) = ln(N) + 1/2 need the latter: that bar prices ONE // region's maximally-discriminative evidence (ln N is the IDF of content // reaching a single context), so comparing a six-region sum against it is a // dimensional error. Recorded here, beside the count, because this is the // only place the per-region contributions are still separable. const regionPeak = new Map(); const steps: DerivationStep[] = []; let order = 0; for (const pc of pool.values()) { if (pc.item.kind === "anchor") { votes.set(pc.item.id, pc.cost); const premises: DerivationItem[] = []; const seenRi = new Set(); let breadthSum = 0; const spans: Array<[number, number]> = []; for (const c of pc.contributions) { const p0 = c.premises[0].item; if (p0.kind !== "region" || seenRi.has(p0.ri)) continue; seenRi.add(p0.ri); const rv = regionVotes[p0.ri]; // Breadth is a ratio over the query's OWN candidate points of // attention (see below), and a corroborating region is not one of // those — it enters neither side of that ratio, so breadth reads // exactly as it did before such regions existed. Its evidence still // counts everywhere else: it is a premise, and it is a separate // PLACE for cluster counting — corroborating is what it is for. if (!rv.corroborating) { breadthSum += rv.absorbed ?? 1; anchored.add(pc.item.id); } premises.push({ kind: "form", span: [rv.start, rv.end] }); // A vote knows where its own evidence sits: `parts` when it stands on // several separate places (a joint binding), the merged span // otherwise. See RegionVote.parts. if (rv.parts !== undefined) { for (const [s, e] of rv.parts) spans.push([s, e]); } else spans.push([rv.start, rv.end]); } regionSupport.set(pc.item.id, breadthSum); // A span is a PLACE, and the same place reached through two different // votes (a standalone region and one part of a joint binding) is still // one place — listing it twice reports evidence the query does not // separately hold. Measured on test/50's fixture: span [18,21) // appeared twice among the top anchor's five. const seenSpan = new Set(); regionSpans.set( pc.item.id, spans.filter((sp) => { const key = `${sp[0]}:${sp[1]}`; if (seenSpan.has(key)) return false; seenSpan.add(key); return true; }), ); regionAxioms.set(pc.item.id, seenRi.size); let peak = 0; for (const c of pc.contributions) { const p0 = c.premises[0].item; if (p0.kind !== "region") continue; const rv = regionVotes[p0.ri]; const own = rv.wFocus ?? rv.w; if (own > peak) peak = own; } regionPeak.set(pc.item.id, peak); steps.push({ order: order++, move: "pool-vote", premises, conclusion: { kind: "form", span: [-1, -1], node: pc.item.id }, cost: pc.cost, producers: [], }); } else if (pc.item.kind === "anchorFocus") { votesIdf.set(pc.item.id, pc.cost); let bestRv: RegionVote | null = null; for (const c of pc.contributions) { const p0 = c.premises[0].item; if (p0.kind !== "region") continue; const rv = regionVotes[p0.ri]; if (!bestRv || rv.wFocus > bestRv.wFocus) bestRv = rv; } if (bestRv) { support.set(pc.item.id, { start: bestRv.start, end: bestRv.end, w: bestRv.wFocus, }); } } } return { votes, votesIdf, support, regionSupport, regionSpans, regionAxioms, regionPeak, anchored, steps, }; } /** The number of DISTINCT clusters a root's contributing regions form — * see Attention.clusters. Two regions belong to the same cluster iff the * gap between them is strictly less than one river-fold quantum W: at * that distance there is no room for a genuinely separate, independently * perceivable unit of content between them (the same "smallest meaningful * distinction" quantum {@link reachThreshold}'s own doc invokes). A gap * of a full quantum or more means real, separate structure could sit * between the two spans, so they count as independent corroboration. * Strict `<` (not `<=`): verified against gap 3.1's own "gender equality" * root, whose two genuine clusters sit EXACTLY W bytes apart — `<= W` * would wrongly merge them into one and break that pinned requirement. */ function countClusters(spans: readonly [number, number][], W: number): number { if (spans.length === 0) return 0; const sorted = [...spans].sort((a, b) => a[0] - b[0]); let clusters = 1; let curEnd = sorted[0][1]; for (let i = 1; i < sorted.length; i++) { const [s, e] = sorted[i]; if (s - curEnd < W) { curEnd = Math.max(curEnd, e); } else { clusters++; curEnd = e; } } return clusters; } export function commitVotes( ctx: MindContext, pooled: { votes: Map; votesIdf: Map; support: Map; regionSupport: Map; regionSpans: Map>; regionAxioms: Map; regionPeak: Map; anchored: Set; steps: DerivationStep[]; }, sat: SaturationInfo, regions: readonly Region[], regionVoter: ReadonlyArray<{ id: number; score: number; w: number } | null>, N: number, td?: TraceDraft, cfg?: ClimbConsensusCfg, ): AttentionRead { const { votes, votesIdf, support, regionSupport, regionSpans, regionAxioms, regionPeak, anchored, steps, } = pooled; if (votes.size === 0) { traceAttention(ctx, regions, regionVoter, [], steps, td, cfg); return { roots: [], ranked: [] }; } // SCALE-INVARIANT confidence — see Attention.breadth's doc. regions.length // is the query's OWN full candidate count (most never vote at all), the // same denominator the "N of M sub-regions voted" rationale text already // reports; regionSupport is that same accounting read PER ANCHOR. // Corroborating regions are excluded from the denominator for the same // reason they are excluded from the numerator (see poolVotes): they are // not candidate points of attention the query wove, so counting them would // silently shrink every anchor's breadth — measured: test/36's genuine // second topic fell 6/11 -> 6/12 and fusion's dispersion gate dropped it, // with nothing else about the climb changed. const totalRegions = Math.max( 1, regions.filter((r) => !r.corroborating).length, ); const ranked = [...votes.entries()] .map(([anchor, vote]) => { const s = support.get(anchor)!; return { anchor, vote, peak: regionPeak.get(anchor) ?? 0, start: s.start, end: s.end, breadth: (regionSupport.get(anchor) ?? 0) / totalRegions, clusters: countClusters( regionSpans.get(anchor) ?? [], ctx.space.maxGroup, ), }; }) .sort((a, b) => b.vote - a.vote); const overlaps = (a: Attention, b: Attention) => a.start < b.end && b.start < a.end; // Read the root cut from the anchors the QUERY pointed at. A vote standing // only on corroborating evidence (a form the query's cut split, recovered // by lookup — Region.corroborating) is evidence for someone else's anchor, // never a point of attention of its own: the query never wove it as an // independent structure, the fold did that. Letting such votes into // idfDesc shifts naturalBreak — they are exact, hence high-IDF, hence they // land at the top of the distribution — and a 2-topic query then elects 3 // roots (test/24:404, and the answered-continuation exclusion probe). const idfDesc = [...votesIdf.entries()] .filter(([anchor]) => anchored.has(anchor)) .map(([, v]) => v) .sort((a, b) => b - a); const rootCut = naturalBreak(idfDesc); // A FURTHER point of attention (beyond the dominant one, which always // grounds) must clear the same absolute significance floor // recallByResonance trusts a climb anchor with — log(N) + 1/2, three-ish // halvings of confidence above pure chance at this corpus scale — not // merely beat whatever its immediate neighbour in the ratio happens to be. // Without it, naturalBreak's ratio is scale-free but not FLOOR-free: on a // large, topic-diverse corpus the steepest ratio in a long noise tail can // sit far below any real signal, admitting scaffolding echoes as if they // were genuine further topics. const floor = consensusFloor(N); const placed: Attention[] = []; const roots: Attention[] = []; const recordAnchor = ( point: Attention, rank: number, status: "root" | "overlap" | "rejected", dominant: boolean, passesNaturalBreak: boolean | undefined, passesConsensusFloor: boolean | undefined, pastLeadingSaturation: boolean | undefined, tiedWithDominant: boolean | undefined, rejectionReasons: AnchorRejectionReason[], ) => { if (!td) return; td.anchors.push({ anchor: point.anchor, rank, pooledVote: point.vote, idfVote: votesIdf.get(point.anchor) ?? 0, candidateBreadth: regions.length, contributingVotes: regionAxioms.get(point.anchor) ?? 0, contributingEvidence: regionSupport.get(point.anchor) ?? 0, breadth: point.breadth, contributingSpans: regionSpans.get(point.anchor) ?? [], clusters: point.clusters, commit: { status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons, }, }); }; for (let rank = 0; rank < ranked.length; rank++) { const point = ranked[rank]; const absorbed = placed.some((p) => overlaps(point, p)); // Commit decisions are recorded LIVE, inside this loop, in the exact // shape the gates below apply them — never reconstructed afterward from // the final `roots` (spec §8's explicit requirement). let status: "root" | "overlap" | "rejected"; let dominant = false; let passesNaturalBreak: boolean | undefined; let passesConsensusFloor: boolean | undefined; let pastLeadingSaturation: boolean | undefined; let tiedWithDominant: boolean | undefined; const rejectionReasons: AnchorRejectionReason[] = []; if (absorbed) { status = "overlap"; } else { const pastLeading = !sat.hasLeading || roots.length === 0 || point.start >= sat.leadingEnd; pastLeadingSaturation = pastLeading; const vote = votesIdf.get(point.anchor) ?? 0; if (roots.length === 0) { // The first non-overlapping root is DOMINANT and bypasses the two // vote thresholds (it always grounds) — only the leading-saturation // gate still applies to it. dominant = true; if (pastLeading) { status = "root"; } else { status = "rejected"; rejectionReasons.push("leading-saturation"); } } else { passesNaturalBreak = vote >= rootCut; passesConsensusFloor = vote >= floor; // CO-DOMINANT — an anchor the estimator cannot separate from the // dominant inherits the dominant's exemption, because that exemption's // only warrant is being TOP, and "top" is not a fact about the corpus // when the ordering moves with the seed. // // The dominant bypasses both vote gates ("it always grounds"); the // runner-up is held to an absolute ln(N)+1/2 floor the dominant never // had to clear. Which of them gets the exemption is then decided by a // sort over ESTIMATED quantities. Measured on test/29 D1's corpus, // 60 seeds per D — true separation 0.54s / 0.75s / 1.04s: // // D s=1/sqrt(D) vote SD (estimated anchor) SD/s flips // 256 0.0625 0.0561 0.90 19/60 // 1024 0.0313 0.0268 0.86 12/60 // 4096 0.0156 0.0074 0.48 2/60 // // The SD tracks 1/sqrt(D) and the flip rate collapses with it, so the // reordering is the ESTIMATOR's, not the corpus's. The loser was then // refused by a floor at 1.599 that neither anchor could ever reach // (corpusN 3) — a coin flip decided which single structure the query // was allowed to have settled on. // // THE BAND IS sqrt(k)*s, NOT s. A vote is a SUM over the anchor's own // contributing regions, so its noise grows as sqrt(k); pricing a summed // margin against one s would be the category error chooseNext's comment // warns about. k is `regionAxioms`, already in hand; s is // `estimatorNoise(D)`, already derived. No constant is introduced. // Verified conservative: measured SD/(sqrt(k)*s) never exceeded 0.72. // // BOUNDED BY CONSTRUCTION: admission requires indistinguishability from // an anchor ALREADY admitted, so it can only admit what the ordinary // rule would have admitted had the noise fallen the other way. It is // N-independent for the same reason — a statement about the estimator, // not about corpus size. const tieBand = Math.sqrt( Math.max(1, regionAxioms.get(point.anchor) ?? 1), ) * estimatorNoise(ctx.store.D); const dominantVote = votesIdf.get(roots[0].anchor) ?? 0; tiedWithDominant = dominantVote - vote < tieBand; if ( ((passesNaturalBreak && passesConsensusFloor) || tiedWithDominant) && pastLeading ) { status = "root"; } else { status = "rejected"; if (!passesNaturalBreak) rejectionReasons.push("below-natural-break"); if (!passesConsensusFloor) { rejectionReasons.push("below-consensus-floor"); } if (!pastLeading) rejectionReasons.push("leading-saturation"); } } if (status === "root") { roots.push(point); } else { recordAnchor( point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons, ); continue; } } recordAnchor( point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons, ); placed.push(point); } traceAttention( ctx, regions, regionVoter, roots, steps, td, cfg ? { ...cfg, naturalBreak: rootCut, consensusFloor: floor } : undefined, ranked, ); return { roots, ranked }; } export function detectSaturated( ctx: MindContext, regions: ReadonlyArray<{ start: number; end: number; chunk?: boolean }>, saturated: ReadonlyArray, ): SaturationInfo { // Intervals are built from CHUNK regions only. collectRegions emits the // tree in POST-ORDER — a parent region arrives AFTER its children and // shares its first child's `start` — so the raw array is not monotone in // byte position, and a saturated parent would fuse with a later saturated // chunk into an interval swallowing a NON-saturated child. Chunk regions // (leaf-parents) are disjoint and already in byte order, and saturation // masking exists to drop canonicalFailed CHUNK votes (see poolVotes), so // chunks are both the sufficient and the safe basis. A region without a // `chunk` flag (a bare {start,end} from a direct caller) is treated as a // chunk. const intervals: Array<{ start: number; end: number }> = []; let intStart = -1; let intEnd = -1; let totalLen = 0; for (let ri = 0; ri < regions.length; ri++) { const r = regions[ri]; totalLen = Math.max(totalLen, r.end); if (r.chunk === false) continue; if (saturated[ri]) { if (intStart === -1) intStart = r.start; intEnd = r.end; } else { if (intStart !== -1) { intervals.push({ start: intStart, end: intEnd }); intStart = -1; } } } if (intStart !== -1) { intervals.push({ start: intStart, end: intEnd }); } const leading = intervals.length > 0 && intervals[0].start === 0 ? intervals[0] : null; const hasLeading = leading !== null && leading.end >= ctx.space.maxGroup && leading.end < totalLen; const leadingEnd = leading !== null ? leading.end : 0; return { leadingEnd, hasLeading, intervals }; } /** Set equality of two climb root lists (the "same conclusion" test the * contrastive margin skips rivals by). */ function sameRoots(a: readonly number[], b: readonly number[]): boolean { if (a.length !== b.length) return false; const s = new Set(a); for (const x of b) if (!s.has(x)) return false; return true; } export function canonicalChunkId( ctx: MindContext, regionBytes: Uint8Array, N: number, reachMemo?: Map, ): number | null { const len = Math.min(regionBytes.length, ctx.space.maxGroup); // WHICH window anchors a region is decided by reach, not by position. This // used to return at the FIRST offset that matched, which was indistinguishable // from correct while every region was exactly W bytes — there was only one // offset. A content-defined segment is longer, and its first window is // whatever happens to start it: for "is frigi" that is " is ", pure // scaffolding, which reaches every context, saturates, and makes the whole // region ABSTAIN. The region's own content ("frigi") never got a say, and // CAST lost a point of attention it needed (test/29 D1/D2). // // So scan every offset and prefer an anchor that still discriminates: not // saturated, and among those the one reaching the FEWEST contexts // (commonality.md, corpus-global). Only when every window in the region // saturates does the old generalising choice stand — there is then no // discriminative anchor to find, and abstaining is the honest outcome. let discId: number | null = null; let discReached = Infinity; let fallback: number | null = null; for (let off = 0; off + len <= regionBytes.length; off++) { const ids = leafIdRun(ctx, regionBytes, off, off + len); // An unknown byte disqualifies THIS window, not the region. This used to // abandon the whole region on the first unseen byte, which was // indistinguishable from correct while regions were exactly W bytes — there // was one window, so failing it was failing the region. A content-defined // segment holds several windows, and a single unknown byte near its start // was silently costing the region its anchor entirely. if (ids === null) continue; const flatId = ctx.store.findBranch(ids); if (flatId === null) continue; if (len < 2) return flatId; // Within one window, the widest reach is still the right CANONICAL // identity — a chunk's anchor should be its most general stable form. let bestId = flatId; let bestReach = edgeAncestors(ctx, flatId, N, reachMemo); for (let k2 = 1; k2 < len; k2++) { const shortIds = ids.slice(0, len - k2); const shortId = ctx.store.findBranch(shortIds); if (shortId === null) continue; const shortReach = edgeAncestors(ctx, shortId, N, reachMemo); if ( shortReach.saturated || shortReach.contextsReached > bestReach.contextsReached ) { bestId = shortId; bestReach = shortReach; } } if (fallback === null) fallback = bestId; if (!bestReach.saturated && bestReach.contextsReached < discReached) { discId = bestId; discReached = bestReach.contextsReached; // Nothing can discriminate better than reaching ONE context, so the scan // stops there rather than pricing the rest of the segment's windows. if (discReached <= 1) break; } } return discId ?? fallback; } export function naturalBreak(votes: number[]): number { if (votes.length <= 1) return votes[0] ?? 0; let breakAt = 1; let steepest = Infinity; for (let i = 1; i < votes.length; i++) { if (votes[i - 1] <= 0) break; const ratio = votes[i] / votes[i - 1]; if (ratio < steepest) { steepest = ratio; breakAt = i; } } return votes[breakAt - 1]; } // ═══════════════════════════════════════════════════════════════════════════ // Cross-region attention — DIRECT region-to-region interaction. // // voteRegions climbs each region INDEPENDENTLY; poolVotes then ADDS those // independent votes. Additive pooling is a soft conjunction, but it can only // ever surface a context at least one region already votes for. Two regions // whose individual climbs land on DIFFERENT contexts leave their JOINT context // — the learnt whole that contains BOTH — with no vote at all, and no amount // of pooling can recover it. ("red" climbs to `red square`, "circle" to // `circle`; nothing votes for `red circle`, the only fact holding both.) // // This is the attention counterpart of the bridge, and it ascends by the SAME // content-addressed junction walk (junction.ts): "which learnt whole contains // region A then region B?" is a bounded DAG ascent from the two forms' // canonical identities — NOT a resonance guess on a synthesised gist. Folding // two region vectors cannot even reconstruct the stored joint form: Sema builds // a multi-word gist from BYTE-chunk folds, so isolated word vectors superpose // into a different direction and resonate to `red circle` and `red square` // indistinguishably. The ascent sidesteps this by matching BYTES, not vectors. // // A joint container is EXACT evidence (it literally holds both forms), so it // votes at full strength — the exact-first discipline voteRegions gives a // content-addressed chunk. Each junction search is a bounded walk with NO // ANN query, and searches are capped at k. Three further disciplines make // the composition ORDER-FREE, N-ARY, and CORPUS-INDEPENDENT: // // • ORDER-FREE — a junction is evidence the forms were LEARNT TOGETHER; // which one the query mentioned first is a fact about the query, not the // learnt whole. The walk tests both byte orders at no extra walk cost // (see junctionContainersFrom's `unordered`). // • N-ARY — binding is not intrinsically pairwise. A pair's containers are // FILTERED by the remaining candidate forms: the container covering the // MOST of the query's composable forms wins, so three cross-cutting // attributes (each pair ambiguous) still resolve to their unique triple — // at the cost of one cached byte read + indexOf per (container, extra), // never an extra walk. // • CORPUS-INDEPENDENT — candidates are ANY voted region, not just // recognised sites. A word never learnt standalone has no site, but its // stored chunks still vote and their BYTES still compose: the ascent // matches byte containment, so a fragment pair evidences the same joint // container the whole word would. Contiguous shards of one word cannot // pair (the adjacency skip), and a pair covered by a single KNOWN region // is skipped — that whole form already votes directly, and re-deriving it // from its own pieces would only double-count. // // EXPLAINING AWAY (the aliasing complement of corpus independence): a chunk // of the query can straddle the byte grid so that it exists verbatim in the // WRONG deposit (" cir" of "red then circle" is a stored chunk of `blue // circle`, never of `red circle` — a pure alignment accident) and its // independent climb then votes for a context the query gives no reason to // believe. When a junction binds, any individual vote whose bytes the joint // container LITERALLY CONTAINS yet whose climb disagrees with the junction's // is superseded: the exact joint evidence explains those bytes, so their // disagreeing vote is grid aliasing, not signal. Votes whose bytes the // container does not hold (a genuine second topic) are untouched. // ═══════════════════════════════════════════════════════════════════════════ // ── Structural-resonance — the FINAL approximate tier ────────────────────── // // Reached only when every DAG junction tier (exact, single-synonym, double- // synonym) found no container. Composes a hypothetical structural gist from // ALREADY-EXISTING structural vectors — the two endpoint regions' own gists // (or, per variant, a halo sibling's stored gist occupying the same slot) // plus the REAL middle-query structure between them — and asks the ANN index // what already-learnt whole resembles that composition. It never perceives // concatenated endpoint bytes and never fabricates a rewritten query string; // see {@link composeStructuralGist}. export type CrossRegionTier = | "exact" | "single-synonym" | "double-synonym" | "structural-resonance"; export interface StructuralVariant { left: StructuralPart; right: StructuralPart; kind: | "exact-exact" | "left-synonym" | "right-synonym" | "double-synonym"; semanticConfidence: number; leftSiblingId?: number; rightSiblingId?: number; } export interface StructuralResonanceProposal { id: number; annScore: number; semanticConfidence: number; effectiveScore: number; variant: StructuralVariant["kind"]; leftSiblingId?: number; rightSiblingId?: number; } const VARIANT_KIND_ORDER: Record = { "exact-exact": -1, "left-synonym": 0, "right-synonym": 1, "double-synonym": 2, }; /** A lightweight, cost-free descriptor of one candidate structural variant — * enough to rank it, but with no bytes read and no vector materialized. */ interface StructuralVariantSpec { kind: "left-synonym" | "right-synonym" | "double-synonym"; semanticConfidence: number; leftSiblingId?: number; rightSiblingId?: number; } /** Same deterministic ordering the old implementation applied to already- * materialized variants (§8): semantic confidence desc, then kind * (left-synonym, right-synonym, double-synonym), then sibling ids asc. */ function compareStructuralVariantSpecs( a: StructuralVariantSpec, b: StructuralVariantSpec, ): number { return b.semanticConfidence - a.semanticConfidence || VARIANT_KIND_ORDER[a.kind] - VARIANT_KIND_ORDER[b.kind] || (a.leftSiblingId ?? -1) - (b.leftSiblingId ?? -1) || (a.rightSiblingId ?? -1) - (b.rightSiblingId ?? -1); } /** Every single- and double-synonym combination, as cost-free descriptors — * no `read`, `gistOf`, `perceive` or `StructuralPart` allocation. Both * sibling lists are already bounded by `haloQueryK`, so the O(haloQueryK²) * cross-product here is cheap; only the SELECTED specs go on to pay for * sibling reconstruction. */ function buildStructuralVariantSpecs( sides: JunctionSynonymSides, ): StructuralVariantSpec[] { const specs: StructuralVariantSpec[] = []; for (const left of sides.leftSiblings) { specs.push({ kind: "left-synonym", semanticConfidence: left.score, leftSiblingId: left.id, }); } for (const right of sides.rightSiblings) { specs.push({ kind: "right-synonym", semanticConfidence: right.score, rightSiblingId: right.id, }); } for (const left of sides.leftSiblings) { for (const right of sides.rightSiblings) { specs.push({ kind: "double-synonym", semanticConfidence: Math.min(left.score, right.score), leftSiblingId: left.id, rightSiblingId: right.id, }); } } specs.sort(compareStructuralVariantSpecs); return specs; } /** A sibling gist cached in the shared, climb-wide memo alongside the * COMPLETE stored byte length it was reconstructed from — the length is * required to tell whether a cache hit is still valid under a probe whose * `maxSiblingBytes` bound is smaller than the one that first cached it. */ interface CachedSiblingGist { gist: Vec; length: number; } /** A halo sibling's structural gist, bounded to `maxBytes` of stored content * and reused across the whole climb. `positiveMemo` (shared across every * probe in the climb, passed in by the caller) remembers only successfully * reconstructed complete gists together with their complete byte length — * a sibling rejected here for being too large for THIS pair's phrase-scale * bound may still be admissible for a larger-spanning pair later, so a * rejection is never memoized globally, and a sibling cached by a LARGER * probe is only reused here when its length still fits THIS probe's * (possibly smaller) bound — eligibility must never depend on which probe * happened to cache the sibling first. `localMemo` is scoped to one * `buildStructuralVariants` call, where every variant shares the same * bound, so a `null` there is safe to reuse. */ function loadBoundedSiblingGist( ctx: MindContext, id: number, maxBytes: number, positiveMemo: Map, localMemo: Map, ): Vec | null { if (localMemo.has(id)) { return localMemo.get(id) ?? null; } const cached = positiveMemo.get(id); if (cached !== undefined) { const result = cached.length <= maxBytes ? cached.gist : null; localMemo.set(id, result); return result; } const length = ctx.store.contentLen(id, maxBytes + 1); if (length <= 0 || length > maxBytes) { localMemo.set(id, null); return null; } const bytes = read(ctx, id, maxBytes + 1); if (bytes.length === 0 || bytes.length > maxBytes) { localMemo.set(id, null); return null; } const gist = gistOf(ctx, bytes); positiveMemo.set(id, { gist, length }); localMemo.set(id, gist); return gist; } /** Build, bound and order every mandatory structural variant (§7-8): the * exact/exact composition is always kept; up to `ctx.cfg.haloQueryK` * synonym variants (single- and double-synonym combined, one shared * budget) are appended, ordered by confidence, then kind, then sibling id. * Variant selection is entirely lightweight (see {@link * buildStructuralVariantSpecs}); a sibling's bytes are read and perceived * only for specs actually retained, and at most once per sibling id per * climb via `siblingGistMemo`. */ export function buildStructuralVariants( ctx: MindContext, ra: Region, rb: Region, sides: JunctionSynonymSides, siblingGistMemo: Map, ): { variants: StructuralVariant[]; exactLeft: StructuralPart; exactRight: StructuralPart; } { const leftLen = ra.end - ra.start; const rightLen = rb.end - rb.start; const exactLeft: StructuralPart = { v: ra.v, len: leftLen }; const exactRight: StructuralPart = { v: rb.v, len: rightLen }; const variants: StructuralVariant[] = [ { left: exactLeft, right: exactRight, kind: "exact-exact", semanticConfidence: 1, }, ]; // Same phrase-scale bound the cross-region junction ladder uses // (`maxInterior`): a sibling whose complete stored content exceeds it is // not materialized as a structural-resonance endpoint, keeping sibling // reconstruction phrase-scale even for a large deposit or conversation // root that merely appeared in a halo result. const maxSiblingBytes = (leftLen + rightLen) * ctx.space.maxGroup; const specs = buildStructuralVariantSpecs(sides); const localGistMemo = new Map(); let retainedSynonyms = 0; for (const spec of specs) { if (retainedSynonyms >= ctx.cfg.haloQueryK) break; let left = exactLeft; let right = exactRight; if (spec.leftSiblingId !== undefined) { const gist = loadBoundedSiblingGist( ctx, spec.leftSiblingId, maxSiblingBytes, siblingGistMemo, localGistMemo, ); if (gist === null) continue; left = { v: gist, len: leftLen }; } if (spec.rightSiblingId !== undefined) { const gist = loadBoundedSiblingGist( ctx, spec.rightSiblingId, maxSiblingBytes, siblingGistMemo, localGistMemo, ); if (gist === null) continue; right = { v: gist, len: rightLen }; } variants.push({ left, right, kind: spec.kind, semanticConfidence: spec.semanticConfidence, leftSiblingId: spec.leftSiblingId, rightSiblingId: spec.rightSiblingId, }); retainedSynonyms++; } return { variants, exactLeft, exactRight }; } /** Deterministic best-of tie-break for two proposals ranked for the SAME * candidate id — effectiveScore, then annScore, then semanticConfidence, * then variant kind, then sibling ids (§10). */ function betterProposal( a: StructuralResonanceProposal, b: StructuralResonanceProposal, ): boolean { if (a.effectiveScore !== b.effectiveScore) { return a.effectiveScore > b.effectiveScore; } if (a.annScore !== b.annScore) return a.annScore > b.annScore; if (a.semanticConfidence !== b.semanticConfidence) { return a.semanticConfidence > b.semanticConfidence; } if (VARIANT_KIND_ORDER[a.variant] !== VARIANT_KIND_ORDER[b.variant]) { return VARIANT_KIND_ORDER[a.variant] < VARIANT_KIND_ORDER[b.variant]; } if ((a.leftSiblingId ?? -1) !== (b.leftSiblingId ?? -1)) { return (a.leftSiblingId ?? -1) < (b.leftSiblingId ?? -1); } return (a.rightSiblingId ?? -1) < (b.rightSiblingId ?? -1); } /** The final approximate tier: compose every retained structural variant, * ANN-query each, merge proposals by candidate id, and validate the winner * through the SAME structural gates every other tier answers to (saturation, * roots, IDF, contrastive margin). Returns null when nothing survives. */ /** {@link structuralResonance}, charged to its own profiling phase — it is * the halo-mediated arm of the cross-region ladder and the one part of it * that resonates. */ async function meteredStructuralResonance( ...args: Parameters ): Promise< ReturnType extends Promise ? R : never > { const ctx = args[0]; return ctx.meter ? await ctx.meter.time( "climb.structuralResonance", () => structuralResonance(...args), ) : await structuralResonance(...args); } export async function structuralResonance( ctx: MindContext, query: Uint8Array, ra: Region, rb: Region, sides: JunctionSynonymSides, siblingGistMemo: Map, k: number, N: number, reachMemo: Map, /** Each side's OWN individual climb roots (from voteRegions), when it cast * one — the self-evidence backstop structural-resonance needs and the * exact tier gets for free from literal byte containment (§11's whole * premise: recover a JOINT context neither side votes for alone). A * candidate whose reach is exactly one side's own conclusion is not new * evidence of a joint whole; it is that side's resonance rediscovering * itself through a synthetic gist still dominated by its own direction. */ ownRootsA: readonly number[] | undefined, ownRootsB: readonly number[] | undefined, trace?: StructuralResonanceTrace, ): Promise< | { proposal: StructuralResonanceProposal; reach: AncestorReach; idf: number } | null > { const { variants } = buildStructuralVariants( ctx, ra, rb, sides, siblingGistMemo, ); if (trace) trace.variantBudget = ctx.cfg.haloQueryK; const middleBytes = query.subarray(ra.end, rb.start); const middlePart: StructuralPart | null = middleBytes.length === 0 ? null : { v: perceive(ctx, middleBytes).v, len: middleBytes.length }; const proposals = new Map(); for (const variant of variants) { const parts: StructuralPart[] = [variant.left]; if (middlePart) parts.push(middlePart); parts.push(variant.right); const synthetic = composeStructuralGist(ctx.space, parts); const hits = await ctx.store.resonate(synthetic, k); if (trace) { trace.variants.push({ kind: variant.kind, semanticConfidence: variant.semanticConfidence, leftSiblingId: variant.leftSiblingId, rightSiblingId: variant.rightSiblingId, annHitsReturned: hits.length, }); } for (const hit of hits) { const candidate: StructuralResonanceProposal = { id: hit.id, annScore: hit.score, semanticConfidence: variant.semanticConfidence, effectiveScore: hit.score * variant.semanticConfidence, variant: variant.kind, leftSiblingId: variant.leftSiblingId, rightSiblingId: variant.rightSiblingId, }; const prev = proposals.get(hit.id); if (prev === undefined || betterProposal(candidate, prev)) { proposals.set(hit.id, candidate); } } } if (trace) trace.mergedProposals = proposals.size; if (proposals.size === 0) { if (trace) { trace.noiseFloor = estimatorNoise(ctx.store.D); trace.outcome = "empty"; } return null; } const sorted = [...proposals.values()].sort((a, b) => b.effectiveScore - a.effectiveScore || a.id - b.id ); // One shared shape for every `examined` entry (spec §5): only `outcome` // varies across the six exit points below, so build it once instead of // repeating the six-field literal at each site. const recordExamined = ( p: StructuralResonanceProposal, outcome: StructuralResonanceCandidateTrace["outcome"], ) => { if (!trace) return; trace.examined.push({ node: p.id, variant: p.variant, leftSiblingId: p.leftSiblingId, rightSiblingId: p.rightSiblingId, annScore: p.annScore, semanticConfidence: p.semanticConfidence, effectiveScore: p.effectiveScore, outcome, }); }; let selected: StructuralResonanceProposal | null = null; let selectedReach: AncestorReach | null = null; let selectedIdf = 0; let rival: StructuralResonanceProposal | null = null; for (const p of sorted) { const reach = edgeAncestors(ctx, p.id, N, reachMemo); if (reach.saturated || reach.roots.length === 0) { recordExamined(p, reach.saturated ? "saturated" : "no-roots"); continue; } const idf = Math.log(N / Math.max(1, reach.contextsReached)); if (idf <= 0) { recordExamined(p, "nonpositive-idf"); continue; } // Self-evidence backstop (see the param doc above): a candidate that is // exactly one side's own already-voted conclusion carries no JOINT // evidence — skip it as if it never survived. if ( (ownRootsA && sameRoots(reach.roots, ownRootsA)) || (ownRootsB && sameRoots(reach.roots, ownRootsB)) ) { recordExamined(p, "same-as-endpoint"); continue; } if (selected === null) { selected = p; selectedReach = reach; selectedIdf = idf; recordExamined(p, "selected"); } else if (!sameRoots(reach.roots, selectedReach!.roots)) { rival = p; recordExamined(p, "contrastive-rival"); break; } else { recordExamined(p, "same-as-selected"); } } if (selected === null || selectedReach === null) { if (trace) { trace.noiseFloor = estimatorNoise(ctx.store.D); trace.outcome = "no-valid-proposal"; } return null; } const margin = rival ? selected.effectiveScore - rival.effectiveScore : selected.effectiveScore; if (trace) { trace.contrastiveMargin = margin; trace.noiseFloor = estimatorNoise(ctx.store.D); } if (margin <= estimatorNoise(ctx.store.D)) { if (trace) trace.outcome = "margin-rejected"; return null; } if (trace) trace.outcome = "accepted"; return { proposal: selected, reach: selectedReach, idf: selectedIdf }; } async function crossRegionVotes( ctx: MindContext, query: Uint8Array, regions: readonly Region[], rvs: { votes: readonly RegionVote[]; saturated: readonly boolean[] }, k: number, N: number, reachMemo: Map, td?: TraceDraft, ): Promise<{ votes: RegionVote[]; superseded: Set }> { // Candidate regions: every region that ALREADY CAST ITS OWN VOTE in // voteRegions — individually idf > 0, genuinely discriminative on its own, // just not necessarily for the SAME context as its partner. This is the // exact shape of the binding problem: "red" alone votes for `red square`, // "circle" alone for `circle` — each independently informative, disagreeing // on the conclusion — and only their CONJUNCTION resolves to the one // context, `red circle`, that actually holds both. // // A region that never voted (idf == 0 — e.g. a repeated system-prompt // prefix shared by every deposit) carries NO individual signal, and must be // excluded here too: ascending from a non-discriminative fragment's seeds // can still land on some deeper, incidentally-unique DESCENDANT container — // its rarity would come entirely from context OUTSIDE the fragments // actually composed, manufacturing confidence the query gave no reason to // have. Requiring a prior individual vote is the same discipline the noise // drop already applies to single regions, extended to compositions — with // one graded relaxation: a KNOWN region that did NOT vote (saturated, or // idf ≤ 0) may still serve as the WEAK side of a pair whose other side DID // vote. Saturation is an abstention about where the region CLIMBS; its // content-addressed identity is still exact, and the junction asks a // different question — "which whole holds both?" — whose conclusion the // container's own idf gate below still guards. Two non-voting regions // never pair (that is exactly the shared-prefix trap above), so at least // one side is always individually discriminative. // // Only MAXIMAL spans compose: a span contained in another candidate is a // fragment of that candidate's evidence, never independent of it — but // containment alone does not establish that relation. An APPROXIMATE // container (a fold segment whose gist merely resonated) does not hold the // evidence of an EXACT one (a recognised site, content-addressed): its // bytes straddle the site rather than explain it, so calling the site a // fragment of it discards the only exact reading of those bytes. Measured: // on `blue then square` the segment `blue t` swallowed the site `blue`, // leaving one candidate and no pair, while on `red then circle` the // segment happened to end at `red `'s edge and the same query shape // composed — the outcome turned on where a cut fell. This is the same // discipline the between-region gate below already states: an approximate // region climbing "somewhere" is ordinary noise, not evidence. // // Shared across every cross-region probe in this climb: a sibling // successfully reconstructed while probing one pair must not be read and // perceived again while probing another pair in the same climb. const siblingGistMemo = new Map(); const votedSpans = new Set(); for (const rv of rvs.votes) votedSpans.add(`${rv.start},${rv.end}`); const seen = new Set(); const eligible: number[] = []; const strong = new Set(); for (let ri = 0; ri < regions.length; ri++) { const r = regions[ri]; const key = `${r.start},${r.end}`; const isStrong = votedSpans.has(key); if ((!isStrong && !r.known) || seen.has(key)) continue; seen.add(key); eligible.push(ri); if (isStrong) strong.add(ri); } const cand = eligible.filter((x) => !eligible.some((y) => y !== x && regions[y].start <= regions[x].start && regions[x].end <= regions[y].end && regions[y].end - regions[y].start > regions[x].end - regions[x].start && (regions[y].known || !regions[x].known) ) ); const none = { votes: [], superseded: new Set() }; if (td) { td.crossRegionSummary = { eligibleRegions: eligible.length, maximalRegions: cand.length, probeLimit: k, probesAttempted: 0, // updated below as probes accrue stopReason: cand.length < 2 ? "insufficient-regions" : undefined, }; } if (cand.length < 2) return none; cand.sort((x, y) => regions[x].start - regions[y].start || regions[x].end - regions[y].end ); const dec = (b: Uint8Array): string => new TextDecoder().decode(b).replace(/\s+/g, " ").trim(); const cache = walkCache(ctx); // One junctionSeeds per candidate for the WHOLE pairing loop — a candidate // recurs in up to |cand|−1 pairs, and its seeds are a pure function of its // bytes. const seedsMemo = new Map(); const seedsOf = (ri: number): number[] => { let s = seedsMemo.get(ri); if (s === undefined) { const r = regions[ri]; s = junctionSeeds(ctx, query.subarray(r.start, r.end)); seedsMemo.set(ri, s); } return s; }; const overlapsSpan = ( e: Region, s: { start: number; end: number }, ): boolean => e.start < s.end && s.start < e.end; const out: RegionVote[] = []; const superseded = new Set(); // A candidate consumed by one junction does not seed another: its evidence // is already composed at full joint strength, and re-pairing it would vote // the same container (or a sub-container of it) twice. const consumed = new Set(); let probes = 0; // When atoms themselves are hubs (atomIsHub — a single byte reaches ≥ √N // contexts, bounded-reads.md's own predicate), the corpus is large enough // that the cross-region junction walks are dominated by the drift through // common content's ancestry. Each of k candidate pairs otherwise spends its // own √N·W budget (profiled: 160,210 junction pops, 31% of think at N = // 325,608), and a cumulative dialogue multiplies bounded work into tens of // seconds. The structural walk is therefore given ONE k·W allowance per // evidence tier, shared across every pair — k pairs × W phrase-scale levels, // the minimal exact check; a pair whose container is not reached within it // falls through to the resonance tier (the ANN proposes what the shallow walk // no longer exhaustively scans, exact-vs-approximate.md). // // Below atomIsHub the store is small and atoms still discriminate, so the // walks keep exhaustive exact traversal (per-walk √N·W) — the shared budget // would otherwise be smaller than the structures the tests deliberately // build. The gate is the SAME derived predicate the climb already uses for // byte atoms, not a separate corpus-size knob: an earlier `N > (k·W)³` cube // never engaged at real scale (96³ = 884,736 > 325,608), and a "share one // √N·W" experiment was too tight below ~10³ contexts (test/36, test/14) — // both are the same mistake of pricing the gate on corpus size instead of on // the atom-hub scale. const corpusScale = atomIsHub(ctx, N); const exactBudget = corpusScale ? { n: k * ctx.space.maxGroup } : undefined; const synonymBudget = corpusScale ? { n: k * ctx.space.maxGroup } : undefined; for (let a = 0; a < cand.length && probes < k; a++) { if (consumed.has(cand[a])) continue; const ra = regions[cand[a]]; for (let b = a + 1; b < cand.length && probes < k; b++) { if (consumed.has(cand[b])) continue; const rb = regions[cand[b]]; if (!strong.has(cand[a]) && !strong.has(cand[b])) continue; if (ra.end >= rb.start) continue; // overlap or adjacent — nothing between // In a cumulative conversation, an old↔old interaction cannot explain // the user turn currently being answered; it was already available // before that turn existed. Keep old↔current pairs (the current turn may // refer to a prior answer), but do not repeatedly spend the junction // budget recomposing two regions wholly before the current boundary. if (ctx.currentTurnStart > 0 && rb.end <= ctx.currentTurnStart) continue; // Candidates strictly BETWEEN ra and rb (cand is sorted by start, so // that is exactly cand[a+1 .. b-1]) that already cast their OWN vote — // genuine, individually-corroborated evidence about what fills the gap // — gate the container search below: a joint container is binding // evidence only when it is CONSISTENT with that evidence, i.e. its own // bytes actually contain what the between-region says. This is the // n-ary composition's normal shape (a between-attribute's bytes DO // recur inside the joint container, credited as an "extra" below) as // opposed to a container that silently substitutes something else for // it (e.g. bridging past "Italy" to a container whose interior is // "Japan" — a different, contradicting learnt whole). // Only a KNOWN (content-addressed, exact) between-region qualifies — // an approximate region's resonance climbing "somewhere" is ordinary // noise (any ANN query returns SOME nearest neighbour), not evidence // this specific gap already means something specific. const between: number[] = []; for (let m = a + 1; m < b; m++) { if ( strong.has(cand[m]) && !consumed.has(cand[m]) && regions[cand[m]].known ) between.push(cand[m]); } // A single KNOWN region covering both: the whole form is already a // stored identity that votes directly; its pieces add nothing. if ( regions.some((r) => r.known && r.start <= ra.start && rb.end <= r.end) ) continue; probes++; if (td?.crossRegionSummary) { td.crossRegionSummary.probesAttempted = probes; } // Trace-only per-probe bookkeeping (spec §2-§7) — built incrementally // as the ladder runs, pushed exactly once at whichever exit fires // below. `pushProbe` is called at every continue/success exit for // THIS pair so the invariant `probes.length === probesAttempted` // holds regardless of which tier settled it. const probe: CrossRegionProbeTrace | undefined = td ? { leftRegionIndex: cand[a], rightRegionIndex: cand[b], betweenRegionIndices: [...between], exact: { attempted: false, candidatesReturned: 0 }, singleSynonym: { attempted: false, candidatesReturned: 0 }, doubleSynonym: { attempted: false, candidatesReturned: 0 }, outcome: "structural-rejected", } : undefined; let probePushed = false; const pushProbe = ( outcome: CrossRegionProbeTrace["outcome"], ) => { if (!td || !probe || probePushed) return; probe.outcome = outcome; td.crossRegionProbes.push(probe); probePushed = true; }; const left = query.subarray(ra.start, ra.end); const right = query.subarray(rb.start, rb.end); // Phrase-scale contract, exactly as the bridge: the glue between the two // forms may be up to W× the content it joins. const maxInterior = (left.length + right.length) * ctx.space.maxGroup; const cap = left.length + right.length + maxInterior; // The graded ladder (spec §1): exact DAG junction, then single-synonym, // then double-synonym, then — only when every DAG tier found nothing — // structural-resonance. `sides` (the two halo sibling lists) is loaded // ONCE and reused by junctionSynonyms AND structural-resonance, so no // ladder rung repeats a halo ANN query an earlier rung already paid for. const sides = await loadJunctionSynonymSides(ctx, left, right); let tier: CrossRegionTier = "exact"; let containers: Array = junctionContainersFrom( ctx, left, right, cap, seedsOf(cand[a]), seedsOf(cand[b]), exactBudget, true, ); if (probe) { probe.exact = { attempted: true, candidatesReturned: containers.length, }; } if (containers.length === 0) { // Tiers 2-4 — synonym containers (junctionSynonyms itself runs // single-synonym first, falling to double-synonym only when // single-synonym found nothing — see junction.ts). const syn = await junctionSynonyms( ctx, left, right, maxInterior, true, sides, synonymBudget, ); if (probe) { const singleAttempted = sides.leftSiblings.length > 0 || sides.rightSiblings.length > 0; const singleReturned = syn[0]?.tier === "single-synonym" ? syn.length : 0; const doubleAttempted = singleAttempted && singleReturned === 0 && sides.leftSiblings.length > 0 && sides.rightSiblings.length > 0; const doubleReturned = syn[0]?.tier === "double-synonym" ? syn.length : 0; probe.singleSynonym = { attempted: singleAttempted, candidatesReturned: singleReturned, }; probe.doubleSynonym = { attempted: doubleAttempted, candidatesReturned: doubleReturned, }; } if (syn.length > 0) { containers = syn; tier = syn[0].tier; } } // Tier 5 — structural-resonance ANN, the FINAL approximate proposal // path. Only reached when every DAG tier found NOTHING, and only when // there is no already-corroborated region between the endpoints (a // between-region with its own vote is evidence the gap already means // something specific — an ANN guess must not override it). let structuralPick: | { proposal: StructuralResonanceProposal; reach: AncestorReach; idf: number; } | null = null; if (containers.length === 0) { // Structural-resonance composes each side's OWN gist directly (no // byte-containment truth backs it, unlike the DAG tiers) — so, unlike // the DAG ladder (which tolerates one approximate side because byte // containment cannot lie), the ANN tier requires BOTH sides to be // KNOWN (content-addressed, exact identities): an approximate chunk // fragment's own resonance is noise at any tier, and composing noise // into a synthetic gist only manufactures a plausible-looking but // spurious ANN neighbour, not evidence of a genuine joint whole. // PHRASE-SCALE CONTRACT — the same one the DAG tiers hold their glue // to (see maxInterior above): a junction, exact or approximate, is a // whole the two forms nearly exhaust, not two arbitrary landmarks // anywhere in a long, multi-topic query. Without this, structural- // resonance would pair opposite ends of an unrelated scaffolding- // dominated query and manufacture a plausible-looking ANN neighbour // for a "gap" that never was a phrase. // BOTH sides must be independently DISCRIMINATIVE (individually // voted — `strong`, not merely a content-addressed `known` chunk): // a shared, non-discriminative scaffolding run (a repeated system // preamble) can be `known` without ever being distinctive evidence // of anything, and composing its own gist into a synthetic query // manufactures a plausible-looking but spurious ANN neighbour. The // DAG tiers can tolerate one merely-`known` side because byte // containment cannot lie; structural-resonance has no such // backstop, so both sides earn their place here the same way an // ordinary approximate region earns its individual vote. const gap = rb.start - ra.end; const reasons: NonNullable< StructuralResonanceTrace["ineligibleReasons"] > = []; if (between.length > 0) reasons.push("between-region"); if (!strong.has(cand[a]) || !strong.has(cand[b])) { reasons.push("not-both-strong"); } if (!ra.known || !rb.known) reasons.push("not-both-known"); if (gap > maxInterior) reasons.push("gap-too-large"); let resonanceTrace: StructuralResonanceTrace | undefined; if (reasons.length > 0) { if (probe) { resonanceTrace = { variantBudget: ctx.cfg.haloQueryK, variants: [], mergedProposals: 0, examined: [], noiseFloor: estimatorNoise(ctx.store.D), outcome: "ineligible", ineligibleReasons: reasons, }; probe.resonance = resonanceTrace; } } else { if (probe) { // `outcome`/`noiseFloor` are required fields with no natural // "unset" value; structuralResonance (called just below) always // overwrites both before returning, on every one of its exit // paths — these are never read in their initial form. resonanceTrace = { variantBudget: ctx.cfg.haloQueryK, variants: [], mergedProposals: 0, examined: [], noiseFloor: 0, outcome: "empty", }; probe.resonance = resonanceTrace; } const ownRootsA = rvs.votes.find((v) => v.start === ra.start && v.end === ra.end )?.roots; const ownRootsB = rvs.votes.find((v) => v.start === rb.start && v.end === rb.end )?.roots; structuralPick = await meteredStructuralResonance( ctx, query, ra, rb, sides, siblingGistMemo, k, N, reachMemo, ownRootsA, ownRootsB, resonanceTrace, ); } if (structuralPick === null) { pushProbe( reasons.length > 0 ? "resonance-ineligible" : "resonance-rejected", ); continue; } tier = "structural-resonance"; } let best: (Junction | SynonymJunction) | null = null; let bestExtras: number[] = []; let bestCov = -1; let reach: AncestorReach; let idf: number; let confidence: number; if (structuralPick !== null) { // A resonance proposal is NOT a Junction — there is no container to // read bytes from, so the self-evidence/contradiction/N-ary // machinery below (byte-verified against a real container) does not // apply; per spec §13, no N-ary extra-region coverage for resonance // proposals. best = { id: structuralPick.proposal.id, interior: new Uint8Array(0) }; bestExtras = []; bestCov = rb.end - ra.start; reach = structuralPick.reach; idf = structuralPick.idf; confidence = structuralPick.proposal.effectiveScore; } else { // Aggregate structural-tier trace (spec §4) — one per DAG tier that // returned at least one container (exact, single-synonym or // double-synonym); only aggregate counts and the final outcome are // recorded, never every candidate. const structuralTrace: CrossRegionStructuralTrace | undefined = probe ? { tier: tier as "exact" | "single-synonym" | "double-synonym", selfEvidenceRejected: 0, contradictionRejected: 0, passedGuards: 0, outcome: "all-rejected", } : undefined; if (probe) probe.structural = structuralTrace; // N-ARY selection: the container covering the MOST remaining candidate // forms wins (then tightest interior, then lowest id). Reads are // cache hits — every container's bytes were already read by the walk. // // SELF-EVIDENCE GUARD: a junction is BINDING evidence only when the // container joins forms the query mentions APART. When the container's // own joined occurrence (left..right including its interior) is a // literal substring of the query, the query already spells that phrase // out contiguously — perception already voted with it, and grid shards // of one phrase pairing "around" a gap chunk would merely rediscover // the phrase they are shards of, then explain away its rivals. for (const c of containers) { const bytes = cachedRead(ctx, cache, c.id, cap); const li = indexOf(bytes, left, 0); const ri = indexOf(bytes, right, 0); if (li >= 0 && ri >= 0) { const joined = bytes.subarray( Math.min(li, ri), Math.max(li + left.length, ri + right.length), ); if (indexOf(query, joined, 0) >= 0) { if (structuralTrace) structuralTrace.selfEvidenceRejected++; continue; // query says it itself } } // CONTRADICTION GUARD: a between-region already carrying its own // vote must actually recur in this container's bytes — otherwise // the container is a different learnt whole that happens to share // ra/rb, and letting it stand in for the gap would silently // override evidence the query itself already resolved there. if ( between.some((bi) => indexOf( bytes, query.subarray(regions[bi].start, regions[bi].end), 0, ) < 0 ) ) { if (structuralTrace) structuralTrace.contradictionRejected++; continue; } if (structuralTrace) structuralTrace.passedGuards++; let cov = left.length + right.length; const extras: number[] = []; for (const ei of cand) { if (ei === cand[a] || ei === cand[b] || consumed.has(ei)) continue; const e = regions[ei]; if (overlapsSpan(e, ra) || overlapsSpan(e, rb)) continue; const eb = query.subarray(e.start, e.end); if (indexOf(bytes, eb, 0) >= 0) { extras.push(ei); cov += eb.length; } } if ( cov > bestCov || (cov === bestCov && best !== null && (c.interior.length < best.interior.length || (c.interior.length === best.interior.length && c.id < best.id))) ) { best = c; bestExtras = extras; bestCov = cov; } } if (best === null) { // every container was self-evidence / contradiction — outcome // stays "all-rejected". pushProbe("structural-rejected"); continue; } const r = edgeAncestors(ctx, best.id, N, reachMemo); if (r.saturated || r.roots.length === 0) { if (structuralTrace) { structuralTrace.outcome = r.saturated ? "saturated" : "no-roots"; } pushProbe("structural-rejected"); continue; } const df = Math.log(N / Math.max(1, r.contextsReached)); if (df <= 0) { if (structuralTrace) structuralTrace.outcome = "nonpositive-idf"; pushProbe("structural-rejected"); continue; } if (structuralTrace) { structuralTrace.outcome = "accepted"; structuralTrace.selectedNode = best.id; } reach = r; idf = df; // Confidence used by voting (spec §13): exact junction = 1; // single/double-synonym = the sibling(s)' score(s), carried on the // SynonymJunction the ladder selected. confidence = "confidence" in best ? best.confidence : 1; } // MUTUAL-EXPLANATION WEIGHT — the same formula for every tier, with // `confidence` collapsed to certainty (1) for exact evidence: under // that collapse this is byte-for-byte the old exact-only formula // (min(1,ratio)·min(1,1/ratio)). For structural-resonance, // `confidence` is already annScore·semanticConfidence — never // multiplied a second time. const lenR = Math.max(1, bestCov); const ratio = Math.sqrt( Math.max(1, ctx.store.contentLen(best.id, lenR * ctx.store.D)) / lenR, ); const mutual = Math.min(1, confidence * ratio) * Math.min(1, confidence / ratio); const w = (mutual * idf) / reach.roots.length; let spanStart = ra.start; let spanEnd = rb.end; for (const ei of bestExtras) { spanStart = Math.min(spanStart, regions[ei].start); spanEnd = Math.max(spanEnd, regions[ei].end); } // CONSUMPTION IS FOR CONTAINER-BACKED EVIDENCE ONLY. Consuming a // candidate says "its evidence is already composed at full joint // strength, re-pairing it would vote the same container twice" — a // claim only a real container can make. A structural-resonance pick // has none (see above: it is NOT a Junction), so consuming its // endpoints locks up candidates on the strength of an ANN guess and // stops genuine evidence from ever composing them. Measured: on // `greet reply-greet then red then circle` the pair // `reply-greet` ▸ `red` resonated to `red square` and consumed `red`, // after which `red` ▸ `circle` was never probed and the exact junction // `red circle` — a stored whole, sitting right there — went unfound. // This is spec §15's asymmetry (only exact DAG evidence may explain // ordinary votes away) applied to the other way a tier can silence // evidence. Both votes now stand and pooling decides between them, // which is what the mechanism market is for. if (structuralPick === null) { consumed.add(cand[a]); consumed.add(cand[b]); for (const ei of bestExtras) consumed.add(ei); } // EXPLAINING AWAY — see the block comment above the function. Byte // containment in the joint container is the relatedness test (the // vote's bytes are literally part of the learnt whole), and FULL root // disjointness is the disagreement test: a vote sharing even one root // with the junction corroborates it and keeps its say elsewhere. // Counted BEFORE pushing the junction's own vote below: each ORIGINAL // region this ascent explains away is evidence the junction speaks // for, not evidence lost — `absorbed` (RegionVote's breadth-accounting // field) must credit the junction with all of it, not just the ONE // pooled axiom it collapses to. // Only EXACT DAG evidence may explain away ordinary votes (spec §15). // Single-synonym, double-synonym, and structural-resonance may ADD // supporting evidence but never remove it: their evidence is itself // approximate (a sibling substitution, or an ANN guess), so treating // their byte-containment the way exact containment is treated would // let an approximation override a genuine, independently-voted region. let explainedAway = 0; // Exact set of ORIGINAL region indices this junction explained away — // recorded live as `superseded.add` fires (spec §3's explicit rule: // never inferred from `absorbed` afterward). const explainedAwayIndices: number[] = []; if (tier === "exact") { const containerBytes = cachedRead(ctx, cache, best.id, cap); const jointRoots = new Set(reach.roots); for (const rv of rvs.votes) { if (rv.roots.some((r) => jointRoots.has(r))) continue; const bytes = query.subarray(rv.start, rv.end); if (indexOf(containerBytes, bytes, 0) >= 0 && !superseded.has(rv)) { superseded.add(rv); explainedAway++; if (td) { const idx = regions.findIndex((r) => r.start === rv.start && r.end === rv.end ); if (idx >= 0) explainedAwayIndices.push(idx); } } } } // COMPOSING TWO SPLIT FORMS DOES NOT WEAVE A POINT OF ATTENTION. // Region.corroborating marks a form the query's own cut SPLIT and // collectRegions recovered by lookup; poolVotes and commitVotes keep // such evidence out of the breadth ratio and out of the root cut's // distribution. This path bypassed both: a junction vote is minted // fresh here and carried nothing, so evidence the query never wove // re-entered the root election as a first-class anchor. // // Measured over the suite: 130 accepted junctions, 44 standing on at // least one corroborating region and 12 standing on NOTHING ELSE (both // endpoints corroborating, all structural-resonance tier). Those 12 // are precisely the leak — the query wove neither endpoint. // // The flag is inherited only when EVERY part is corroborating. One // genuine fold region among the parts means the query did point here, // and the junction anchors on it; that also preserves the case // Region.corroborating's doc calls out as REFUTED to bar (the correct // log record reachable only through a split form still grounds, because // it grounds as evidence for an anchor, not as a topic of its own). // Safe against the explaining-away accounting because that is EXACT // tier only (spec §15) and an all-corroborating junction has no exact // ordinary vote to absorb. const jointCorroborating = [cand[a], cand[b], ...bestExtras] .every((ri) => regions[ri].corroborating === true); out.push({ start: spanStart, end: spanEnd, canonicalFailed: false, // content-addressed: never saturation-masked roots: reach.roots, w, wFocus: w, absorbed: 1 + explainedAway, ...(jointCorroborating ? { corroborating: true } : {}), // The places this junction actually stands on — its two endpoints and // any N-ary extras, NOT the merged span [spanStart, spanEnd], which // swallows the gap and reads as one neighbourhood. See // RegionVote.parts. parts: [cand[a], cand[b], ...bestExtras] .map((ri): readonly [number, number] => [ regions[ri].start, regions[ri].end, ]) .sort((x, y) => x[0] - y[0]), }); pushProbe("accepted"); if (td) { td.crossRegionJunctionVotes.push({ container: best.id, span: [spanStart, spanEnd], roots: [...reach.roots], sourceRegionIndices: [cand[a], cand[b], ...bestExtras], explainedAwayRegionIndices: explainedAwayIndices, absorbed: 1 + explainedAway, tier, probe: td.crossRegionProbes.length - 1, confidence, evidenceBytes: bestCov, mutualWeight: mutual, voteWeightPerRoot: w, }); } const label = [cand[a], cand[b], ...bestExtras] .sort((x, y) => regions[x].start - regions[y].start) .map((ri) => dec(query.subarray(regions[ri].start, regions[ri].end))) .join(" ▸ "); const tierNote = tier === "exact" ? `junction node ${best.id}` + (best.interior.length === 0 ? " (adjacent)" : ` (interior "${dec(best.interior)}")`) + ", by content-addressed ascent" : tier === "structural-resonance" ? `structurally-composed ANN proposal, node ${best.id} — the query ` + `structurally composed the endpoint regions, the real middle-` + `query structure, and the selected halo-sibling endpoint ` + `direction(s) (variant ${structuralPick!.proposal.variant}, ` + `annScore ${structuralPick!.proposal.annScore.toFixed(3)} × ` + `semanticConfidence ${ structuralPick!.proposal.semanticConfidence.toFixed(3) } = effectiveScore ${ structuralPick!.proposal.effectiveScore.toFixed(3) }); it did not concatenate endpoint bytes or rewrite the query` : `${tier} junction node ${best.id}` + (best.interior.length === 0 ? " (adjacent)" : ` (interior "${dec(best.interior)}")`) + `, by halo-sibling DAG ascent (confidence ${confidence.toFixed(3)})`; ctx.trace?.step( "crossRegion", [{ text: label, role: "pair" }], reach.roots.map((r) => ({ text: dec(read(ctx, r)).slice(0, 60), node: r, role: "joint-context", })), `${label} → ${tierNote} → ${reach.roots.length} context(s)` + (superseded.size > 0 ? `; ${superseded.size} aliasing vote(s) explained away` : ""), ); break; // ra is consumed — move to the next unconsumed candidate } } if (td) td.supersededOrdinaryVotes = superseded.size; if (td?.crossRegionSummary) { td.crossRegionSummary.stopReason = probes >= k ? "probe-limit" : "pairs-exhausted"; } return { votes: out, superseded }; } /** Emit the "climbConsensus" step — the human-readable note this always * produced, now paired (when `ctx.trace` and `cfg` are both present) with * the structured {@link ClimbConsensusData} payload on the SAME step's * `data` field. Every exit of {@link computeAttention} funnels through * here, so instrumentation and the existing rationale text can never drift * apart — see the instrumentation spec's §9 "every exit path". */ export function traceAttention( ctx: MindContext, regions: ReadonlyArray<{ start: number; end: number }>, regionVoter: ReadonlyArray<{ id: number; score: number; w: number } | null>, roots: ReadonlyArray, steps: ReadonlyArray = [], td?: TraceDraft, cfg?: ClimbConsensusCfg, ranked: ReadonlyArray = roots, ): void { if (!ctx.trace) return; const voters: RationaleItem[] = []; for (let i = 0; i < regions.length; i++) { const rv = regionVoter[i]; if (rv == null) continue; const item = rNode(ctx, rv.id, "sub-region", rv.score); item.text = `${item.text} (df-w ${rv.w.toFixed(2)})`; voters.push(item); } const t = ctx.trace.enter("climbConsensus", voters); // The pooled-evidence decision, one DerivationStep per anchor — the same // shape {@link GraphSearch}'s own cover steps take (see traceDerivation). if (steps.length > 0) traceDerivation(ctx, steps); const data: ClimbConsensusData | undefined = (td && cfg) ? { version: 1, cache: { hit: false, detailAvailable: true }, config: { annK: cfg.k, crossRegionProbeLimit: cfg.k, mode: cfg.mode, ...(cfg.N !== undefined ? { corpusN: cfg.N } : {}), dimension: ctx.store.D, ...(cfg.N !== undefined ? { hubBound: hubBound(ctx) } : {}), estimatorNoise: estimatorNoise(ctx.store.D), ...(cfg.naturalBreak !== undefined ? { naturalBreak: cfg.naturalBreak } : {}), ...(cfg.consensusFloor !== undefined ? { consensusFloor: cfg.consensusFloor } : {}), }, candidates: { perceived: cfg.perceivedCount, recognised: cfg.totalRegions - cfg.perceivedCount, total: cfg.totalRegions, }, ...(td.regions.length > 0 ? { regions: td.regions } : {}), ...(cfg.reachMemo ? { reaches: serialiseReaches(cfg.reachMemo) } : {}), ...(td.crossRegionSummary ? { crossRegion: { eligibleRegions: td.crossRegionSummary.eligibleRegions, maximalRegions: td.crossRegionSummary.maximalRegions, probeLimit: td.crossRegionSummary.probeLimit, probesAttempted: td.crossRegionSummary.probesAttempted, junctionVotes: td.crossRegionJunctionVotes, supersededOrdinaryVotes: td.supersededOrdinaryVotes, probes: td.crossRegionProbes, stopReason: td.crossRegionSummary.stopReason ?? "pairs-exhausted", }, } : {}), ...(td.saturation ? { saturation: td.saturation } : {}), ...(td.pooling ? { pooling: td.pooling } : {}), ...(td.anchors.length > 0 ? { anchors: td.anchors } : {}), result: { roots: [...roots], ranked: [...ranked] }, } : undefined; t.done( roots.map((r) => rNode(ctx, r.anchor, "anchor", r.vote)), roots.length === 0 ? `${regions.length} sub-regions climbed the DAG, but none agreed on a context` : roots.length === 1 ? `${voters.length} of ${regions.length} sub-regions voted; IDF-weighted consensus picked one context (vote ${ roots[0].vote.toFixed(2) })` : `${voters.length} of ${regions.length} sub-regions voted; consensus ordered ${roots.length} INDEPENDENT points of attention (votes ${ roots.map((r) => r.vote.toFixed(2)).join(", ") })`, data, ); }