/** * Shared hybrid search primitive over the Neo4j knowledge graph. * * Pre-Task-675 there were two BM25 implementations over the same fulltext * index — one in the memory MCP tool, one in the admin Hono route — with * divergent filter semantics (soft-delete primitive) and divergent ranking * (agent ran hybrid vector+BM25, UI ran BM25-only). collapses both * into this lib; memory MCP + admin route now share a single code path so * `/graph` UI ranking matches what the agent sees. * * The fulltext index `knowledge_fulltext` was renamed to `entity_search` * and its label union expanded from 3 labels (KnowledgeDocument | Section | * Chunk) to the full operator-meaningful label set (~40 labels) on every * textual property the schema's writers assign. Task 416 split that single * `entity_search` into an admin index plus a public twin; the public twin * and its whole read path were retired once public agents became toolless, * so the lib now targets the single `entity_search_admin` index via * FULLTEXT_INDEX_ADMIN below. The doctrine test at * `__tests__/fulltext-coverage.test.ts` parses that declaration and asserts * the universal label union. * * QUERY --> EMBED --> VECTOR SEARCH (per index) --> ┐ * │ ├--> MERGE --> EXPAND --> RESULTS * └--> BM25 FULL-TEXT SEARCH -------------------> ┘ * * Hybrid merge: normalise BM25 scores to [0,1] via min-max, then combine * combined = 0.7 * vector_score + 0.3 * normalised_bm25_score * Dedup by nodeId, keeping the higher combined score. * * Trashed nodes (`:Trashed` label + `deletedAt` property) are excluded * from every path via `notTrashed()` from `graph-trash`. * * The lib is STATELESS. Callers pass a `Session` and, for hybrid, an * `embed` function. This keeps the lib independent of Ollama config and * free of circular imports against each caller's own getSession/embed * (memory MCP and the Hono ui build each have their own copies for * cross-build-boundary reasons — see neo4j-store.ts:68 comment). * * Log emission is the caller's responsibility — the lib throws or returns * `mode` in the result so the caller can format its own `[graph-search]` * line with the prefix and fields it wants. */ import { type Session } from "neo4j-driver"; import { DEFAULT_LAYERS, type DedupLayer } from "./dedup.js"; import { type RetrievalClass } from "./route.js"; export type { DedupLayer, RetrievalClass }; export { DEFAULT_LAYERS }; /** * Name of the fulltext index (Task 416 admin index; the public twin was * retired with the public read path). Mirrors the `CREATE FULLTEXT INDEX` * declaration in `platform/neo4j/schema.cypher`. The doctrine test asserts * the name matches this constant — drift between schema and this constant * breaks BM25 silently. `entity_search_admin` carries `n.compiledTruth`. */ export declare const FULLTEXT_INDEX_ADMIN = "entity_search_admin"; /** Detect the public-agent read shape (`allowedScopes` exactly `['public']`). * The BM25 index picker that used this predicate was retired with the public * read path. Its one remaining consumer — the `projectPublicProperties` swap * in the memory MCP's memory-search.ts — is now unreachable: every live read * passes `allowedScopes: undefined`, so this returns false on every call. Kept * as that projection's gate for when a public read surface is restored. */ export declare function isPublicOnlyScope(allowedScopes: readonly string[] | undefined): boolean; export interface SearchHit { nodeId: string; labels: string[]; properties: Record; score: number; } export interface SearchResult extends SearchHit { /** * granular component scores so the /data UI can render * `vector N.NN · bm25 N.NN · combined N.NN` per row. `vectorScore` is the * raw cosine in [0,1] from the per-label vector index; `bm25Score` is the * post-merge normalised value in [0,1] used in the combine (NOT the raw * Lucene score, which has no operator-meaningful unit). Either may be 0 * when only one half of the hybrid hit. In the degraded (bm25-only) path * `vectorScore` is always 0 and `bm25Score` is normalised over the * local hit set. */ vectorScore: number; bm25Score: number; related: Array<{ /** * neighbour `elementId`. Required by the /graph canvas to * render edges in pipeline-collapse mode (search response IS the canvas * data). The MCP memory-search tool ignores it; the field is additive. */ nodeId: string; relationship: string; direction: string; labels: string[]; properties: Record; }>; } export interface ScoredNode { nodeId: string; labels: string[]; properties: Record; vectorScore: number; bm25Score: number; /** * true when this node entered the merge map via the BM25 path * with a raw Lucene score > 0 (literal-token match against the universal * fulltext index), or via the keyword-subscriptions property-lookup path. * Set BEFORE bm25 normalisation, so a single-hit set whose normalised * value collapses to 0.0 still carries the flag. * * Read by the vector-threshold filter as the carve-out predicate: rows * with weak cosine survive when `bm25Hit` is true (operator's literal * tokens override the semantic similarity floor). Without this flag the * naive `bm25Score > 0` check would drop every bottom-of-set BM25 row * because min-max normalisation pins the lowest score to 0. */ bm25Hit: boolean; } /** * `"hybrid"` = weighted-sum fusion path (the baseline; the * brief calls this `weighted-sum`). * `"rrf"` = RRF fusion path (Task 308, flagged via `MAXY_GS_RRF`). * `"bm25"` = degraded path after embed failure with `degradeOnEmbedFailure`. * * UI callers (e.g. `platform/ui/app/graph/page.tsx`) pattern-match on * `"bm25"` to render the Ollama-down banner; non-`"bm25"` values collapse * to the normal banner, so adding `"rrf"` is wire-compatible. */ export type SearchMode = "hybrid" | "rrf" | "bm25"; export interface Bm25OnlyParams { query: string; accountId?: string; limit: number; allowedScopes?: string[]; agentSlug?: string; keywords?: string[]; keywordMatch?: "any" | "all"; /** * gate BM25 hits to nodes carrying at least one of these labels. * Mirrors hybrid()'s vector-half label filter so the Ollama-down fallback * honours the same gate the operator applied via /graph chips. Empty array * is treated as "no gate" (matches hybrid's `labels && labels.length > 0` * guard); admin route enforces non-empty `labels` as a precondition. */ labels?: string[]; /** * Task 485 — per-visitor slice ringfence. When set, the scope filter * widens from `(scope IS NULL OR scope IN $allowedScopes)` to * `(sliceToken = $sliceToken OR scope IS NULL OR scope IN $allowedScopes)` * so a gated public-agent read sees its visitor's user-scoped nodes in * addition to public-scope rows. Skipped on every admin and open-mode * public call; the env var `ACCESS_SLICE_TOKEN` populates it on the * memory MCP server side. */ sliceToken?: string; } /** * Build the scope-and-slice WHERE fragment + params. Centralises the three * cypher composition sites (bm25, vector, prop) so the gated/non-gated * behaviour does not drift between them. Returns empty strings when no * filter applies. Exported so deterministic-read tools outside the ranking * stack (e.g. memory-lookup-by-name) reuse the exact same scope predicate * rather than re-deriving it — drift here is an account-isolation gap. */ export declare function buildScopeAndSliceClause(allowedScopes: readonly string[] | undefined, sliceToken: string | undefined, alias?: string): { clause: string; params: Record; }; export interface HybridParams extends Bm25OnlyParams { expandHops?: number; keywordSubscriptions?: string[]; /** * When true, a failing `embed()` does NOT throw — the lib falls back to * `bm25Only()` and returns `mode: "bm25"`. Admin-route callers set this * true so Ollama-down degrades to BM25-only; MCP tool callers leave it * false so embed failure surfaces loudly. */ degradeOnEmbedFailure?: boolean; /** * minimum raw vector cosine in [0,1] required for a row to * pass the merge step. Rows below the threshold are dropped UNLESS they * also entered via the BM25 path (`bm25Hit === true`) — that carve-out * preserves literal-token matches whose embedding happened to score * mediocre. Counts surface in HybridResponse so callers can render an * "N suppressed — show all" affordance. * * Undefined ⇒ no threshold. The lib itself takes no opinion on the value; * every search *surface* opts into the shared `DEFAULT_VECTOR_THRESHOLD` * (exported below) as its default, so the same query yields the same node * set across chat memory-search, /graph and /data (Task 635). A caller that * genuinely wants no floor — the graph-subgraph pivot-narrowing mask — just * passes nothing. * Zero ⇒ explicit "show all" override (the operator's "show all" click on * /data and /graph). * * Calibration: see `DEFAULT_VECTOR_THRESHOLD` below. */ vectorThreshold?: number; /** * Routing hint from the inbound-gateway classifier (Task 304). When * the `MAXY_GS_ROUTE` flag is on, the lib picks a label filter and * fusion-weight plan per class. Omitted ⇒ `"general"` (unflagged * callers see no change). */ retrievalClass?: RetrievalClass; /** Per-call override: enable Reciprocal Rank Fusion. */ enableRrf?: boolean; /** Per-call override: enable Haiku multi-query expansion. */ enableExpansion?: boolean; /** Per-call override: enable compiledTruth + backlink boosts. */ enableBoosts?: boolean; /** Per-call override: enable 4-layer dedup (default layers below). */ enableDedup?: boolean; /** Per-call override: enable retrieval-class routing. */ enableRoute?: boolean; /** * Layers passed to the 4-layer dedup when `enableDedup` is true. * Defaults to `["nodeId","slug","canonicalName","contentHash"]`. */ dedupLayers?: DedupLayer[]; /** RRF smoothing constant. Default 60 (gbrain default). */ rrfK?: number; } /** * Shared default vector-cosine floor applied by every search *surface* — chat * memory-search, /graph and /data — so the same query yields the same node set * across all of them (Task 635). `hybrid()` keeps the `vectorThreshold === * undefined ⇒ no floor` contract; surfaces opt in by passing this constant as * their default. Callers that genuinely want no floor (the graph-subgraph * pivot-narrowing mask) pass nothing. * * Calibration (Real Agent laptop, project_embedding index, account b3b638d9): * the `brochure` query splits 13 Project nodes into 6 keepers in * [0.8473, 0.8623] and 7 noise rows in [0.7159, 0.8143] — a 0.033-wide gap at * ~0.83 with no row inside it. 0.82 sits in that gap: drops all 7 noise rows, * keeps all 6 keepers. BM25-positive rows bypass the floor regardless (the * `bm25Hit` carve-out above), so proper-noun matches survive a tight floor. * Recalibrate by picking a value in the gap between the lowest in-scope keeper * and the highest noise row. */ export declare const DEFAULT_VECTOR_THRESHOLD = 0.82; export interface HybridResponse { mode: SearchMode; results: SearchResult[]; /** Populated when degradeOnEmbedFailure fired. Caller logs it. */ embedError?: string; /** * milliseconds spent on the batched expand round-trip alone * (separate from embed + vector + BM25). The /graph admin route emits * this as `expand-ms=N` so a regression on the post-Task-747 batching * surfaces in server.log without needing a profiler. Zero when * `expandHops === 0` or no merged results. */ expandMs: number; /** * operator-facing threshold accounting. * * `rawMerged` count of nodes in the merge map BEFORE the threshold * filter ran (vector + BM25 + keyword-subscriptions * union, deduped by nodeId). * `suppressed` count of nodes the threshold filter dropped. * Always 0 when `vectorThreshold` is undefined. * `bm25Bypass` count of nodes that would have been dropped by the * vector-cosine floor BUT survived because `bm25Hit` * was true. Subset of the rendered set. * `threshold` the actual threshold applied (echoes the param); * null when no filter ran. Drives the "show all" * affordance: if null, no banner; if non-null and * `suppressed > 0`, banner with count. */ rawMerged: number; suppressed: number; bm25Bypass: number; threshold: number | null; /** * Per-stage counts surfaced for the caller's `[graph-search:hybrid]` * log line (Task 308). Always present; values reflect whichever * enhancements ran on this call. `expansions` is 1 when expansion * was off, `boosted - fused` is 0 when boosts were no-ops, etc. */ stageCounts: { expansions: number; vector: number; bm25: number; fused: number; boosted: number; deduped: number; final: number; }; } export type EmbedFn = (text: string) => Promise; /** * Lucene-escape special characters per Neo4j's fulltext query grammar. * `/[+\-&|!(){}[\]^"~*?:\\/]/g` — matches memory-search.ts:127 and * neo4j-store.ts:2521 verbatim; consolidation preserves escape set. */ export declare function escapeLucene(query: string): string; /** * Normalise BM25 scores to [0, 1] using min-max within the result set. * If all scores are equal (or single result), returns 1.0 for all — * the range-zero branch prevents divide-by-zero and keeps the combined * score meaningful. */ export declare function normaliseBm25Scores(scores: number[]): number[]; export declare function discoverIndexes(session: Session): Promise>; /** Clear the index cache — call after schema changes. */ export declare function clearIndexCache(): void; /** * BM25 full-text search against the universal `entity_search` index. * Returns [] when the index doesn't exist — matches memory-search.ts * graceful-fallback semantics so a fresh account or a Pi whose index is * in the POPULATING window doesn't 500 the caller. */ export declare function bm25Only(session: Session, params: Bm25OnlyParams): Promise; /** * Hybrid vector + BM25 search with graph expansion. * * Sequence: * 1. Resolve flags + route plan. Route may short-circuit (`retrievalClass=none`). * 2. Multi-query expansion (Task 308 flag): Haiku produces 3-5 paraphrases. * Each paraphrase is embedded and run through vector+BM25 in parallel. * 3. embed(query) — may throw if Ollama is unreachable. When * `degradeOnEmbedFailure=true`, caller receives bm25Only() result * with `mode: "bm25"`; otherwise the throw propagates. * 4. Vector search per label (one query per vector index discovered at * boot). Nodes-by-label filter short-circuits when the requested * labels have no index. * 5. BM25 search on the universal `entity_search` index — * same filter semantics as vector half (scope/agent/keyword/trashed). * 6. Keyword subscriptions (when set): BM25 per keyword + property * lookup against `node.keywords` array. Both bypass the agentSlug * filter — subscriptions are scope-inclusive by design (matches * memory-search.ts comment L319). Runs once, not per expansion. * 7. Fusion: weighted-sum (`vw * vector + bw * bm25_norm`) by default, * RRF when `MAXY_GS_RRF` is on. Route plan supplies fusion weights. * 8. Boosts (Task 308 flag): compiledTruth bump + backlink log-bump. * 9. Dedup (Task 308 flag): 4-layer dedup over the union, default layers * `[nodeId, slug, canonicalName, contentHash]`. * 10. Threshold + sort + slice. Counts (rawMerged / suppressed / bm25Bypass) * reflect the pre-threshold and post-threshold populations. * 11. Graph expand (1 hop by default, 0 to skip) — notTrashed filter + * scope + agent clauses mirrored. */ export declare function hybrid(session: Session, embed: EmbedFn, params: HybridParams): Promise; //# sourceMappingURL=index.d.ts.map