/**
 * Hybrid Retrieval System
 * Combines vector search + BM25 full-text search with RRF fusion
 */

import type { MemoryStore, MemorySearchResult } from "./store.js";
import type { Embedder } from "./embedder.js";
import { filterNoise } from "./noise-filter.js";
import * as envConfig from "./env-config.js";
import { extractErrorSignatures } from "./error-signature.js";
import { shouldSkipRetrieval } from "./adaptive-retrieval.js";
import { expandQueryWithAliases } from "./aliases.js";
import { expandQuery } from "./query-expander.js";
import { detectLang, tokenizeFts } from "./language-hook.js";
import { type AccessTracker, computeHotnessScore, parseAccessMetadata } from "./access-tracker.js";
import { readUtility } from "./memory-utility.js";
import { weibullDecay, resolveTier, isDecayExempt, adjustHalfLifeForEmotion, computeArousalBoost } from "./decay-engine.js";
import { logInfo, logWarn } from "./stderr-log.js";
import { extractBoundaryMetadata, isDurableMemoryScope, isTranscriptScope } from "./memory-boundaries.js";
import type { TraceCollector } from "./retrieval-trace.js";
import { extractTopicTag } from "./topic-tag.js";
import { filterInterference } from "./rif.js";
import { FrequencyTracker } from "./frequency-tracker.js";
import { applyConfidenceWeight } from "./confidence-tracker.js";
import { deduplicateByVersionGroup } from "./version-manager.js";
import { deduplicateByClusterInsight } from "./consolidation-engine.js";
import { isActiveMemory, recordAccess as recordEvolutionAccess, parseEvolution, computeDecayScore } from "./memory-evolution.js";
import {
  isMultiVectorEnabled,
  extractMultiVectorText,
  textOverlapScore,
  blendMultiVectorScores,
  adaptiveBlendConfig,
  tokenize,
} from "./multi-vector.js";
import { parseTemporalQuery, matchesTemporalConstraint, type TemporalConstraint } from "./temporal-parser.js";
import { parseSourceIds } from "./kg-store.js";
import type { KGStore } from "./kg-store.js";
import { isKGModeEnabled } from "./kg-extractor.js";
import { detectEntities } from "./query-entity-detector.js";
import { buildGraph, pprTraverse } from "./ppr-traversal.js";
import { logInfo } from "./stderr-log.js";
import { AUDIT_RETRIEVED_CAP, type AuditLogger } from "./audit-log.js";
import { parseEmotion, isEmotionScoringEnabled } from "./memory-schema.js";
import { parseNarrative, isNarrativeModeEnabled } from "./narrative-schema.js";
import type { EmotionMetadata } from "./memory-schema.js";
import { detectEmotion } from "./emotion-detector.js";
import { shouldReconstruct, reconstruct as runReconstruction } from "./context-reconstructor.js";
import { recordReconstructionUsage as runUsageShadow } from "./usage-tracker.js";
import type {
  ReconstructionLLMClient,
  ReconstructionOutput,
  CandidateExpansionDeps,
} from "./context-reconstructor.js";

// ============================================================================
// Types & Configuration
// ============================================================================

export interface RetrievalConfig {
  mode: "hybrid" | "vector";
  vectorWeight: number;
  bm25Weight: number;
  minScore: number;
  rerank: "cross-encoder" | "lightweight" | "none";
  candidatePoolSize: number;
  /** Recency boost half-life in days (default: 14). Set 0 to disable. */
  recencyHalfLifeDays: number;
  /** Max recency boost factor (default: 0.10) */
  recencyWeight: number;
  /** Filter noise from results (default: true) */
  filterNoise: boolean;
  /** Reranker API key (enables cross-encoder reranking) */
  rerankApiKey?: string;
  /** Reranker model (default: jina-reranker-v3) */
  rerankModel?: string;
  /** Reranker API endpoint (default: https://api.jina.ai/v1/rerank). */
  rerankEndpoint?: string;
  /** Reranker provider format. Determines request/response shape and auth header.
   *  - "jina" (default): Authorization: Bearer, string[] documents, results[].relevance_score
   *  - "siliconflow": same format as jina (alias, for clarity)
   *  - "voyage": Authorization: Bearer, string[] documents, data[].relevance_score
   *  - "pinecone": Api-Key header, {text}[] documents, data[].score */
  rerankProvider?: "jina" | "siliconflow" | "voyage" | "pinecone" | "vllm";
  /**
   * Length normalization: penalize long entries that dominate via sheer keyword
   * density. Formula: score *= 1 / (1 + log2(charLen / anchor)).
   * anchor = reference length (default: 500 chars). Entries shorter than anchor
   * get a slight boost; longer entries get penalized progressively.
   * Set 0 to disable. (default: 300)
   */
  lengthNormAnchor: number;
  /**
   * Hard cutoff after rerank: discard results below this score.
   * Applied after all scoring stages (rerank, recency, importance, length norm).
   * Higher = fewer but more relevant results. (default: 0.35)
   */
  hardMinScore: number;
  /**
   * Time decay half-life in days. Entries older than this lose score.
   * Different from recencyBoost (additive bonus for new entries):
   * this is a multiplicative penalty for old entries.
   * Formula: score *= 0.5 + 0.5 * exp(-ageDays / halfLife)
   * At halfLife days: ~0.68x. At 2*halfLife: ~0.59x. At 4*halfLife: ~0.52x.
   * Set 0 to disable. (default: 60)
   */
  timeDecayHalfLifeDays: number;
  /**
   * Hotness blend weight. Blends access-frequency hotness into final score.
   * Formula: final = score * (1-alpha) + hotness * alpha.
   * Zero-access rows keep their score unchanged (bonus-only cold start).
   * 0 = disabled (default), 0.15 = recommended.
   */
  hotnessWeight: number;
  /**
   * Utility weight — 价值回溯（memory-utility.ts）算出的效用值对最终分的影响强度。
   * Formula: score *= 1 + utilityWeight * utility，utility ∈ [-0.8, 0.8]。
   * 与 hotness/frequency 的区别在信号来源：那两个数「被读过多少次」，
   * 这个数「被用到的那些任务成没成」——前者是热度，后者是因果贡献。
   * 没有 utility 的条目分数原样不动（bonus-only 冷启动，不惩罚没数据的）。
   *
   * 0 = disabled（默认）。**默认关是有意的影子期**：目前生产库里带 join key 的
   * observation 从 0 开始积累，样本不足时开启只会放大噪声。
   * 升级条件写在 open-loops「MemOS 价值回溯」条目里，不在这行注释里——
   * 注释不会有人定期回来读，那个条目每窗口都会被推到眼前。
   */
  utilityWeight: number;
  /**
   * Per-category score thresholds. When a result's category matches a key,
   * that threshold is used instead of hardMinScore. Categories not listed
   * fall back to hardMinScore. (default: see DEFAULT_CATEGORY_MIN_SCORES)
   */
  categoryMinScores?: Record<string, number>;
  /** Enable Retrieval Interference Filter (RIF) — demotes near-duplicate weak results. Default: false. */
  enableRIF?: boolean;
  /** RIF cosine similarity threshold for "near-duplicate" (default: 0.85). */
  rifThreshold?: number;
  /** RIF score ratio: demote if score < ratio * stronger result's score (default: 0.80). */
  rifScoreRatio?: number;
  /**
   * Source diversity: ensure top-k results span multiple scopes/sessions.
   * When > 0 and candidates come from >= 3 distinct scopes, applies round-robin
   * selection (one per scope first, then fill remaining slots by score).
   * 0 = disabled (default). Recommended: 0.5 (moderate diversity).
   */
  sourceDiversity?: number;
  /**
   * Adaptive candidate pool multiplier for aggregation queries.
   * When a query contains counting/listing signals ("how many", "all the", etc.),
   * candidatePoolSize is multiplied by this factor.
   * 1 = disabled (default). Recommended: 1.5.
   */
  adaptivePoolMultiplier?: number;
  /**
   * Enable multi-hop retrieval: after first-pass results, extract entities
   * and run focused follow-up queries to improve cross-session coverage.
   * Costs 1-3 extra embedding calls per retrieve. Default: false.
   */
  multiHop?: boolean;
  /**
   * Maximum number of entity-focused follow-up queries in multi-hop mode.
   * Default: 3.
   */
  multiHopMaxQueries?: number;
}

export interface RetrievalContext {
  query: string;
  limit: number;
  scopeFilter?: string[];
  category?: string;
  /** Origin of retrieval call. Access reinforcement is only applied for "manual"
   *  calls to prevent auto-recall from strengthening noise memories.
   *  Defaults to "manual" when unset (backward-compatible). */
  source?: "manual" | "auto-recall" | "cli";
  /** When false (default), archived records are excluded from results. */
  includeArchived?: boolean;
  /** Optional trace collector for per-stage pipeline observability. */
  trace?: TraceCollector;
  /** Enable KG graph traversal (PPR) as an additional retrieval signal. */
  graph?: boolean;
  /** Override config-level multiHop for this single retrieval call. */
  multiHop?: boolean;
  /** MP-1: Filter results by topicTag stored in metadata. */
  topicTag?: string;
  /** Request constructive retrieval reconstruction */
  reconstruct?: boolean;
  /** F3: Query memories valid at a specific point in time (ms timestamp). */
  validAt?: number;
  /** F3: When true, include expired memories (demoted 80%). Default: false. */
  includeExpired?: boolean;
}

export interface RetrievalResult extends MemorySearchResult {
  sources: {
    vector?: { score: number; rank: number };
    bm25?: { score: number; rank: number };
    graph?: { score: number; rank: number };
    fused?: { score: number };
    reranked?: { score: number };
    /** HP-narrative: true when this result was pulled as a narrative sibling */
    narrativeSibling?: boolean;
  };
}

/**
 * Phase 4: Extended retrieval result set with optional first-class reconstruction.
 * Backward-compatible — callers treating this as RetrievalResult[] still work.
 */
export type RetrievalResultSet = RetrievalResult[] & {
  reconstruction?: ReconstructionOutput;
  /**
   * P-omitted: 因 limit 上限被截断、未返回的达标结果数（提高 limit 可见）。
   * 仅统计 finalizeResults 的 limit 截断，不含质量过滤（min-score/noise/MMR）丢弃——
   * 那些提高 limit 也拿不到，报告出来反而误导调用方。
   */
  omitted?: { count: number; reason: "limit" };
};

// ============================================================================
// Default Configuration
// ============================================================================

/**
 * Per-category minimum scores. Lower threshold = easier to recall.
 * profile/preferences: low bar because identity & preferences are almost always relevant.
 * cases/patterns: higher bar because they are more specific and noisy matches are costly.
 */
export const DEFAULT_CATEGORY_MIN_SCORES: Record<string, number> = {
  profile: 0.25,
  preferences: 0.25,
  entities: 0.30,
  events: 0.35,
  cases: 0.40,
  patterns: 0.45,
};

// P1.2（2026-06-25）：默认 vector-only。canary eval 实测个人记忆库 vector 召回质量全面优于
// hybrid（robustness 带 scope WITH 6/12 + 不带 scope WITHOUT 10/12，完胜 hybrid 的 3/9；BM25 在
// 高度同质的个人记忆里是净噪声，会把 vector 本应 top 的目标如 dbcb317a 挤掉）。改频率层救低频
// 目标实测是死结（动 hotness 必伤靠 hotness 上位的记忆）。hybrid 路径代码全保留，mode 改回
// "hybrid" 即可切换（注：multiHopExpand 仍硬编码走 hybrid，默认 off，config.retrieval.multiHop
// 开了会破纯 vector）。数据盲区：canary 全是语义 query，未覆盖 BM25 强项的精确关键词/token 召回
// （变量名 / commit hash / alias-map 昵称）——待补 exact-token canary 验证。
export const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = {
  mode: "vector",
  vectorWeight: 0.7,
  bm25Weight: 0.3,
  minScore: 0.3,
  rerank: "cross-encoder",
  // ⚠️ 2026-08-23 查实：**这里超过 20 的部分从来没生效过**。两条候选腿最终都落到
  // store.ts 的 `clampInt(limit, 1, 20)`（vectorSearch:588 / bm25Search:643），
  // 路径是 retriever.ts:1109 算出 ceil(candidatePoolSize × multiplier) →
  // runVectorSearch/runBM25Search(:1341/:1373) → store.*Search(limit) → 被压成 20。
  // 那个 clamp 从初始提交 c10607a（2026-03-03）就在，而 07-16 调参（9374a4f）只改了本文件、
  // 没碰 store.ts —— 所以这个 30 从落地当天就是空转。retrieval-profiles.ts 的 24/26/28 同理。
  // **不要顺手去改那个 clamp**：动它等于扩候选池，而扩池方案已于 2026-08-22 跑 shadow 后被
  // 实测否决（RecallNest memory:pivot `rejected-recall-widen-candidate-pool`，主犯认定订正为
  // applyLayerAdmission 79.4% 而非 hard_min 13.5%）。要推翻它得先拿出那条否决记录要求的新证据。
  candidatePoolSize: 30,
  sourceDiversity: 1,
  recencyHalfLifeDays: 14,
  recencyWeight: 0.10,
  filterNoise: true,
  rerankModel: "jina-reranker-v3",
  rerankEndpoint: "https://api.jina.ai/v1/rerank",
  // 2026-08-06：500 → 800。不是拍的——writing profile 早就在用 800
  // （retrieval-profiles.ts），这里只是把一个已在生产跑着的值推广到 default。
  // 实测（eval/ghost-scan.ts，14,949 条 active 全库快照）：结构性幽灵
  // 775 → 245（5.18% → 1.64%），救回 530 条，中位长度 1,356 字符那批正落在
  // 打折最重的区间。挤占效应见 eval/lengthnorm-shadow.ts：20 条真实 query
  // 里 14 条 top-6 完全不变，6 条掉出在 limit 放大到 20 后全部复现
  // ——即没有任何条目被删掉，只是挤出窗口，而那 6 条的重排来自
  // applySourceDiversity + MMR（候选数 ≤ limit 时它根本不轮转，
  // anchor 放大让更多候选过阈值才把这个既有行为激活），与本参数无关。
  lengthNormAnchor: 800,
  hardMinScore: 0.35,
  timeDecayHalfLifeDays: 60,
  hotnessWeight: 0,
  utilityWeight: 0,
};

/**
 * P1.2+: 检测 query 是否含「精确 token」——大写常量 / commit hash / 文件名 / snake/camelCase /
 * 低元音 ID（如 rwmjhb）。这类 token vectorOnly 抓不到（embedding 做语义相似非精确匹配，跟语言
 * 无关：实测纯/中文/中英混 query 都漏靠 token 区分的目标），需 BM25 兜底。刻意避开元音正常的普通词
 * （mem0/recallnest/GitHub），避免日常语义 query 误触发 BM25 噪声。
 */
export function hasExactToken(query: string): boolean {
  // 【为什么中文 query 不触发本函数 —— 这是实测后的决定，不是 ASCII-only 的遗漏】
  // 2026-08-01 用临时开关对 5 条真实中文 query 做过 A/B：强制让中文触发 BM25 补偿后，
  // 3/4 的 query 零新增（BM25 捞回的条目 vector 早已召回，被 vecIds 去重滤掉），
  // 唯一有新增的那次补进 3 条、其中 1 条是当天无关对话，且补偿结果是**插在队首**的。
  // 中文分词本身没问题（babel-memory 已注册，detectLang 正确返回 zh，
  // "双机契约" → "双机 契约"），所以这不是 tokenizer 问题 ——
  // 是 BM25 对中文**语义类** query 的边际收益本就接近零，放开只会拿噪声占前三名。
  // 若将来要重开，先复现那组 A/B，别只凭"中文没走 BM25"这个现象下判断。
  for (const t of query.split(/[\s，。、；：（）()【】「」"'`?？!！]+/)) {
    if (t.length < 5) continue;
    // 全大写 + 下划线常量（必须含下划线，避免 JSON/HTTP/GitHub 等普通缩写/驼峰误触发）
    if (/^[A-Z][A-Z0-9]*_[A-Z0-9_]+$/.test(t)) return true;                   // DEDUP_HARD_THRESHOLD
    // snake_case 标识符（含下划线）
    if (/^[a-z][a-z0-9]*_[a-z0-9_]+$/i.test(t)) return true;                  // capture_engine
    // 文件名（基名 + 代码扩展名）
    if (/^[\w./-]+\.(ts|tsx|js|jsx|py|md|sh|json|sql|go|rs|java|cpp|yml|yaml|toml)$/i.test(t)) return true;  // capture-heuristic.ts
    // hex commit hash（7-40，含数字+字母，纯 hex）
    if (/^[0-9a-f]{7,40}$/i.test(t) && /[0-9]/.test(t) && /[a-f]/i.test(t)) return true;  // dd12fb5
    // 注：camelCase（GitHub/TypeScript）与低元音 ID（rwmjhb/pytest/nginx）两条宽规则已移除——
    // 前者误伤日常技术 query 引回 BM25 噪声；rwmjhb 类纯小写 ID 即便触发，fts 中英混分词也召回不到
    // （根因在 tokenizer，是独立 backlog），靠中英一起 query 的中文锚点已能补回。
  }
  return false;
}

// ============================================================================
// Utility Functions
// ============================================================================

function clampInt(value: number, min: number, max: number): number {
  if (!Number.isFinite(value)) return min;
  return Math.min(max, Math.max(min, Math.floor(value)));
}

function clamp01(value: number, fallback: number): number {
  if (!Number.isFinite(value)) return Number.isFinite(fallback) ? fallback : 0;
  return Math.min(1, Math.max(0, value));
}

/**
 * F3: Filter results by temporal validity window.
 * - Expired memories (validUntil < now) are demoted by 80% or excluded.
 * - validAt: only return memories valid at a specific point in time.
 */
function filterByValidity(
  results: RetrievalResult[],
  now: number,
  options: { validAt?: number; includeExpired?: boolean },
): RetrievalResult[] {
  const { validAt, includeExpired } = options;
  const checkTime = validAt ?? now;

  return results.reduce<RetrievalResult[]>((acc, r) => {
    const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);

    // Point-in-time query: skip memories not yet valid or already expired at checkTime
    if (validAt != null) {
      if (evo.validFrom > checkTime) return acc; // not yet valid
      if (evo.validUntil != null && evo.validUntil < checkTime) {
        if (!includeExpired) return acc;
        // Include but demote 80%
        acc.push({ ...r, score: r.score * 0.2 });
        return acc;
      }
      acc.push(r);
      return acc;
    }

    // Default: check if expired relative to now
    if (evo.validUntil != null && evo.validUntil < now) {
      if (!includeExpired) return acc;
      acc.push({ ...r, score: r.score * 0.2 });
      return acc;
    }

    acc.push(r);
    return acc;
  }, []);
}

/** P0.1: Short query detection — ≤ 4 tokens (CJK chars count as 1 token each). */
const SHORT_QUERY_TOKEN_THRESHOLD = 4;

function isShortQuery(query: string): boolean {
  return tokenize(query).length <= SHORT_QUERY_TOKEN_THRESHOLD;
}

/** P0.1: Minimum score discount for short queries to widen the candidate pool. */
const SHORT_QUERY_MIN_SCORE_FACTOR = 0.6;

/** P0.1: Anchor boost factor — how much a perfect anchor match boosts score. */
const ANCHOR_BOOST_MAX = 0.25;

/**
 * P0.1: Boost scores for candidates whose metadata.anchor matches the query.
 * Uses textOverlapScore (token coverage + density) — same as multi-vector blend.
 * Short queries benefit most because anchor text is also short (≤80 chars).
 */
function applyAnchorBoost(results: RetrievalResult[], query: string): RetrievalResult[] {
  if (results.length === 0) return results;
  return results.map(r => {
    const meta = parseMetadata(r.entry.metadata);
    const anchor = meta.anchor;
    if (typeof anchor !== "string" || !anchor) return r;
    const overlap = textOverlapScore(query, anchor);
    if (overlap <= 0) return r;
    // Boost proportional to overlap quality, capped at ANCHOR_BOOST_MAX
    const boost = 1 + overlap * ANCHOR_BOOST_MAX;
    return { ...r, score: r.score * boost };
  });
}

/** Multiplicative boost applied to a case on exact error-signature match. */
const ERROR_SIGNATURE_BOOST_FACTOR = 1.5;

/**
 * A1 (P3-A): error-signature exact-match boost. When RECALLNEST_ERROR_SIGNATURE_BOOST
 * is on, case memories whose metadata.error_signature overlaps the query's extracted
 * signatures get a strong multiplicative boost ("上次一模一样的报错"精确召回). Flag off
 * (default) returns the input unchanged — bit-identical, so the pure-move eval baseline holds.
 * Only affects category="cases"; read and query side share extractErrorSignatures so the
 * signatures compare apples-to-apples.
 */
export function applyErrorSignatureBoost(results: RetrievalResult[], query: string): RetrievalResult[] {
  if (!envConfig.errorSignatureBoost()) return results;
  const querySignatures = extractErrorSignatures({ problem: query });
  if (querySignatures.length === 0) return results;
  const querySet = new Set(querySignatures.map(s => s.toLowerCase()));
  return results.map(r => {
    if (r.entry.category !== "cases") return r;
    const sigs = parseMetadata(r.entry.metadata).error_signature;
    if (!Array.isArray(sigs) || sigs.length === 0) return r;
    const hit = sigs.some(s => typeof s === "string" && querySet.has(s.toLowerCase()));
    if (!hit) return r;
    return { ...r, score: Math.min(1, Math.max(0, r.score * ERROR_SIGNATURE_BOOST_FACTOR)) };
  });
}

/**
 * Emotion scoring stage: boost memories matching query emotional tone.
 * No-op when query is neutral or memory lacks emotion data.
 */
export function applyEmotionWeight(
  results: Array<{ score: number; metadata: string; [k: string]: unknown }>,
  queryEmotion: EmotionMetadata | null,
): typeof results {
  if (!queryEmotion || Math.abs(queryEmotion.valence) < 0.2) {
    return results;
  }
  return results.map(r => {
    const memEmotion = parseEmotion(r.metadata);
    if (!memEmotion) return r;
    const alignment = 1 - Math.abs(queryEmotion.valence - memEmotion.valence) / 2;
    const boost = 1.0 + 0.15 * alignment;
    return { ...r, score: r.score * boost };
  });
}

// ============================================================================
// Source Diversity — round-robin across scopes for multi-session coverage
// ============================================================================

/** Minimum distinct sources to activate diversity logic. */
const SOURCE_DIVERSITY_MIN_SOURCES = 3;

/**
 * Ensures top-k results span multiple scopes/sessions via round-robin.
 * Each distinct source gets its top-1 result first, remaining slots fill by score.
 */
function applySourceDiversity(results: RetrievalResult[], limit: number): RetrievalResult[] {
  if (results.length <= limit) return results;

  // Group by scope
  const byScope = new Map<string, RetrievalResult[]>();
  for (const r of results) {
    const key = r.entry.scope || "__default__";
    const bucket = byScope.get(key);
    if (bucket) bucket.push(r);
    else byScope.set(key, [r]);
  }

  if (byScope.size < SOURCE_DIVERSITY_MIN_SOURCES) {
    return results.slice(0, limit);
  }

  // Round-robin: each scope contributes its top-1
  const selected: RetrievalResult[] = [];
  const usedIds = new Set<string>();

  // Sort scopes by their best result score (highest first)
  const scopeEntries = [...byScope.entries()].sort(
    (a, b) => (b[1][0]?.score ?? 0) - (a[1][0]?.score ?? 0),
  );

  for (const [, items] of scopeEntries) {
    if (selected.length >= limit) break;
    const top = items[0];
    if (top && !usedIds.has(top.entry.id)) {
      selected.push(top);
      usedIds.add(top.entry.id);
    }
  }

  // Fill remaining slots by global score
  for (const r of results) {
    if (selected.length >= limit) break;
    if (!usedIds.has(r.entry.id)) {
      selected.push(r);
      usedIds.add(r.entry.id);
    }
  }

  return selected;
}

// ============================================================================
// Adaptive Candidate Pool — detect aggregation queries
// ============================================================================

const AGGREGATION_EN_RE = /\b(?:how many|how much|all the|every|list all|total number|count)\b/i;
const AGGREGATION_CN_RE = /(?:多少[个件条只]?|一共|总共|所有的?|列举|有几[个件条只])/u;

function isAggregationQuery(query: string): boolean {
  return AGGREGATION_EN_RE.test(query) || AGGREGATION_CN_RE.test(query);
}

// ============================================================================
// Multi-hop Entity Extraction — extract entities from retrieval results
// ============================================================================

const MULTI_HOP_STOP_WORDS = new Set([
  "the", "a", "an", "is", "are", "was", "were", "be", "been",
  "have", "has", "had", "will", "would", "can", "could", "should",
  "do", "does", "did", "not", "no", "but", "and", "or", "if",
  "this", "that", "these", "those", "it", "its", "user", "assistant",
  "my", "your", "his", "her", "our", "their", "i", "you", "he", "she",
  "we", "they", "me", "him", "us", "them", "who", "what", "where",
  "when", "how", "why", "which", "about", "with", "from", "for",
  "into", "also", "just", "very", "some", "more", "most", "other",
]);

const COMMON_CJK_STOP = /^(用户|助手|可以|需要|已经|没有|什么|这个|那个|因为|所以|但是|如果)$/;

/**
 * Extract salient entities from retrieval result texts.
 * Focuses on proper nouns and CJK named entities.
 * Returns unique entities sorted by frequency (most common first).
 */
function extractEntitiesFromResults(results: RetrievalResult[]): string[] {
  const freq = new Map<string, number>();

  for (const r of results) {
    const text = r.entry.text;

    // Capitalized multi-word entities: "Adobe Premiere Pro", "Sony A7III"
    const capPattern = /\b([A-Z][a-zA-Z0-9]*(?:[\s\-][A-Z][a-zA-Z0-9]*)*)\b/g;
    let match: RegExpExecArray | null;
    while ((match = capPattern.exec(text)) !== null) {
      const entity = match[1].trim();
      if (entity.length < 2 || MULTI_HOP_STOP_WORDS.has(entity.toLowerCase())) continue;
      freq.set(entity, (freq.get(entity) ?? 0) + 1);
    }

    // CJK named entities: sequences of 2-8 CJK chars (simple heuristic)
    const cjkPattern = /([\p{Script=Han}]{2,8})/gu;
    while ((match = cjkPattern.exec(text)) !== null) {
      const entity = match[1];
      if (COMMON_CJK_STOP.test(entity)) continue;
      freq.set(entity, (freq.get(entity) ?? 0) + 1);
    }
  }

  // Sort by frequency descending, then by length descending (prefer specific entities)
  return [...freq.entries()]
    .sort((a, b) => b[1] - a[1] || b[0].length - a[0].length)
    .map(([entity]) => entity);
}

function parseMetadata(raw?: string): Record<string, unknown> {
  try {
    return JSON.parse(raw || "{}") as Record<string, unknown>;
  } catch {
    return {};
  }
}

// ============================================================================
// Rerank Provider Adapters
// ============================================================================

type RerankProvider = "jina" | "siliconflow" | "voyage" | "pinecone" | "vllm";

interface RerankItem { index: number; score: number }

/** Build provider-specific request headers and body */
function buildRerankRequest(
  provider: RerankProvider,
  apiKey: string,
  model: string,
  query: string,
  documents: string[],
  topN: number,
): { headers: Record<string, string>; body: Record<string, unknown> } {
  switch (provider) {
    case "pinecone":
      return {
        headers: {
          "Content-Type": "application/json",
          "Api-Key": apiKey,
          "X-Pinecone-API-Version": "2024-10",
        },
        body: {
          model,
          query,
          documents: documents.map(text => ({ text })),
          top_n: topN,
          rank_fields: ["text"],
        },
      };
    case "voyage":
      return {
        headers: {
          "Content-Type": "application/json",
          "Authorization": `Bearer ${apiKey}`,
        },
        body: {
          model,
          query,
          documents,
          // Voyage uses top_k (not top_n) to limit reranked outputs.
          top_k: topN,
        },
      };
    case "vllm":
      // Docker Model Runner / vLLM: no auth required, runs locally
      return {
        headers: {
          "Content-Type": "application/json",
        },
        body: {
          model,
          query,
          documents,
          top_n: topN,
        },
      };
    case "siliconflow":
    case "jina":
    default:
      return {
        headers: {
          "Content-Type": "application/json",
          "Authorization": `Bearer ${apiKey}`,
        },
        body: {
          model,
          query,
          documents,
          top_n: topN,
        },
      };
  }
}

/** Parse provider-specific response into unified format */
function parseRerankResponse(
  provider: RerankProvider,
  data: Record<string, unknown>,
): RerankItem[] | null {
  const parseItems = (
    items: unknown,
    scoreKeys: Array<"score" | "relevance_score">,
  ): RerankItem[] | null => {
    if (!Array.isArray(items)) return null;
    const parsed: RerankItem[] = [];
    for (const raw of items as Array<Record<string, unknown>>) {
      const index = typeof raw?.index === "number" ? raw.index : Number(raw?.index);
      if (!Number.isFinite(index)) continue;
      let score: number | null = null;
      for (const key of scoreKeys) {
        const value = raw?.[key];
        const n = typeof value === "number" ? value : Number(value);
        if (Number.isFinite(n)) {
          score = n;
          break;
        }
      }
      if (score === null) continue;
      parsed.push({ index, score });
    }
    return parsed.length > 0 ? parsed : null;
  };

  switch (provider) {
    case "pinecone": {
      // Pinecone: usually { data: [{ index, score, ... }] }
      // Also tolerate results[] with score/relevance_score for robustness.
      return (
        parseItems(data.data, ["score", "relevance_score"]) ??
        parseItems(data.results, ["score", "relevance_score"])
      );
    }
    case "voyage": {
      // Voyage: usually { data: [{ index, relevance_score }] }
      // Also tolerate results[] for compatibility across gateways.
      return (
        parseItems(data.data, ["relevance_score", "score"]) ??
        parseItems(data.results, ["relevance_score", "score"])
      );
    }
    case "vllm":
    case "siliconflow":
    case "jina":
    default: {
      // Jina / SiliconFlow / vLLM: usually { results: [{ index, relevance_score }] }
      // Also tolerate data[] for compatibility across gateways.
      return (
        parseItems(data.results, ["relevance_score", "score"]) ??
        parseItems(data.data, ["relevance_score", "score"])
      );
    }
  }
}

// Cosine similarity for reranking fallback
function cosineSimilarity(a: number[], b: number[]): number {
  if (a.length !== b.length) {
    throw new Error("Vector dimensions must match for cosine similarity");
  }

  let dotProduct = 0;
  let normA = 0;
  let normB = 0;

  for (let i = 0; i < a.length; i++) {
    dotProduct += a[i] * b[i];
    normA += a[i] * a[i];
    normB += b[i] * b[i];
  }

  const norm = Math.sqrt(normA) * Math.sqrt(normB);
  return norm === 0 ? 0 : dotProduct / norm;
}

// ============================================================================
// Memory Retriever
// ============================================================================

export class MemoryRetriever {
  private accessTracker?: AccessTracker;
  private kgStore?: KGStore;
  private frequencyTracker?: FrequencyTracker;
  private auditLogger?: AuditLogger;
  private llmClient?: ReconstructionLLMClient;
  /**
   * Session-scoped suppression list (dopamine-inspired "do not disturb").
   * IDs in this set get a score penalty during retrieval but are NOT deleted.
   * Call clearSessionSuppression() to reset (e.g. at session end).
   */
  private sessionSuppressed = new Set<string>();

  constructor(
    private store: MemoryStore,
    private embedder: Embedder,
    private config: RetrievalConfig = DEFAULT_RETRIEVAL_CONFIG
  ) {}

  /** Attach an AccessTracker to enable reinforcement-based decay. */
  setAccessTracker(tracker: AccessTracker): void {
    this.accessTracker = tracker;
  }

  /** Attach a KGStore to enable graph-based retrieval (PPR). */
  setKGStore(kgStore: KGStore): void {
    this.kgStore = kgStore;
  }

  /** P0.2: Attach a FrequencyTracker for hit-count based boosting. */
  setFrequencyTracker(tracker: FrequencyTracker): void {
    this.frequencyTracker = tracker;
  }

  /** F-1: Attach an AuditLogger for recording retrieve operations. */
  setAuditLogger(logger: AuditLogger): void {
    this.auditLogger = logger;
  }

  /** Attach an LLM client for constructive retrieval reconstruction. */
  setLLMClient(client: ReconstructionLLMClient): void {
    this.llmClient = client;
  }

  /**
   * Shadow-record which memories a reconstruction actually cited via [src:ID].
   * 影子统计：写 metadata.usage（injection-vs-use 双计数），第一阶段不参与 ranking/forget。
   * best-effort：不传 scope（影子写入不做权限过滤，避免跨 scope 扩展候选触发越权 throw）；
   * 内部 allSettled 不抛错，绝不阻断主检索/resume 流程。
   */
  async recordReconstructionUsage(citedIds: string[]): Promise<void> {
    await runUsageShadow(this.store, citedIds);
  }

  /** Suppress a memory ID for the current session (score × 0.3). */
  suppressForSession(id: string): void {
    this.sessionSuppressed.add(id);
  }

  /** Clear all session suppressions. */
  clearSessionSuppression(): void {
    this.sessionSuppressed.clear();
  }

  /** Resolve the minimum score for a given category, falling back to hardMinScore.
   *  P0.1: When shortQuery is true, thresholds are lowered to widen the candidate pool. */
  private minScoreFor(category: string, shortQuery = false): number {
    const map = this.config.categoryMinScores ?? DEFAULT_CATEGORY_MIN_SCORES;
    const base = map[category] ?? this.config.hardMinScore;
    return shortQuery ? base * SHORT_QUERY_MIN_SCORE_FACTOR : base;
  }

  async retrieve(context: RetrievalContext): Promise<RetrievalResultSet> {
    const { query: rawQuery, limit, scopeFilter, category, includeArchived, trace, graph } = context;
    const safeLimit = clampInt(limit, 1, 20);

    // Adaptive retrieval: skip trivial queries to save embedding API calls
    if (shouldSkipRetrieval(rawQuery)) {
      return [];
    }

    // Step 2: alias expansion — append canonical entity tokens for matching colloquial phrases.
    // 不改原 query，仅追加，保留 vector embedding 原意；BM25 拿到更多锚点。
    const query = expandQueryWithAliases(rawQuery);

    let results: RetrievalResult[];

    // 分派前必须先让 store 完成惰性初始化，否则下面读到的 hasFtsSupport 是初始化前的
    // false，hybrid 模式的**首次**检索会静默落回 vector 路径（同一实例第二次才正常）。
    // 该缺陷自 c10607a 初始版本就在，长期未暴露有两个原因：生产 mode 一直是 "vector"，
    // 第一个条件短路、根本读不到第二个；而 eval 的 createFreshEvalComponentsFactory 每
    // case 新建组件，每个 case 都停在"首次"，于是 eval 从未测到过 hybrid 路径。
    // 注：supplementExactTokenWithBm25 那里的顺序依赖是被意识到的（见其注释"hasFtsSupport
    // 在 vectorOnly 触发 store init 后为真"），漏的只是这个分派点。
    // `?.` 是必需的：测试大量使用鸭子类型的 fake store，它们没有 ready()。
    await this.store.ready?.();

    // For vector-only mode, use legacy behavior
    if (this.config.mode === "vector" || !this.store.hasFtsSupport) {
      results = await this.vectorOnlyRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace);
    } else {
      // Hybrid retrieval with vector + BM25 + RRF fusion (+ optional PPR graph)
      results = await this.hybridRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace, graph);
    }

    // 0-hit fallback: minScore filter 把所有候选过滤光时，放宽阈值（=0）重试一次。
    // 只在原 path 0 hit 触发；正常召回不受影响。
    if (results.length === 0) {
      if (this.config.mode === "vector" || !this.store.hasFtsSupport) {
        results = await this.vectorOnlyRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace, 0);
      } else {
        results = await this.hybridRetrieval(query, safeLimit, scopeFilter, category, includeArchived, trace, graph, 0);
      }
    }

    // P-omitted: 快照 finalize 阶段因 limit 截断的达标结果数（提高 limit 可见）。
    // 必须在后续 validity/topicTag filter（以及 BM25 补偿等换数组操作）丢失 ResultSet 属性之前读取。
    let omittedByLimit = (results as RetrievalResultSet).omitted?.count ?? 0;

    // P1.2+: exact-token BM25 补偿——query 含精确 token（大写常量/commit hash/.ts 文件名/camelCase/
    // 低元音 ID 如 rwmjhb）时，vector 漏掉的精确命中用 BM25 补回（vectorOnly 做语义相似、不做精确
    // 字符串匹配，是 embedding 本质，跟语言无关）。普通词（mem0/recallnest/GitHub）不触发，日常
    // 语义召回不受影响。
    if (hasExactToken(rawQuery)) {
      results = await this.supplementExactTokenWithBm25(query, results, safeLimit, scopeFilter, category, includeArchived);
    }

    // LME-2: Multi-hop retrieval — extract entities from first-pass results,
    // run focused follow-up queries, merge to improve cross-session coverage.
    const useMultiHop = context.multiHop ?? this.config.multiHop ?? false;
    if (useMultiHop && results.length > 0) {
      results = await this.multiHopExpand(results, context);
    }

    // MP-1: Topic Tag post-filter — only keep results matching the requested topicTag
    if (context.topicTag && results.length > 0) {
      const tag = context.topicTag.toLowerCase();
      results = results.filter(r => {
        const entryTag = extractTopicTag(r.entry.metadata);
        return entryTag?.toLowerCase() === tag;
      });
    }

    // F3: Temporal validity filter — expired memories excluded by default
    if (results.length > 0) {
      results = filterByValidity(results, Date.now(), {
        validAt: context.validAt,
        includeExpired: context.includeExpired,
      });
    }

    // P0.2: Record frequency hits for returned results (manual queries only)
    // Moved after topicTag filter so filtered-out entries are not reinforced.
    if (this.frequencyTracker && results.length > 0 && context.source !== "auto-recall") {
      this.frequencyTracker.recordHits(results.map(r => r.entry.id));
    }

    // A-3: Record evolution access counts (async, non-blocking)
    // Moved after topicTag filter so filtered-out entries are not reinforced.
    if (results.length > 0 && context.source !== "auto-recall" && this.store.update) {
      Promise.resolve().then(async () => {
        for (const r of results) {
          try {
            const updated = recordEvolutionAccess(r.entry.metadata);
            await this.store.update!(r.entry.id, { metadata: updated });
          } catch (err) { console.error("[recallnest] Evolution access tracking failed:", err instanceof Error ? err.message : String(err)); }
        }
      });
    }

    // Record access for returned results (async, non-blocking).
    // Only reinforce on manual retrieval — auto-recall must not strengthen noise.
    // Tier 3.2: pass raw vector similarity scores for novelty gating — only novel
    // retrievals (high query-result distance) trigger access reinforcement.
    if (this.accessTracker && results.length > 0 && context.source !== "auto-recall") {
      this.accessTracker.recordAccess(
        results.map(r => r.entry.id),
        results.map(r => r.sources.vector?.score),
      );
    }

    // F-1: Audit log — record retrieve operation (non-blocking, silent on failure)
    try {
      if (this.auditLogger) {
        const retrieved = results.slice(0, AUDIT_RETRIEVED_CAP).map((r) => {
          const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
          const boundary = extractBoundaryMetadata(r.entry.metadata);
          return {
            id: r.entry.id.slice(0, 8),
            rev: evo.version,
            st: evo.status,
            ...(boundary?.layer ? { ly: boundary.layer } : {}),
            ...(boundary?.authority ? { au: boundary.authority } : {}),
          };
        });
        this.auditLogger.log({
          operation: "retrieve",
          scope: context.scopeFilter?.[0],
          actor: context.source || "manual",
          details: `query="${context.query.slice(0, 80)}" hits=${results.length}`,
          ...(retrieved.length > 0 ? { retrieved } : {}),
          // Only when the list is short of the truth — a truncated list must look truncated.
          ...(results.length > retrieved.length ? { retrievedTotal: results.length } : {}),
        });
      }
    } catch {
      // Audit must never block retrieval
    }

    // Phase 4: Constructive retrieval — first-class reconstruction (no metadata hack)
    const resultSet = results as RetrievalResultSet;
    // P-omitted: 把快照的 limit 截断数挂回最终 resultSet（validity filter 已丢原数组属性）。
    if (omittedByLimit > 0) {
      resultSet.omitted = { count: omittedByLimit, reason: "limit" };
    }
    const constructiveFlag = envConfig.constructiveRetrieval();
    if (shouldReconstruct({
      flagEnabled: constructiveFlag,
      callerOptIn: context.reconstruct === true,
      resultCount: results.length,
      llmAvailable: this.llmClient
        ? ("isAvailable" in this.llmClient ? (this.llmClient as { isAvailable(): boolean }).isAvailable() : true)
        : false,
    })) {
      try {
        const expansionDeps = this.buildExpansionDeps(context);
        const reconstruction = await runReconstruction(
          { query: context.query, results, mode: "search" },
          this.llmClient!,
          expansionDeps,
        );
        resultSet.reconstruction = reconstruction;
        // Shadow usage：记录被 reconstruction 真正引用的记忆（use 信号），不动 ranking/forget。
        await this.recordReconstructionUsage(reconstruction.sources.map(s => s.id));
      } catch {
        // Silent degradation — return raw results
      }
    }

    return resultSet;
  }

  /**
   * P1.2+: exact-token BM25 补偿。vectorOnly 主路径后，若 query 含精确 token，用 BM25 精确匹配把
   * vector 漏掉的命中补回。hasFtsSupport 在 vectorOnly 触发 store init 后为真；fake store 为假时跳过。
   * bm25 精确命中提到结果前（exact-token 意图=精确优先），vector 语义结果跟后，去重后 slice limit。
   */
  private async supplementExactTokenWithBm25(
    query: string,
    vectorResults: RetrievalResult[],
    limit: number,
    scopeFilter?: string[],
    category?: string,
    includeArchived?: boolean,
  ): Promise<RetrievalResult[]> {
    if (!this.store.hasFtsSupport) return vectorResults;
    const ftsQuery = tokenizeFts(expandQuery(query), detectLang(query));
    const bm25 = await this.runBM25Search(ftsQuery, Math.max(limit, 5), scopeFilter, category, includeArchived);
    const vecIds = new Set(vectorResults.map(r => r.entry.id));
    const supplements = bm25
      .filter(b => !vecIds.has(b.entry.id))
      .filter(b => isActiveMemory(b.entry.metadata))   // 排掉 superseded/archived/consolidated（补偿绕过了 finalize 的尾部过滤，这里补回 active 校验）
      .slice(0, 3)
      .map(b => ({
        entry: b.entry,
        score: b.score,
        sources: { bm25: { score: b.score, rank: b.rank } },
      } as RetrievalResult));
    if (supplements.length === 0) return vectorResults;
    return [...supplements, ...vectorResults].slice(0, limit);
  }

  /**
   * LME-2: Multi-hop expansion — extract entities from first-pass results,
   * run entity-focused follow-up queries, merge with original results.
   * Improves cross-session coverage for aggregation/counting queries.
   */
  private async multiHopExpand(
    firstPass: RetrievalResult[],
    context: RetrievalContext,
  ): Promise<RetrievalResult[]> {
    const maxQueries = this.config.multiHopMaxQueries ?? 3;
    const entities = extractEntitiesFromResults(firstPass);
    if (entities.length === 0) return firstPass;

    // Pick top entities that aren't already well-covered in the query
    const queryLower = context.query.toLowerCase();
    const novelEntities = entities
      .filter(e => !queryLower.includes(e.toLowerCase()))
      .slice(0, maxQueries);

    if (novelEntities.length === 0) return firstPass;

    // Run follow-up queries in parallel (entity + original query context)
    const followUpPromises = novelEntities.map(entity =>
      this.hybridRetrieval(
        `${entity} ${context.query}`,
        context.limit,
        context.scopeFilter,
        context.category,
        context.includeArchived,
        undefined, // no trace for follow-up
        context.graph,
      ).catch(() => [] as RetrievalResult[]),
    );
    const followUpResults = await Promise.all(followUpPromises);

    // Merge: deduplicate by memory ID, keep highest score
    const merged = new Map<string, RetrievalResult>();
    for (const r of firstPass) {
      merged.set(r.entry.id, r);
    }
    for (const batch of followUpResults) {
      for (const r of batch) {
        const existing = merged.get(r.entry.id);
        if (!existing || r.score > existing.score) {
          merged.set(r.entry.id, r);
        }
      }
    }

    // Sort by score descending, limit
    return [...merged.values()]
      .sort((a, b) => b.score - a.score)
      .slice(0, context.limit);
  }

  private async vectorOnlyRetrieval(
    query: string,
    limit: number,
    scopeFilter?: string[],
    category?: string,
    includeArchived?: boolean,
    trace?: TraceCollector,
    minScoreOverride?: number,
  ): Promise<RetrievalResult[]> {
    // Temporal reasoning for vector-only mode
    const temporal = parseTemporalQuery(query);
    const searchQuery = temporal.constraint ? temporal.cleanedQuery : query;

    trace?.startStage("vector_search", 0);
    const queryVector = await this.embedder.embedQuery(searchQuery);
    // P0.1: Lower minScore for short queries; minScoreOverride 用于 0-hit fallback retry
    const shortQ = isShortQuery(searchQuery);
    const vectorMinScore = minScoreOverride !== undefined
      ? minScoreOverride
      : (shortQ
          ? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
          : this.config.minScore);
    // P-omitted: over-fetch a wider candidate pool so finalizeResults can detect limit-capped
    // omissions. Previously fetched exactly `limit`, so vector-only omitted was always 0
    // (hybrid already over-fetches candidatePoolSize). Cost is minimal — ANN with a larger K
    // plus a few more local scoring passes, no extra API; P1.2's vector-only savings come from
    // skipping hybrid BM25/fusion/rerank, not from fetch size.
    const fetchLimit = Math.max(limit, this.config.candidatePoolSize ?? limit);
    const results = await this.store.vectorSearch(queryVector, fetchLimit, vectorMinScore, scopeFilter);
    trace?.endStage(results.length, results.map(r => r.score));

    // Filter by category if specified
    const afterCategory = category
      ? results.filter(r => r.entry.category === category)
      : results;

    // Filter archived records (default: exclude)
    const afterArchive = includeArchived
      ? afterCategory
      : afterCategory.filter(r => parseMetadata(r.entry.metadata).archived !== true);

    // Temporal filter
    const filtered = temporal.constraint
      ? afterArchive.filter(r => matchesTemporalConstraint(r.entry.timestamp, temporal.constraint!))
      : afterArchive;

    const mapped = filtered.map((result, index) => ({
      ...result,
      sources: {
        vector: { score: result.score, rank: index + 1 },
      },
    } as RetrievalResult));

    // P0.1: Anchor boost for vector-only path
    const anchorBoosted = applyAnchorBoost(mapped, searchQuery);

    const scored = this.applySharedScoring(anchorBoosted, query, trace);

    // Session suppression: temporarily penalize "do not disturb" memories (vector-only path)
    const afterSuppression = this.applySessionSuppression(scored);

    return this.finalizeResults(afterSuppression, {
      shortQ,
      minScoreOverride,
      includeArchived,
      limit,
      trace,
    });
  }

  private async hybridRetrieval(
    query: string,
    limit: number,
    scopeFilter?: string[],
    category?: string,
    includeArchived?: boolean,
    trace?: TraceCollector,
    graph?: boolean,
    minScoreOverride?: number,
  ): Promise<RetrievalResult[]> {
    // Adaptive pool: widen candidate pool for aggregation queries ("how many", "all the")
    const multiplier = (this.config.adaptivePoolMultiplier ?? 1) > 1 && isAggregationQuery(query)
      ? (this.config.adaptivePoolMultiplier ?? 1)
      : 1;
    const candidatePoolSize = Math.max(
      Math.ceil(this.config.candidatePoolSize * multiplier),
      limit * 2,
    );

    // Temporal reasoning: extract time constraint and clean query for semantic search
    const temporal = parseTemporalQuery(query);
    const searchQuery = temporal.constraint ? temporal.cleanedQuery : query;

    // Compute query embedding once, reuse for vector search + reranking
    const queryVector = await this.embedder.embedQuery(searchQuery);

    // Run vector, BM25, and optionally PPR searches in parallel
    const useGraph = graph && isKGModeEnabled() && this.kgStore;
    const pprPromise = useGraph
      ? this.runPPRSearch(query, scopeFilter?.[0], trace)
      : Promise.resolve([] as RetrievalResult[]);

    // Pre-tokenize expanded query for BM25 so CJK text produces meaningful FTS matches
    const expandedQuery = expandQuery(searchQuery);
    const ftsQuery = tokenizeFts(expandedQuery, detectLang(expandedQuery));

    trace?.startStage("vector_search", 0);
    const [vectorResults, bm25Results, pprResults] = await Promise.all([
      this.runVectorSearch(queryVector, candidatePoolSize, scopeFilter, category, includeArchived),
      this.runBM25Search(ftsQuery, candidatePoolSize, scopeFilter, category, includeArchived),
      pprPromise,
    ]);
    trace?.endStage(vectorResults.length, vectorResults.map(r => r.score));

    trace?.startStage("bm25_search", 0);
    trace?.endStage(bm25Results.length, bm25Results.map(r => r.score));

    // Fuse results using weighted score fusion (async: validates BM25-only entries exist in store)
    const shortQ = isShortQuery(searchQuery);
    trace?.startStage("rrf_fusion", vectorResults.length + bm25Results.length + pprResults.length);
    const fusedResults = await this.fuseResults(vectorResults, bm25Results, pprResults, shortQ);
    trace?.endStage(fusedResults.length, fusedResults.map(r => r.score));

    // Temporal filter: when query has a time constraint, remove non-matching candidates
    trace?.startStage("temporal_filter", fusedResults.length);
    const temporalFiltered = temporal.constraint
      ? fusedResults.filter(r => matchesTemporalConstraint(r.entry.timestamp, temporal.constraint!))
      : fusedResults;
    trace?.endStage(temporalFiltered.length, temporalFiltered.map(r => r.score));

    // Multi-vector L0/L1 blend: re-score candidates using metadata abstracts/overviews
    trace?.startStage("multi_vector_blend", temporalFiltered.length);
    const multiVecBlended = this.applyMultiVectorBlend(temporalFiltered, searchQuery);
    trace?.endStage(multiVecBlended.length, multiVecBlended.map(r => r.score));

    // P0.1: Anchor boost — short query terms matching short anchor text
    trace?.startStage("anchor_boost", multiVecBlended.length);
    const anchorBoosted = applyAnchorBoost(multiVecBlended, searchQuery);
    trace?.endStage(anchorBoosted.length, anchorBoosted.map(r => r.score));

    // Apply minimum score threshold
    trace?.startStage("min_score_filter", anchorBoosted.length);
    // P0.1: Lower minScore for short queries to widen the candidate pool;
    // minScoreOverride 用于 retrieve() 顶层的 0-hit fallback retry。
    const effectiveMinScore = minScoreOverride !== undefined
      ? minScoreOverride
      : (isShortQuery(searchQuery)
          ? this.config.minScore * SHORT_QUERY_MIN_SCORE_FACTOR
          : this.config.minScore);
    const filtered = anchorBoosted.filter(r => r.score >= effectiveMinScore);
    trace?.endStage(filtered.length, filtered.map(r => r.score));

    // Rerank if enabled
    trace?.startStage("rerank", filtered.length);
    const reranked = this.config.rerank !== "none"
      ? await this.rerankResults(query, queryVector, filtered.slice(0, limit * 2))
      : filtered;
    trace?.endStage(reranked.length, reranked.map(r => r.score));

    const scored = this.applySharedScoring(reranked, query, trace);

    return this.finalizeResults(scored, {
      shortQ,
      minScoreOverride,
      includeArchived,
      limit,
      trace,
    });
  }

  /**
   * Shared scoring pipeline — the 12 stages identical across the vector-only and
   * hybrid paths. Order is load-bearing (Codex §3.6 "顺序即语义"); do not reorder.
   * `query` feeds the emotion stage; `trace` carries per-stage trace spans.
   * Path-specific stages stay in the caller: anchor / min_score / rerank / multi-vector
   * blend run BEFORE this; vector-only session suppression runs AFTER this.
   */
  private applySharedScoring(
    results: RetrievalResult[],
    query: string,
    trace?: TraceCollector,
  ): RetrievalResult[] {
    trace?.startStage("recency_boost", results.length);
    const boosted = this.applyRecencyBoost(results);
    trace?.endStage(boosted.length, boosted.map(r => r.score));

    trace?.startStage("importance_weight", boosted.length);
    const weighted = this.applyImportanceWeight(boosted);
    trace?.endStage(weighted.length, weighted.map(r => r.score));

    // 价值回溯：紧跟 importance 之后，因为两者回答的是同一类问题（这条本身有多值得），
    // 只是 importance 说的是「存的时候认为多重要」，utility 说的是「用起来结果怎么样」。
    trace?.startStage("utility_weight", weighted.length);
    const utilityWeighted = this.applyUtilityWeight(weighted);
    trace?.endStage(utilityWeighted.length, utilityWeighted.map(r => r.score));

    trace?.startStage("confidence_weight", utilityWeighted.length);
    const confidenceWeighted = this.applyConfidence(utilityWeighted);
    trace?.endStage(confidenceWeighted.length, confidenceWeighted.map(r => r.score));

    let afterEmotionVector = confidenceWeighted;
    if (isEmotionScoringEnabled()) {
      const queryEmotion = detectEmotion(query);
      const adapted = confidenceWeighted.map(r => ({ ...r, metadata: r.entry.metadata ?? "{}" }));
      const emotioned = applyEmotionWeight(adapted, queryEmotion);
      afterEmotionVector = confidenceWeighted.map((r, i) => ({ ...r, score: emotioned[i]?.score ?? r.score }));
    }

    trace?.startStage("boundary_weight", afterEmotionVector.length);
    const boundaryWeighted = this.applyBoundaryWeight(afterEmotionVector);
    trace?.endStage(boundaryWeighted.length, boundaryWeighted.map(r => r.score));

    trace?.startStage("asset_type_weight", boundaryWeighted.length);
    const assetWeighted = this.applyAssetTypeWeight(boundaryWeighted);
    trace?.endStage(assetWeighted.length, assetWeighted.map(r => r.score));

    trace?.startStage("length_norm", assetWeighted.length);
    const lengthNormalized = this.applyLengthNormalization(assetWeighted);
    trace?.endStage(lengthNormalized.length, lengthNormalized.map(r => r.score));

    trace?.startStage("time_decay", lengthNormalized.length);
    const timeDecayed = this.applyTimeDecay(lengthNormalized);
    trace?.endStage(timeDecayed.length, timeDecayed.map(r => r.score));

    // B-1/E-1: Evolution decay blend — boost memories with high composite decay score
    const evolutionBlended = this.applyEvolutionDecayBlend(timeDecayed);

    // E-1: Access count boost — memories retrieved more often get a score nudge
    const accessBoosted = this.applyAccessCountBoost(evolutionBlended);

    // Hotness blend: boost frequently + recently accessed memories
    trace?.startStage("hotness_blend", accessBoosted.length);
    const hotnessBlended = this.applyHotnessBlend(accessBoosted);
    trace?.endStage(hotnessBlended.length, hotnessBlended.map(r => r.score));

    // P0.2: Frequency boost — repeatedly retrieved memories score higher
    trace?.startStage("frequency_boost", hotnessBlended.length);
    const frequencyBoosted = this.applyFrequencyBoost(hotnessBlended);
    trace?.endStage(frequencyBoosted.length, frequencyBoosted.map(r => r.score));

    // A1 error-signature 精确匹配 boost（flag 门控，默认 off → no-op、bit-identical）
    return applyErrorSignatureBoost(frequencyBoosted, query);
  }

  /**
   * Shared finalization tail — identical across both paths: sort → hard min →
   * evolution-status filter → noise → RIF → MMR → version/cluster dedup →
   * source diversity → limit. `shortQ` must be the value derived from the
   * path's searchQuery (do not recompute from the raw query here).
   */
  private finalizeResults(
    results: RetrievalResult[],
    opts: {
      shortQ: boolean;
      minScoreOverride?: number;
      includeArchived?: boolean;
      limit: number;
      trace?: TraceCollector;
    },
  ): RetrievalResult[] {
    const { shortQ, minScoreOverride, includeArchived, limit, trace } = opts;
    const rescored = [...results].sort((a, b) => b.score - a.score);

    trace?.startStage("hard_min_score", rescored.length);
    const hardFiltered = minScoreOverride !== undefined
      ? rescored
      : rescored.filter(r => r.score >= this.minScoreFor(r.entry.category, shortQ));
    trace?.endStage(hardFiltered.length, hardFiltered.map(r => r.score));

    // Evolution status filter: exclude superseded/archived/consolidated memories
    const evolutionFiltered = includeArchived
      ? hardFiltered
      : hardFiltered.filter(r => isActiveMemory(r.entry.metadata));

    // LA-1: 资格层准入 —— 默认让 durable 层回答，evidence 留给 deja / 显式溯源。
    // 放在各类 dedup 之前：先定"谁有资格进候选集"，再谈多样性与去重。
    const layerAdmitted = this.applyLayerAdmission(evolutionFiltered, trace);

    trace?.startStage("noise_filter", layerAdmitted.length);
    const denoised = this.config.filterNoise
      ? filterNoise(layerAdmitted, r => r.entry.text)
      : layerAdmitted;
    trace?.endStage(denoised.length, denoised.map(r => r.score));

    // RIF: demote near-duplicate weak results to improve diversity
    const afterRif = this.config.enableRIF
      ? filterInterference(denoised, this.config.rifThreshold ?? 0.85, this.config.rifScoreRatio ?? 0.80)
      : denoised;

    // MMR deduplication: avoid top-k filled with near-identical memories
    trace?.startStage("mmr_diversity", afterRif.length);
    const deduplicated = this.applyMMRDiversity(afterRif);
    trace?.endStage(deduplicated.length, deduplicated.map(r => r.score));

    // Tier 3.3: Version group dedup — keep only the top-ranked version per group
    const versionDeduped = deduplicateByVersionGroup(deduplicated);

    // LC-P2: Cluster insight dedup — prefer cluster summary over individual source memories
    const clusterDeduped = deduplicateByClusterInsight(versionDeduped);

    // Source diversity: round-robin across scopes for multi-session coverage
    const diversified = (this.config.sourceDiversity ?? 0) > 0
      ? applySourceDiversity(clusterDeduped, limit)
      : clusterDeduped;

    trace?.startStage("final_limit", diversified.length);
    const final = diversified.slice(0, limit);
    trace?.endStage(final.length, final.map(r => r.score));

    // P-omitted: 从 source-diversity 之前的候选池(clusterDeduped)算 limit 截断数。
    // 若启用 sourceDiversity，applySourceDiversity 会先把 diversified 压到 limit，
    // 用 diversified.length 会让 omitted 恒为 0（Codex 复审 P3-1，与 vector-only 同类）。
    // 注意：clusterDeduped 仍可能含将被 retrieve 层 validity filter 过滤的 expired 项，
    // 故 omitted 是“上界估计”，输出层用 "up to N" 措辞反映（P3-2）。
    const omittedByLimit = Math.max(0, clusterDeduped.length - final.length);
    if (omittedByLimit > 0) {
      (final as RetrievalResultSet).omitted = { count: omittedByLimit, reason: "limit" };
    }

    return final;
  }

  private async runVectorSearch(
    queryVector: number[],
    limit: number,
    scopeFilter?: string[],
    category?: string,
    includeArchived?: boolean,
  ): Promise<Array<MemorySearchResult & { rank: number }>> {
    let results: MemorySearchResult[];
    try {
      results = await this.store.vectorSearch(queryVector, limit, 0.1, scopeFilter);
    } catch (err) {
      // Fail-open: log warning and continue with empty results (backport from v1.0.30)
      logWarn("vectorSearch failed, continuing with empty results:", err);
      results = [];
    }

    // Filter by category if specified
    const afterCategory = category
      ? results.filter(r => r.entry.category === category)
      : results;

    // Filter archived records (default: exclude)
    const filtered = includeArchived
      ? afterCategory
      : afterCategory.filter(r => parseMetadata(r.entry.metadata).archived !== true);

    return filtered.map((result, index) => ({
      ...result,
      rank: index + 1,
    }));
  }

  private async runBM25Search(
    query: string,
    limit: number,
    scopeFilter?: string[],
    category?: string,
    includeArchived?: boolean,
  ): Promise<Array<MemorySearchResult & { rank: number }>> {
    const results = await this.store.bm25Search(query, limit, scopeFilter);

    // Filter by category if specified
    const afterCategory = category
      ? results.filter(r => r.entry.category === category)
      : results;

    // Filter archived records (default: exclude)
    const filtered = includeArchived
      ? afterCategory
      : afterCategory.filter(r => parseMetadata(r.entry.metadata).archived !== true);

    return filtered.map((result, index) => ({
      ...result,
      rank: index + 1,
    }));
  }

  /**
   * Run PPR graph traversal: detect entities -> BFS neighborhood -> PPR -> map to MemoryEntry.
   * Returns RetrievalResult[] with graph source scores.
   */
  private async runPPRSearch(
    query: string,
    scope?: string,
    trace?: TraceCollector,
  ): Promise<RetrievalResult[]> {
    if (!this.kgStore) return [];

    try {
      trace?.startStage("ppr_entity_detect", 0);
      const detected = await detectEntities(query, this.kgStore, scope);
      trace?.endStage(detected.entities.length, []);

      if (detected.entities.length === 0) return [];

      // BFS neighborhood from KG
      trace?.startStage("ppr_neighborhood", detected.entities.length);
      const hopLimit = detected.isMultiHop ? 3 : 2;
      const neighborhood = await this.kgStore.getNeighborhood(detected.entities, hopLimit, scope);
      trace?.endStage(neighborhood.length, []);

      if (neighborhood.length === 0) return [];

      // Run PPR
      trace?.startStage("ppr_traverse", neighborhood.length);
      const graph = buildGraph(neighborhood);
      const pprResults = pprTraverse(graph, detected.entities, { hopLimit, topK: 20 });
      trace?.endStage(pprResults.length, pprResults.map(r => r.score));

      if (pprResults.length === 0) return [];

      // Collect all source_memory_ids from triples related to PPR-scored entities
      const entityScoreMap = new Map(pprResults.map(r => [r.entity, r.score]));
      const memoryScoreMap = new Map<string, number>();

      for (const nr of neighborhood) {
        for (const triple of nr.triples) {
          const subjectScore = entityScoreMap.get(triple.subject) ?? 0;
          const objectScore = entityScoreMap.get(triple.object) ?? 0;
          const tripleScore = Math.max(subjectScore, objectScore) * triple.confidence;

          if (tripleScore > 0) {
            // All contributing memories share the fact's score — not just the
            // first contributor (multi-source triples after mention counting)
            for (const mid of parseSourceIds(triple.source_memory_ids, triple.source_memory_id)) {
              const existing = memoryScoreMap.get(mid) ?? 0;
              memoryScoreMap.set(mid, Math.max(existing, tripleScore));
            }
          }
        }
      }

      // Fetch memory entries for the top scored memories
      const sortedMemories = [...memoryScoreMap.entries()]
        .sort((a, b) => b[1] - a[1])
        .slice(0, 20);

      const results: RetrievalResult[] = [];
      let rank = 0;
      for (const [memId, score] of sortedMemories) {
        const entry = await this.store.getById(memId);
        if (!entry) continue;
        rank++;
        results.push({
          entry,
          score,
          sources: {
            graph: { score, rank },
          },
        });
      }

      logInfo(`[PPR] ${detected.entities.length} seed entities -> ${pprResults.length} PPR nodes -> ${results.length} memories`);
      return results;
    } catch (err) {
      logWarn(`[PPR] graph traversal failed, continuing without graph results: ${String(err)}`);
      return [];
    }
  }

  private async fuseResults(
    vectorResults: Array<MemorySearchResult & { rank: number }>,
    bm25Results: Array<MemorySearchResult & { rank: number }>,
    graphResults: RetrievalResult[] = [],
    shortQuery: boolean = false,
  ): Promise<RetrievalResult[]> {
    // Adaptive weights: short queries (≤4 tokens) favor BM25 for exact keyword matching.
    // For normal queries, use config weights directly.
    const vW = shortQuery
      ? this.config.vectorWeight * 0.7
      : this.config.vectorWeight;
    const bW = shortQuery
      ? Math.min(this.config.bm25Weight * 1.5, 0.6)
      : this.config.bm25Weight;
    const totalW = vW + bW;

    // Create maps for quick lookup
    const vectorMap = new Map<string, MemorySearchResult & { rank: number }>();
    const bm25Map = new Map<string, MemorySearchResult & { rank: number }>();
    const graphMap = new Map<string, RetrievalResult>();

    vectorResults.forEach(result => {
      vectorMap.set(result.entry.id, result);
    });

    bm25Results.forEach(result => {
      bm25Map.set(result.entry.id, result);
    });

    graphResults.forEach(result => {
      graphMap.set(result.entry.id, result);
    });

    // Get all unique document IDs
    const allIds = new Set([...vectorMap.keys(), ...bm25Map.keys(), ...graphMap.keys()]);

    // Calculate fused scores using configurable weights
    const fusedResults: RetrievalResult[] = [];

    for (const id of allIds) {
      const vectorResult = vectorMap.get(id);
      const bm25Result = bm25Map.get(id);
      const graphResult = graphMap.get(id);

      // FIX(#15): BM25-only results may be "ghost" entries whose vector data was
      // deleted but whose FTS index entry lingers until the next index rebuild.
      // Validate that the entry actually exists in the store before including it.
      if (!vectorResult && !graphResult && bm25Result) {
        try {
          const exists = await this.store.hasId(id);
          if (!exists) continue; // Skip ghost entry
        } catch {
          // If hasId fails, keep the result (fail-open)
        }
      }

      // Use the result with more complete data (prefer vector > graph > BM25)
      const baseResult = vectorResult || graphResult || bm25Result!;

      const vectorScore = vectorResult ? vectorResult.score : 0;
      const bm25Score = bm25Result ? bm25Result.score : 0;
      const graphScore = graphResult?.sources.graph?.score ?? 0;

      let fusedScore: number;
      if (vectorResult && bm25Result) {
        // Both signals agree: weighted average + 5% dual-match confirmation bonus.
        // Graph hit adds up to 15% on top.
        fusedScore = clamp01(
          ((vW * vectorScore + bW * bm25Score) / totalW) * 1.05 + (graphScore * 0.15),
          0.1,
        );
      } else if (vectorResult) {
        // Vector only: semantic match, graph as bonus
        fusedScore = clamp01(vectorScore + (graphScore * 0.15), 0.1);
      } else if (graphResult) {
        // Graph only: entity-relationship match, BM25 as bonus
        fusedScore = clamp01(graphScore + (bm25Score > 0 ? 0.10 : 0), 0.1);
      } else {
        // BM25 only: exact keyword match without semantic confirmation.
        // Short queries get a small boost since BM25 is more reliable for exact terms.
        const bm25OnlyBoost = shortQuery ? 1.15 : 1.0;
        fusedScore = clamp01(bm25Score * bm25OnlyBoost, 0.1);
      }

      fusedResults.push({
        entry: baseResult.entry,
        score: fusedScore,
        sources: {
          vector: vectorResult ? { score: vectorResult.score, rank: vectorResult.rank } : undefined,
          bm25: bm25Result ? { score: bm25Result.score, rank: bm25Result.rank } : undefined,
          graph: graphResult?.sources.graph,
          fused: { score: fusedScore },
        },
      });
    }

    // Sort by fused score descending
    return fusedResults.sort((a, b) => b.score - a.score);
  }

  /**
   * Rerank results using cross-encoder API (Jina, Pinecone, or compatible).
   * Falls back to cosine similarity if API is unavailable or fails.
   */
  private async rerankResults(query: string, queryVector: number[], results: RetrievalResult[]): Promise<RetrievalResult[]> {
    if (results.length === 0) {
      return results;
    }

    // Try cross-encoder rerank via configured provider API
    const provider = this.config.rerankProvider || "jina";
    const needsApiKey = provider !== "vllm";
    if (this.config.rerank === "cross-encoder" && (!needsApiKey || this.config.rerankApiKey)) {
      try {
        const model = this.config.rerankModel || "jina-reranker-v3";
        const endpoint = this.config.rerankEndpoint || "https://api.jina.ai/v1/rerank";
        const documents = results.map(r => r.entry.text);

        // Build provider-specific request
        const { headers, body } = buildRerankRequest(provider, this.config.rerankApiKey || "", model, query, documents, results.length);

        // Timeout: 5 seconds to prevent stalling retrieval pipeline
        const controller = new AbortController();
        const timeout = setTimeout(() => controller.abort(), 5000);

        let response: Response;
        try {
          response = await fetch(endpoint, {
            method: "POST",
            headers,
            body: JSON.stringify(body),
            signal: controller.signal,
          });
        } finally {
          clearTimeout(timeout);
        }

        if (response.ok) {
          const data = await response.json() as Record<string, unknown>;

          // Parse provider-specific response into unified format
          const parsed = parseRerankResponse(provider, data);

          if (!parsed) {
            logWarn("Rerank API: invalid response shape, falling back to cosine");
          } else {
            // Build a Set of returned indices to identify unreturned candidates
            const returnedIndices = new Set(parsed.map(r => r.index));

            const reranked = parsed
              .filter(item => item.index >= 0 && item.index < results.length)
              .map(item => {
                const original = results[item.index];
                // Blend: 60% cross-encoder score + 40% original fused score
                const blendedScore = clamp01(
                  item.score * 0.6 + original.score * 0.4,
                  original.score * 0.5,
                );
                return {
                  ...original,
                  score: blendedScore,
                  sources: {
                    ...original.sources,
                    reranked: { score: item.score },
                  },
                };
              });

            // Keep unreturned candidates with their original scores (slightly penalized)
            const unreturned = results
              .filter((_, idx) => !returnedIndices.has(idx))
              .map(r => ({ ...r, score: r.score * 0.8 }));

            return [...reranked, ...unreturned].sort((a, b) => b.score - a.score);
          }
        } else {
          const errText = await response.text().catch(() => "");
          logWarn(`Rerank API returned ${response.status}: ${errText.slice(0, 200)}, falling back to cosine`);
        }
      } catch (error) {
        if (error instanceof Error && error.name === "AbortError") {
          logWarn("Rerank API timed out (5s), falling back to cosine");
        } else {
          logWarn("Rerank API failed, falling back to cosine:", error);
        }
      }
    }

    // Fallback: lightweight cosine similarity rerank
    try {
      const reranked = results.map(result => {
        const cosineScore = cosineSimilarity(queryVector, result.entry.vector);
        const combinedScore = (result.score * 0.7) + (cosineScore * 0.3);

        return {
          ...result,
          score: clamp01(combinedScore, result.score),
          sources: {
            ...result.sources,
            reranked: { score: cosineScore },
          },
        };
      });

      return reranked.sort((a, b) => b.score - a.score);
    } catch (error) {
      logWarn("Reranking failed, returning original results:", error);
      return results;
    }
  }

  /**
   * Multi-vector L0/L1 blend: re-score candidates using text overlap
   * between query and L0 abstract / L1 overview stored in metadata.
   *
   * Short/conceptual queries get higher L0/L1 weight (topic matching).
   * Detailed queries stay dominated by L2 main vector score.
   * Feature-gated: RECALLNEST_MULTI_VECTOR=true.
   */
  private applyMultiVectorBlend(results: RetrievalResult[], query: string): RetrievalResult[] {
    if (!isMultiVectorEnabled() || results.length === 0) return results;

    const qTokenCount = tokenize(query).length;
    const blendConfig = adaptiveBlendConfig(qTokenCount);

    return results.map(r => {
      const { l0, l1 } = extractMultiVectorText(r.entry.metadata);

      const l0Score = l0 ? textOverlapScore(query, l0) : null;
      const l1Score = l1 ? textOverlapScore(query, l1) : null;

      // Skip blending if no L0/L1 available
      if (l0Score === null && l1Score === null) return r;

      const blendedScore = blendMultiVectorScores(r.score, l0Score, l1Score, blendConfig);
      return { ...r, score: blendedScore };
    });
  }

  /**
   * Apply recency boost: newer memories get a small score bonus.
   * This ensures corrections/updates naturally outrank older entries
   * when semantic similarity is close.
   * Formula: boost = exp(-ageDays / halfLife) * weight
   */
  private applyRecencyBoost(results: RetrievalResult[]): RetrievalResult[] {
    const { recencyHalfLifeDays, recencyWeight } = this.config;
    if (!recencyHalfLifeDays || recencyHalfLifeDays <= 0 || !recencyWeight) {
      return results;
    }

    const now = Date.now();
    const boosted = results.map(r => {
      const ts = (r.entry.timestamp && r.entry.timestamp > 0) ? r.entry.timestamp : now;
      const ageDays = (now - ts) / 86_400_000;
      const boost = Math.exp(-ageDays / recencyHalfLifeDays) * recencyWeight;
      return {
        ...r,
        score: clamp01(r.score + boost, r.score),
      };
    });

    return boosted.sort((a, b) => b.score - a.score);
  }

  /**
   * Apply importance weighting: memories with higher importance get a score boost.
   * This ensures critical memories (importance=1.0) outrank casual ones (importance=0.5)
   * when semantic similarity is close.
   * Formula: score *= (baseWeight + (1 - baseWeight) * importance)
   * With baseWeight=0.5: importance=1.0 → ×1.0, importance=0.7 → ×0.85, importance=0.5 → ×0.75, importance=0.0 → ×0.5
   */
  private applyImportanceWeight(results: RetrievalResult[]): RetrievalResult[] {
    const baseWeight = 0.5;
    const weighted = results.map(r => {
      const importance = r.entry.importance ?? 0.5;
      const factor = baseWeight + (1 - baseWeight) * importance;
      return {
        ...r,
        score: clamp01(r.score * factor, r.score * baseWeight),
      };
    });
    return weighted.sort((a, b) => b.score - a.score);
  }

  /**
   * Viewpoint confidence: corrected/contradicted memories are penalized,
   * confirmed memories are untouched. Same multiplicative pattern as importance.
   * Formula: score *= (0.5 + 0.5 * confidence)
   */
  private applyConfidence(results: RetrievalResult[]): RetrievalResult[] {
    const weighted = results.map(r => ({
      ...r,
      score: applyConfidenceWeight(r.score, r.entry),
    }));
    return weighted.sort((a, b) => b.score - a.score);
  }

  private applyAssetTypeWeight(results: RetrievalResult[]): RetrievalResult[] {
    const weighted = results.map(r => {
      const metadata = parseMetadata(r.entry.metadata);
      const isAsset = metadata.source === "asset" || r.entry.scope.startsWith("asset:");
      if (!isAsset) return r;

      const assetType = String(metadata.assetType || (r.entry.scope.startsWith("asset:brief:") ? "memory-brief" : "pinned-memory"));
      const factor = assetType === "memory-brief"
        ? 0.88
        : assetType === "pinned-memory"
          ? 1.04
          : 1.0;

      return {
        ...r,
        score: clamp01(r.score * factor, r.score * 0.75),
      };
    });

    return weighted.sort((a, b) => b.score - a.score);
  }

  /** LA-1: 一条结果是否属于 evidence 层（transcript 碎片）。判据与 applyBoundaryWeight 保持一致。 */
  private isEvidenceLayer(r: RetrievalResult): boolean {
    const boundary = extractBoundaryMetadata(r.entry.metadata);
    if (boundary) return boundary.layer === "evidence";
    return isTranscriptScope(r.entry.scope);
  }

  /**
   * LA-1: 资格层准入 —— 决定"哪一层来回答这个 query"，而非调整分数。
   *
   * 与 applyBoundaryWeight 的区别是本方法存在的全部理由：那个改分数（排序层），
   * 会让来源盖过相关性（`retriever-boundary-weight.test.ts` 明确禁止这点）；
   * 本方法只挑选候选集，层内相关性排序原封不动。
   *
   * durable 命中不足 `layerAdmissionMin` 时回退到全量候选池 —— 新主题在结论层
   * 往往是空的，宁缺毋滥不能变成什么都不给。
   */
  private applyLayerAdmission(
    results: RetrievalResult[],
    trace?: TraceCollector,
  ): RetrievalResult[] {
    const mode = envConfig.layerAdmission();
    if (mode === "off" || results.length === 0) return results;

    const durable = results.filter(r => !this.isEvidenceLayer(r));
    const dropped = results.length - durable.length;
    const thin = durable.length < envConfig.layerAdmissionMin();

    if (mode === "observe") {
      logInfo(
        `[layer-admission] observe: durable=${durable.length} evidence=${dropped} ` +
        `total=${results.length} action=${thin ? "fallback(keep-all)" : "would-drop-evidence"}`,
      );
      return results;
    }

    if (thin) return results;

    trace?.startStage("layer_admission", results.length);
    trace?.endStage(durable.length, durable.map(r => r.score));
    return durable;
  }

  private applyBoundaryWeight(results: RetrievalResult[]): RetrievalResult[] {
    const weighted = results.map(r => {
      const boundary = extractBoundaryMetadata(r.entry.metadata);
      let factor = 1.0;

      if (boundary?.layer === "durable") {
        factor = boundary.authority === "structured-memory" ? 1.03 : 1.01;
      } else if (boundary?.layer === "evidence") {
        factor = boundary.authority === "transcript-ingest" ? 0.97 : 0.99;
      } else if (!boundary) {
        if (isDurableMemoryScope(r.entry.scope)) factor = 1.01;
        if (isTranscriptScope(r.entry.scope)) factor = 0.99;
      }

      return {
        ...r,
        score: clamp01(r.score * factor, r.score * 0.85),
      };
    });

    return weighted.sort((a, b) => b.score - a.score);
  }

  /**
   * Length normalization: penalize long entries that dominate search results
   * via sheer keyword density and broad semantic coverage.
   * Short, focused entries (< anchor) get a slight boost.
   * Long, sprawling entries (> anchor) get penalized.
   * Formula: score *= 1 / (1 + log2(charLen / anchor))
   */
  private applyLengthNormalization(results: RetrievalResult[]): RetrievalResult[] {
    const anchor = this.config.lengthNormAnchor;
    if (!anchor || anchor <= 0) return results;

    const normalized = results.map(r => {
      const charLen = r.entry.text.length;
      const ratio = charLen / anchor;
      // No penalty for entries at or below anchor length.
      // Gentle logarithmic decay for longer entries:
      //   anchor (500) → 1.0, 800 → 0.75, 1000 → 0.67, 1500 → 0.56, 2000 → 0.50
      // This prevents long, keyword-rich entries from dominating top-k
      // while keeping their scores reasonable.
      const logRatio = Math.log2(Math.max(ratio, 1)); // no boost for short entries
      const factor = 1 / (1 + 0.5 * logRatio);
      return {
        ...r,
        score: clamp01(r.score * factor, r.score * 0.3),
      };
    });

    return normalized.sort((a, b) => b.score - a.score);
  }

  /**
   * Time decay: Weibull stretched-exponential penalty for old entries.
   * Borrowed from memory-lancedb-pro v1.1.0 decay engine.
   *
   * Tier-aware: Core memories decay slower (β=0.8, floor=0.85),
   * Working memories at standard rate (β=1.0, floor=0.65),
   * Peripheral memories decay faster (β=1.3, floor=0.45).
   *
   * Combined with access-tracker reinforcement: frequently recalled
   * memories get extended half-life regardless of tier.
   */
  private applyTimeDecay(results: RetrievalResult[]): RetrievalResult[] {
    const baseHalfLife = this.config.timeDecayHalfLifeDays;
    if (!baseHalfLife || baseHalfLife <= 0) return results;

    const now = Date.now();
    const decayed = results.map(r => {
      // HP-7: Exempt core/pinned/recently-accessed (and procedural categories) from decay
      if (isDecayExempt(r.entry.metadata, r.entry.importance, r.entry.category)) return r;

      const ts = (r.entry.timestamp && r.entry.timestamp > 0) ? r.entry.timestamp : now;
      const ageDays = (now - ts) / 86_400_000;

      // Use access-reinforced half-life if tracker is attached
      let halfLife = this.accessTracker
        ? this.accessTracker.computeEffectiveHalfLife(baseHalfLife, r.entry.metadata)
        : baseHalfLife;

      // HP-emo: Emotion-adjusted half-life — strong emotion slows forgetting
      const emotion = isEmotionScoringEnabled() ? parseEmotion(r.entry.metadata) : null;
      if (emotion) {
        halfLife = adjustHalfLifeForEmotion(halfLife, emotion);
      }

      // Resolve tier from metadata (defaults to peripheral)
      const tier = resolveTier(r.entry.metadata, r.entry.importance);

      // Weibull decay with tier-specific shape and floor
      const factor = weibullDecay(ageDays, halfLife, tier);
      // HP-emo: Arousal boost — flashbulb memory effect (1.0–1.1x)
      const arousalFactor = emotion ? computeArousalBoost(emotion) : 1.0;
      return {
        ...r,
        score: clamp01(r.score * arousalFactor * factor, r.score * 0.3),
      };
    });

    return decayed.sort((a, b) => b.score - a.score);
  }

  /**
   * B-1/E-1: Evolution decay blend — factor in composite decay score
   * (time × frequency × importance) as a scoring signal.
   * Weight increased from 0.05 to 0.10 with B-3 LLM importance assessment.
   */
  private applyEvolutionDecayBlend(results: RetrievalResult[]): RetrievalResult[] {
    const EVOLUTION_BLEND_WEIGHT = 0.10;
    return results.map(r => {
      // HP-7: Exempt entries (incl. procedural categories) get max decay score (1.0) → no penalty
      if (isDecayExempt(r.entry.metadata, r.entry.importance, r.entry.category)) {
        return {
          ...r,
          score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + 1.0 * EVOLUTION_BLEND_WEIGHT, 0),
        };
      }
      const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
      const decay = computeDecayScore(evo, r.entry.importance, undefined, r.entry.metadata);
      return {
        ...r,
        score: clamp01(r.score * (1 - EVOLUTION_BLEND_WEIGHT) + decay * EVOLUTION_BLEND_WEIGHT, 0),
      };
    });
  }

  /**
   * E-1: Access count boost — memories with higher access_count get a score nudge.
   * Complements frequency-tracker (query→memory mapping) by operating at
   * the individual memory level (total retrieval hits across all queries).
   * Formula: score *= 1 + log2(1 + accessCount) × 0.03, capped at ×1.15.
   */
  private applyAccessCountBoost(results: RetrievalResult[]): RetrievalResult[] {
    return results.map(r => {
      const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
      if (evo.accessCount <= 0) return r;
      const boost = Math.min(0.15, Math.log2(1 + evo.accessCount) * 0.03);
      return {
        ...r,
        score: clamp01(r.score * (1 + boost), 0),
      };
    });
  }

  /**
   * Phase 4: Build candidate expansion deps from internal stores.
   * Each dep is only provided if the corresponding store/feature is available.
   */
  private buildExpansionDeps(context: RetrievalContext): CandidateExpansionDeps {
    const deps: CandidateExpansionDeps = {};
    const scope = context.scopeFilter?.[0];

    // KG neighbor expansion — requires KG mode + attached KG store
    if (isKGModeEnabled() && this.kgStore) {
      const kgStore = this.kgStore;
      const store = this.store;
      deps.expandViaKG = async (results) => {
        // Extract unique source_memory_ids from KG triples linked to result entities
        const entityNames: string[] = [];
        for (const r of results.slice(0, 5)) {
          try {
            const meta = JSON.parse(r.entry.metadata || "{}");
            if (meta.kg_entities && Array.isArray(meta.kg_entities)) {
              entityNames.push(...meta.kg_entities);
            }
          } catch (err) { console.error("[recallnest] KG entity extraction failed:", err instanceof Error ? err.message : String(err)); }
        }
        if (entityNames.length === 0) return [];
        const unique = [...new Set(entityNames)].slice(0, 5);
        const neighborhood = await kgStore.getNeighborhood(unique, 1, scope);
        const sourceIds = new Set<string>();
        const existingIds = new Set(results.map(r => r.entry.id));
        for (const n of neighborhood) {
          for (const t of n.triples) {
            // Expand through ALL contributing memories of each fact, not just
            // the first contributor (multi-source triples after mention counting)
            for (const mid of parseSourceIds(t.source_memory_ids, t.source_memory_id)) {
              if (!existingIds.has(mid)) sourceIds.add(mid);
            }
          }
        }
        // Fetch entries by ID
        const expanded: RetrievalResult[] = [];
        for (const id of [...sourceIds].slice(0, 5)) {
          const entry = await store.getById(id);
          if (entry) {
            expanded.push({ entry, score: 0.5, sources: {} });
          }
        }
        return expanded;
      };
    }

    // Evolution chain expansion — walk supersedes/supersededBy links
    if (this.store.getById) {
      const store = this.store;
      deps.expandViaEvolution = async (results) => {
        const expanded: RetrievalResult[] = [];
        const existingIds = new Set(results.map(r => r.entry.id));
        for (const r of results.slice(0, 5)) {
          const evo = parseEvolution(r.entry.metadata, r.entry.timestamp);
          for (const linkedId of [evo.supersedes, evo.supersededBy].filter(Boolean) as string[]) {
            if (existingIds.has(linkedId)) continue;
            const entry = await store.getById(linkedId);
            if (entry) {
              existingIds.add(linkedId);
              expanded.push({ entry, score: r.score * 0.6, sources: {} });
            }
          }
        }
        return expanded;
      };
    }

    // Cluster member expansion — get sourceMemories from cluster insights
    if (this.store.getById) {
      const store = this.store;
      deps.expandViaClusters = async (results) => {
        const expanded: RetrievalResult[] = [];
        const existingIds = new Set(results.map(r => r.entry.id));
        for (const r of results.slice(0, 5)) {
          try {
            const meta = JSON.parse(r.entry.metadata || "{}");
            if ((meta.cluster_insight || meta.cross_memory_pattern) && meta.evolution?.sourceMemories) {
              for (const memberId of (meta.evolution.sourceMemories as string[]).slice(0, 3)) {
                if (existingIds.has(memberId)) continue;
                const entry = await store.getById(memberId);
                if (entry) {
                  existingIds.add(memberId);
                  expanded.push({ entry, score: r.score * 0.5, sources: {} });
                }
              }
            }
          } catch (err) { console.error("[recallnest] Cluster member expansion failed:", err instanceof Error ? err.message : String(err)); }
        }
        return expanded;
      };
    }

    // Narrative sibling expansion — reuse existing expandNarrativeSiblings logic
    if (isNarrativeModeEnabled()) {
      deps.expandViaNarrative = async (results) => {
        const expanded = await this.expandNarrativeSiblings(results);
        // Return only the new siblings (not in original results)
        const existingIds = new Set(results.map(r => r.entry.id));
        return expanded.filter(r => !existingIds.has(r.entry.id));
      };
    }

    return deps;
  }

  /**
   * HP-narrative: Expand retrieval results with narrative siblings.
   * When a retrieved memory has narrative metadata, pull other memories
   * sharing the same generalEventId to provide temporal context.
   *
   * Siblings are added with a dampened score (70% of the triggering result)
   * and capped at 3 siblings per general event to avoid flooding.
   */
  async expandNarrativeSiblings(results: RetrievalResult[]): Promise<RetrievalResult[]> {
    if (!isNarrativeModeEnabled() || results.length === 0) return results;

    // Collect unique generalEventIds from results
    const eventIdToScore = new Map<string, number>();
    const existingIds = new Set(results.map(r => r.entry.id));

    for (const r of results) {
      const narrative = parseNarrative(r.entry.metadata);
      if (!narrative) continue;
      const existing = eventIdToScore.get(narrative.generalEventId);
      if (!existing || r.score > existing) {
        eventIdToScore.set(narrative.generalEventId, r.score);
      }
    }

    if (eventIdToScore.size === 0) return results;

    // Search store for siblings — use scope-based listing and filter by generalEventId
    const siblings: RetrievalResult[] = [];
    const MAX_SIBLINGS_PER_EVENT = 3;

    try {
      const allEntries = await this.store.list(undefined, undefined, 500, 0);
      for (const entry of allEntries) {
        if (existingIds.has(entry.id)) continue;
        const narrative = parseNarrative(entry.metadata);
        if (!narrative) continue;
        const triggerScore = eventIdToScore.get(narrative.generalEventId);
        if (triggerScore === undefined) continue;

        // Count how many siblings we've already added for this event
        const eventSiblingCount = siblings.filter(s => {
          const sn = parseNarrative(s.entry.metadata);
          return sn?.generalEventId === narrative.generalEventId;
        }).length;
        if (eventSiblingCount >= MAX_SIBLINGS_PER_EVENT) continue;

        siblings.push({
          entry,
          score: triggerScore * 0.7,
          sources: { narrativeSibling: true },
        });
        existingIds.add(entry.id);
      }
    } catch {
      // Non-blocking — if store listing fails, return original results
    }

    return [...results, ...siblings].sort((a, b) => b.score - a.score);
  }

  /**
   * Blend access-frequency hotness into retrieval scores.
   *
   * Formula: final = score * (1 - alpha) + hotness * alpha
   * where hotness = sigmoid(log1p(accessCount)) * exp(-decayRate * ageDays)
   * Zero-access rows keep their incoming score; accessed rows retain this formula.
   *
   * No-op when hotnessWeight is 0 or AccessTracker is absent.
   */
  /**
   * 价值回溯加权：被用在成功任务里的记忆上浮，被用在失败任务里的下沉。
   *
   * 这是 utility 唯一的消费点——它**只喂排序**，不进 tier、不进 decay 豁免、
   * 更不写回 importance（那一列是阈值语义，≥0.95 会触发永久免衰减，
   * 详见 memory-utility.ts 抬头的三条理由）。
   *
   * No-op when utilityWeight is 0（默认）或条目没有 utility。
   */
  private applyUtilityWeight(results: RetrievalResult[]): RetrievalResult[] {
    const raw = this.config.utilityWeight;
    const weight = Math.min(1, Math.max(0, Number.isFinite(raw) ? raw : 0));
    if (weight <= 0) return results;

    const weighted = results.map((r) => {
      const utility = readUtility(r.entry.metadata);
      if (utility === null) return r; // 没数据 ≠ 中性差评，原样放行
      const factor = 1 + weight * utility;
      return {
        ...r,
        score: clamp01(r.score * factor, 0),
      };
    });

    return weighted.sort((a, b) => b.score - a.score);
  }

  private applyHotnessBlend(results: RetrievalResult[]): RetrievalResult[] {
    const raw = this.config.hotnessWeight;
    const alpha = Math.min(1, Math.max(0, Number.isFinite(raw) ? raw : 0));
    if (alpha <= 0 || !this.accessTracker) return results;

    const blended = results.map((r) => {
      const { accessCount, lastAccessedAt } = parseAccessMetadata(r.entry.metadata);
      const hotness = computeHotnessScore(
        accessCount,
        lastAccessedAt || r.entry.timestamp,
      );
      if (accessCount === 0) return r;
      return {
        ...r,
        score: clamp01(r.score * (1 - alpha) + hotness * alpha, 0),
      };
    });

    return blended.sort((a, b) => b.score - a.score);
  }

  /**
   * P0.2: Frequency boost — memories that get repeatedly retrieved score higher.
   * Unlike hotness (access-tracker based, decay-oriented), this is pure hit-count
   * boost designed to surface "always relevant" memories like daily patrol rules.
   */
  private applyFrequencyBoost(results: RetrievalResult[]): RetrievalResult[] {
    if (!this.frequencyTracker || results.length === 0) return results;
    return results.map(r => {
      const multiplier = this.frequencyTracker!.getBoostMultiplier(r.entry.id);
      if (multiplier <= 1.0) return r;
      return { ...r, score: Math.min(1, r.score * multiplier) };
    });
  }

  /**
   * MMR-inspired diversity filter: greedily select results that are both
   * relevant (high score) and diverse (low similarity to already-selected).
   *
   * Uses cosine similarity between memory vectors. If two memories have
   * cosine similarity > threshold (default 0.92), the lower-scored one
   * is demoted to the end rather than removed entirely.
   *
   * This prevents top-k from being filled with near-identical entries
   * (e.g. 3 similar "SVG style" memories) while keeping them available
   * if the pool is small.
   */
  /**
   * Session-level "do not disturb": penalize suppressed memories (score × 0.3).
   * Inspired by dopamine-mediated transient inhibition — memory isn't deleted,
   * just temporarily quieted so it doesn't dominate irrelevant sessions.
   */
  private applySessionSuppression(results: RetrievalResult[]): RetrievalResult[] {
    if (this.sessionSuppressed.size === 0) return results;
    return results.map(r =>
      this.sessionSuppressed.has(r.entry.id)
        ? { ...r, score: r.score * 0.3 }
        : r,
    ).sort((a, b) => b.score - a.score);
  }

  private applyMMRDiversity(results: RetrievalResult[], similarityThreshold = 0.85): RetrievalResult[] {
    if (results.length <= 1) return results;

    const selected: RetrievalResult[] = [];
    const deferred: RetrievalResult[] = [];

    for (const candidate of results) {
      // Check if this candidate is too similar to any already-selected result
      const tooSimilar = selected.some(s => {
        // Both must have vectors to compare.
        // LanceDB returns Arrow Vector objects (not plain arrays),
        // so use .length directly and Array.from() for conversion.
        const sVec = s.entry.vector;
        const cVec = candidate.entry.vector;
        if (!sVec?.length || !cVec?.length) return false;
        const sArr = Array.from(sVec as Iterable<number>);
        const cArr = Array.from(cVec as Iterable<number>);
        const sim = cosineSimilarity(sArr, cArr);
        return sim > similarityThreshold;
      });

      if (tooSimilar) {
        deferred.push(candidate);
      } else {
        selected.push(candidate);
      }
    }
    // Append deferred results at the end (available but deprioritized)
    return [...selected, ...deferred];
  }

  // Update configuration
  updateConfig(newConfig: Partial<RetrievalConfig>): void {
    this.config = { ...this.config, ...newConfig };
  }

  // Get current configuration
  getConfig(): RetrievalConfig {
    return { ...this.config };
  }

  // Test retrieval system
  async test(query = "test query"): Promise<{
    success: boolean;
    mode: string;
    hasFtsSupport: boolean;
    error?: string;
  }> {
    try {
      const results = await this.retrieve({
        query,
        limit: 1,
      });

      return {
        success: true,
        mode: this.config.mode,
        hasFtsSupport: this.store.hasFtsSupport,
      };
    } catch (error) {
      return {
        success: false,
        mode: this.config.mode,
        hasFtsSupport: this.store.hasFtsSupport,
        error: error instanceof Error ? error.message : String(error),
      };
    }
  }
}

// ============================================================================
// Factory Function
// ============================================================================

export function createRetriever(
  store: MemoryStore,
  embedder: Embedder,
  config?: Partial<RetrievalConfig>
): MemoryRetriever {
  const fullConfig = { ...DEFAULT_RETRIEVAL_CONFIG, ...config };
  return new MemoryRetriever(store, embedder, fullConfig);
}