{ "repo": "mempalace", "repo_path": "/path/to/mempalace", "dataset": "/path/to/Memtrace/benchmarks/fair/dataset.json", "n_dataset_rows": 1000, "n_query_cases": 3000, "limit": 10, "query_variants": [ "exact", "split", "typo" ], "adapters_requested": [ "chromadb", "gitnexus", "memtrace" ], "notes": { "cgc": "CGC is excluded from this hybrid benchmark because current CGC source has graph + exact/full-text/substring search, but no BM25/vector/RRF retrieval path.", "gitnexus_embedding_fix": "GitNexus was rerun with local @ladybugdb/core/@ladybugdb/core-darwin-arm64 upgraded to 0.16.0. 0.15.4 reproduced vector-index WAL corruption/segfaults on this machine. Django embeddings were generated by invoking GitNexus embedding pipeline directly because the GitNexus CLI skips embeddings above 50,000 nodes." }, "tools": { "memtrace_find_code": { "adapter": "memtrace_find_code", "description": "Memtrace find_code, hybrid BM25 + vector + graph/RRF \u2014 backend=memdb-current", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 44.17, "acc_at_5_pct": 92.57, "acc_at_10_pct": 98.67, "recall_at_10_pct": 98.67, "precision_at_10": 0.2017, "mrr": 0.625, "conditional_acc_at_1_pct": 44.17, "avg_latency_ms": 35.1, "median_latency_ms": 34.9, "p95_latency_ms": 37.62, "p99_latency_ms": 39.65, "avg_tokens": 1518, "wall_seconds": 106.72, "peak_rss_mb": 878.8, "index_time_s": 0, "embedding_time_s": 0, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 96.6, "acc_at_10_pct": 100, "mrr": 0.9798 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 16.9, "acc_at_10_pct": 98.7, "mrr": 0.4472 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 19, "acc_at_10_pct": 97.3, "mrr": 0.448 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_main_no_args_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 6, "top_paths": [ "mempalace/tests/test_cli.py", "mempalace/mempalace/backends/chroma.py", "mempalace/tests/test_onboarding.py" ], "rank": 1, "latency_ms": 137.0246410369873, "tokens": 1489 }, { "id": "q1:split", "variant": "split", "query": "test main no args prints help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 6, "top_paths": [ "mempalace/mempalace/backends/chroma.py", "mempalace/tests/test_cli.py", "mempalace/mempalace/split_mega_files.py" ], "rank": 2, "latency_ms": 35.80188751220703, "tokens": 1478 }, { "id": "q1:typo", "variant": "typo", "query": "test_main_no_ars_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 4, "top_paths": [ "mempalace/tests/test_cli.py", "mempalace/tests/test_onboarding.py", "mempalace/tests/test_searcher.py" ], "rank": 1, "latency_ms": 37.85991668701172, "tokens": 1490 } ] }, "gitnexus_query": { "adapter": "gitnexus_query", "description": "GitNexus query tool, BM25 + semantic + RRF + process grouping", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 11.67, "acc_at_5_pct": 79.8, "acc_at_10_pct": 94.57, "recall_at_10_pct": 94.57, "precision_at_10": 0.1292, "mrr": 0.3462, "conditional_acc_at_1_pct": 11.67, "avg_latency_ms": 367.07, "median_latency_ms": 389.78, "p95_latency_ms": 437.56, "p99_latency_ms": 502.05, "avg_tokens": 357, "wall_seconds": 1103.73, "peak_rss_mb": 32.9, "index_time_s": 0, "embedding_time_s": 0, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 15.8, "acc_at_10_pct": 98.4, "mrr": 0.3881 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 7.4, "acc_at_10_pct": 97.7, "mrr": 0.3218 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 11.8, "acc_at_10_pct": 87.6, "mrr": 0.3286 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_main_no_args_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 10, "top_paths": [ "mempalace/cli.py", "mempalace/split_mega_files.py", "benchmarks/convomem_bench.py" ], "rank": 7, "latency_ms": 399.23715591430664, "tokens": 411 }, { "id": "q1:split", "variant": "split", "query": "test main no args prints help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 10, "top_paths": [ "mempalace/cli.py", "mempalace/split_mega_files.py", "benchmarks/convomem_bench.py" ], "rank": 7, "latency_ms": 402.8947353363037, "tokens": 410 }, { "id": "q1:typo", "variant": "typo", "query": "test_main_no_ars_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 10, "top_paths": [ "benchmarks/convomem_bench.py", "mempalace/onboarding.py", "mempalace/hooks_cli.py" ], "rank": 6, "latency_ms": 423.85101318359375, "tokens": 396 } ] }, "chromadb_vector": { "adapter": "chromadb_vector", "description": "ChromaDB vector-only baseline, 800-char source chunks", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 59.73, "acc_at_5_pct": 82.93, "acc_at_10_pct": 85.47, "recall_at_10_pct": 85.47, "precision_at_10": 0.1816, "mrr": 0.6949, "conditional_acc_at_1_pct": 59.73, "avg_latency_ms": 57.24, "median_latency_ms": 56.33, "p95_latency_ms": 61.15, "p99_latency_ms": 70.84, "avg_tokens": 1936, "wall_seconds": 171.78, "peak_rss_mb": 878.8, "index_time_s": 19.61, "embedding_time_s": 19.61, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 62.3, "acc_at_10_pct": 87.9, "mrr": 0.7248 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 64, "acc_at_10_pct": 90, "mrr": 0.7414 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 52.9, "acc_at_10_pct": 78.5, "mrr": 0.6184 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_main_no_args_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 2, "top_paths": [ "mempalace/tests/test_cli.py", "mempalace/tests/test_miner.py" ], "rank": 1, "latency_ms": 101.43399238586426, "tokens": 2000 }, { "id": "q1:split", "variant": "split", "query": "test main no args prints help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 3, "top_paths": [ "mempalace/tests/test_cli.py", "mempalace/tests/test_miner.py", "mempalace/tests/test_repair.py" ], "rank": 1, "latency_ms": 64.90111351013184, "tokens": 2000 }, { "id": "q1:typo", "variant": "typo", "query": "test_main_no_ars_prints_help", "target": "test_main_no_args_prints_help", "expected_file": "mempalace/tests/test_cli.py", "paths_count": 6, "top_paths": [ "mempalace/tests/test_query_sanitizer.py", "mempalace/tests/test_repair.py", "mempalace/tests/test_miner.py" ], "rank": 4, "latency_ms": 64.06092643737793, "tokens": 1921 } ] } } }