{ "repo": "django", "repo_path": "/path/to/django", "dataset": "/path/to/Memtrace/benchmarks/fair/dataset_1k_django.json", "n_dataset_rows": 1000, "n_query_cases": 3000, "limit": 10, "query_variants": [ "exact", "split", "typo" ], "adapters_requested": [ "chromadb", "gitnexus", "memtrace" ], "notes": { "cgc": "CGC is excluded from this hybrid benchmark because current CGC source has graph + exact/full-text/substring search, but no BM25/vector/RRF retrieval path.", "gitnexus_embedding_fix": "GitNexus was rerun with local @ladybugdb/core/@ladybugdb/core-darwin-arm64 upgraded to 0.16.0. 0.15.4 reproduced vector-index WAL corruption/segfaults on this machine. Django embeddings were generated by invoking GitNexus embedding pipeline directly because the GitNexus CLI skips embeddings above 50,000 nodes." }, "tools": { "memtrace_find_code": { "adapter": "memtrace_find_code", "description": "Memtrace find_code, hybrid BM25 + vector + graph/RRF \u2014 backend=memdb-current", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 48.87, "acc_at_5_pct": 84.03, "acc_at_10_pct": 88.5, "recall_at_10_pct": 88.5, "precision_at_10": 0.1933, "mrr": 0.6319, "conditional_acc_at_1_pct": 48.87, "avg_latency_ms": 490.71, "median_latency_ms": 483.89, "p95_latency_ms": 518.57, "p99_latency_ms": 560.77, "avg_tokens": 1526, "wall_seconds": 1473.95, "peak_rss_mb": 1097.4, "index_time_s": 0, "embedding_time_s": 0, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 79.7, "acc_at_10_pct": 92.4, "mrr": 0.8463 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 31.1, "acc_at_10_pct": 86, "mrr": 0.5061 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 35.8, "acc_at_10_pct": 87.1, "mrr": 0.5433 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_sql_table_creation_suffix_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 6, "top_paths": [ "django/tests/backends/postgresql/test_creation.py", "django/django/db/backends/base/creation.py", "django/django/db/backends/mysql/creation.py" ], "rank": 1, "latency_ms": 6634.70196723938, "tokens": 1632 }, { "id": "q1:split", "variant": "split", "query": "test sql table creation suffix with encoding and template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 4, "top_paths": [ "django/tests/backends/postgresql/test_creation.py", "django/django/db/backends/base/creation.py", "django/django/db/backends/mysql/creation.py" ], "rank": 1, "latency_ms": 510.2694034576416, "tokens": 1651 }, { "id": "q1:typo", "variant": "typo", "query": "test_sql_table_creation_suffx_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 7, "top_paths": [ "django/django/db/backends/base/creation.py", "django/django/db/backends/mysql/creation.py", "django/tests/backends/postgresql/test_creation.py" ], "rank": 3, "latency_ms": 505.7220458984375, "tokens": 1611 } ] }, "gitnexus_query": { "adapter": "gitnexus_query", "description": "GitNexus query tool, BM25 + semantic + RRF + process grouping", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 38.63, "acc_at_5_pct": 69.9, "acc_at_10_pct": 72.8, "recall_at_10_pct": 72.8, "precision_at_10": 0.1438, "mrr": 0.5176, "conditional_acc_at_1_pct": 38.63, "avg_latency_ms": 757.47, "median_latency_ms": 850.12, "p95_latency_ms": 894.28, "p99_latency_ms": 954.01, "avg_tokens": 200, "wall_seconds": 2274.8, "peak_rss_mb": 33.5, "index_time_s": 0, "embedding_time_s": 0, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 49.2, "acc_at_10_pct": 84.6, "mrr": 0.6311 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 37.9, "acc_at_10_pct": 78.1, "mrr": 0.5308 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 28.8, "acc_at_10_pct": 55.7, "mrr": 0.391 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_sql_table_creation_suffix_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 7, "top_paths": [ "django/db/backends/base/creation.py", "django/db/backends/oracle/operations.py", "django/tests/backends/postgresql/test_creation.py" ], "rank": 3, "latency_ms": 982.2330474853516, "tokens": 386 }, { "id": "q1:split", "variant": "split", "query": "test sql table creation suffix with encoding and template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 7, "top_paths": [ "django/db/backends/base/creation.py", "django/db/backends/oracle/operations.py", "django/tests/backends/postgresql/test_creation.py" ], "rank": 3, "latency_ms": 872.17116355896, "tokens": 382 }, { "id": "q1:typo", "variant": "typo", "query": "test_sql_table_creation_suffx_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 6, "top_paths": [ "django/db/backends/base/creation.py", "django/tests/backends/postgresql/test_creation.py", "django/tests/backends/oracle/test_creation.py" ], "rank": 2, "latency_ms": 873.3339309692383, "tokens": 300 } ] }, "chromadb_vector": { "adapter": "chromadb_vector", "description": "ChromaDB vector-only baseline, 800-char source chunks", "n_queries": 3000, "query_variants": [ "exact", "split", "typo" ], "coverage_pct": 100, "acc_at_1_pct": 28.9, "acc_at_5_pct": 48.9, "acc_at_10_pct": 53.33, "recall_at_10_pct": 53.33, "precision_at_10": 0.1121, "mrr": 0.3718, "conditional_acc_at_1_pct": 28.9, "avg_latency_ms": 57.08, "median_latency_ms": 56.46, "p95_latency_ms": 59.24, "p99_latency_ms": 64.64, "avg_tokens": 1837, "wall_seconds": 171.31, "peak_rss_mb": 1097.4, "index_time_s": 291.36, "embedding_time_s": 291.36, "by_variant": { "exact": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 33.7, "acc_at_10_pct": 60.4, "mrr": 0.4281 }, "split": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 32.9, "acc_at_10_pct": 59.5, "mrr": 0.4196 }, "typo": { "n_queries": 1000, "coverage_pct": 100, "acc_at_1_pct": 20.1, "acc_at_10_pct": 40.1, "mrr": 0.2676 } }, "sample_results": [ { "id": "q1:exact", "variant": "exact", "query": "test_sql_table_creation_suffix_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 8, "top_paths": [ "django/tests/backends/postgresql/test_creation.py", "django/db/backends/mysql/creation.py", "django/tests/schema/tests.py" ], "rank": 1, "latency_ms": 106.16803169250488, "tokens": 2000 }, { "id": "q1:split", "variant": "split", "query": "test sql table creation suffix with encoding and template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 9, "top_paths": [ "django/tests/backends/postgresql/test_creation.py", "django/db/backends/mysql/creation.py", "django/db/backends/base/schema.py" ], "rank": 1, "latency_ms": 62.212228775024414, "tokens": 2000 }, { "id": "q1:typo", "variant": "typo", "query": "test_sql_table_creation_suffx_with_encoding_and_template", "target": "test_sql_table_creation_suffix_with_encoding_and_template", "expected_file": "django/tests/backends/postgresql/test_creation.py", "paths_count": 9, "top_paths": [ "django/tests/backends/postgresql/test_creation.py", "django/tests/schema/tests.py", "django/db/backends/mysql/creation.py" ], "rank": 1, "latency_ms": 58.22920799255371, "tokens": 2000 } ] } } }