#!/usr/bin/env node import { writeFileSync, mkdirSync } from 'fs'; import { isAbsolute, join } from 'path'; import Database from 'better-sqlite3'; import { initializeDatabase, closeDatabase } from '../src/infrastructure/database/database/init.js'; import { HybridSearchFactory } from '../src/domains/search/factories/hybrid-search.factory.js'; import { loadBenchmarkQueries, loadBenchmarkCorpus, type BenchmarkQuery, type BenchmarkCorpusEntry, } from '../src/test/helpers/search-quality-benchmark-fixtures.js'; import { mergeCandidateIds } from '../src/test/helpers/search-quality-candidate-builder.js'; interface CliOptions { benchmarkDir: string; limit: number; randomNegatives: number; help: boolean; } function parseArgs(): CliOptions { const args = process.argv.slice(2); const options: CliOptions = { benchmarkDir: join(process.cwd(), 'tests', 'fixtures', 'search-quality', 'benchmark-v3'), limit: 30, randomNegatives: 10, help: false, }; for (let index = 0; index < args.length; index++) { const arg = args[index]; if (arg === '--benchmark-dir' && args[index + 1]) { options.benchmarkDir = isAbsolute(args[index + 1]) ? args[index + 1] : join(process.cwd(), args[index + 1]); index++; continue; } if (arg === '--limit' && args[index + 1]) { options.limit = parseInt(args[index + 1], 10); index++; continue; } if (arg === '--random-negatives' && args[index + 1]) { options.randomNegatives = parseInt(args[index + 1], 10); index++; continue; } if (arg === '--help' || arg === '-h') { options.help = true; } } return options; } function printHelp(): void { console.log(` Search quality labeling candidate generator Reads queries.json, runs hybrid search for each query, merges top results with random negatives, and writes label-candidates.json. Does NOT write ground-truth.json (human labeling only). Usage: npm run quality:benchmark:candidates npm run quality:benchmark:candidates -- --benchmark-dir tests/fixtures/search-quality/benchmark-v3 --limit 30 --random-negatives 10 Options: --benchmark-dir Benchmark directory (default: tests/fixtures/search-quality/benchmark-v3) --limit Max number of queries to process (default: 30) --random-negatives Number of random negative candidate IDs per query (default: 10) --help, -h Show this help `); } function buildMemoryIdToBenchmarkIdMap(corpus: BenchmarkCorpusEntry[]): Map { const map = new Map(); for (const entry of corpus) { map.set(entry.source_memory_id, entry.benchmark_id); } return map; } function hashString(value: string): number { let hash = 2166136261; for (let index = 0; index < value.length; index++) { hash ^= value.charCodeAt(index); hash = Math.imul(hash, 16777619); } return hash >>> 0; } function sampleDeterministicNegatives( allBenchmarkIds: string[], exclude: Set, count: number, seedSource: string ): string[] { const pool = allBenchmarkIds.filter((id) => !exclude.has(id)); if (pool.length === 0 || count <= 0) return []; return [...pool] .sort((left, right) => { const leftHash = hashString(`${seedSource}:${left}`); const rightHash = hashString(`${seedSource}:${right}`); return leftHash - rightHash || left.localeCompare(right); }) .slice(0, count); } export interface LabelCandidateEntry { query_id: string; query: string; candidate_benchmark_ids: string[]; } async function main(): Promise { const options = parseArgs(); if (options.help) { printHelp(); return; } let db: Database.Database | null = null; try { const queries = loadBenchmarkQueries(options.benchmarkDir); const limitedQueries = queries.slice(0, options.limit); let corpus: BenchmarkCorpusEntry[] = []; try { corpus = loadBenchmarkCorpus(options.benchmarkDir); } catch { console.warn('Corpus not found or empty; random negatives will be skipped.'); } const memoryToBenchmark = buildMemoryIdToBenchmarkIdMap(corpus); const allBenchmarkIds = corpus.map((e) => e.benchmark_id); db = await initializeDatabase(); const searchEngine = HybridSearchFactory.createDefaultEngine(db); const entries: LabelCandidateEntry[] = []; for (const q of limitedQueries) { const hybridTop: string[] = []; try { const result = await searchEngine.search(db!, { query: q.query, limit: 20 }); for (const item of result.items) { const bid = memoryToBenchmark.get(item.id); if (bid) hybridTop.push(bid); } } catch (err) { console.warn(`Search failed for query_id=${q.query_id}: ${err instanceof Error ? err.message : String(err)}`); } const excludeSet = new Set(hybridTop); const negatives = sampleDeterministicNegatives( allBenchmarkIds, excludeSet, options.randomNegatives, q.query_id ); const candidate_benchmark_ids = mergeCandidateIds([hybridTop, negatives]); entries.push({ query_id: q.query_id, query: q.query, candidate_benchmark_ids, }); } const outPath = join(options.benchmarkDir, 'label-candidates.json'); mkdirSync(options.benchmarkDir, { recursive: true }); writeFileSync(outPath, `${JSON.stringify(entries, null, 2)}\n`, 'utf-8'); console.log(`Queries processed: ${entries.length}`); console.log(`Written: ${outPath}`); } finally { if (db) { closeDatabase(db); } } } main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; });