#!/usr/bin/env node
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
import { basename, join, resolve } from 'path';
import Database from 'better-sqlite3';
import { initializeDatabase, closeDatabase } from '../src/infrastructure/database/database/init.js';
import { DatabaseUtils } from '../src/shared/utils/database.js';
import {
buildBenchmarkCorpus,
type BenchmarkCorpusEntry,
type BenchmarkSourceMemory,
} from '../src/test/helpers/search-quality-benchmark-builder.js';
import { loadBenchmarkCorpus, type BenchmarkManifest } from '../src/test/helpers/search-quality-benchmark-fixtures.js';
interface CliOptions {
outputDir: string;
dryRun: boolean;
help: boolean;
}
function parseArgs(): CliOptions {
const args = process.argv.slice(2);
const options: CliOptions = {
outputDir: join('tests', 'fixtures', 'search-quality', 'benchmark-v3'),
dryRun: false,
help: false,
};
for (let index = 0; index < args.length; index++) {
const arg = args[index];
if (arg === '--output-dir' && args[index + 1]) {
options.outputDir = args[index + 1];
index++;
continue;
}
if (arg === '--dry-run') {
options.dryRun = true;
continue;
}
if (arg === '--help' || arg === '-h') {
options.help = true;
}
}
return options;
}
function printHelp(): void {
console.log(`
Search quality benchmark corpus exporter
Requires an already-initialized Memento database (memory_item table must exist).
Use DB_PATH to point to your existing database if it is not in the default location.
Usage:
npm run quality:benchmark:export
DB_PATH=/path/to/memory.db npm run quality:benchmark:export
npm run quality:benchmark:export -- --dry-run
npm run quality:benchmark:export -- --output-dir tests/fixtures/search-quality/benchmark-v3
Options:
--output-dir
Export directory (default: tests/fixtures/search-quality/benchmark-v3)
--dry-run Print summary without writing files
--help, -h Show this help
Environment:
DB_PATH Path to SQLite database (default: from config, often ./data/memory.db)
`);
}
function loadJsonArrayLength(filePath: string): number {
if (!existsSync(filePath)) {
return 0;
}
const parsed = JSON.parse(readFileSync(filePath, 'utf-8')) as unknown;
return Array.isArray(parsed) ? parsed.length : 0;
}
function deriveBenchmarkVersion(outputDir: string): string {
const directoryName = basename(resolve(outputDir));
const match = directoryName.match(/^benchmark-(.+)$/);
return match?.[1] ?? directoryName;
}
function createManifest(outputDir: string, corpus: BenchmarkCorpusEntry[]): BenchmarkManifest {
return {
benchmark_version: deriveBenchmarkVersion(outputDir),
created_at: new Date().toISOString(),
corpus_size: corpus.length,
query_count: loadJsonArrayLength(join(outputDir, 'queries.json')),
ground_truth_count: loadJsonArrayLength(join(outputDir, 'ground-truth.json')),
source: 'full-memory-snapshot',
labeling_policy: 'binary-human-labeled',
strict_ci: true,
ground_truth_reviewed: false,
notes: [
'Exported from current memory_item snapshot',
'Human-labeled queries and ground truth must be added separately',
'Set ground_truth_reviewed=true only after manual review is complete',
],
};
}
async function loadSourceMemories(db: Database.Database): Promise {
return DatabaseUtils.all(
db,
`SELECT id, type, content, tags, created_at
FROM memory_item
ORDER BY created_at ASC, id ASC`
) as BenchmarkSourceMemory[];
}
function writeCorpus(outputDir: string, corpus: BenchmarkCorpusEntry[]): void {
mkdirSync(outputDir, { recursive: true });
writeFileSync(
join(outputDir, 'corpus.jsonl'),
corpus.map((entry) => JSON.stringify(entry)).join('\n') + (corpus.length > 0 ? '\n' : ''),
'utf-8'
);
}
function writeManifest(outputDir: string, manifest: BenchmarkManifest): void {
mkdirSync(outputDir, { recursive: true });
writeFileSync(join(outputDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
}
async function main(): Promise {
const options = parseArgs();
if (options.help) {
printHelp();
return;
}
let db: Database.Database | null = null;
try {
db = await initializeDatabase();
const hasMemoryItem = db
.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='memory_item'`)
.get();
if (!hasMemoryItem) {
console.error(
'Error: memory_item table not found. The database must be initialized first.\n' +
' - Use DB_PATH to point to an existing Memento database, e.g.:\n' +
' DB_PATH=./data/memory.db npm run quality:benchmark:export\n' +
' - Or run "npm run db:init -w @memento/core" first (then export will have 0 memories until you add data).'
);
process.exitCode = 1;
return;
}
const sourceRows = await loadSourceMemories(db);
const existingCorpus = existsSync(join(options.outputDir, 'corpus.jsonl'))
? loadBenchmarkCorpus(options.outputDir)
: [];
const corpus = buildBenchmarkCorpus(sourceRows, existingCorpus);
const manifest = createManifest(options.outputDir, corpus);
console.log(`source memories: ${sourceRows.length}`);
console.log(`benchmark corpus: ${corpus.length}`);
console.log(`dropped empty content: ${sourceRows.length - corpus.length}`);
console.log(`output directory: ${options.outputDir}`);
if (options.dryRun) {
console.log('dry run complete (no files written)');
return;
}
writeCorpus(options.outputDir, corpus);
writeManifest(options.outputDir, manifest);
console.log(`wrote: ${join(options.outputDir, 'corpus.jsonl')}`);
console.log(`wrote: ${join(options.outputDir, 'manifest.json')}`);
} finally {
if (db) {
closeDatabase(db);
}
}
}
function printDatabaseHint(error: unknown): void {
const msg = error instanceof Error ? error.message : String(error);
if (msg.includes('no such table') && msg.includes('memory_item')) {
console.error(
'\nHint: Use an already-initialized Memento database (memory_item must exist).\n' +
' DB_PATH=/path/to/your/memory.db npm run quality:benchmark:export\n' +
'Example (main repo data): DB_PATH=../data/memory.db npm run quality:benchmark:export'
);
}
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
printDatabaseHint(error);
process.exitCode = 1;
});