"use client" import { useState, useEffect, useRef } from "react" import { useRouter } from "next/navigation" import Link from "next/link" import { getProviders, getBenchmarks, getModels, startRun, getCompletedRuns, type RunSummary, type PhaseId, PHASE_ORDER, type SelectionMode, type SampleType, type SamplingConfig, type Provider, } from "@/lib/api" import { SingleSelect } from "@/components/single-select" type Tab = "new" | "advanced" export default function NewRunPage() { const router = useRouter() const [activeTab, setActiveTab] = useState("new") const [loading, setLoading] = useState(true) const [submitting, setSubmitting] = useState(false) const [error, setError] = useState(null) const [providers, setProviders] = useState([]) const [benchmarks, setBenchmarks] = useState<{ name: string; displayName: string }[]>([]) const [models, setModels] = useState({}) const [completedRuns, setCompletedRuns] = useState([]) const [form, setForm] = useState({ provider: "", benchmark: "", runId: "", judgeModel: "gpt-4o", answeringModel: "gpt-4o", selectionMode: "full" as SelectionMode, sampleType: "consecutive" as SampleType, perCategory: "2", limit: "", concurrency: { default: undefined as number | undefined, ingest: undefined as number | undefined, indexing: undefined as number | undefined, search: undefined as number | undefined, answer: undefined as number | undefined, evaluate: undefined as number | undefined, }, }) const [advancedForm, setAdvancedForm] = useState({ sourceRunId: "", newRunId: "", fromPhase: "search" as PhaseId, }) const [editingRunId, setEditingRunId] = useState(false) const [editingAdvancedRunId, setEditingAdvancedRunId] = useState(false) const [editingJudgeModel, setEditingJudgeModel] = useState(false) const [editingAnsweringModel, setEditingAnsweringModel] = useState(false) const [editingConcurrency, setEditingConcurrency] = useState(false) const [showAdvancedConcurrencyNew, setShowAdvancedConcurrencyNew] = useState(false) const [showAdvancedConcurrencyAdvanced, setShowAdvancedConcurrencyAdvanced] = useState(false) const [editingPhase, setEditingPhase] = useState(null) const runIdInputRef = useRef(null) const advancedRunIdInputRef = useRef(null) const concurrencyInputRef = useRef(null) const phaseInputRefs = useRef>({}) useEffect(() => { if (editingConcurrency && concurrencyInputRef.current) { concurrencyInputRef.current.focus() concurrencyInputRef.current.select() } }, [editingConcurrency]) useEffect(() => { if (editingPhase && phaseInputRefs.current[editingPhase]) { phaseInputRefs.current[editingPhase]?.focus() phaseInputRefs.current[editingPhase]?.select() } }, [editingPhase]) useEffect(() => { loadOptions() }, []) useEffect(() => { if (editingRunId && runIdInputRef.current) { runIdInputRef.current.focus() runIdInputRef.current.select() } }, [editingRunId]) useEffect(() => { if (editingAdvancedRunId && advancedRunIdInputRef.current) { advancedRunIdInputRef.current.focus() advancedRunIdInputRef.current.select() } }, [editingAdvancedRunId]) const selectedSourceRun = completedRuns.find((r) => r.runId === advancedForm.sourceRunId) useEffect(() => { if (advancedForm.sourceRunId && selectedSourceRun) { const sourceProvider = providers.find((p) => p.name === selectedSourceRun.provider) setForm((f) => ({ ...f, judgeModel: selectedSourceRun.judge, answeringModel: selectedSourceRun.answeringModel, concurrency: { default: sourceProvider?.concurrency?.default ?? 1, ingest: sourceProvider?.concurrency?.ingest, indexing: sourceProvider?.concurrency?.indexing, search: sourceProvider?.concurrency?.search, answer: sourceProvider?.concurrency?.answer, evaluate: sourceProvider?.concurrency?.evaluate, }, })) const timestamp = new Date().toISOString().slice(0, 10).replace(/-/g, "") const random = Math.random().toString(36).slice(2, 6) setAdvancedForm((prev) => ({ ...prev, newRunId: `${selectedSourceRun.provider}-${selectedSourceRun.benchmark}-${timestamp}-${random}`, })) setEditingJudgeModel(false) setEditingAnsweringModel(false) } }, [advancedForm.sourceRunId, selectedSourceRun, providers]) useEffect(() => { setEditingJudgeModel(false) setEditingAnsweringModel(false) if (selectedSourceRun) { const canChangeJudge = ["indexing", "search", "answer", "evaluate"].includes( advancedForm.fromPhase ) const canChangeAnswering = ["indexing", "search", "answer"].includes(advancedForm.fromPhase) if (!canChangeJudge) { setForm((f) => ({ ...f, judgeModel: selectedSourceRun.judge })) } if (!canChangeAnswering) { setForm((f) => ({ ...f, answeringModel: selectedSourceRun.answeringModel })) } } }, [advancedForm.fromPhase, selectedSourceRun]) const canChangeJudgeModel = ["indexing", "search", "answer", "evaluate"].includes( advancedForm.fromPhase ) const canChangeAnsweringModel = ["indexing", "search", "answer"].includes(advancedForm.fromPhase) const selectedProvider = providers.find((p) => p.name === form.provider) useEffect(() => { if (selectedProvider) { setForm((f) => ({ ...f, concurrency: { default: selectedProvider.concurrency?.default ?? 1, ingest: selectedProvider.concurrency?.ingest, indexing: selectedProvider.concurrency?.indexing, search: selectedProvider.concurrency?.search, answer: selectedProvider.concurrency?.answer, evaluate: selectedProvider.concurrency?.evaluate, }, })) } }, [form.provider, providers]) async function loadOptions() { try { const [providersRes, benchmarksRes, modelsRes, runsRes] = await Promise.all([ getProviders(), getBenchmarks(), getModels(), getCompletedRuns(), ]) setProviders(providersRes.providers) setBenchmarks(benchmarksRes.benchmarks) setModels(modelsRes.models) setCompletedRuns(runsRes) if (providersRes.providers.length > 0) { const firstProvider = providersRes.providers[0] const defaultConcurrency = firstProvider.concurrency?.default ?? 1 setForm((f) => ({ ...f, provider: firstProvider.name, concurrency: { default: defaultConcurrency, ingest: firstProvider.concurrency?.ingest, indexing: firstProvider.concurrency?.indexing, search: firstProvider.concurrency?.search, answer: firstProvider.concurrency?.answer, evaluate: firstProvider.concurrency?.evaluate, }, })) } if (benchmarksRes.benchmarks.length > 0) { setForm((f) => ({ ...f, benchmark: benchmarksRes.benchmarks[0].name })) } } catch (e) { setError(e instanceof Error ? e.message : "Failed to load options") } finally { setLoading(false) } } function generateRunId() { const timestamp = new Date().toISOString().slice(0, 10).replace(/-/g, "") const random = Math.random().toString(36).slice(2, 6) return `${form.provider}-${form.benchmark}-${timestamp}-${random}` } const displayRunId = form.runId || (form.provider && form.benchmark ? generateRunId() : "run-id") async function handleSubmit(e: React.FormEvent) { e.preventDefault() if (activeTab === "advanced") { if (!advancedForm.sourceRunId || !selectedSourceRun) { setError("Please select a source run") return } if (!advancedForm.newRunId) { setError("Please enter a new run ID") return } } const runId = activeTab === "advanced" ? advancedForm.newRunId : form.runId || generateRunId() const fromPhase = activeTab === "advanced" ? advancedForm.fromPhase : undefined const sourceRunId = activeTab === "advanced" ? advancedForm.sourceRunId : undefined const provider = activeTab === "advanced" && selectedSourceRun ? selectedSourceRun.provider : form.provider const benchmark = activeTab === "advanced" && selectedSourceRun ? selectedSourceRun.benchmark : form.benchmark const judgeModel = activeTab === "advanced" && !canChangeJudgeModel && selectedSourceRun ? selectedSourceRun.judge : form.judgeModel const answeringModel = activeTab === "advanced" && !canChangeAnsweringModel && selectedSourceRun ? selectedSourceRun.answeringModel : form.answeringModel let sampling: SamplingConfig | undefined if (activeTab === "new") { console.log("Form state:", { selectionMode: form.selectionMode, perCategory: form.perCategory, sampleType: form.sampleType, }) if (form.selectionMode === "full") { sampling = { mode: "full" } } else if (form.selectionMode === "sample") { const perCategoryValue = parseInt(form.perCategory) || 2 // Default to 2 if not set sampling = { mode: "sample", sampleType: form.sampleType, perCategory: perCategoryValue, } } else if (form.selectionMode === "limit" && form.limit) { sampling = { mode: "limit", limit: parseInt(form.limit), } } } console.log("Submitting with sampling config:", sampling) // Only send concurrency if not all defaults (1) const hasNonDefaultConcurrency = (form.concurrency.default !== undefined && form.concurrency.default !== 1) || form.concurrency.ingest !== undefined || form.concurrency.indexing !== undefined || form.concurrency.search !== undefined || form.concurrency.answer !== undefined || form.concurrency.evaluate !== undefined const concurrency = hasNonDefaultConcurrency ? { ...(form.concurrency.default !== undefined && { default: form.concurrency.default }), ...(form.concurrency.ingest !== undefined && { ingest: form.concurrency.ingest }), ...(form.concurrency.indexing !== undefined && { indexing: form.concurrency.indexing }), ...(form.concurrency.search !== undefined && { search: form.concurrency.search }), ...(form.concurrency.answer !== undefined && { answer: form.concurrency.answer }), ...(form.concurrency.evaluate !== undefined && { evaluate: form.concurrency.evaluate }), } : undefined try { setSubmitting(true) setError(null) await startRun({ provider, benchmark, runId, judgeModel, answeringModel, sampling, concurrency, force: activeTab === "new", fromPhase, sourceRunId, }) router.push(`/runs/${encodeURIComponent(runId)}`) } catch (e) { setError(e instanceof Error ? e.message : "Failed to start run") setSubmitting(false) } } const allModels = [...Object.values(models).flat()] as { alias: string; displayName: string }[] const providerOptions = providers.map((p) => ({ value: p.name, label: p.displayName })) const benchmarkOptions = benchmarks.map((b) => ({ value: b.name, label: b.displayName })) const modelOptions = allModels.map((m) => ({ value: m.alias, label: m.displayName || m.alias })) const runOptions = completedRuns.map((r) => ({ value: r.runId, label: r.runId, sublabel: `${r.provider} · ${r.benchmark}${r.summary.total ? ` · ${r.summary.total}q` : ""}${r.accuracy !== null ? ` · ${(r.accuracy * 100).toFixed(0)}%` : ""}`, })) if (loading) { return (
) } return (
Runs / {activeTab === "new" ? "New Run" : "Advanced"}
{activeTab === "advanced" && ( <>

Create a new run using data from a completed run. The new run will copy checkpoint data up to the selected phase.

setAdvancedForm({ ...advancedForm, sourceRunId: value })} placeholder="Choose a source run..." wide />
{advancedForm.sourceRunId && ( <>
{!editingAdvancedRunId ? ( ) : ( setAdvancedForm({ ...advancedForm, newRunId: e.target.value }) } onBlur={() => setEditingAdvancedRunId(false)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingAdvancedRunId(false) } }} className="w-full px-3 py-2 text-sm bg-[#222222] border border-[#444444] rounded text-text-primary placeholder-text-muted focus:outline-none focus:border-accent font-mono lowercase" /> )}
{PHASE_ORDER.map((phase) => { const isSelected = advancedForm.fromPhase === phase const isDisabled = phase === "ingest" return ( ) })}

Will copy data up to this phase from source run, then execute this phase and subsequent phases

Source run settings

Provider:{" "} {selectedSourceRun?.provider}
Benchmark:{" "} {selectedSourceRun?.benchmark}
Judge:{" "} {canChangeJudgeModel ? ( ) : ( {form.judgeModel} )}
Answering:{" "} {canChangeAnsweringModel ? ( ) : ( {form.answeringModel} )}
{(editingJudgeModel || editingAnsweringModel) && (
{editingJudgeModel && canChangeJudgeModel && (
{ setForm({ ...form, judgeModel: value }) setEditingJudgeModel(false) }} placeholder="Select model" dropUp />
)} {editingAnsweringModel && canChangeAnsweringModel && (
{ setForm({ ...form, answeringModel: value }) setEditingAnsweringModel(false) }} placeholder="Select model" dropUp />
)}
)}
Concurrent requests{!showAdvancedConcurrencyAdvanced && ":"} {!showAdvancedConcurrencyAdvanced && (editingConcurrency ? ( setForm({ ...form, concurrency: { ...form.concurrency, default: e.target.value ? parseInt(e.target.value) : undefined, }, }) } onBlur={() => setEditingConcurrency(false)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingConcurrency(false) } }} min="1" /> ) : ( ))}
{showAdvancedConcurrencyAdvanced && (

Override source run concurrency settings

{(["ingest", "indexing", "search", "answer", "evaluate"] as const).map( (phase) => (
{phase}: {editingPhase === phase ? ( { phaseInputRefs.current[phase] = el }} type="number" className="w-16 px-2 py-0.5 text-sm bg-[#222222] border border-[#444444] rounded text-text-primary focus:outline-none focus:border-accent" value={form.concurrency[phase] ?? ""} onChange={(e) => setForm({ ...form, concurrency: { ...form.concurrency, [phase]: e.target.value ? parseInt(e.target.value) : undefined, }, }) } onBlur={() => setEditingPhase(null)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingPhase(null) } }} placeholder={String(form.concurrency.default ?? 1)} min="1" /> ) : ( )}
) )}
)}
)} )} {activeTab === "new" && ( <>
{!editingRunId ? ( ) : ( setForm({ ...form, runId: e.target.value })} onBlur={() => setEditingRunId(false)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingRunId(false) } }} className="w-full px-3 py-2 text-sm bg-[#222222] border border-[#444444] rounded text-text-primary placeholder-text-muted focus:outline-none focus:border-accent font-mono lowercase" /> )}
setForm({ ...form, provider: value })} placeholder="Select provider" />
setForm({ ...form, benchmark: value })} placeholder="Select benchmark" />
setForm({ ...form, judgeModel: value })} placeholder="Select model" />
setForm({ ...form, answeringModel: value })} placeholder="Select model" />
{(["full", "sample", "limit"] as SelectionMode[]).map((mode) => { const isSelected = form.selectionMode === mode const labels = { full: "Full", sample: "Sample", limit: "Limit" } return ( ) })}
{form.selectionMode === "sample" && (
setForm({ ...form, perCategory: e.target.value })} placeholder="2" min="1" /> per category
{(["consecutive", "random"] as SampleType[]).map((type) => { const isSelected = form.sampleType === type return ( ) })}
)} {form.selectionMode === "limit" && (
setForm({ ...form, limit: e.target.value })} placeholder="e.g. 100" min="1" />
)}
Concurrent requests{!showAdvancedConcurrencyNew && ":"} {!showAdvancedConcurrencyNew && (editingConcurrency ? ( setForm({ ...form, concurrency: { ...form.concurrency, default: e.target.value ? parseInt(e.target.value) : undefined, }, }) } onBlur={() => setEditingConcurrency(false)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingConcurrency(false) } }} min="1" /> ) : ( ))}
{showAdvancedConcurrencyNew && (

Process multiple items simultaneously for faster execution

{(["ingest", "indexing", "search", "answer", "evaluate"] as const).map( (phase) => (
{phase}: {editingPhase === phase ? ( { phaseInputRefs.current[phase] = el }} type="number" className="w-16 px-2 py-0.5 text-sm bg-[#222222] border border-[#444444] rounded text-text-primary focus:outline-none focus:border-accent" value={form.concurrency[phase] ?? ""} onChange={(e) => setForm({ ...form, concurrency: { ...form.concurrency, [phase]: e.target.value ? parseInt(e.target.value) : undefined, }, }) } onBlur={() => setEditingPhase(null)} onKeyDown={(e) => { if (e.key === "Enter" || e.key === "Escape") { setEditingPhase(null) } }} placeholder={String(form.concurrency.default ?? 1)} min="1" /> ) : ( )}
) )}
)}
)} {error && (
{error}
)}
Cancel
) }