import { useMemo, useState, useEffect } from "react"; import { Gauge, Loader2, Zap, Check, X, RotateCcw, Download, Plus, ListPlus } from "lucide-react"; import { useTranslation } from "@/lib/i18n"; import { useConfigStore } from "@/store/config-store"; import { cn } from "@/lib/utils"; import type { Provider, Model } from "@/types"; // Model returned by /api/pi/provider-models. Kept local to this page — these // are stored separately from the provider's configured/enabled models. interface FetchedModel { id: string; name?: string; contextWindow?: number; maxTokens?: number; reasoning?: boolean; vision?: boolean; audio?: boolean; cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number }; source?: string; } // Per-model speed-test result. interface ModelResult { status: "idle" | "testing" | "done"; runs: number; success: number; latencies: number[]; lastMessage?: string; } const RUNS_PER_MODEL = 3; // sequential calls per model to derive a rate // Two speed profiles. Slow mode spaces requests out and retries harder to // avoid tripping upstream rate limits (HTTP 429). const SPEED_PROFILES = { normal: { betweenCalls: 600, betweenModels: 800, maxRetries: 2, backoff: 3000 }, slow: { betweenCalls: 2000, betweenModels: 4000, maxRetries: 4, backoff: 6000 }, } as const; type SpeedMode = keyof typeof SPEED_PROFILES; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); // LocalStorage key: the speed-test model catalog is kept entirely separate // from the app's configured models (models.json / enabledModels). const STORE_KEY = "speedtest:model-catalog"; function avg(nums: number[]): number { if (nums.length === 0) return 0; return Math.round(nums.reduce((a, b) => a + b, 0) / nums.length); } // Load/save the fetched-model catalog (map of providerId → FetchedModel[]). function loadCatalog(): Record { try { const raw = localStorage.getItem(STORE_KEY); return raw ? JSON.parse(raw) : {}; } catch { return {}; } } function saveCatalog(catalog: Record) { try { localStorage.setItem(STORE_KEY, JSON.stringify(catalog)); } catch { /* ignore quota errors */ } } // LocalStorage key: per-provider speed-test results, so they survive // navigating away and back (the route unmounts this page). const RESULTS_KEY = "speedtest:model-results"; // LocalStorage key: last selected provider on this page. const LAST_PROVIDER_KEY = "speedtest:last-provider"; type AllResults = Record>; function loadResults(): AllResults { try { const raw = localStorage.getItem(RESULTS_KEY); if (!raw) return {}; const all = JSON.parse(raw) as AllResults; // Drop entries stuck in "testing" (page left mid-run) — they never finished. for (const pid of Object.keys(all)) { const kept = Object.fromEntries( Object.entries(all[pid] ?? {}).filter(([, r]) => r.status === "done") ); if (Object.keys(kept).length === 0) delete all[pid]; else all[pid] = kept; } return all; } catch { return {}; } } function saveResults(all: AllResults) { try { localStorage.setItem(RESULTS_KEY, JSON.stringify(all)); } catch { /* ignore quota errors */ } } export function ModelSpeedTestPage() { const { t } = useTranslation(); const { allProviders, auth } = useConfigStore(); // Resolve a usable API key for a provider (models.json apiKey or auth.json). const keyOf = (p: Provider) => p.apiKey ?? auth?.[p.id]?.key ?? ""; // Custom providers only (built-in providers are excluded from speed tests). // A provider is listed once it has an endpoint — models come from the // locally-stored speed-test catalog, not the configured model list. const testableProviders = useMemo( () => allProviders.filter((p) => p.type === "custom" && (p.baseUrl ?? "").trim() !== ""), [allProviders] ); const [selectedId, setSelectedId] = useState( () => localStorage.getItem(LAST_PROVIDER_KEY) ?? testableProviders[0]?.id ?? null ); const selected = testableProviders.find((p) => p.id === selectedId) ?? testableProviders[0] ?? null; useEffect(() => { if (selected?.id) localStorage.setItem(LAST_PROVIDER_KEY, selected.id); }, [selected?.id]); // Speed-test model catalog, persisted in localStorage. const [catalog, setCatalog] = useState>({}); useEffect(() => { setCatalog(loadCatalog()); }, []); const models = selected ? (catalog[selected.id] ?? []) : []; // Results persisted per provider so they survive route changes; the map // below is the current provider's slice. const [allResults, setAllResults] = useState(() => loadResults()); useEffect(() => { saveResults(allResults); }, [allResults]); const results = useMemo( () => new Map(Object.entries(allResults[selected?.id ?? ""] ?? {})), [allResults, selected] ); // setResults scoped to the selected provider (drop-in for the old state setter). const setResults = ( next: Map | ((prev: Map) => Map) ) => { const pid = selected?.id ?? ""; setAllResults((prevAll) => { const prevMap = new Map(Object.entries(prevAll[pid] ?? {})); const nextMap = typeof next === "function" ? next(prevMap) : next; return { ...prevAll, [pid]: Object.fromEntries(nextMap) }; }); }; const [running, setRunning] = useState(false); const [speedMode, setSpeedMode] = useState("normal"); const [fetching, setFetching] = useState(false); const [fetchError, setFetchError] = useState(null); const [fetchInfo, setFetchInfo] = useState(null); // Add-to-provider support: models already configured under this provider. const { addModel } = useConfigStore(); const configuredIds = useMemo( () => new Set((selected?.models ?? []).map((m) => m.id)), [selected] ); const toModelDef = (m: FetchedModel): Model => { const input: Model["input"] = ["text"]; if (m.vision) input.push("image"); if (m.audio) input.push("audio"); return { id: m.id, name: m.name, reasoning: m.reasoning ?? false, input, contextWindow: m.contextWindow ?? 262144, maxTokens: m.maxTokens ?? 32768, cost: m.cost ? { input: m.cost.input ?? 0, output: m.cost.output ?? 0, cacheRead: m.cost.cacheRead ?? 0, cacheWrite: m.cost.cacheWrite ?? 0, } : { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, }; }; // Models that passed 100% and are not configured yet — the batch-add set. const passedNew = useMemo( () => models.filter((m) => { const r = results.get(m.id); return !!r && r.runs > 0 && r.success === r.runs && !configuredIds.has(m.id); }), [models, results, configuredIds] ); const addToProvider = async (m: FetchedModel) => { if (!selected || configuredIds.has(m.id)) return; // Added disabled by default — the user enables it on the providers page. addModel(selected.id, toModelDef(m)); }; const addAllPassed = async () => { if (!selected || running || passedNew.length === 0) return; passedNew.forEach((m) => addModel(selected.id, toModelDef(m))); }; const setResult = (modelId: string, patch: Partial) => { setResults((prev) => { const next = new Map(prev); const cur = next.get(modelId) ?? { status: "idle", runs: 0, success: 0, latencies: [] }; next.set(modelId, { ...cur, ...patch }); return next; }); }; // One-click: fetch all models from the provider endpoint, store locally. const fetchModels = async () => { if (!selected || fetching) return; setFetching(true); setFetchError(null); setFetchInfo(null); try { const res = await fetch("/api/pi/provider-models", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ baseUrl: (selected.baseUrl ?? "").trim(), apiKey: keyOf(selected), providerId: selected.id, }), }); const data = await res.json(); if (data.error) { setFetchError(data.error); return; } const fetched: FetchedModel[] = data.models ?? []; const next = { ...loadCatalog(), [selected.id]: fetched }; saveCatalog(next); setCatalog(next); setResults(new Map()); // stale results no longer match the new list setFetchInfo(t("speed_test.fetched_count", String(fetched.length))); } catch { setFetchError(t("speed_test.fetch_failed")); } finally { setFetching(false); } }; const clearModels = () => { if (!selected || fetching || running) return; const next = { ...loadCatalog() }; delete next[selected.id]; saveCatalog(next); setCatalog(next); setResults(new Map()); setFetchInfo(null); }; const testModelOnce = async (provider: Provider, model: FetchedModel) => { const res = await fetch("/api/pi/model-test", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ baseUrl: (provider.baseUrl ?? "").trim(), modelId: model.id, apiKey: keyOf(provider), apiType: provider.api ?? undefined, }), }); return res.json() as Promise<{ success: boolean; latencyMs?: number; message?: string; status?: number }>; }; // Returns true when a result indicates a rate-limit response. const isRateLimited = (r: { status?: number; message?: string }) => r.status === 429 || /\b429\b|rate.?limit|too many/i.test(r.message ?? ""); const testOneModel = async (provider: Provider, model: FetchedModel) => { const profile = SPEED_PROFILES[speedMode]; setResults((prev) => { const next = new Map(prev); next.set(model.id, { status: "testing", runs: 0, success: 0, latencies: [] }); return next; }); let success = 0; const latencies: number[] = []; let lastMessage: string | undefined; for (let i = 0; i < RUNS_PER_MODEL; i++) { if (i > 0) await sleep(profile.betweenCalls); let data: { success: boolean; latencyMs?: number; message?: string; status?: number }; let attempt = 0; // Retry with backoff specifically on 429 so bursty limits recover. // eslint-disable-next-line no-constant-condition while (true) { try { data = await testModelOnce(provider, model); } catch { data = { success: false, message: "network error" }; } if (data.success || !isRateLimited(data) || attempt >= profile.maxRetries) break; attempt++; await sleep(profile.backoff * attempt); } if (data!.success) { success++; if (typeof data!.latencyMs === "number") latencies.push(data!.latencyMs); } else { lastMessage = data!.message; } setResult(model.id, { status: "testing", runs: i + 1, success, latencies, lastMessage }); } setResult(model.id, { status: "done", runs: RUNS_PER_MODEL, success, latencies, lastMessage }); }; const runAll = async () => { if (!selected || running || models.length === 0) return; const profile = SPEED_PROFILES[speedMode]; setRunning(true); setResults(new Map()); try { for (let i = 0; i < models.length; i++) { if (i > 0) await sleep(profile.betweenModels); const model = models[i]; if (model) await testOneModel(selected, model); } } finally { setRunning(false); } }; const resetResults = () => { if (running) return; setResults(new Map()); }; return (
MODEL BENCHMARK // LATENCY MATRIX

{t("speed_test.title")}

{t("speed_test.subtitle")}

{t("speed_test.runs_note", String(RUNS_PER_MODEL))}
{testableProviders.length === 0 ? (

{t("speed_test.no_provider")}

{t("speed_test.no_provider_desc")}

) : (
{/* Left: provider picker */}

{t("speed_test.providers")} ({testableProviders.length})

{testableProviders.map((p) => { const count = (catalog[p.id] ?? []).length; return ( ); })}
{/* Right: fetch + speed results */}
{selected && ( <>

{selected.name}

{models.length} {t("speed_test.models")}

{fetchError && (
{fetchError}
)} {fetchInfo && !fetchError && (
{fetchInfo}
)} {models.length === 0 ? (

{t("speed_test.empty_catalog")}

{t("speed_test.empty_catalog_desc")}

) : (
{models.map((m) => { const r = results.get(m.id); const rate = r && r.runs > 0 ? Math.round((r.success / r.runs) * 100) : null; const avgMs = r ? avg(r.latencies) : 0; const minMs = r && r.latencies.length ? Math.min(...r.latencies) : 0; const maxMs = r && r.latencies.length ? Math.max(...r.latencies) : 0; return ( ); })}
{t("speed_test.col_model")} {t("speed_test.col_success_rate")} {t("speed_test.col_avg_latency")} {t("speed_test.col_range")} {t("speed_test.col_status")} {t("speed_test.col_action")}
{m.id} {rate === null ? ( ) : ( = 100 ? "text-emerald-400" : rate > 0 ? "text-amber-400" : "text-red-400" )}> {rate}% ({r!.success}/{r!.runs}) )} {avgMs > 0 ? `${avgMs} ms` : } {minMs > 0 ? `${minMs}–${maxMs}` : "—"} {!r || r.status === "idle" ? ( {t("speed_test.pending")} ) : r.status === "testing" ? ( {t("speed_test.testing")} {r.runs}/{RUNS_PER_MODEL} ) : r.success === r.runs ? ( {t("speed_test.ok")} ) : r.success > 0 ? ( {t("speed_test.partial")} ) : ( {r.lastMessage ? r.lastMessage.slice(0, 40) : t("speed_test.fail")} )} {rate === 100 && (configuredIds.has(m.id) ? ( {t("speed_test.exists")} ) : ( ))}
)} )}
)}
); }