import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import { normalizeContext, type Api, type Model } from "@earendil-works/pi-ai"; import { spawn } from "node:child_process"; import { debug } from "./debug.ts"; import { parseClassifierStderr, formatDiagnostic } from "./diagnostics.ts"; import { readWindowsCredential } from "./windows-credential.ts"; // ── Classifier model — union type, no type-cast lies ───────── /** A model from pi's registry — full auth, provider support. */ interface RegistryClassifier { readonly kind: "registry"; readonly model: Model; } /** A direct HTTP endpoint — no auth, OpenAI-compatible only. */ interface EndpointClassifier { readonly kind: "endpoint"; readonly id: string; readonly baseUrl: string; } export interface JevClassifier { readonly kind: "jev"; readonly id: string; readonly provider: "typesafe" | "openrouter"; readonly model: string; readonly endpoint: string; readonly credentialTarget?: string; } /** Union: registry, raw OpenAI-compatible endpoint, or native Jev decision model. */ export type ClassifierModel = RegistryClassifier | EndpointClassifier | JevClassifier; function classifierBaseUrl(cm: ClassifierModel): string { if (cm.kind === "registry") return cm.model.baseUrl; return cm.kind === "jev" ? cm.endpoint : cm.baseUrl; } function classifierId(cm: ClassifierModel): string { return cm.kind === "registry" ? cm.model.id : cm.id; } function isOpenAiCompatibleEndpoint(cm: ClassifierModel): boolean { if (cm.kind === "jev") return false; if (cm.kind === "endpoint") return true; // endpoint config implies OpenAI-compatible const api = cm.model.api; return ( api === "openai-completions" || api === "openai-responses" || api === "openai-codex-responses" || api === "azure-openai-responses" || api === "mistral-conversations" ); } const DEFAULT_SYSTEM_PROMPT = "You are a routing classifier. Follow the requested output format exactly."; const DEFAULT_MAX_TOKENS = 8; const MAX_CLASSIFIER_PROMPT_CHARS = 8_000; export const DEFAULT_JEV_CREDENTIAL_TARGET = "pi-bifrost/jev-api-key"; export function jevClassifierForReference( reference: string, credentialTarget = DEFAULT_JEV_CREDENTIAL_TARGET, ): JevClassifier | undefined { if (reference.startsWith("typesafe/jev-")) { return { kind: "jev", id: reference, provider: "typesafe", model: reference.slice("typesafe/".length), endpoint: "https://api.typesafe.ai/v1/systemone", credentialTarget, }; } if (reference.startsWith("openrouter/~typesafe/jev-") || reference.startsWith("openrouter/typesafe/jev-")) { return { kind: "jev", id: reference, provider: "openrouter", model: reference.slice("openrouter/".length), endpoint: "https://openrouter.ai/api/alpha/decisions", }; } return undefined; } export function isJevModelReference(reference: string): boolean { return jevClassifierForReference(reference) !== undefined; } export type ClassifierAttempt = | { readonly status: "accepted"; readonly tier: string } | { readonly status: "rejected" } | { readonly status: "failed" }; export interface ClassifierOptions { systemPrompt?: string; maxTokens?: number; temperature?: number; method?: "direct" | "subprocess" | "auto"; tierDescriptions?: Record; signal?: AbortSignal; /** Minimum accepted confidence (0-1). Responses below this are treated as * a reject, so routing falls through to regex/fallback and the current * model stays stable. 0 = accept any confidently-tagged tier. */ confidenceThreshold?: number; /** Internal seam for deterministic Credential Manager tests. */ credentialReader?: (target: string) => Promise; } /** Parse `tier` or `tier ` without inventing missing confidence. */ export function parseClassification(text: string, categories: readonly string[]): { tier?: string; confidence?: number; } { const t = text.trim(); const confMatch = t.match(/(\d?\d(?:\.\d+)?)\s*%?$/u); let confidence: number | undefined; let body = t; if (confMatch) { const num = Number(confMatch[1]); const parsed = num > 1 ? num / 100 : num; if (parsed >= 0 && parsed <= 1) { confidence = parsed; body = t.slice(0, confMatch.index ?? 0).trim(); } } const tier = extractCategory(body, categories); return { tier, confidence }; } export function categoryLabel(category: string): string { return category; } function boundedPrompt(userPrompt: string): string { return userPrompt.length <= MAX_CLASSIFIER_PROMPT_CHARS ? userPrompt : `${userPrompt.slice(0, 4_000)}\n...[middle omitted]...\n${userPrompt.slice(-4_000)}`; } export function classificationPrompt( categories: readonly string[], userPrompt: string, tierDescriptions?: Record, requireConfidence = false, ): string { const categoryList = tierDescriptions ? categories.map((c) => { const desc = tierDescriptions[c]; return desc ? `${categoryLabel(c)}: ${desc}` : categoryLabel(c); }).join("\n") : categories.map(categoryLabel).join(", "); const request = boundedPrompt(userPrompt); const output = requireConfidence ? " " : ""; return ( `Categories:\n${categoryList}\n\n` + `Choose exactly one category. Output ${output}. No explanation.\n\n` + `Request: ${request}\n\n` + `Category:` ); } export function extractCategory(text: string, categories: readonly string[]): string | undefined { // Strip punctuation and whitespace — LLM may output "frontier." or "frontier\n". const needle = text.trim().toLowerCase().replace(/[^\p{L}\p{N}]+$/gu, "").replace(/^[^\p{L}\p{N}]+/gu, ""); return categories.find((cat) => cat.toLowerCase() === needle); } function piCommand(): { command: string; args: string[] } { // Reuse the current Node binary and script path for subprocess. // Falls back to bare "pi" if argv[1] is unavailable (e.g. bundled executable). const script = process.argv[1]; if (script) return { command: process.execPath, args: [script] }; return { command: "pi", args: [] }; } export function parseAttempt( content: string, categories: readonly string[], confidenceThreshold: number, ): ClassifierAttempt { const parsed = parseClassification(content, categories); if (!parsed.tier) return { status: "rejected" }; if (confidenceThreshold > 0 && (parsed.confidence === undefined || parsed.confidence < confidenceThreshold)) { return { status: "rejected" }; } return { status: "accepted", tier: parsed.tier }; } interface DirectClassification { outcome: ClassifierAttempt; /** True once a provider/endpoint accepted a direct attempt. */ attempted: boolean; /** Extra judgments from Jev speculative fan-out: isTrivial noul, effort score. */ jevExtras?: { isTrivial?: number; effort?: number; }; } export function jevRequest( classifierModel: JevClassifier, categories: readonly string[], prompt: string, tierDescriptions?: Record, systemPrompt?: string, ): object { return { state: boundedPrompt(prompt), model: classifierModel.model, questions: { category: { type: "choice", instructions: { task: "Choose the single pi-bifrost routing category that best fits the request.", priority: "Classify by the hardest and most consequential requested work.", ...(systemPrompt ? { additional_policy: systemPrompt } : {}), }, criteria: Object.fromEntries( categories.map((category) => [category, tierDescriptions?.[category] ?? null]), ), }, isTrivial: { type: "noul", instructions: "The request can be fully satisfied with a one-line command, a simple lookup, or a single trivial file change.", }, effort: { type: "score", instructions: "How much cognitive reasoning depth does this task require?", criteria: [ "Simple search, lookup, or trivial one-line change", "Standard implementation, debugging, or refactoring", "Complex multi-file architecture, distributed systems, or security analysis", ], }, }, }; } async function classifyWithJev( ctx: ExtensionContext, classifierModel: JevClassifier, categories: readonly string[], prompt: string, options: ClassifierOptions, ): Promise { try { const providerAuth = classifierModel.provider === "openrouter" ? await ctx.modelRegistry.getProviderAuth("openrouter") : undefined; const apiKey = classifierModel.provider === "typesafe" ? await (options.credentialReader ?? readWindowsCredential)( classifierModel.credentialTarget ?? DEFAULT_JEV_CREDENTIAL_TARGET, ) : providerAuth?.auth.apiKey; const authHeaders = providerAuth?.auth.headers ?? {}; if (!apiKey && !("Authorization" in authHeaders) && !("authorization" in authHeaders)) { return { attempted: false, outcome: { status: "failed" } }; } const headers: Record = { "Content-Type": "application/json", ...authHeaders }; if (apiKey && !("Authorization" in headers) && !("authorization" in headers)) { headers.Authorization = `Bearer ${apiKey}`; } const response = await fetch(classifierModel.endpoint, { method: "POST", headers, body: JSON.stringify(jevRequest( classifierModel, categories, prompt, options.tierDescriptions, options.systemPrompt, )), signal: options.signal ?? ctx.signal ?? AbortSignal.timeout(30_000), }); if (!response.ok) { console.error(`[bifrost] classifier HTTP ${response.status} from ${classifierModel.endpoint}`); return { attempted: true, outcome: { status: "failed" } }; } const data = (await response.json()) as { answers?: { category?: { choice?: string; confidence?: number }; isTrivial?: { noul?: number }; effort?: { score?: number }; }; }; const answer = data.answers?.category; if (!answer?.choice) return { attempted: true, outcome: { status: "failed" } }; const tier = extractCategory(answer.choice, categories); const threshold = options.confidenceThreshold ?? 0; const outcome: ClassifierAttempt = !tier || (threshold > 0 && (answer.confidence === undefined || answer.confidence < threshold)) ? { status: "rejected" } : { status: "accepted", tier }; const isTrivialNoul = data.answers?.isTrivial?.noul; const effortScore = data.answers?.effort?.score; debug("classifier", "jev.done", { model: classifierModel.id, tier, confidence: answer.confidence, threshold, status: outcome.status, isTrivial: isTrivialNoul, effort: effortScore, }); return { attempted: true, outcome, ...(isTrivialNoul !== undefined || effortScore !== undefined ? { jevExtras: { ...(isTrivialNoul !== undefined ? { isTrivial: isTrivialNoul } : {}), ...(effortScore !== undefined ? { effort: effortScore } : {}), }, } : {}), }; } catch { return { attempted: true, outcome: { status: "failed" } }; } } async function classifyWithDirectHttp( ctx: ExtensionContext, classifierModel: ClassifierModel, categories: readonly string[], prompt: string, options: ClassifierOptions = {}, ): Promise { const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS; const temperature = options.temperature ?? 0; const threshold = options.confidenceThreshold ?? 0; const userPrompt = classificationPrompt(categories, prompt, options.tierDescriptions, threshold > 0); if (classifierModel.kind === "jev") { return classifyWithJev(ctx, classifierModel, categories, prompt, options); } if (classifierModel.kind === "registry") { try { const provider = ctx.modelRegistry.getProvider(classifierModel.model.provider); if (!provider) return { attempted: false, outcome: { status: "failed" } }; // Pre-check auth so an unconfigured provider keeps the attempted:false // semantics (subprocess fallback) instead of surfacing as a failed attempt. const auth = await ctx.modelRegistry.getProviderAuth(classifierModel.model.provider); if (!auth) return { attempted: false, outcome: { status: "failed" } }; const stream = ctx.modelRegistry.streamSimple( classifierModel.model, normalizeContext({ systemPrompt, messages: [{ role: "user", content: userPrompt, timestamp: Date.now() }], }), { maxTokens, temperature, // Omitted reasoning disables thinking; models that cannot disable it get the lowest level. ...(classifierModel.model.thinkingLevelMap?.off === null ? { reasoning: "minimal" as const } : {}), signal: options.signal ?? ctx.signal, cacheRetention: "none", }, ); try { const response = await stream.result(); const content = response.content .filter((c: { type: string; text?: string }): c is { type: "text"; text: string } => c.type === "text") .map((c: { text: string }) => c.text) .join("\n") .trim(); if (!content) { debug("classifier", "registry.empty_response", { model: classifierId(classifierModel) }); return { attempted: true, outcome: { status: "failed" } }; } const parsed = parseClassification(content, categories); const outcome = parseAttempt(content, categories, threshold); debug("classifier", "registry.done", { model: classifierId(classifierModel), raw: content.slice(0, 100), tier: parsed.tier, confidence: parsed.confidence, threshold, status: outcome.status, }); return { attempted: true, outcome }; } catch { return { attempted: true, outcome: { status: "failed" } }; } } catch { return { attempted: false, outcome: { status: "failed" } }; } } if (!isOpenAiCompatibleEndpoint(classifierModel)) { return { attempted: false, outcome: { status: "failed" } }; } const body = { model: classifierId(classifierModel), messages: [ { role: "system", content: systemPrompt }, { role: "user", content: userPrompt }, ], max_tokens: maxTokens, temperature: temperature, stream: false, }; const base = classifierBaseUrl(classifierModel); const baseUrl = base.endsWith("/") ? base : `${base}/`; const url = new URL("chat/completions", baseUrl).toString(); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: options.signal ?? ctx.signal ?? AbortSignal.timeout(30_000), }); if (!response.ok) { console.error( `[bifrost] classifier HTTP ${response.status} from ${classifierBaseUrl(classifierModel)}`, ); return { attempted: true, outcome: { status: "failed" } }; } const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }>; }; const content = data.choices?.[0]?.message?.content?.trim(); if (!content) { debug("classifier", "http.empty_response", { url: classifierBaseUrl(classifierModel) }); return { attempted: true, outcome: { status: "failed" } }; } const parsed = parseClassification(content, categories); const outcome = parseAttempt(content, categories, threshold); debug("classifier", "http.done", { model: classifierId(classifierModel), raw: content.slice(0, 100), tier: parsed.tier, confidence: parsed.confidence, threshold, status: outcome.status, }); return { attempted: true, outcome }; } catch { return { attempted: true, outcome: { status: "failed" } }; } } async function classifyWithSubprocess( _ctx: ExtensionContext, classifierModel: ClassifierModel, categories: readonly string[], prompt: string, options: ClassifierOptions = {}, ): Promise { // Subprocess only works with registry models (needs provider/id for --model). if (classifierModel.kind !== "registry") return { status: "failed" }; const model = classifierModel.model; debug("classifier", "subprocess.start", { model: `${model.provider}/${model.id}` }); const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; const threshold = options.confidenceThreshold ?? 0; const userPrompt = classificationPrompt(categories, prompt, options.tierDescriptions, threshold > 0); const { command, args } = piCommand(); const piArgs = [ ...args, "--no-extensions", "--no-prompt-templates", "--no-context-files", "--no-approve", "--no-session", "--print", "--system-prompt", systemPrompt, "-p", userPrompt, "--model", `${model.provider}/${model.id}`, "--thinking", "off", ]; return new Promise((resolve) => { const child = spawn(command, piArgs, { stdio: ["ignore", "pipe", "pipe"], env: { PATH: process.env.PATH, HOME: process.env.HOME, USERPROFILE: process.env.USERPROFILE, TEMP: process.env.TEMP, TMPDIR: process.env.TMPDIR, }, }); let stdout = ""; let stderr = ""; let settled = false; const finish = (result: ClassifierAttempt) => { if (settled) return; settled = true; clearTimeout(timer); options.signal?.removeEventListener("abort", abort); resolve(result); }; const abort = () => { child.kill("SIGTERM"); finish({ status: "failed" }); }; const timer = setTimeout(() => { console.error(`[bifrost] classifier subprocess timed out`); abort(); }, 30_000); const MAX_CHUNK = 2000; child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => { if (stdout.length < MAX_CHUNK) stdout += chunk; }); child.stderr.on("data", (chunk: string) => { if (stderr.length < MAX_CHUNK) stderr += chunk; }); options.signal?.addEventListener("abort", abort, { once: true }); if (options.signal?.aborted) abort(); child.on("error", (err: Error) => { console.error(`[bifrost] classifier subprocess error: ${err}`); finish({ status: "failed" }); }); child.on("close", (code: number | null) => { if (settled) return; if (code !== 0) { const modelId = `${model.provider}/${model.id}`; const diagnostic = parseClassifierStderr(stderr, modelId, code); debug("classifier", "subprocess.error", { model: modelId, exitCode: code, diagnosticCode: diagnostic.code, stderr: stderr.slice(0, 200), }); console.error(`[bifrost] ${formatDiagnostic(diagnostic)}`); finish({ status: "failed" }); return; } const parsed = parseClassification(stdout, categories); const outcome = parseAttempt(stdout, categories, threshold); debug("classifier", "subprocess.done", { model: `${model.provider}/${model.id}`, raw: stdout.trim().slice(0, 100), tier: parsed.tier, confidence: parsed.confidence, threshold, status: outcome.status, }); finish(outcome); }); }); } export async function classifyWithLLM( ctx: ExtensionContext, classifierModel: ClassifierModel, categories: readonly string[], prompt: string, options: ClassifierOptions = {}, ): Promise { const method = options.method ?? "auto"; if (method === "direct" || method === "auto") { const direct = await classifyWithDirectHttp( ctx, classifierModel, categories, prompt, options, ); // A completed direct request must not be repeated through a subprocess. if (direct.attempted) return { ...direct.outcome, jevExtras: direct.jevExtras }; } if (method === "subprocess" || method === "auto") { return classifyWithSubprocess(ctx, classifierModel, categories, prompt, options); } return { status: "failed" }; }