import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { afterAll, afterEach, beforeEach, describe, expect, mock, test, } from "bun:test"; import { z } from "zod"; const WORKSPACE_DIR = process.env.VELLUM_WORKSPACE_DIR!; const CONFIG_PATH = join(WORKSPACE_DIR, "config.json"); function ensureTestDir(): void { const dirs = [ WORKSPACE_DIR, join(WORKSPACE_DIR, "data"), join(WORKSPACE_DIR, "data", "memory"), join(WORKSPACE_DIR, "data", "logs"), ]; for (const dir of dirs) { if (!existsSync(dir)) { mkdirSync(dir, { recursive: true }); } } } afterAll(() => { mock.restore(); }); import { VELLUM_MANAGED_CONNECTION_NAME } from "../../providers/vellum-model-routing.js"; import { getDefaultProvider, resolveDefaultConnectionName, setDefaultProvider, } from "../default-provider.js"; import { invalidateConfigCache, loadRawConfig } from "../loader.js"; import { getSchemaAtPath } from "../schema-utils.js"; import { DefaultProviderSchema, LLMSchema } from "../schemas/llm.js"; function readConfig(): Record { return JSON.parse(readFileSync(CONFIG_PATH, "utf-8")); } describe("LLMSchema.defaultProvider", () => { test("accepts a bare provider", () => { expect(() => LLMSchema.parse({ defaultProvider: { provider: "anthropic" } }), ).not.toThrow(); }); test("accepts a provider pinned to a connection", () => { const parsed = LLMSchema.parse({ defaultProvider: { provider: "vellum", connectionName: "x" }, }); expect(parsed.defaultProvider).toEqual({ provider: "vellum", connectionName: "x", }); }); // The strict write-side schema; the `.catch` only applies when reading // persisted config. test("rejects an unknown provider", () => { expect(() => DefaultProviderSchema.parse({ provider: "not-a-provider" }), ).toThrow(); }); test("rejects an empty connectionName", () => { expect(() => DefaultProviderSchema.parse({ provider: "anthropic", connectionName: "", }), ).toThrow(); }); test("rejects a missing provider", () => { expect(() => DefaultProviderSchema.parse({ connectionName: "x" }), ).toThrow(); }); test("existing configs without the field still parse", () => { const parsed = LLMSchema.parse({}); expect(parsed.defaultProvider).toBeUndefined(); }); // Guards the loader-recovery hazard described at `DefaultProviderField`: // a nested failure must never strand a `{ connectionName }` fragment. test("an invalid defaultProvider is dropped atomically", () => { const result = LLMSchema.safeParse({ profiles: { "my-profile": {} }, activeProfile: "my-profile", defaultProvider: { provider: "not-a-provider", connectionName: "x" }, }); expect(result.success).toBe(true); if (!result.success) { return; } expect(result.data.defaultProvider).toBeUndefined(); expect(result.data.activeProfile).toBe("my-profile"); }); test("a non-object defaultProvider is dropped, not fatal", () => { const result = LLMSchema.safeParse({ defaultProvider: "anthropic" }); expect(result.success).toBe(true); if (!result.success) { return; } expect(result.data.defaultProvider).toBeUndefined(); }); // The config-schema API must see through the field's wrappers to the // object shape (the introspection hazard described at `DefaultProviderField`). test("schema introspection reaches the object shape and provider enum", () => { const atField = getSchemaAtPath(LLMSchema, "defaultProvider"); expect(atField).not.toBeNull(); const atProvider = getSchemaAtPath(LLMSchema, "defaultProvider.provider"); expect(atProvider).not.toBeNull(); expect(atProvider?.safeParse("anthropic").success).toBe(true); expect(atProvider?.safeParse("not-a-provider").success).toBe(false); }); test("JSON Schema emission includes the field's object shape", () => { // Same options `handleGetConfigSchema` uses. const json = z.toJSONSchema(LLMSchema, { unrepresentable: "any", io: "input", }) as { properties?: Record }>; }; const field = json.properties?.defaultProvider; expect(field?.properties?.provider).toBeDefined(); }); }); describe("resolveDefaultConnectionName", () => { test("an explicit pin wins", () => { expect( resolveDefaultConnectionName({ provider: "anthropic", connectionName: "my-connection", }), ).toBe("my-connection"); }); test("vellum resolves to the managed connection name", () => { expect(resolveDefaultConnectionName({ provider: "vellum" })).toBe( VELLUM_MANAGED_CONNECTION_NAME, ); }); test("chatgpt resolves to the canonical subscription connection name", () => { expect(resolveDefaultConnectionName({ provider: "chatgpt" })).toBe( "chatgpt-subscription", ); }); test("every other provider resolves to its personal connection", () => { expect(resolveDefaultConnectionName({ provider: "anthropic" })).toBe( "anthropic-personal", ); expect(resolveDefaultConnectionName({ provider: "openai" })).toBe( "openai-personal", ); expect(resolveDefaultConnectionName({ provider: "gemini" })).toBe( "gemini-personal", ); expect(resolveDefaultConnectionName({ provider: "fireworks" })).toBe( "fireworks-personal", ); expect(resolveDefaultConnectionName({ provider: "openrouter" })).toBe( "openrouter-personal", ); }); test("an explicit pin wins even for vellum", () => { expect( resolveDefaultConnectionName({ provider: "vellum", connectionName: "pinned", }), ).toBe("pinned"); }); }); describe("getDefaultProvider / setDefaultProvider", () => { beforeEach(() => { ensureTestDir(); if (existsSync(CONFIG_PATH)) { rmSync(CONFIG_PATH, { force: true }); } invalidateConfigCache(); }); afterEach(() => { invalidateConfigCache(); }); test("getDefaultProvider returns null when absent", () => { expect(getDefaultProvider()).toBeNull(); }); test("set/get round-trip through the raw config", () => { setDefaultProvider({ provider: "openai", connectionName: "openai-work" }); expect(getDefaultProvider()).toEqual({ provider: "openai", connectionName: "openai-work", }); const raw = loadRawConfig(); const llm = raw.llm as Record; expect(llm.defaultProvider).toEqual({ provider: "openai", connectionName: "openai-work", }); }); test("setDefaultProvider validates the provider before writing", () => { expect(() => setDefaultProvider({ provider: "not-a-provider", }), ).toThrow(); expect(existsSync(CONFIG_PATH)).toBe(false); }); test("getDefaultProvider accepts a pre-loaded config without re-reading disk", () => { setDefaultProvider({ provider: "gemini" }); const config = { llm: { defaultProvider: { provider: "anthropic" } } }; // @ts-expect-error partial AssistantConfig stub for the pure-read path expect(getDefaultProvider(config)).toEqual({ provider: "anthropic" }); }); test("writes invalidate the config cache so getConfigReadOnly sees the new value", () => { setDefaultProvider({ provider: "fireworks" }); const after = readConfig(); expect((after.llm as Record).defaultProvider).toEqual({ provider: "fireworks", }); }); });