import { describe, expect, test } from "bun:test"; import { resolveRoutingIdentity } from "../../providers/connection-resolution.js"; import { resolveModelIntent } from "../../providers/model-intents.js"; import { CALL_SITE_DEFAULTS } from "../call-site-defaults.js"; import { CODE_DEFAULT_PROFILE_ENTRIES, getEffectiveProfile, getEffectiveProfiles, getEffectiveProfilesForProvider, PROFILE_IMPLS, resolveDefaultProfileForProvider, } from "../default-profile-catalog.js"; import { DEFAULT_PROFILE_KEYS, DEFAULT_PROFILE_PROVIDERS, OS_BETA_PROFILE_KEY, } from "../default-profile-names.js"; import { resolveCallSiteConfig, resolveDefaultProfileKey, } from "../llm-resolver.js"; import { type DefaultProviderConfig, type LLMCallSite, LLMCallSiteEnum, LLMSchema, type ProfileEntry, } from "../schemas/llm.js"; /** Thin managed-source stubs: a workspace that carries only ownership * markers for the defaults, no profile content. */ function managedStubs(): Record { return Object.fromEntries( DEFAULT_PROFILE_KEYS.map((key) => [key, { source: "managed" as const }]), ); } describe("getEffectiveProfiles", () => { test("the code catalog materializes a full body for every default profile", () => { for (const key of [...DEFAULT_PROFILE_KEYS, OS_BETA_PROFILE_KEY]) { const body = CODE_DEFAULT_PROFILE_ENTRIES[key]; expect(body).toBeDefined(); expect(typeof body.model).toBe("string"); expect(body.provider).toBe("vellum"); expect(body.provider_connection).toBeUndefined(); expect(body.source).toBe("managed"); } }); test("every vellum-column body translates to a managed dispatch target", () => { for (const key of [...DEFAULT_PROFILE_KEYS, OS_BETA_PROFILE_KEY]) { const body = CODE_DEFAULT_PROFILE_ENTRIES[key]; const identity = resolveRoutingIdentity(body.provider, body.model); expect(identity?.connectionName).toBe("vellum"); expect(typeof identity?.expectedProvider).toBe("string"); } }); test("the managed Balanced profile routes GLM 5.2 through Fireworks", () => { const balanced = CODE_DEFAULT_PROFILE_ENTRIES.balanced; expect(balanced.model).toBe("accounts/fireworks/models/glm-5p2"); expect(resolveRoutingIdentity(balanced.provider, balanced.model)).toEqual({ connectionName: "vellum", expectedProvider: "fireworks", }); }); test("the managed Quality profile routes GPT-5.6 Sol through OpenAI", () => { const quality = CODE_DEFAULT_PROFILE_ENTRIES["quality-optimized"]; expect(quality.model).toBe("gpt-5.6-sol"); expect(resolveRoutingIdentity(quality.provider, quality.model)).toEqual({ connectionName: "vellum", expectedProvider: "openai", }); }); test("defaults absent from the workspace resolve from the catalog; os-beta stays flag-gated", () => { const effective = getEffectiveProfiles(undefined); expect(Object.keys(effective).sort()).toEqual( [...DEFAULT_PROFILE_KEYS].sort(), ); expect(getEffectiveProfile({}, "balanced")?.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); expect(getEffectiveProfile({}, OS_BETA_PROFILE_KEY)).toBeUndefined(); }); test("a managed-source entry resolves to the code body with only label/status/topP from disk", () => { const workspace: Record = { balanced: { source: "managed", label: "Balanced (Managed)", status: "disabled", topP: 0.7, // Stale content drift on disk must lose to the code default body. provider: "anthropic", model: "claude-sonnet-4-6", maxTokens: 1, }, }; const entry = getEffectiveProfile(workspace, "balanced"); expect(entry).toBeDefined(); expect(entry?.label).toBe("Balanced (Managed)"); expect(entry?.status).toBe("disabled"); expect(entry?.topP).toBe(0.7); expect(entry?.model).toBe(CODE_DEFAULT_PROFILE_ENTRIES.balanced.model); expect(String(entry?.provider)).toBe( String(CODE_DEFAULT_PROFILE_ENTRIES.balanced.provider), ); expect(entry?.maxTokens).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.maxTokens, ); }); test("a user-owned profile sharing a default name wins verbatim", () => { const workspace: Record = { balanced: { source: "user", provider: "anthropic", model: "claude-sonnet-4-6", maxTokens: 1000, }, }; expect(getEffectiveProfile(workspace, "balanced")).toBe(workspace.balanced); }); test("custom profiles pass through untouched", () => { const workspace: Record = { ...managedStubs(), "my-custom": { provider: "anthropic", model: "claude-sonnet-4-6" }, }; const effective = getEffectiveProfiles(workspace); expect(effective["my-custom"]).toBe(workspace["my-custom"]); expect(effective.balanced.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); }); test("an injected catalog changes the effective view without any workspace change", () => { const workspace = managedStubs(); const stubCatalog: Record = { balanced: { ...CODE_DEFAULT_PROFILE_ENTRIES.balanced, provider: "anthropic", model: "claude-sonnet-4-6", }, }; const before = getEffectiveProfiles(workspace); const after = getEffectiveProfiles(workspace, stubCatalog); expect(before.balanced.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); expect(after.balanced.model).toBe("claude-sonnet-4-6"); }); }); describe("resolver integration", () => { test("resolution serves default-profile content from the code catalog, not the workspace body", () => { // The headline milestone test: with only thin managed stubs on disk, // resolution comes entirely from the code catalog — shipping a release // with a new catalog body changes resolution with no workspace // migration. const llm = LLMSchema.parse({ activeProfile: "balanced", profiles: managedStubs(), }); const resolved = resolveCallSiteConfig("mainAgent", llm); expect(resolved.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); expect(String(resolved.provider)).toBe("vellum"); expect(resolved.provider_connection).toBeUndefined(); }); test("an empty workspace resolves every call-site default from the catalog", () => { const llm = LLMSchema.parse({ activeProfile: "balanced" }); for (const [callSite, dflt] of Object.entries(CALL_SITE_DEFAULTS)) { if (dflt.profile == null) { continue; } const expected = CODE_DEFAULT_PROFILE_ENTRIES[dflt.profile]; const resolved = resolveCallSiteConfig(callSite as LLMCallSite, llm); // A site-level model pin legitimately overrides the profile's model. expect(resolved.model).toBe((dflt.model ?? expected.model) as string); } }); test("thin managed stubs and fully seeded bodies resolve identically at every call site", () => { const seededProfiles = Object.fromEntries( Object.entries(CODE_DEFAULT_PROFILE_ENTRIES).filter( ([name]) => name !== OS_BETA_PROFILE_KEY, ), ); const seeded = LLMSchema.parse({ activeProfile: "balanced", profiles: seededProfiles, }); const stubbed = LLMSchema.parse({ activeProfile: "balanced", profiles: managedStubs(), }); for (const callSite of Object.keys(CALL_SITE_DEFAULTS)) { expect(resolveCallSiteConfig(callSite as LLMCallSite, stubbed)).toEqual( resolveCallSiteConfig(callSite as LLMCallSite, seeded), ); } }); test("a disabled managed default does not divert resolution to the custom-* clone", () => { const llm = LLMSchema.parse({ profiles: { balanced: { source: "managed", status: "disabled" }, "custom-balanced": { source: "user", provider: "anthropic", model: "claude-sonnet-4-6", maxTokens: 1000, }, }, }); // The fallback anchor is the code-owned intent: a legacy disabled stub // does not suppress it, and the user-mutable custom-* clone never // captures the call site. expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("balanced"); const resolved = resolveCallSiteConfig("mainAgent", llm); expect(resolved.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); expect(resolved.model).not.toBe("claude-sonnet-4-6"); }); test("a user profile cannot shadow latency-optimized", () => { // `latency-optimized` is code-owned: it fronts every live-voice turn, so // a same-named workspace entry never governs what it resolves to. The // entry stays on disk and stays a valid `activeProfile` reference. const llm = LLMSchema.parse({ profiles: { "latency-optimized": { source: "user", provider: "anthropic", model: "claude-opus-4-6", provider_connection: "anthropic-personal", maxTokens: 32000, effort: "high", }, }, }); const body = CODE_DEFAULT_PROFILE_ENTRIES["latency-optimized"]!; for (const callSite of [ "voiceFrontDoor", "voiceProgressNarration", ] as const) { const resolved = resolveCallSiteConfig(callSite, llm); expect(resolved.model).toBe(body.model as string); expect(resolved.model).not.toBe("claude-opus-4-6"); expect(String(resolved.provider)).toBe("vellum"); } // Listing follows resolution: the catalog body is what surfaces. expect(getEffectiveProfiles(llm.profiles)["latency-optimized"]?.model).toBe( body.model, ); expect(() => LLMSchema.parse({ activeProfile: "latency-optimized", profiles: llm.profiles, }), ).not.toThrow(); }); }); describe("schema validation", () => { test("voiceFrontDoor is the only front callsite", () => { expect( LLMCallSiteEnum.options.filter((callSite) => callSite.includes("Front")), ).toEqual(["voiceFrontDoor"]); }); test("always-available default names are valid references; os-beta only when materialized", () => { expect(() => LLMSchema.parse({ activeProfile: "balanced" })).not.toThrow(); expect(() => LLMSchema.parse({ advisorProfile: "quality-optimized" }), ).not.toThrow(); expect(() => LLMSchema.parse({ callSites: { mainAgent: { profile: "cost-optimized" } }, }), ).not.toThrow(); // Flag-gated: valid only while its workspace stub exists. expect(() => LLMSchema.parse({ activeProfile: OS_BETA_PROFILE_KEY }), ).toThrow(); expect(() => LLMSchema.parse({ activeProfile: OS_BETA_PROFILE_KEY, profiles: { [OS_BETA_PROFILE_KEY]: { source: "managed" } }, }), ).not.toThrow(); expect(() => LLMSchema.parse({ callSites: { mainAgent: { profile: "no-such" } } }), ).toThrow(); }); test("defaultProvider accepts any default-capable API-key provider and drops the rest", () => { expect( LLMSchema.parse({ defaultProvider: { provider: "together" } }) .defaultProvider, ).toEqual({ provider: "together" }); // Endpoint-supplied and keyless providers have no code-resolvable // default profile implementation; the catch drops them atomically. for (const provider of ["litellm", "openai-compatible", "ollama"]) { expect( LLMSchema.parse({ defaultProvider: { provider } }).defaultProvider, ).toBeUndefined(); } }); }); describe("resolveDefaultProfileForProvider", () => { const dp = ( provider: DefaultProviderConfig["provider"], connectionName?: string, ) => ({ provider, ...(connectionName ? { connectionName } : {}) }); test("every matrix column materializes every default profile key", () => { for (const provider of DEFAULT_PROFILE_PROVIDERS) { for (const key of DEFAULT_PROFILE_KEYS) { const entry = resolveDefaultProfileForProvider( undefined, key, dp(provider), ); expect(entry).toBeDefined(); expect(typeof entry?.model).toBe("string"); expect(entry?.provider).toBeDefined(); // Identity columns stamp no connection; BYOK columns always do. if (entry?.provider === "vellum" || entry?.provider === "chatgpt") { expect(entry?.provider_connection).toBeUndefined(); } else { expect(entry?.provider_connection).toBeDefined(); } expect(entry?.source).toBe("managed"); } } }); test("the chatgpt column resolves Codex-pinned models with no connection stamp", () => { const byKey: Record = { balanced: "gpt-5.6-luna", "quality-optimized": "gpt-5.6-sol", "cost-optimized": "gpt-5.6-luna", "latency-optimized": "gpt-5.6-luna", }; const effective = getEffectiveProfilesForProvider(undefined, dp("chatgpt")); for (const [key, model] of Object.entries(byKey)) { const entry = effective[key]; expect(entry?.provider).toBe("chatgpt"); expect(entry?.model).toBe(model); expect(entry?.provider_connection).toBeUndefined(); expect(entry?.source).toBe("managed"); } // Cost and Speed both opt fully out of reasoning. expect(effective["cost-optimized"]?.effort).toBe("none"); expect(effective["latency-optimized"]?.effort).toBe("none"); expect(effective.balanced?.thinking?.enabled).toBe(true); }); test("a provider without a named matrix column materializes from the shared BYOK templates", () => { const entry = resolveDefaultProfileForProvider( undefined, "balanced", dp("together"), ); expect(entry?.provider).toBe("together"); expect(entry?.provider_connection).toBe("together-personal"); // No intent table for `together`: the intent falls back to the // provider's catalog defaultModel. expect(entry?.model).toBe(resolveModelIntent("together", "balanced")); expect(entry?.source).toBe("managed"); }); test("maxTokens clamps to the resolved model's catalog output cap", () => { // atlascloud has no intent table: every intent resolves to its catalog // defaultModel, which caps output at 8192, below the balanced template's // 16000. const balanced = resolveDefaultProfileForProvider( undefined, "balanced", dp("atlascloud"), ); expect(balanced?.maxTokens).toBe(8192); // minimax's defaultModel caps output at 16384, below the quality // template's 32000. const quality = resolveDefaultProfileForProvider( undefined, "quality-optimized", dp("minimax"), ); expect(quality?.maxTokens).toBe(16384); }); test("maxTokens stays at the template value when the model's cap allows it", () => { const entry = resolveDefaultProfileForProvider( undefined, "balanced", dp("anthropic"), ); expect(entry?.maxTokens).toBe(PROFILE_IMPLS.balanced.anthropic.maxTokens); }); test("BYOK columns resolve the intent to a provider-specific model and personal connection", () => { const entry = resolveDefaultProfileForProvider( undefined, "balanced", dp("anthropic"), ); expect(entry?.provider).toBe("anthropic"); expect(entry?.provider_connection).toBe("anthropic-personal"); expect(entry?.model).toBe(resolveModelIntent("anthropic", "balanced")); }); test("the vellum column keeps its underlying dispatch provider and managed connection", () => { const entry = resolveDefaultProfileForProvider( undefined, "balanced", dp("vellum"), ); // `vellum` is a routing identity: balanced dispatches through fireworks. expect(entry?.provider).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.provider, ); expect(entry?.provider_connection).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.provider_connection, ); expect(entry?.model).toBe( CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string, ); }); test("an explicit connectionName wins over the convention", () => { const entry = resolveDefaultProfileForProvider( undefined, "balanced", dp("openai", "work-openai"), ); expect(entry?.provider).toBe("openai"); expect(entry?.provider_connection).toBe("work-openai"); }); test("a user-source workspace entry shadows the provider-resolved default", () => { const workspace: Record = { balanced: { source: "user", provider: "openai", model: "gpt-5.5" }, }; const entry = resolveDefaultProfileForProvider( workspace, "balanced", dp("anthropic"), ); expect(entry).toBe(workspace.balanced); }); test("a managed-source stub contributes only label/status/topP over the provider-resolved body", () => { const workspace: Record = { balanced: { source: "managed", label: "Balanced (BYOK)", status: "disabled", topP: 0.7, model: "stale-model-should-be-ignored", }, }; const entry = resolveDefaultProfileForProvider( workspace, "balanced", dp("gemini"), ); expect(entry?.label).toBe("Balanced (BYOK)"); expect(entry?.status).toBe("disabled"); expect(entry?.topP).toBe(0.7); expect(entry?.provider).toBe("gemini"); expect(entry?.model).toBe(resolveModelIntent("gemini", "balanced")); expect(entry?.provider_connection).toBe("gemini-personal"); }); test("a null defaultProvider falls back to the vellum code bodies", () => { for (const key of DEFAULT_PROFILE_KEYS) { expect(resolveDefaultProfileForProvider(undefined, key, null)).toEqual( getEffectiveProfile(undefined, key) as ProfileEntry, ); } }); test("non-matrix names pass through like getEffectiveProfile", () => { const workspace: Record = { "custom-mine": { source: "user", provider: "openai", model: "gpt-5.4" }, }; expect( resolveDefaultProfileForProvider( workspace, "custom-mine", dp("anthropic"), ), ).toBe(workspace["custom-mine"]); expect( resolveDefaultProfileForProvider(workspace, "no-such", dp("anthropic")), ).toBeUndefined(); }); test("os-beta stays flag-gated and provider-independent", () => { expect( resolveDefaultProfileForProvider( undefined, OS_BETA_PROFILE_KEY, dp("anthropic"), ), ).toBeUndefined(); const workspace: Record = { [OS_BETA_PROFILE_KEY]: { source: "managed" }, }; const entry = resolveDefaultProfileForProvider( workspace, OS_BETA_PROFILE_KEY, dp("anthropic"), ); // The os-beta body never varies with the default provider. expect(entry?.provider).toBe( CODE_DEFAULT_PROFILE_ENTRIES[OS_BETA_PROFILE_KEY].provider, ); }); test("agrees with getEffectiveProfile for the vellum default provider", () => { for (const key of DEFAULT_PROFILE_KEYS) { expect( resolveDefaultProfileForProvider(undefined, key, dp("vellum")), ).toEqual(getEffectiveProfile(undefined, key) as ProfileEntry); } }); });