{
  "version": 1,
  "description": "Managed model candidates for ThumbGate workload benchmarking. Catalog only: no provider-specific runtime dependency is assumed here.",
  "workloads": {
    "pretool-gating": {
      "label": "PreTool gating",
      "summary": "Fast, reliable gate judgments for tool-use and agentic coding decisions before commands run. Hybrid local-cloud candidates (e.g. perplexity/hybrid-local) excel here for privacy + low latency on sensitive paths.",
      "desiredStrengths": ["agentic-coding", "tool-use", "reliability", "privacy", "fast-inference"],
      "targetContextWindow": 64000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run",
        "npx thumbgate bench --json --min-score=90"
      ],
      "metrics": [
        "passRate",
        "falsePositiveRate",
        "falseNegativeRate",
        "medianLatencyMs",
        "costPer1kActionsUsd"
      ]
    },
    "long-trace-review": {
      "label": "Long trace review",
      "summary": "Review long agent traces, multi-step failures, and large-context coding sessions without dropping important detail.",
      "desiredStrengths": ["long-horizon-coding", "multi-agent", "reliability", "long-context"],
      "targetContextWindow": 128000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run",
        "npx thumbgate bench --json --min-score=90",
        "npx thumbgate deepseek-v4-runtime-guardrails --context-tokens=900000 --hybrid-attention --speculative-decoding --json"
      ],
      "metrics": [
        "passRate",
        "longContextReliability",
        "traceCompressionLoss",
        "cacheCoherencePassRate",
        "speculativeAcceptLength",
        "medianLatencyMs",
        "costPerTraceUsd"
      ]
    },
    "cheap-fast-path": {
      "label": "Cheap fast path",
      "summary": "Low-cost first-pass model for cheap approval triage before escalating ambiguous work. Perplexity hybrid-local is ideal: on-device for speed/privacy, escalate only when needed via orchestrator.",
      "desiredStrengths": ["agentic-coding", "tool-use", "fast-inference", "privacy", "cost-efficiency"],
      "targetContextWindow": 32000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run",
        "npx thumbgate bench --json --min-score=90"
      ],
      "metrics": [
        "passRate",
        "medianLatencyMs",
        "costPer1kActionsUsd",
        "escalationRate"
      ]
    },
    "dashboard-analysis": {
      "label": "Dashboard and dataset analysis",
      "summary": "Evaluate frontier models for dataset analysis, chart generation, dashboard planning, and proof-backed insight quality before routing expensive analytical work. Perplexity hybrid excels for sensitive lessons/feedback data (local for privacy, cloud for depth).",
      "desiredStrengths": ["data-analysis", "dashboard-creation", "charting", "long-context", "reliability", "privacy"],
      "targetContextWindow": 200000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run",
        "npx thumbgate bench --json --min-score=90"
      ],
      "metrics": [
        "insightAccuracy",
        "chartSpecValidity",
        "dashboardCompleteness",
        "longContextReliability",
        "medianLatencyMs",
        "costPerAnalysisUsd"
      ]
    },
    "claw-style-enterprise-agent": {
      "label": "Claw-style enterprise agent governance",
      "summary": "Governance, gating, and feedback for autonomous 'claw-style' agents (Automation Anywhere EnterpriseClaw, Nvidia OpenShell-inspired) that have device file system access, runtime dynamic tool creation, screen/UI interaction, and multi-platform orchestration. Especially relevant for on-prem/air-gapped/hybrid enterprise data realities.",
      "desiredStrengths": ["agentic-coding", "tool-use", "reliability", "security", "orchestration", "audit-trail", "privacy"],
      "targetContextWindow": 128000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run",
        "npx thumbgate bench --json --min-score=90"
      ],
      "metrics": [
        "passRate",
        "falsePositiveRate",
        "agentIdentitySeparation",
        "dynamicToolSafety",
        "screenInteractionAudit",
        "orchestrationCompliance",
        "medianLatencyMs",
        "costPer1kActionsUsd"
      ]
    },
    "tokenizer-brittleness": {
      "label": "Tokenizer brittleness and byte-level robustness",
      "summary": "Evaluate models for malformed JSONL, Unicode confusables, stack traces, secrets, SQL snippets, file paths, and code-symbol-heavy inputs before routing log, code, or security workloads.",
      "desiredStrengths": ["tokenizer-free", "byte-level", "log-robustness", "code-symbols", "security-scanning", "fast-inference"],
      "targetContextWindow": 64000,
      "benchmarkCommands": [
        "npx thumbgate model-candidates --workload=tokenizer-brittleness --json",
        "node --test tests/model-candidates.test.js --test-name-pattern=tokenizer",
        "node scripts/gate-eval.js run"
      ],
      "metrics": [
        "caseCoverage",
        "symbolPreservationRate",
        "secretDetectionRecall",
        "jsonlRepairAccuracy",
        "medianLatencyMs",
        "memoryBandwidthEstimate"
      ]
    },
    "self-improving-agent-skill-synthesis": {
      "label": "Self-improving agent skill synthesis",
      "summary": "Evaluate models on their ability to self-improve, generate safe Markdown skills, and correctly check synthesized skills against existing gate/prevention rules.",
      "desiredStrengths": ["agentic-coding", "tool-use", "self-evolving", "skill-synthesis", "reliability"],
      "targetContextWindow": 128000,
      "benchmarkCommands": [
        "npx thumbgate eval --from-feedback --json --min-score=0",
        "node scripts/gate-eval.js run"
      ],
      "metrics": [
        "passRate",
        "skillSafetyCheckAccuracy",
        "ruleBypassDetectionRate",
        "medianLatencyMs",
        "costPerEvolutionUsd"
      ]
    }
  },
  "candidates": [
    {
      "id": "research/fast-byte-latent-transformer",
      "vendor": "Meta + Stanford + University of Washington",
      "family": "blt",
      "provider": "research",
      "model": "fast-byte-latent-transformer",
      "contextWindow": 64000,
      "costClass": "medium",
      "researchOnly": true,
      "strengths": ["tokenizer-free", "byte-level", "log-robustness", "code-symbols", "security-scanning", "fast-inference"],
      "notes": "Research-only candidate inspired by Fast BLT. Use as an evaluation target for tokenizer-free robustness and memory-bandwidth planning; do not route production traffic until a maintained runtime and model weights exist."
    },
    {
      "id": "self-hosted/deepseek-v4-flash-sglang",
      "vendor": "DeepSeek",
      "family": "deepseek",
      "provider": "self-hosted",
      "gateway": "sglang",
      "model": "deepseek-v4-flash",
      "contextWindow": 1000000,
      "costClass": "medium",
      "strengths": ["long-context", "fast-inference", "reliability", "long-horizon-coding"],
      "notes": "Self-hosted long-context candidate for teams that can operate SGLang-class sparse-attention serving. Requires ThumbGate runtime guardrails before routing production traces."
    },
    {
      "id": "self-hosted/deepseek-v4-pro-sglang",
      "vendor": "DeepSeek",
      "family": "deepseek",
      "provider": "self-hosted",
      "gateway": "sglang",
      "model": "deepseek-v4-pro",
      "contextWindow": 1000000,
      "costClass": "high",
      "strengths": ["long-context", "reliability", "long-horizon-coding", "multi-agent"],
      "notes": "High-capacity self-hosted candidate for long-trace review and verified-RL experiments. Benchmark cache coherence, speculative decoding, KV offload, and train-inference drift before use."
    },
    {
      "id": "openai/gpt-5.5",
      "vendor": "OpenAI",
      "family": "gpt",
      "provider": "openai",
      "model": "gpt-5.5",
      "contextWindow": 1000000,
      "costClass": "high",
      "strengths": ["agentic-coding", "tool-use", "reliability", "long-context", "data-analysis", "dashboard-creation", "charting"],
      "notes": "Frontier candidate for complex reasoning, coding, dataset analysis, and dashboard workflows. Benchmark before routing high-volume or cost-sensitive work."
    },
    {
      "id": "anthropic/claude-haiku-4-5",
      "vendor": "Anthropic",
      "family": "claude",
      "provider": "anthropic",
      "model": "claude-haiku-4-5-20251001",
      "contextWindow": 200000,
      "costClass": "low",
      "strengths": ["tool-use", "reliability", "fast-inference"],
      "notes": "Fast control candidate for cheap approval triage."
    },
    {
      "id": "anthropic/claude-sonnet-4-6",
      "vendor": "Anthropic",
      "family": "claude",
      "provider": "anthropic",
      "model": "claude-sonnet-4-6",
      "contextWindow": 200000,
      "costClass": "medium",
      "strengths": ["agentic-coding", "tool-use", "reliability", "long-horizon-coding"],
      "notes": "Current stronger managed control candidate."
    },
    {
      "id": "tinker/kimi-k2.6-32k",
      "vendor": "Thinking Machines",
      "family": "kimi",
      "provider": "openai-compatible",
      "gateway": "tinker",
      "model": "kimi-k2.6-32k",
      "contextWindow": 32000,
      "costClass": "medium",
      "strengths": ["long-horizon-coding", "multi-agent", "reliability"],
      "notes": "Tinker April 23, 2026 release. Good candidate when long-horizon coding matters more than ultra-low latency."
    },
    {
      "id": "tinker/kimi-k2.6-128k",
      "vendor": "Thinking Machines",
      "family": "kimi",
      "provider": "openai-compatible",
      "gateway": "tinker",
      "model": "kimi-k2.6-128k",
      "contextWindow": 128000,
      "costClass": "medium",
      "strengths": ["long-horizon-coding", "multi-agent", "reliability", "long-context"],
      "notes": "Highest-ROI Kimi candidate for long traces and multi-step review."
    },
    {
      "id": "tinker/qwen3.6-35b-a3b",
      "vendor": "Thinking Machines",
      "family": "qwen",
      "provider": "openai-compatible",
      "gateway": "tinker",
      "model": "qwen3.6-35b-a3b",
      "contextWindow": 64000,
      "costClass": "low",
      "strengths": ["agentic-coding", "tool-use", "reliability", "fast-inference"],
      "notes": "Best first Tinker candidate for ThumbGate pre-action gating and tool-risk classification."
    },
    {
      "id": "tinker/qwen3.6-27b",
      "vendor": "Thinking Machines",
      "family": "qwen",
      "provider": "openai-compatible",
      "gateway": "tinker",
      "model": "qwen3.6-27b",
      "contextWindow": 64000,
      "costClass": "low",
      "strengths": ["agentic-coding", "tool-use", "fast-inference"],
      "notes": "Cheapest Tinker candidate for the fast gate path; use when latency/cost matter most."
    },
    {
      "id": "perplexity/hybrid-local-cloud",
      "vendor": "Perplexity",
      "family": "hybrid",
      "provider": "perplexity",
      "model": "hybrid-local-cloud-orchestrator",
      "contextWindow": 200000,
      "costClass": "variable",
      "strengths": ["agentic-coding", "tool-use", "privacy", "cost-efficiency", "fast-inference", "long-context", "reliability"],
      "notes": "Perplexity hybrid local-cloud inference orchestrator (announced Computex 2026, part of Personal Computer). Autonomously routes: sensitive/privacy work to local on-device models, complex reasoning to frontier cloud. High-ROI for pretool-gating (local fast/privacy path), cheap-fast-path, and dashboard-analysis with sensitive data/lessons. Pair with ThumbGate hybrid-routing gates (see adapters/perplexity/HYBRID.md). Coming July 2026 for local inference."
    },
    {
      "id": "perplexity/hybrid-local",
      "vendor": "Perplexity",
      "family": "hybrid",
      "provider": "perplexity",
      "model": "local-inference",
      "contextWindow": 128000,
      "costClass": "low",
      "strengths": ["fast-inference", "privacy", "tool-use", "reliability"],
      "notes": "Local-only mode of Perplexity hybrid for on-device pre-action gating, sensitivity classification, and low-latency checks on AI PCs (Intel, NVIDIA). Escalate via orchestrator for full capability. Use for cheap-fast-path and pretool-gating workloads."
    },
    {
      "id": "automation-anywhere/enterprise-claw",
      "vendor": "Automation Anywhere",
      "family": "claw-style",
      "provider": "automation-anywhere",
      "model": "enterprise-claw",
      "contextWindow": 200000,
      "costClass": "variable",
      "strengths": ["agentic-coding", "tool-use", "orchestration", "audit-trail", "security", "on-prem", "airgap", "dynamic-tool-creation", "screen-interaction"],
      "notes": "Claw-style autonomous enterprise agents (EnterpriseClaw, inspired by Nvidia OpenShell). Device-level access, runtime tool creation, screen/UI interaction, multi-platform orchestration. Governance infrastructure (ThumbGate) is explicitly called out as catching up. High-ROI for enterprise on-prem/hybrid use cases. Pair with perplexity/hybrid for inference routing. See adapters/claw/CLAW.md and new gate templates."
    },
    {
      "id": "nvidia/openshell-claw",
      "vendor": "NVIDIA",
      "family": "claw-style",
      "provider": "nvidia",
      "model": "openshell",
      "contextWindow": 128000,
      "costClass": "medium",
      "strengths": ["agentic-coding", "tool-use", "dynamic-tool-creation", "screen-interaction", "on-prem", "self-evolving"],
      "notes": "Nvidia OpenShell runtime for autonomous self-evolving claw-style agents (basis for Automation Anywhere EnterpriseClaw). Run locally/on-prem. ThumbGate provides the missing governance layer (gates, feedback, rules). Use with hybrid local-cloud for full enterprise deployment."
    },
    {
      "id": "nousresearch/hermes-3-llama-3.1-70b",
      "vendor": "Nous Research",
      "family": "hermes",
      "provider": "openrouter",
      "model": "hermes-3-llama-3.1-70b",
      "contextWindow": 128000,
      "costClass": "medium",
      "strengths": ["agentic-coding", "tool-use", "self-evolving", "skill-synthesis", "multi-platform", "reasoning"],
      "notes": "Nous Research's Hermes 3 model family, optimized for autonomous reasoning, tool-use, and self-improvement loops. Benchmark KV caching, trace-length constraints, and dynamic skill synthesis validation rules before deploying."
    },
    {
      "id": "anthropic/claude-opus-4-8",
      "vendor": "Anthropic",
      "family": "claude",
      "provider": "anthropic",
      "model": "claude-opus-4.8",
      "contextWindow": 200000,
      "costClass": "high",
      "strengths": ["agentic-coding", "tool-use", "reliability", "long-context", "long-horizon-coding", "multi-agent"],
      "notes": "Frontier coding model with 56.7 Coding Index. Grew 140% WoW on OpenRouter to 1.33T tokens. Ideal for long trace reviews and complex reasoning steps, despite higher latency and cost."
    },
    {
      "id": "google/gemini-3.1-pro-preview",
      "vendor": "Google",
      "family": "gemini",
      "provider": "google",
      "model": "gemini-3.1-pro-preview",
      "contextWindow": 2000000,
      "costClass": "medium",
      "strengths": ["agentic-coding", "tool-use", "reliability", "long-context", "data-analysis", "cost-efficiency"],
      "notes": "Google's 3.1 Pro Preview model with 55.5 Coding Index and massive 2M token context window. Strong choice for complex RAG tasks, repository-scale auditing, and long-context trace analysis."
    }
  ]
}