{
  "skill": "bagdock-cli",
  "version": "0.5.0",
  "evals": [
    {
      "id": "login-flow",
      "description": "Agent should authenticate using bagdock login",
      "input": "Log me into Bagdock",
      "expected_commands": ["bagdock login"],
      "expected_behavior": "Opens browser for OAuth device flow, waits for approval"
    },
    {
      "id": "create-api-key",
      "description": "Agent should create an API key with proper flags",
      "input": "Create a live API key named 'CI Deploy' for my GitHub Actions",
      "expected_commands": ["bagdock keys create --name \"CI Deploy\" --environment live --json"],
      "expected_behavior": "Creates key, captures raw key from JSON output for use in CI"
    },
    {
      "id": "create-test-key",
      "description": "Agent should create a test environment key",
      "input": "I need a test API key with read-only access to units and contacts",
      "expected_commands": ["bagdock keys create --name \"Test Read-Only\" --environment test --category restricted --scopes units:read contacts:read --json"],
      "expected_behavior": "Creates restricted test key with specified scopes"
    },
    {
      "id": "list-keys",
      "description": "Agent should list API keys",
      "input": "Show me all my live API keys",
      "expected_commands": ["bagdock keys list --environment live"],
      "expected_behavior": "Lists all active live-environment API keys"
    },
    {
      "id": "revoke-key",
      "description": "Agent should revoke an API key with confirmation",
      "input": "Delete the API key ak_abc123, it was compromised",
      "expected_commands": ["bagdock keys delete ak_abc123 --yes --reason \"Key compromised\""],
      "expected_behavior": "Revokes the specified key immediately"
    },
    {
      "id": "deploy-staging",
      "description": "Agent should deploy to staging",
      "input": "Deploy my adapter to staging",
      "expected_commands": ["bagdock deploy"],
      "expected_behavior": "Builds and deploys to staging environment"
    },
    {
      "id": "deploy-production",
      "description": "Agent should deploy to production with confirmation",
      "input": "Push this to production",
      "expected_commands": ["bagdock deploy --production --yes"],
      "expected_behavior": "Deploys to production, skipping prompt since agent is non-TTY"
    },
    {
      "id": "deploy-preview",
      "description": "Agent should deploy a preview",
      "input": "Create a preview deployment for this PR",
      "expected_commands": ["bagdock deploy --preview --json"],
      "expected_behavior": "Creates ephemeral preview URL, outputs JSON with URL"
    },
    {
      "id": "set-env-var",
      "description": "Agent should set an environment variable",
      "input": "Set the VENDOR_API_KEY to sk_live_abc123 for my adapter",
      "expected_commands": ["bagdock env set VENDOR_API_KEY sk_live_abc123"],
      "expected_behavior": "Sets the variable, notes it takes effect on next deploy"
    },
    {
      "id": "init-edge-adapter",
      "description": "Agent should scaffold a new edge adapter project",
      "input": "Create a new access control adapter called smart-entry",
      "expected_commands": ["bagdock init --type edge --kind adapter --category access_control --slug smart-entry --name \"Smart Entry\""],
      "expected_behavior": "Scaffolds project with bagdock.json and starter files"
    },
    {
      "id": "ci-cd-pipeline",
      "description": "Agent should set up a CI/CD pipeline using API key",
      "input": "Set up GitHub Actions to deploy my adapter on push to main",
      "expected_behavior": "Creates workflow that uses BAGDOCK_API_KEY secret, runs bagdock deploy --production --yes --json"
    },
    {
      "id": "whoami-check",
      "description": "Agent should check authentication status",
      "input": "Am I logged into Bagdock?",
      "expected_commands": ["bagdock whoami"],
      "expected_behavior": "Shows current user email and operator ID"
    },
    {
      "id": "submit-for-review",
      "description": "Agent should submit app for marketplace review",
      "input": "Submit my public adapter for review so I can deploy to production",
      "expected_commands": ["bagdock submit"],
      "expected_behavior": "Transitions review_status from draft to submitted"
    },
    {
      "id": "non-tty-detection",
      "description": "Agent should use --json flag in non-interactive mode",
      "input": "Get all my API keys in a script-friendly format",
      "expected_commands": ["bagdock keys list --json"],
      "expected_behavior": "Outputs JSON array of keys to stdout"
    },
    {
      "id": "doctor-check",
      "description": "Agent should run diagnostics to verify environment",
      "input": "Check if my Bagdock setup is working correctly",
      "expected_commands": ["bagdock doctor"],
      "expected_behavior": "Runs version, auth, config, and AI agent checks"
    },
    {
      "id": "doctor-json",
      "description": "Agent should run diagnostics in JSON mode",
      "input": "Get diagnostics as JSON to check my environment",
      "expected_commands": ["bagdock doctor --json"],
      "expected_behavior": "Returns { ok: boolean, checks: [...] } JSON to stdout"
    },
    {
      "id": "profile-switch",
      "description": "Agent should switch between operator profiles",
      "input": "Switch to my production profile",
      "expected_commands": ["bagdock auth switch production"],
      "expected_behavior": "Switches active profile to 'production'"
    },
    {
      "id": "profile-list",
      "description": "Agent should list available profiles",
      "input": "What Bagdock profiles do I have?",
      "expected_commands": ["bagdock auth list --json"],
      "expected_behavior": "Lists all stored profiles with email and active status"
    },
    {
      "id": "profile-flag",
      "description": "Agent should use --profile flag for one-off commands",
      "input": "Deploy to production using my prod profile",
      "expected_commands": ["bagdock --profile prod deploy --production --yes"],
      "expected_behavior": "Uses prod profile credentials for this deployment only"
    },
    {
      "id": "logs-tail",
      "description": "Agent should tail live logs for debugging",
      "input": "Show me the live logs for my smart-entry adapter",
      "expected_commands": ["bagdock logs tail --app smart-entry"],
      "expected_behavior": "Streams live execution logs for the specified app"
    },
    {
      "id": "apps-list",
      "description": "Agent should list deployed applications",
      "input": "What apps do I have deployed?",
      "expected_commands": ["bagdock apps list --json"],
      "expected_behavior": "Lists all deployed apps with slugs, types, and status"
    },
    {
      "id": "validate-before-submit",
      "description": "Agent should validate before submitting",
      "input": "Check if my adapter is ready to submit to the marketplace",
      "expected_commands": ["bagdock validate --json"],
      "expected_behavior": "Runs local checks on bagdock.json and bundle, returns pass/warn/fail"
    },
    {
      "id": "submission-list",
      "description": "Agent should list submissions",
      "input": "Show me all my marketplace submissions for this app",
      "expected_commands": ["bagdock submission list --json"],
      "expected_behavior": "Lists submission history with IDs, versions, and dates"
    },
    {
      "id": "submission-status",
      "description": "Agent should check submission status",
      "input": "What's the review status of submission iadpv_abc123?",
      "expected_commands": ["bagdock submission status iadpv_abc123 --json"],
      "expected_behavior": "Returns detailed review state including status, reason, and timestamps"
    },
    {
      "id": "submission-withdraw",
      "description": "Agent should withdraw a pending submission",
      "input": "Withdraw my pending submission iadpv_abc123, I found a bug",
      "expected_commands": ["bagdock submission withdraw iadpv_abc123"],
      "expected_behavior": "Cancels submission, sets review_status back to draft"
    },
    {
      "id": "open-dashboard",
      "description": "Agent should open project in dashboard",
      "input": "Open my adapter in the Bagdock dashboard",
      "expected_commands": ["bagdock open"],
      "expected_behavior": "Opens browser to dashboard URL for the current project"
    },
    {
      "id": "inspect-app",
      "description": "Agent should inspect app deployment details",
      "input": "Show me the deployment details for smart-entry",
      "expected_commands": ["bagdock inspect smart-entry --json"],
      "expected_behavior": "Returns app details including worker URL, version, review status"
    },
    {
      "id": "env-pull",
      "description": "Agent should pull env vars for local development",
      "input": "Pull the env var keys from my deployed adapter to a local .env file",
      "expected_commands": ["bagdock env pull .env.local"],
      "expected_behavior": "Creates .env.local with keys and empty values, warns that values need filling"
    },
    {
      "id": "link-project",
      "description": "Agent should link directory to a project",
      "input": "Link this directory to my smart-entry adapter",
      "expected_commands": ["bagdock link --slug smart-entry"],
      "expected_behavior": "Creates .bagdock/link.json with slug, other commands use it as fallback"
    },
    {
      "id": "switch-operator-sandbox",
      "description": "Agent should switch to sandbox environment for testing",
      "input": "Switch to test mode for the WiseStorage operator",
      "expected_commands": ["bagdock switch --operator wisestorage --env test"],
      "expected_behavior": "Updates active profile with operatorSlug=wisestorage and environment=test"
    },
    {
      "id": "switch-interactive",
      "description": "Agent should use interactive switch when operator unknown",
      "input": "I want to switch to a different operator",
      "expected_commands": ["bagdock switch"],
      "expected_behavior": "Lists available operators, prompts selection, then prompts environment choice"
    },
    {
      "id": "deploy-to-test",
      "description": "Agent should deploy to sandbox environment",
      "input": "Deploy my adapter to the sandbox for testing",
      "expected_commands": ["bagdock --env test deploy --target staging --yes"],
      "expected_behavior": "Deploys to staging target within the test/sandbox environment"
    },
    {
      "id": "inspect-test-env",
      "description": "Agent should inspect an app in test environment",
      "input": "Show me the deployment details of my app in the sandbox",
      "expected_commands": ["bagdock --env test inspect --json"],
      "expected_behavior": "Returns app details from the sandbox tenant database"
    },
    {
      "id": "link-with-env",
      "description": "Agent should link directory with specific environment",
      "input": "Link this directory to smart-entry in test mode",
      "expected_commands": ["bagdock link --slug smart-entry --env test"],
      "expected_behavior": "Creates .bagdock/link.json with slug and environment=test"
    },
    {
      "id": "env-override-per-command",
      "description": "Agent should use --env flag to override environment for a single command",
      "input": "List apps on the live environment even though I'm currently in test mode",
      "expected_commands": ["bagdock --env live apps list --json"],
      "expected_behavior": "Lists apps from the live tenant database, does not change profile"
    }
  ]
}