| model | arm | tool | calls | errors | rejected | wrong-tool |
|---|---|---|---:|---:|---:|---:|
| anthropic-cc_claude-sonnet-5-5 | extension | grep | 39 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | extension | edit | 39 | 3 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | extension | read | 15 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | extension | find | 4 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | extension | bash | 4 | 0 | 0 | 2 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find | grep | 40 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find | edit | 31 | 3 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find | read | 11 | 1 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find | bash | 10 | 0 | 0 | 7 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find-grep | bash | 77 | 2 | 0 | 2 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find-grep | edit | 23 | 1 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find-grep | read | 12 | 2 | 1 | 0 |
| anthropic-cc_claude-sonnet-5-5 | no-ls-find-grep | bash (tests) | 2 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | stock | bash | 73 | 6 | 0 | 7 |
| anthropic-cc_claude-sonnet-5-5 | stock | edit | 15 | 0 | 0 | 0 |
| anthropic-cc_claude-sonnet-5-5 | stock | read | 2 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | grep | 57 | 2 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | read | 46 | 3 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | edit | 40 | 2 | 1 | 0 |
| openai-codex_gpt-6-luna | extension | find | 26 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | bash (tests) | 13 | 13 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | bash | 7 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | extension | ls | 4 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find | grep | 62 | 2 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find | read | 43 | 8 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find | edit | 38 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find | bash (tests) | 13 | 13 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find | bash | 4 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | read | 76 | 14 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | bash | 57 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | edit | 48 | 1 | 1 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | bash (tests) | 15 | 15 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | ast_search | 11 | 1 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | write | 1 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | no-ls-find-grep | nu | 1 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | stock | bash | 49 | 0 | 0 | 7 |
| openai-codex_gpt-6-luna | stock | edit | 49 | 1 | 0 | 0 |
| openai-codex_gpt-6-luna | stock | read | 47 | 0 | 0 | 0 |
| openai-codex_gpt-6-luna | stock | bash (tests) | 13 | 13 | 0 | 1 |