/** * Codex EVAL-tier transport — the runner + trace parser that * `measureTriggerRate`/`runEval` dispatch to via the `ModelOutputParser` seam. * * SCHEMA: CONFIRMED against real `codex exec --json` (codex-cli 0.139.0, ChatGPT * auth). The stream is the thread/item model: * * {"type":"thread.started","thread_id":"…"} * {"type":"turn.started"} * {"type":"item.started","item":{"id":"item_0","type":"command_execution",…}} // mid-flight * {"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"…","aggregated_output":"…","exit_code":0}} * {"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"…"}} * {"type":"turn.completed","usage":{"input_tokens":…,"cached_input_tokens":…,"output_tokens":…}} * * So: assistant text = `item.completed` with `item.type:"agent_message"` → * `item.text`; a tool call = `item.type:"command_execution"` → `item.command`; * usage rides `turn.completed`. We count `item.completed` ONLY (an `item.started` * carries the same `id` mid-flight — counting both double-counts). * * THE SKILL FINDING: Codex has NO discrete "skill selected" event (its CLI has no * Skill-tool concept). When a skill triggers, the model READS the skill's * `SKILL.md` via a `command_execution` (`sed/cat … skills//SKILL.md`) and * usually says so in an `agent_message`. So "did skill X fire" on Codex is not a * clean trace event like Claude's `Skill` tool_use — it's detected by the * SKILL.md read (`codexSkillFired`). Best-effort by nature (a cached skill might * not be re-read); pair with a behavioral/judged check for certainty. */ import type { HarnessLiveDriver } from "../../core/live-driver.js"; import type { ParsedModelRun, AgentRunArgs, RunOut, EvalDriver } from "../../eval.js"; import type { ToolCall } from "../../core/harness-driver.js"; /** Parse `codex exec --json` stdout into the common trace fields (confirmed schema). */ export declare function parseCodexEvalRun(out: { stdout: string; }): ParsedModelRun; /** * The error message if the run errored or was rate-limited (an `error` / * `turn.failed` event), else null. CRITICAL for the eval tier: an errored turn * must NOT be scored as a clean "skill didn't fire" miss — dogfooding hit a Codex * usage limit ("You've hit your usage limit…") whose `error` event left an empty * trace that `codexSkillFired` read as recall 0. A caller should skip/retry an * errored run, not count it. (The Claude path has `isRateLimited` + backoff; this * is the Codex equivalent detector.) */ export declare function codexRunError(out: { stdout: string; }): string | null; /** * Did Codex activate skill `name` on this run? Detected by the SKILL.md read — * Codex has no discrete skill-selection event, so when a skill triggers the model * reads its `…//SKILL.md` via a `command_execution`. Best-effort (a cached * skill might not be re-read); for the trigger-rate `fired` predicate over Codex. */ export declare function codexSkillFired(run: { toolCalls: readonly ToolCall[]; }, name: string): boolean; /** * Materialize a (Claude-shaped) plugin dir's skills into `/.codex/skills/` — * where Codex actually discovers them (validated live: codex reads * `/.codex/skills//SKILL.md`). This is the Codex analog of Claude's * `--plugin-dir`: `measureTriggerRate` hands the runner a `pluginDir` (the * stubbed/packaged skills), and the Codex runner installs them here before the * turn. Pure fs — unit-testable without a binary. */ export declare function installCodexSkills(pluginDir: string, cwd: string): number; /** * Refuse an `effort` this adapter cannot honour, LOUDLY. * * The Claude runner pins the reasoning budget through `--effort` plus the env var * it sits under. Codex exposes no mapping we have measured — its config carries a * `model_reasoning_effort` key, but nothing here has driven the real binary * through it, so claiming support would assert something unverified. * * The alternative — forwarding `task`/`cwd`/`timeoutMs` and dropping `effort` on * the floor, as this runner does today — is the silent CC-only path the * harness-parity rule forbids: a spec would declare `effort: "low"`, the lock * would RECORD `low`, and the run would happen at whatever Codex defaults to. * A stale number is recoverable; a confidently mislabelled one is not. * * Pure so the deferral itself is tested rather than living inside the ignored * subprocess region. */ export declare function refuseCodexEffort(effort: string | number | undefined): void; /** * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills` * (Codex's discovery path, vs Claude's `--plugin-dir`), then drive a real * `codex exec --json` turn. The seam `measureTriggerRate(spec, { evalDriver: * codexEvalDriver })` dispatches through. */ export declare function codexEvalAgentRunner(args: AgentRunArgs): Promise; /** * Why Codex trigger-rate is EXPERIMENTAL (not a supported measurement). Codex * has NO skill-selection event, so `codexSkillFired` infers firing from whether * the model READ `skills//SKILL.md` — which is wrong in BOTH directions: a * cached/already-in-context skill isn't re-read (false NEGATIVE), and an * exploratory read while listing isn't a real fire (false POSITIVE). So the * number can be off either way. Deterministic `vigiles audit` is fully supported * on Codex; only this behavioral tier is experimental until a live run measures * the oracle's accuracy vs ground truth. See docs/harness-testing-codex.md. */ export declare const CODEX_TRIGGER_RATE_EXPERIMENTAL: string; /** * The Codex eval driver — pass to `measureTriggerRate(spec, { evalDriver: * codexEvalDriver })` to run a trigger-rate eval natively on `codex exec`. Pair * the spec's `fired` with `codexSkillFired` (Codex has no Skill-tool event). */ export declare const codexEvalDriver: EvalDriver; /** * The Codex {@link HarnessLiveDriver} — the EXECUTING tiers' side of the * adapter, reached through `codexAdapter.liveDriver()`. * * It is the object `scan-behavioral.ts:buildProbe` used to build from * `harness === "codex"`: the same four answers, now carried by the adapter that * knows them instead of switched on by a name in the application layer. */ export declare const codexLiveDriver: HarnessLiveDriver; /** * Spawn real `codex exec --json` for the eval tier (real model, the user's codex * auth — NOT the mock). CONFIRMED flags (codex 0.139.0): `--json` for the event * stream, `--skip-git-repo-check` for a bare cwd, the approvals/sandbox bypass so * the turn runs unattended, `-C ` for the working dir, prompt as the trailing * positional, and stdin = /dev/null (`stdio: ["ignore",…]`) — codex otherwise * blocks on "Reading additional input from stdin…". Needs ChatGPT/API auth + * network egress to the model backend. */ export declare function codexEvalRunner(args: { task: string; cwd: string; timeoutMs: number; }): { code: number; stdout: string; }; //# sourceMappingURL=eval.d.ts.map