{ "schemaVersion": "ucr.full-study/1", "executedAt": "2026-08-10T14:54:24.099Z", "evidenceClass": "executable-smoke", "plan": { "schemaVersion": "ucr.full-study-plan/1", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "seedHash": "41b25ff01f365e97e60404d48067231087bfdf3680c3243e04039cea20f40b62", "clients": [ { "id": "codex", "model": "gpt-5.6-sol", "modelFamily": "openai", "version": "codex-cli 0.147.0", "modelVersion": "gpt-5.6-sol", "lifecycleFamily": "process-hook" }, { "id": "claude-code", "model": "claude-sonnet-5", "modelFamily": "anthropic", "version": "2.1.225 (Claude Code)", "modelVersion": "claude-sonnet-5", "lifecycleFamily": "process-hook" }, { "id": "gemini", "model": "gemini-2.5-pro", "modelFamily": "google", "version": "gemini-cli 0.28.1", "modelVersion": "gemini-2.5-pro", "lifecycleFamily": "process-hook" } ], "directions": [ "codex->codex", "codex->claude-code", "codex->gemini", "claude-code->codex", "claude-code->claude-code", "claude-code->gemini", "gemini->codex", "gemini->claude-code", "gemini->gemini" ], "pairedRepetitionsPerCell": 1, "negativeRepetitionsPerCell": 1, "includeSameClient": true, "budgets": { "contextTokens": 160, "retries": 1, "toolCalls": 40, "timeoutMs": 600000 }, "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "primaryPairsPerDirection": 11, "preregisteredPairsPerArm": 357, "minimumSubgroupPairs": 30, "minimumNegativeSamplesPerDirection": 1053, "negativeConfidence": { "samples": 1053, "maximumRate": 0.01, "z": 3.2607674853887727, "comparisons": 45, "familyAlpha": 0.05, "method": "bonferroni-familywise-wilson-zero-failure-upper-bound" }, "negativeSamplesPerDirectionPerArm": 11, "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialCount": 693 }, "coverage": { "passed": true, "checks": { "effectivenessDesign": true, "metricMapping": true, "competitiveBaselines": true, "productionStages": true }, "design": { "passed": true, "checks": { "benchmarkFamilies": true, "benchmarkArms": true, "clients": true, "modelFamilies": true, "bidirectional": true, "sameClient": true, "crossClient": true, "sameSession": true, "crossSession": true, "crossProject": true, "concurrentAgents": true, "pairedControls": true, "isolatedWorkspaces": true, "materialVariants": true, "executionTopology": true }, "coverage": { "families": 1, "arms": 1, "clients": 3, "modelFamilies": 3, "directions": 9, "materialVariants": 485 } }, "competitionCoverage": 1, "productionCoverage": 1, "metricSources": { "applicabilityPrecision": "full-effectiveness", "applicabilityPrecisionIntervalLow": "full-effectiveness", "preActionDelivery": "full-effectiveness", "preActionDeliveryIntervalLow": "full-effectiveness", "irrelevantDelivery": "full-effectiveness", "irrelevantDeliveryIntervalHigh": "full-effectiveness", "staleDelivery": "full-effectiveness", "staleDeliveryIntervalHigh": "full-effectiveness", "recurrenceReduction": "full-effectiveness", "recurrenceIntervalLow": "full-effectiveness", "naturalCorrectnessDelta": "full-effectiveness", "naturalCorrectnessIntervalLow": "full-effectiveness", "severeUnquarantined": "full-effectiveness", "emptyP95Overhead": "full-effectiveness", "reconstructionTokenReduction": "full-effectiveness", "firstSuccessorTokenReduction": "full-effectiveness", "firstSuccessorTokenIntervalLow": "full-effectiveness", "latencyOverheadP95": "full-effectiveness", "knownMistakeRecurrence": "full-effectiveness", "contradictoryDelivery": "full-effectiveness", "contradictoryDeliveryIntervalHigh": "full-effectiveness", "negativeDeliveryIntervalHigh": "full-effectiveness", "consumerSchemaTokensP95": "full-effectiveness", "captureModelCallsP95": "full-effectiveness", "writerIntegrity": "writer-conformance", "crossClientPassed": "full-effectiveness", "benchmarkFamilyCoverage": "full-effectiveness", "benchmarkArmCoverage": "full-effectiveness", "directionalCorrectnessIntervalLow": "full-effectiveness", "familyCorrectnessIntervalLow": "full-effectiveness", "directionalTokenOverheadHigh": "full-effectiveness", "causalChainIntegrity": "full-effectiveness", "trialIntegrity": "full-effectiveness", "independentGrading": "full-effectiveness", "competitivePassed": "live-competitive", "competitiveCoverage": "live-competitive", "competitiveReproducibility": "live-competitive" }, "missingMetrics": [] }, "poweredDesign": false, "complete": false, "trialsPlanned": 693, "trialsExecuted": 2, "failures": [], "ledger": { "schemaVersion": "ucr.evidence-ledger/2", "evidenceClass": "executable-smoke", "run": { "schemaVersion": "ucr.evidence-run/2", "runId": "full-study-52538fbf4906eaac", "evidenceClass": "executable-smoke", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "sourceTreeHash": "73f83b7752a494dc71d2c2d626b12e9922c7dbdc2f6f879f16ebb29b23bca5ab", "runner": { "name": "ucr-full-study", "node": "v25.6.0", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "poweredDesign": false, "minimumSubgroupPairs": 30 }, "startedAt": "2026-08-10T14:54:24.094Z", "runHash": "59fa8a7873938dec8649f33d94016bb8375bd58beca06cb35959df337a83eb35" }, "rows": [ { "study": "full-effectiveness", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialId": "codex->claude-code:cross-model-001:0:empty", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 37, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "empty", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "4f5777767e446baf7e658dd317134ebde2f555950387b8ac1a67e66459a5aeb8", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": false, "eligible": false, "selected": false, "delivered": false, "deliveryPhase": null, "stale": false, "contradictory": false, "contextOverheadRatio": 0, "reconstructionTokens": 2, "totalTokens": 43406, "latencyMs": 52268, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 0, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "4c2201a9c0803014ddbf184c166346ae33eba38dfa27b8c1baf37f3d963d6473", "workspaceStateHash": "18f947e182e1322e8bf253c5dec168c4004627f48da2059f16e188b526ed8a7b", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "086393eda728218852b32af603f9f7a272b55258ebade49903f30032fc6c924d", "runnerExitCode": 0, "runnerOutputHash": "d54525dd86015204fd23c31bbbe4731a32781f10ebef256825491117e3a36485", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-52538fbf4906eaac", "sequence": 0, "previousHash": "59fa8a7873938dec8649f33d94016bb8375bd58beca06cb35959df337a83eb35", "rowHash": "039116de82a5e6e3db5970af90cf206fa5a2bfc7334ca860b493ea645146334a" }, { "study": "full-effectiveness", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialId": "codex->claude-code:cross-model-001:0:runtime", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 340, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "runtime", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "90bc20152c4ca188e4703921bb2a20cb5dddf276e8b99f98ca953604448f538a", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": true, "eligible": true, "selected": true, "delivered": true, "deliveryPhase": "pre-action", "stale": false, "contradictory": false, "contextOverheadRatio": 113, "reconstructionTokens": 2, "totalTokens": 43434, "latencyMs": 58410, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 226, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "106b54ecb3284a495d2226dd60d0f62fcf52587cac5eae35fc702c2d65796ef1", "workspaceStateHash": "1242c720484a99f5edcb41d8315bc57410bec278e455b90fb3b9fc90dbef0505", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "c032fe82150f8b71d6d2a287c37c8cb262fbb0e2e6d85f3b336278af74c6eafa", "runnerExitCode": 0, "runnerOutputHash": "4d651d27e91d440a3a6c98911866cebddee0765374774bbbaaa7ba3875ab94ae", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-52538fbf4906eaac", "sequence": 1, "previousHash": "039116de82a5e6e3db5970af90cf206fa5a2bfc7334ca860b493ea645146334a", "rowHash": "44d54c87248fa495db458c9a856fbf50e74a19bfcdcc00301481b9b27f46bb25" } ], "rowCount": 2, "chainHead": "44d54c87248fa495db458c9a856fbf50e74a19bfcdcc00301481b9b27f46bb25", "transcriptsPublished": false, "endedAt": "2026-08-10T14:54:24.095Z", "ledgerHash": "235d0b546f92f312450a24531b37cb0be5a0f512855858d083efd870ca12f1ad", "signature": "Vb45jMkHld0JGCWcKAy8LLLHYMGP3KR2QHfJ+qlq/Ik6MMgWW350hdtmwxK29BxXouXH71DRqT9ejXjKo8/gAQ==" }, "ledgerKeyId": "ucr-ed25519-77b546af455c7ecd3c9c", "ledgerVerification": { "valid": true, "diagnostics": [], "validSignature": true, "ledgerHash": "235d0b546f92f312450a24531b37cb0be5a0f512855858d083efd870ca12f1ad", "rowCount": 2 }, "derived": { "schemaVersion": "ucr.derived-metrics/2", "metrics": { "applicabilityPrecision": null, "applicabilityPrecisionIntervalLow": null, "preActionDelivery": null, "preActionDeliveryIntervalLow": null, "irrelevantDelivery": null, "irrelevantDeliveryIntervalHigh": null, "staleDelivery": null, "staleDeliveryIntervalHigh": null, "recurrenceReduction": null, "recurrenceIntervalLow": null, "naturalCorrectnessDelta": null, "naturalCorrectnessIntervalLow": null, "severeUnquarantined": 0, "emptyP95Overhead": null, "reconstructionTokenReduction": null, "firstSuccessorTokenReduction": null, "firstSuccessorTokenIntervalLow": null, "latencyOverheadP95": null, "knownMistakeRecurrence": null, "contradictoryDelivery": null, "contradictoryDeliveryIntervalHigh": null, "negativeDeliveryIntervalHigh": null, "consumerSchemaTokensP95": null, "captureModelCallsP95": null, "writerIntegrity": false, "crossClientPassed": false, "benchmarkFamilyCoverage": null, "benchmarkArmCoverage": null, "directionalCorrectnessIntervalLow": null, "familyCorrectnessIntervalLow": null, "directionalTokenOverheadHigh": null, "causalChainIntegrity": false, "trialIntegrity": false, "independentGrading": false, "competitivePassed": false, "competitiveCoverage": null, "competitiveReproducibility": false }, "intervals": { "recurrence": { "pairs": 0, "mean": null, "low": null, "high": null }, "correctness": { "pairs": 0, "mean": null, "low": null, "high": null }, "reconstruction": { "pairs": 0, "mean": null, "low": null, "high": null }, "firstSuccessorTokens": { "pairs": 0, "mean": null, "low": null, "high": null } }, "sources": { "applicabilityPrecision": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "applicabilityPrecisionIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDeliveryIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessDelta": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "severeUnquarantined": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "emptyP95Overhead": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "reconstructionTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "latencyOverheadP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "knownMistakeRecurrence": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "negativeDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "consumerSchemaTokensP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "captureModelCallsP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "writerIntegrity": { "requiredClass": "conformance", "eligibleLedgerHashes": [] }, "crossClientPassed": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkFamilyCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkArmCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "familyCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalTokenOverheadHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "causalChainIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "trialIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "independentGrading": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "competitivePassed": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveCoverage": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveReproducibility": { "requiredClass": "superiority", "eligibleLedgerHashes": [] } }, "inputLedgerHashes": [ "235d0b546f92f312450a24531b37cb0be5a0f512855858d083efd870ca12f1ad" ], "rejectedLedgers": 0, "derivedHash": "a2aaea46ba70a41862db4c3097c48606e5519f91fc2b2b809e4101c0491beba7" }, "costDiagnostics": { "passed": false, "thresholds": { "maximumTokenOverhead": 0.05, "maximumLatencyOverhead": 0.05 }, "groups": [ { "dimension": "direction", "value": "codex->claude-code", "pairs": 1, "tokenOverhead": 0.000645072109846565, "latencyOverheadP95": 0.11750975740414785, "regressions": [ "latency" ], "passed": false }, { "dimension": "family", "value": "cross-model-handoff", "pairs": 1, "tokenOverhead": 0.000645072109846565, "latencyOverheadP95": 0.11750975740414785, "regressions": [ "latency" ], "passed": false }, { "dimension": "client", "value": "claude-code", "pairs": 1, "tokenOverhead": 0.000645072109846565, "latencyOverheadP95": 0.11750975740414785, "regressions": [ "latency" ], "passed": false } ] }, "passed": false, "limitations": [ "qualification or partial execution cannot supply effectiveness metrics" ], "reportHash": "91ec6ef245f2a5d583589ab65e9990047e8e7774a97dcf97e50eb993b20cf8c1" }