{ "schemaVersion": "ucr.full-study/1", "executedAt": "2026-08-10T15:04:31.638Z", "evidenceClass": "executable-smoke", "plan": { "schemaVersion": "ucr.full-study-plan/1", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "seedHash": "41b25ff01f365e97e60404d48067231087bfdf3680c3243e04039cea20f40b62", "clients": [ { "id": "codex", "model": "gpt-5.6-sol", "modelFamily": "openai", "version": "codex-cli 0.147.0", "modelVersion": "gpt-5.6-sol", "lifecycleFamily": "process-hook" }, { "id": "claude-code", "model": "claude-sonnet-5", "modelFamily": "anthropic", "version": "2.1.225 (Claude Code)", "modelVersion": "claude-sonnet-5", "lifecycleFamily": "process-hook" }, { "id": "gemini", "model": "gemini-2.5-pro", "modelFamily": "google", "version": "gemini-cli 0.28.1", "modelVersion": "gemini-2.5-pro", "lifecycleFamily": "process-hook" } ], "directions": [ "codex->codex", "codex->claude-code", "codex->gemini", "claude-code->codex", "claude-code->claude-code", "claude-code->gemini", "gemini->codex", "gemini->claude-code", "gemini->gemini" ], "pairedRepetitionsPerCell": 1, "negativeRepetitionsPerCell": 1, "includeSameClient": true, "budgets": { "contextTokens": 160, "retries": 1, "toolCalls": 40, "timeoutMs": 600000 }, "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "primaryPairsPerDirection": 11, "preregisteredPairsPerArm": 357, "minimumSubgroupPairs": 30, "minimumNegativeSamplesPerDirection": 1053, "negativeConfidence": { "samples": 1053, "maximumRate": 0.01, "z": 3.2607674853887727, "comparisons": 45, "familyAlpha": 0.05, "method": "bonferroni-familywise-wilson-zero-failure-upper-bound" }, "negativeSamplesPerDirectionPerArm": 11, "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialCount": 693 }, "coverage": { "passed": true, "checks": { "effectivenessDesign": true, "metricMapping": true, "competitiveBaselines": true, "productionStages": true }, "design": { "passed": true, "checks": { "benchmarkFamilies": true, "benchmarkArms": true, "clients": true, "modelFamilies": true, "bidirectional": true, "sameClient": true, "crossClient": true, "sameSession": true, "crossSession": true, "crossProject": true, "concurrentAgents": true, "pairedControls": true, "isolatedWorkspaces": true, "materialVariants": true, "executionTopology": true }, "coverage": { "families": 1, "arms": 1, "clients": 3, "modelFamilies": 3, "directions": 9, "materialVariants": 485 } }, "competitionCoverage": 1, "productionCoverage": 1, "metricSources": { "applicabilityPrecision": "full-effectiveness", "applicabilityPrecisionIntervalLow": "full-effectiveness", "preActionDelivery": "full-effectiveness", "preActionDeliveryIntervalLow": "full-effectiveness", "irrelevantDelivery": "full-effectiveness", "irrelevantDeliveryIntervalHigh": "full-effectiveness", "staleDelivery": "full-effectiveness", "staleDeliveryIntervalHigh": "full-effectiveness", "recurrenceReduction": "full-effectiveness", "recurrenceIntervalLow": "full-effectiveness", "naturalCorrectnessDelta": "full-effectiveness", "naturalCorrectnessIntervalLow": "full-effectiveness", "severeUnquarantined": "full-effectiveness", "emptyP95Overhead": "full-effectiveness", "reconstructionTokenReduction": "full-effectiveness", "firstSuccessorTokenReduction": "full-effectiveness", "firstSuccessorTokenIntervalLow": "full-effectiveness", "latencyOverheadP95": "full-effectiveness", "knownMistakeRecurrence": "full-effectiveness", "contradictoryDelivery": "full-effectiveness", "contradictoryDeliveryIntervalHigh": "full-effectiveness", "negativeDeliveryIntervalHigh": "full-effectiveness", "consumerSchemaTokensP95": "full-effectiveness", "captureModelCallsP95": "full-effectiveness", "writerIntegrity": "writer-conformance", "crossClientPassed": "full-effectiveness", "benchmarkFamilyCoverage": "full-effectiveness", "benchmarkArmCoverage": "full-effectiveness", "directionalCorrectnessIntervalLow": "full-effectiveness", "familyCorrectnessIntervalLow": "full-effectiveness", "directionalTokenOverheadHigh": "full-effectiveness", "causalChainIntegrity": "full-effectiveness", "trialIntegrity": "full-effectiveness", "independentGrading": "full-effectiveness", "competitivePassed": "live-competitive", "competitiveCoverage": "live-competitive", "competitiveReproducibility": "live-competitive" }, "missingMetrics": [] }, "poweredDesign": false, "complete": false, "trialsPlanned": 693, "trialsExecuted": 2, "selection": { "directions": "codex->claude-code", "families": "cross-model-handoff", "arms": "runtime,empty", "maxTrials": 2, "selectedTrials": 2, "selectedTrialIdsHash": "6f7f5e0bd8a59e11f69978a7c36109a50dbaa42854bef10f313a6263be46e921" }, "qualification": { "schemaVersion": "ucr.study-qualification/1", "status": "failed", "passed": false, "checks": { "selectionComplete": true, "trialIntegrity": true, "taskCorrect": true, "runtimeDelivered": true, "controlsWithheld": true, "hardNegativesWithheld": true, "pairedCoverage": true, "correctnessNonInferiority": true, "tokenNonInferiority": false }, "failed": [ "tokenNonInferiority" ], "selectedTrials": 2, "executedTrials": 2, "completePairs": 1, "maximumObservedTokenOverhead": 0.30118847803604426 }, "failures": [], "ledger": { "schemaVersion": "ucr.evidence-ledger/2", "evidenceClass": "executable-smoke", "run": { "schemaVersion": "ucr.evidence-run/2", "runId": "full-study-b41b295a817364f0", "evidenceClass": "executable-smoke", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "sourceTreeHash": "04b94602e8f05cf10b372097f7a937202ac4d56d4e3d2369cad98a926e000497", "runner": { "name": "ucr-full-study", "node": "v25.6.0", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "poweredDesign": false, "minimumSubgroupPairs": 30 }, "startedAt": "2026-08-10T15:04:31.633Z", "runHash": "661458c6fbe2ba24297adbe47b1ed49c48d7474d1e24674d0a2b3b34064b5b5c" }, "rows": [ { "study": "full-effectiveness", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialId": "codex->claude-code:cross-model-001:0:empty", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 37, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "empty", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "4f5777767e446baf7e658dd317134ebde2f555950387b8ac1a67e66459a5aeb8", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": false, "eligible": false, "selected": false, "delivered": false, "deliveryPhase": null, "stale": false, "contradictory": false, "contextOverheadRatio": 0, "reconstructionTokens": 2, "totalTokens": 58983, "latencyMs": 73607, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 0, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "c14fc55e509c4eb3b08f8912c304e110d0be7bc21e0f7a1e4b49ea15b1a703d7", "workspaceStateHash": "56dcb01e34f92be97210737301db50575ba3c1f9c502398527220b54e5ccc2fa", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "c5d39c2daa8a0fc5bb37f839710a52d3730be76a280b74b2a7935fed1f6c02dc", "runnerExitCode": 0, "runnerOutputHash": "9d5fd02c3be8a0c6db4062d844ad32291b1e451582251d339c3ec85a005a6000", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-b41b295a817364f0", "sequence": 0, "previousHash": "661458c6fbe2ba24297adbe47b1ed49c48d7474d1e24674d0a2b3b34064b5b5c", "rowHash": "60afdb50fe7763ed3bc3c74baffcd347ad7ab901bbe9f66f6cd10e8a99f15ae5" }, { "study": "full-effectiveness", "planHash": "8d0debfb4abd51a0da45754b1d232c17395fd5348a7c47971d2f69018dadbd93", "trialId": "codex->claude-code:cross-model-001:0:runtime", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 340, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "runtime", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "90bc20152c4ca188e4703921bb2a20cb5dddf276e8b99f98ca953604448f538a", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": true, "eligible": true, "selected": true, "delivered": true, "deliveryPhase": "pre-action", "stale": false, "contradictory": false, "contextOverheadRatio": 186.5, "reconstructionTokens": 2, "totalTokens": 76748, "latencyMs": 79619, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 373, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "7da230d987e97929604b0dcdc4958e766bfaaac90593aee35f23852fb612ceba", "workspaceStateHash": "2657550a9eeabdc97f4b36b36270d739a29510c47d85bd2081c266cf99bfd460", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "d4efa61c267d3bcb53d660007965cc517eeb93090a0b9dbc551c371017534f92", "runnerExitCode": 0, "runnerOutputHash": "b07bd1514dd791d327aa3fc517abe5bb3986c98d51dea211d825a4121a8d35dd", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-b41b295a817364f0", "sequence": 1, "previousHash": "60afdb50fe7763ed3bc3c74baffcd347ad7ab901bbe9f66f6cd10e8a99f15ae5", "rowHash": "a74f8db3a2de748ea4efb34ce3059eed03993dbeda42631726800226019793ca" } ], "rowCount": 2, "chainHead": "a74f8db3a2de748ea4efb34ce3059eed03993dbeda42631726800226019793ca", "transcriptsPublished": false, "endedAt": "2026-08-10T15:04:31.634Z", "ledgerHash": "52159aeaf25ac2da45ad79ba150beb90f117a9d589e31f960f4379d8b7b37cdf", "signature": "B2r0UDdc5etoQgnfPvrC+yU9cZvorm8tWlM4NZFpC7qRNkMXTePBuaELtVXzqqSzCGI5YFD7Y7W10vDIo/57Ag==" }, "ledgerKeyId": "ucr-ed25519-77b546af455c7ecd3c9c", "ledgerVerification": { "valid": true, "diagnostics": [], "validSignature": true, "ledgerHash": "52159aeaf25ac2da45ad79ba150beb90f117a9d589e31f960f4379d8b7b37cdf", "rowCount": 2 }, "derived": { "schemaVersion": "ucr.derived-metrics/2", "metrics": { "applicabilityPrecision": null, "applicabilityPrecisionIntervalLow": null, "preActionDelivery": null, "preActionDeliveryIntervalLow": null, "irrelevantDelivery": null, "irrelevantDeliveryIntervalHigh": null, "staleDelivery": null, "staleDeliveryIntervalHigh": null, "recurrenceReduction": null, "recurrenceIntervalLow": null, "naturalCorrectnessDelta": null, "naturalCorrectnessIntervalLow": null, "severeUnquarantined": 0, "emptyP95Overhead": null, "reconstructionTokenReduction": null, "firstSuccessorTokenReduction": null, "firstSuccessorTokenIntervalLow": null, "latencyOverheadP95": null, "knownMistakeRecurrence": null, "contradictoryDelivery": null, "contradictoryDeliveryIntervalHigh": null, "negativeDeliveryIntervalHigh": null, "consumerSchemaTokensP95": null, "captureModelCallsP95": null, "writerIntegrity": false, "crossClientPassed": false, "benchmarkFamilyCoverage": null, "benchmarkArmCoverage": null, "directionalCorrectnessIntervalLow": null, "familyCorrectnessIntervalLow": null, "directionalTokenOverheadHigh": null, "causalChainIntegrity": false, "trialIntegrity": false, "independentGrading": false, "competitivePassed": false, "competitiveCoverage": null, "competitiveReproducibility": false }, "intervals": { "recurrence": { "pairs": 0, "mean": null, "low": null, "high": null }, "correctness": { "pairs": 0, "mean": null, "low": null, "high": null }, "reconstruction": { "pairs": 0, "mean": null, "low": null, "high": null }, "firstSuccessorTokens": { "pairs": 0, "mean": null, "low": null, "high": null } }, "sources": { "applicabilityPrecision": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "applicabilityPrecisionIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDeliveryIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessDelta": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "severeUnquarantined": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "emptyP95Overhead": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "reconstructionTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "latencyOverheadP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "knownMistakeRecurrence": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "negativeDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "consumerSchemaTokensP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "captureModelCallsP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "writerIntegrity": { "requiredClass": "conformance", "eligibleLedgerHashes": [] }, "crossClientPassed": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkFamilyCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkArmCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "familyCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalTokenOverheadHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "causalChainIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "trialIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "independentGrading": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "competitivePassed": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveCoverage": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveReproducibility": { "requiredClass": "superiority", "eligibleLedgerHashes": [] } }, "inputLedgerHashes": [ "52159aeaf25ac2da45ad79ba150beb90f117a9d589e31f960f4379d8b7b37cdf" ], "rejectedLedgers": 0, "derivedHash": "a2aaea46ba70a41862db4c3097c48606e5519f91fc2b2b809e4101c0491beba7" }, "costDiagnostics": { "passed": false, "thresholds": { "maximumTokenOverhead": 0.05, "maximumLatencyOverhead": 0.05 }, "groups": [ { "dimension": "direction", "value": "codex->claude-code", "pairs": 1, "tokenOverhead": 0.30118847803604426, "latencyOverheadP95": 0.08167701441438993, "regressions": [ "tokens", "latency" ], "passed": false }, { "dimension": "family", "value": "cross-model-handoff", "pairs": 1, "tokenOverhead": 0.30118847803604426, "latencyOverheadP95": 0.08167701441438993, "regressions": [ "tokens", "latency" ], "passed": false }, { "dimension": "client", "value": "claude-code", "pairs": 1, "tokenOverhead": 0.30118847803604426, "latencyOverheadP95": 0.08167701441438993, "regressions": [ "tokens", "latency" ], "passed": false } ] }, "passed": false, "limitations": [ "qualification or partial execution cannot supply effectiveness metrics" ], "reportHash": "9310db9a6c22dd9e66f60f91a092db51afdc3b38fb0c0d18e1c43870fc8de208" }