{ "schemaVersion": "ucr.full-study/1", "executedAt": "2026-08-10T14:23:46.586Z", "evidenceClass": "executable-smoke", "plan": { "schemaVersion": "ucr.full-study-plan/1", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "seedHash": "41b25ff01f365e97e60404d48067231087bfdf3680c3243e04039cea20f40b62", "clients": [ { "id": "codex", "model": "gpt-5.6-sol", "modelFamily": "openai", "version": "codex-cli 0.147.0", "modelVersion": "gpt-5.6-sol", "lifecycleFamily": "process-hook" }, { "id": "claude-code", "model": "claude-sonnet-5", "modelFamily": "anthropic", "version": "2.1.225 (Claude Code)", "modelVersion": "claude-sonnet-5", "lifecycleFamily": "process-hook" }, { "id": "gemini", "model": "gemini-2.5-pro", "modelFamily": "google", "version": "gemini-cli 0.28.1", "modelVersion": "gemini-2.5-pro", "lifecycleFamily": "process-hook" } ], "directions": [ "codex->codex", "codex->claude-code", "codex->gemini", "claude-code->codex", "claude-code->claude-code", "claude-code->gemini", "gemini->codex", "gemini->claude-code", "gemini->gemini" ], "pairedRepetitionsPerCell": 1, "negativeRepetitionsPerCell": 1, "includeSameClient": true, "budgets": { "contextTokens": 160, "retries": 1, "toolCalls": 40, "timeoutMs": 600000 }, "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "primaryPairsPerDirection": 11, "preregisteredPairsPerArm": 357, "minimumSubgroupPairs": 30, "minimumNegativeSamplesPerDirection": 1053, "negativeConfidence": { "samples": 1053, "maximumRate": 0.01, "z": 3.2607674853887727, "comparisons": 45, "familyAlpha": 0.05, "method": "bonferroni-familywise-wilson-zero-failure-upper-bound" }, "negativeSamplesPerDirectionPerArm": 11, "planHash": "304663a636671cd838304ae46a22aca0d42685f543ea17d391d1574fbf952f71", "trialCount": 693 }, "coverage": { "passed": true, "checks": { "effectivenessDesign": true, "metricMapping": true, "competitiveBaselines": true, "productionStages": true }, "design": { "passed": true, "checks": { "benchmarkFamilies": true, "benchmarkArms": true, "clients": true, "modelFamilies": true, "bidirectional": true, "sameClient": true, "crossClient": true, "sameSession": true, "crossSession": true, "crossProject": true, "concurrentAgents": true, "pairedControls": true, "isolatedWorkspaces": true, "materialVariants": true, "executionTopology": true }, "coverage": { "families": 1, "arms": 1, "clients": 3, "modelFamilies": 3, "directions": 9, "materialVariants": 485 } }, "competitionCoverage": 1, "productionCoverage": 1, "metricSources": { "applicabilityPrecision": "full-effectiveness", "applicabilityPrecisionIntervalLow": "full-effectiveness", "preActionDelivery": "full-effectiveness", "preActionDeliveryIntervalLow": "full-effectiveness", "irrelevantDelivery": "full-effectiveness", "irrelevantDeliveryIntervalHigh": "full-effectiveness", "staleDelivery": "full-effectiveness", "staleDeliveryIntervalHigh": "full-effectiveness", "recurrenceReduction": "full-effectiveness", "recurrenceIntervalLow": "full-effectiveness", "naturalCorrectnessDelta": "full-effectiveness", "naturalCorrectnessIntervalLow": "full-effectiveness", "severeUnquarantined": "full-effectiveness", "emptyP95Overhead": "full-effectiveness", "reconstructionTokenReduction": "full-effectiveness", "firstSuccessorTokenReduction": "full-effectiveness", "firstSuccessorTokenIntervalLow": "full-effectiveness", "latencyOverheadP95": "full-effectiveness", "knownMistakeRecurrence": "full-effectiveness", "contradictoryDelivery": "full-effectiveness", "contradictoryDeliveryIntervalHigh": "full-effectiveness", "negativeDeliveryIntervalHigh": "full-effectiveness", "consumerSchemaTokensP95": "full-effectiveness", "captureModelCallsP95": "full-effectiveness", "writerIntegrity": "writer-conformance", "crossClientPassed": "full-effectiveness", "benchmarkFamilyCoverage": "full-effectiveness", "benchmarkArmCoverage": "full-effectiveness", "directionalCorrectnessIntervalLow": "full-effectiveness", "familyCorrectnessIntervalLow": "full-effectiveness", "directionalTokenOverheadHigh": "full-effectiveness", "causalChainIntegrity": "full-effectiveness", "trialIntegrity": "full-effectiveness", "independentGrading": "full-effectiveness", "competitivePassed": "live-competitive", "competitiveCoverage": "live-competitive", "competitiveReproducibility": "live-competitive" }, "missingMetrics": [] }, "poweredDesign": false, "complete": false, "trialsPlanned": 693, "trialsExecuted": 2, "failures": [], "ledger": { "schemaVersion": "ucr.evidence-ledger/2", "evidenceClass": "executable-smoke", "run": { "schemaVersion": "ucr.evidence-run/2", "runId": "full-study-2e7403b40bf6d521", "evidenceClass": "executable-smoke", "benchmarkHash": "597b75c9fb2a20a171bdde67c7a40cc5aef9aa27ee730550a747c750a39243d2", "sourceTreeHash": "6d56b0f65c183dad638e2dbd2c4486281dbbc90567c42953618f060e10e1ea0a", "runner": { "name": "ucr-full-study", "node": "v25.6.0", "planHash": "304663a636671cd838304ae46a22aca0d42685f543ea17d391d1574fbf952f71", "registrationHash": "e171d5da8431aa64f847b8bd5452b7f40facdb4330d0c21d6b6e38d89b486549", "poweredDesign": false, "minimumSubgroupPairs": 30 }, "startedAt": "2026-08-10T14:23:46.581Z", "runHash": "a223fedeb616034e67c2f2ac001c978b006b8631165ac7f74620bfc1cbc28f00" }, "rows": [ { "study": "full-effectiveness", "planHash": "304663a636671cd838304ae46a22aca0d42685f543ea17d391d1574fbf952f71", "trialId": "codex->claude-code:cross-model-001:0:empty", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 37, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "empty", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "4f5777767e446baf7e658dd317134ebde2f555950387b8ac1a67e66459a5aeb8", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": false, "eligible": false, "selected": false, "delivered": false, "deliveryPhase": null, "stale": false, "contradictory": false, "contextOverheadRatio": 0, "reconstructionTokens": 2, "totalTokens": 43791, "latencyMs": 80016, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 0, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "42ea57ca5a1632573b2b1bb87486d40095a63fe0dfd6dae32f26ae1d9731a49d", "workspaceStateHash": "d514523ad74921a7687a7aec2a96f45aef2def87b26f96e09d1ea4bb53ea7952", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "dc2311a0db52f6ed02ff5d92bce43f590191e1ab50e8db9dc357266430a2bfb8", "runnerExitCode": 0, "runnerOutputHash": "2c9a0aab70f544f3d93e61087e292f593c9ad1584b816d782ebfcd4983b3996b", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-2e7403b40bf6d521", "sequence": 0, "previousHash": "a223fedeb616034e67c2f2ac001c978b006b8631165ac7f74620bfc1cbc28f00", "rowHash": "dbf109e9462fb0b1c3ec2e9e9551ddb3165aec3306fbeb0c16c51ba34bcc99a4" }, { "study": "full-effectiveness", "planHash": "304663a636671cd838304ae46a22aca0d42685f543ea17d391d1574fbf952f71", "trialId": "codex->claude-code:cross-model-001:0:runtime", "pairId": "codex->claude-code:cross-model-001:0", "studySequence": 340, "taskId": "cross-model-001", "family": "cross-model-handoff", "arm": "runtime", "direction": "codex->claude-code", "producerClient": "codex", "consumerClient": "claude-code", "producerFamily": "openai", "consumerFamily": "anthropic", "producerModel": "gpt-5.6-sol", "consumerModel": "claude-sonnet-5", "model": "claude-sonnet-5", "modelVersion": "claude-sonnet-5", "permissionsHash": "99cfc916f214c8e56d0093fa298b4e7f8739bd206e7452bbdfe25a9396338a38", "contextBudget": 160, "retryBudget": 1, "toolBudget": 40, "sessionMode": "cross-session", "projectMode": "same-project", "agentMode": "single-successor", "expectedProviderInvocations": 2, "executionTopologyHash": "90bc20152c4ca188e4703921bb2a20cb5dddf276e8b99f98ca953604448f538a", "poweredStratum": false, "correct": true, "severeHarm": false, "mistakeExecuted": false, "knownMistake": false, "quarantinedBeforeNext": false, "applicable": true, "eligible": true, "selected": true, "delivered": true, "deliveryPhase": "pre-action", "stale": false, "contradictory": false, "contextOverheadRatio": 147, "reconstructionTokens": 2, "totalTokens": 62003, "latencyMs": 106034, "phaseAccounting": { "captureModelCalls": 0, "contextTokens": 294, "staticSchemaTokens": 0 }, "costLedger": { "costUsd": null, "source": "provider-cli-native" }, "causalClaim": false, "causalChain": null, "causalChainValid": false, "outcomeHash": "a39c75a8d4f372de758b4545ed3f1c9610ce0fea7cfbdfcf060665fc886f583b", "workspaceStateHash": "6e2c3729c061c8f34d1a58bc510c8844e22fa8829b12ae1af33d8743ec21d931", "changedProtected": [], "trialIntegrityValid": true, "graderVerified": true, "graderReceiptHash": "780a823071198c86498cb41c575cbce225d737da927863eecdacd7ca8b4f765a", "runnerExitCode": 0, "runnerOutputHash": "94d22dd263ee454fde043df6a96b005d7e70f6501b77705c16c509991717d381", "runnerErrorHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "schemaVersion": "ucr.evidence-row/2", "runId": "full-study-2e7403b40bf6d521", "sequence": 1, "previousHash": "dbf109e9462fb0b1c3ec2e9e9551ddb3165aec3306fbeb0c16c51ba34bcc99a4", "rowHash": "7c8e5e755223cb15b9b924b8b2b81ba6877d3a42cbb5b5d122c183f54c5f025a" } ], "rowCount": 2, "chainHead": "7c8e5e755223cb15b9b924b8b2b81ba6877d3a42cbb5b5d122c183f54c5f025a", "transcriptsPublished": false, "endedAt": "2026-08-10T14:23:46.582Z", "ledgerHash": "78c7e808213199debba047880be32b6ac1c7647bf72a23057b9ba8d2fd3304b1", "signature": "Heb8JRlZqE5zAn6SdcW0kiRpjSF+oQR00ahrJbafYiJq5Nrwgyj4sie5DBsEo4tXXIJQceEvb0laAeHgkqMnDg==" }, "ledgerKeyId": "ucr-ed25519-77b546af455c7ecd3c9c", "ledgerVerification": { "valid": true, "diagnostics": [], "validSignature": true, "ledgerHash": "78c7e808213199debba047880be32b6ac1c7647bf72a23057b9ba8d2fd3304b1", "rowCount": 2 }, "derived": { "schemaVersion": "ucr.derived-metrics/2", "metrics": { "applicabilityPrecision": null, "applicabilityPrecisionIntervalLow": null, "preActionDelivery": null, "preActionDeliveryIntervalLow": null, "irrelevantDelivery": null, "irrelevantDeliveryIntervalHigh": null, "staleDelivery": null, "staleDeliveryIntervalHigh": null, "recurrenceReduction": null, "recurrenceIntervalLow": null, "naturalCorrectnessDelta": null, "naturalCorrectnessIntervalLow": null, "severeUnquarantined": 0, "emptyP95Overhead": null, "reconstructionTokenReduction": null, "firstSuccessorTokenReduction": null, "firstSuccessorTokenIntervalLow": null, "latencyOverheadP95": null, "knownMistakeRecurrence": null, "contradictoryDelivery": null, "contradictoryDeliveryIntervalHigh": null, "negativeDeliveryIntervalHigh": null, "consumerSchemaTokensP95": null, "captureModelCallsP95": null, "writerIntegrity": false, "crossClientPassed": false, "benchmarkFamilyCoverage": null, "benchmarkArmCoverage": null, "directionalCorrectnessIntervalLow": null, "familyCorrectnessIntervalLow": null, "directionalTokenOverheadHigh": null, "causalChainIntegrity": false, "trialIntegrity": false, "independentGrading": false, "competitivePassed": false, "competitiveCoverage": null, "competitiveReproducibility": false }, "intervals": { "recurrence": { "pairs": 0, "mean": null, "low": null, "high": null }, "correctness": { "pairs": 0, "mean": null, "low": null, "high": null }, "reconstruction": { "pairs": 0, "mean": null, "low": null, "high": null }, "firstSuccessorTokens": { "pairs": 0, "mean": null, "low": null, "high": null } }, "sources": { "applicabilityPrecision": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "applicabilityPrecisionIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "preActionDeliveryIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "irrelevantDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "staleDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "recurrenceIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessDelta": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "naturalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "severeUnquarantined": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "emptyP95Overhead": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "reconstructionTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenReduction": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "firstSuccessorTokenIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "latencyOverheadP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "knownMistakeRecurrence": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDelivery": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "contradictoryDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "negativeDeliveryIntervalHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "consumerSchemaTokensP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "captureModelCallsP95": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "writerIntegrity": { "requiredClass": "conformance", "eligibleLedgerHashes": [] }, "crossClientPassed": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkFamilyCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "benchmarkArmCoverage": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "familyCorrectnessIntervalLow": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "directionalTokenOverheadHigh": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "causalChainIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "trialIntegrity": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "independentGrading": { "requiredClass": "effectiveness", "eligibleLedgerHashes": [] }, "competitivePassed": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveCoverage": { "requiredClass": "superiority", "eligibleLedgerHashes": [] }, "competitiveReproducibility": { "requiredClass": "superiority", "eligibleLedgerHashes": [] } }, "inputLedgerHashes": [ "78c7e808213199debba047880be32b6ac1c7647bf72a23057b9ba8d2fd3304b1" ], "rejectedLedgers": 0, "derivedHash": "a2aaea46ba70a41862db4c3097c48606e5519f91fc2b2b809e4101c0491beba7" }, "costDiagnostics": { "passed": false, "thresholds": { "maximumTokenOverhead": 0.05, "maximumLatencyOverhead": 0.05 }, "groups": [ { "dimension": "direction", "value": "codex->claude-code", "pairs": 1, "tokenOverhead": 0.41588454248589896, "latencyOverheadP95": 0.32515996800639874, "regressions": [ "tokens", "latency" ], "passed": false }, { "dimension": "family", "value": "cross-model-handoff", "pairs": 1, "tokenOverhead": 0.41588454248589896, "latencyOverheadP95": 0.32515996800639874, "regressions": [ "tokens", "latency" ], "passed": false }, { "dimension": "client", "value": "claude-code", "pairs": 1, "tokenOverhead": 0.41588454248589896, "latencyOverheadP95": 0.32515996800639874, "regressions": [ "tokens", "latency" ], "passed": false } ] }, "passed": false, "limitations": [ "qualification or partial execution cannot supply effectiveness metrics" ], "reportHash": "a1019257d1236130312e568d45b8e382fe29d060663c8ec2d5946339de87880f" }