{ "baseline_variant": "baseline", "caveat": "Proxy byte reductions are reported separately from matched-task token/cost metrics; shifted cost savings require measured primary cost and measured external cost when external tokens are present. Wall time and provider cached-token fields are diagnostic telemetry, not proof of ContextGuard-caused token or cost savings; provider-cache discounts must stay separate from token-reduction claims. Public hosted savings claims must use public_claim_readiness.claim_allowed; unsupported claims are forbidden.", "claim_status": "token_savings_observed_cost_unmeasured", "comparisons": [ { "baseline_corrections_per_successful_task": 0.0, "baseline_failure_rate": 0.0, "baseline_successful_task_count": 1, "baseline_variant": "baseline", "corrections_delta_per_successful_task": 0.0, "cost_savings_pct_with_shift": null, "failure_rate_delta_pp": 0.0, "matched_successful_task_count": 1, "missing_baseline_success_tasks": [], "paired_corrections_task_count": 1, "paired_cost_task_count": 0, "paired_token_task_count": 1, "paired_wall_time_task_count": 1, "quality_gate": "pass", "token_delta_per_successful_task": -240.0, "token_savings_pct": 24.0, "variant": "brief_mode_standard", "variant_corrections_per_successful_task": 0.0, "variant_failure_rate": 0.0, "wall_time_change_pct": -4.0000000000000036, "wall_time_delta_seconds_per_successful_task": -0.40000000000000036 } ], "public_claim_readiness": { "blocking_gate_ids": [ "matched_successful_tasks", "provider_measured_token_cost", "shifted_cost_accounting", "confidence_failure_notes", "provider_export_provenance" ], "claim_allowed": false, "claim_boundary": { "claim_allowed_field": "public_claim_readiness.claim_allowed", "fixed_percent_savings_claim_without_matched_provider_report_forbidden": true, "hosted_api_cost_savings_claim_without_claim_allowed_forbidden": true, "hosted_api_token_savings_claim_without_claim_allowed_forbidden": true, "id": "public_claim_readiness_authoritative_release_gate", "reason": "Public hosted token/cost savings claims are forbidden unless every readiness gate passes and public_claim_readiness.claim_allowed is true.", "reporting_only": true, "requires_confidence_and_failure_notes": true, "requires_matched_successful_tasks": true, "requires_provider_export_provenance": true, "requires_provider_measured_tokens_and_cost": true, "requires_quality_non_inferiority": true, "requires_shifted_cost_accounting": true, "unsupported_claims_forbidden": true }, "gates": [ { "evidence": { "comparison_count": 1, "matched_pair_count": 0, "min_matched_successful_task_count": 1.0, "missing_baseline_success_task_count": 0, "variants": [ "brief_mode_standard" ] }, "id": "matched_successful_tasks", "label": "Matched successful tasks", "passed": false, "reason": "missing_or_regressed_matched_successful_tasks", "required": true, "status": "fail" }, { "evidence": { "matched_pair_count": 0, "required_fields": [ "matched_pair_evidence[*].measurements.baseline.primary_tokens.measured", "matched_pair_evidence[*].measurements.variant.primary_tokens.measured", "matched_pair_evidence[*].measurements.baseline.primary_cost_usd.measured", "matched_pair_evidence[*].measurements.variant.primary_cost_usd.measured" ] }, "id": "provider_measured_token_cost", "label": "Provider-measured token and primary cost", "passed": false, "reason": "missing_provider_measured_primary_tokens_or_cost", "required": true, "status": "fail" }, { "evidence": { "max_corrections_delta_per_successful_task": 0.0, "max_failure_rate_delta_pp": 0.0, "quality_gates": [ "pass" ] }, "id": "quality_non_inferiority", "label": "Quality non-inferiority", "passed": true, "reason": "all_quality_gates_pass", "required": true, "status": "pass" }, { "evidence": { "matched_pair_count": 0, "required_fields": [ "matched_pair_evidence[*].claim_boundary.shifted_cost_claim_allowed", "matched_pair_evidence[*].measurements.baseline.total_cost_with_shift_usd.measured", "matched_pair_evidence[*].measurements.variant.total_cost_with_shift_usd.measured" ] }, "id": "shifted_cost_accounting", "label": "Shifted-cost accounting", "passed": false, "reason": "missing_shifted_cost_claim_accounting", "required": true, "status": "fail" }, { "evidence": { "comparison_failure_fields_present": true, "explicit_note_count": 0, "failed_row_count": 0, "failed_rows_with_notes": 0, "replay_row_count": 0 }, "id": "confidence_failure_notes", "label": "Confidence and failure notes", "passed": false, "reason": "missing_explicit_replay_notes_or_failure_evidence", "required": true, "status": "unknown" }, { "evidence": { "mixed_csv": false, "provider_names": [], "replay_row_count": 0, "report_row_count": 2, "same_run_complete": false, "source_types": [] }, "id": "provider_export_provenance", "label": "Provider-export provenance", "passed": false, "reason": "missing_or_mixed_provider_export_provenance", "required": true, "status": "unknown" } ], "generated_from": "matched_pair_evidence_and_replay_provenance", "passed_required_gate_count": 1, "public_claim_eligible_observed": null, "public_claim_status_observed": "csv_provenance_unknown_requires_original_evidence_or_trusted_ledger", "raw_metric_claim_status_observed": "token_savings_observed_cost_unmeasured", "reason": "replay_evidence_required_for_public_claim", "required_gate_count": 6, "required_gate_ids": [ "matched_successful_tasks", "provider_measured_token_cost", "quality_non_inferiority", "shifted_cost_accounting", "confidence_failure_notes", "provider_export_provenance" ], "schema_version": "contextguard.bench.public-claim-readiness.v1", "status": "csv_provenance_unknown_requires_original_evidence_or_trusted_ledger" }, "row_count": 2, "schema": "context-guard-bench-report-v1", "summary_by_variant": { "baseline": { "artifacts_used_per_successful_task": 0.0, "artifacts_used_successful": 0, "byte_reduction_ratio": 1.0, "byte_savings_pct": 0.0, "bytes_after_successful": 12000, "bytes_before_successful": 12000, "bytes_saved_per_successful_task": 0.0, "bytes_saved_successful": 0, "compression_strategy": "baseline", "corrections_per_successful_task": 0.0, "corrections_successful": 0, "external_cost_successful_usd": 0.0, "external_cost_unknown_successful": 1, "external_tokens_measured_successful": 0, "external_tokens_per_successful_task": null, "external_tokens_successful": 0, "failed_runs": 0, "failure_rate": 0.0, "hook_triggers_successful": 0, "is_baseline_strategy": true, "observed_telemetry": { "byte_savings": "observed", "external_tokens": "unavailable", "primary_cost": "unavailable", "provider_cache": "unavailable", "token_proxy": "inferred", "tokens": "observed", "wall_time": "observed" }, "primary_cost_all_runs_usd": 0.0, "primary_cost_measured_runs": 0, "primary_cost_measured_successful": 0, "primary_cost_per_successful_task_usd": null, "primary_cost_per_task_including_failures_usd": null, "primary_cost_successful_usd": 0.0, "primary_tokens_measured_runs": 1, "primary_tokens_measured_successful": 1, "provider_cached_tokens_all_runs": 0, "provider_cached_tokens_measured_runs": 0, "provider_cached_tokens_measured_successful": 0, "provider_cached_tokens_per_successful_task": 0.0, "provider_cached_tokens_per_task_including_failures": 0.0, "provider_cached_tokens_successful": 0, "runs": 1, "successful_runs": 1, "successful_task_count": 1, "task_count": 1, "token_proxy_saved_per_successful_task": 0.0, "token_proxy_saved_successful": 0, "tokens_per_successful_task": 1000.0, "tokens_per_task_including_failures": 1000.0, "total_cost_with_shift_all_runs_usd": 0.0, "total_cost_with_shift_measured_runs": 0, "total_cost_with_shift_measured_successful": 0, "total_cost_with_shift_per_successful_task_usd": null, "total_cost_with_shift_per_task_including_failures_usd": null, "total_cost_with_shift_successful_usd": 0.0, "total_tokens_all_runs": 1000, "total_tokens_successful": 1000, "turns_successful": 0, "wall_time_seconds_all_runs": 10.0, "wall_time_seconds_measured_runs": 1, "wall_time_seconds_measured_successful": 1, "wall_time_seconds_per_successful_task": 10.0, "wall_time_seconds_per_task_including_failures": 10.0, "wall_time_seconds_successful": 10.0 }, "brief_mode_standard": { "artifacts_used_per_successful_task": 0.0, "artifacts_used_successful": 0, "byte_reduction_ratio": 0.75, "byte_savings_pct": 25.0, "bytes_after_successful": 9000, "bytes_before_successful": 12000, "bytes_saved_per_successful_task": 3000.0, "bytes_saved_successful": 3000, "compression_strategy": "brief_mode_standard", "corrections_per_successful_task": 0.0, "corrections_successful": 0, "external_cost_successful_usd": 0.0, "external_cost_unknown_successful": 1, "external_tokens_measured_successful": 0, "external_tokens_per_successful_task": null, "external_tokens_successful": 0, "failed_runs": 0, "failure_rate": 0.0, "hook_triggers_successful": 0, "is_baseline_strategy": false, "observed_telemetry": { "byte_savings": "observed", "external_tokens": "unavailable", "primary_cost": "unavailable", "provider_cache": "unavailable", "token_proxy": "inferred", "tokens": "observed", "wall_time": "observed" }, "primary_cost_all_runs_usd": 0.0, "primary_cost_measured_runs": 0, "primary_cost_measured_successful": 0, "primary_cost_per_successful_task_usd": null, "primary_cost_per_task_including_failures_usd": null, "primary_cost_successful_usd": 0.0, "primary_tokens_measured_runs": 1, "primary_tokens_measured_successful": 1, "provider_cached_tokens_all_runs": 0, "provider_cached_tokens_measured_runs": 0, "provider_cached_tokens_measured_successful": 0, "provider_cached_tokens_per_successful_task": 0.0, "provider_cached_tokens_per_task_including_failures": 0.0, "provider_cached_tokens_successful": 0, "runs": 1, "successful_runs": 1, "successful_task_count": 1, "task_count": 1, "token_proxy_saved_per_successful_task": 750.0, "token_proxy_saved_successful": 750, "tokens_per_successful_task": 760.0, "tokens_per_task_including_failures": 760.0, "total_cost_with_shift_all_runs_usd": 0.0, "total_cost_with_shift_measured_runs": 0, "total_cost_with_shift_measured_successful": 0, "total_cost_with_shift_per_successful_task_usd": null, "total_cost_with_shift_per_task_including_failures_usd": null, "total_cost_with_shift_successful_usd": 0.0, "total_tokens_all_runs": 760, "total_tokens_successful": 760, "turns_successful": 0, "wall_time_seconds_all_runs": 9.6, "wall_time_seconds_measured_runs": 1, "wall_time_seconds_measured_successful": 1, "wall_time_seconds_per_successful_task": 9.6, "wall_time_seconds_per_task_including_failures": 9.6, "wall_time_seconds_successful": 9.6 } } }