# Iteration Analytics Schema
# Based on REF-015 Self-Refine: Iterative Refinement with Self-Feedback
# Issue: #250

$schema: "https://json-schema.org/draft/2020-12/schema"
$id: "https://aiwg.io/schemas/iteration-analytics/v1"
title: "Iteration Analytics Schema"
description: |
  Iteration quality tracking and adaptive stopping criteria to prevent
  unnecessary refinement cycles per REF-015 Self-Refine and to carry
  LFD-style loop controls for issue #1585.

type: object
required:
  - version
  - quality_tracking
  - stopping_criteria
  - ralph_integration

properties:
  version:
    type: string
    pattern: "^\\d+\\.\\d+\\.\\d+$"
    default: "1.0.0"

  quality_tracking:
    $ref: "#/$defs/QualityTracking"

  stopping_criteria:
    $ref: "#/$defs/StoppingCriteria"

  ralph_integration:
    $ref: "#/$defs/RalphIntegration"

$defs:
  QualityTracking:
    type: object
    description: "Iteration-over-iteration quality measurement"
    properties:
      enabled:
        type: boolean
        default: true

      research_backing:
        type: object
        properties:
          source: { type: string, default: "REF-015" }
          finding: { type: string, default: "Iterative refinement shows diminishing returns after 2-3 iterations" }
          recommendation: { type: string, default: "Stop when improvement < 5% to prevent wasted compute" }

      metrics_schema:
        type: object
        properties:
          iteration: { type: integer }
          timestamp: { type: string, format: "date-time" }
          experiment:
            $ref: "#/$defs/ExperimentRecord"
          scores:
            type: object
            properties:
              tests_passing:
                type: number
                description: "Percentage of tests passing (0-1)"
              code_quality:
                type: number
                description: "Linter score normalized (0-1)"
              coverage:
                type: number
                description: "Test coverage percentage (0-1)"
              complexity:
                type: number
                description: "Inverse complexity score (0-1)"
          quality:
            type: number
            description: "Weighted aggregate quality score"
          tokens_used:
            type: integer
            minimum: 0
            description: "Observable total token count for this iteration."
          execution_time_ms:
            type: integer
            minimum: 0
            description: "Observable wall-clock duration for this iteration."
          tool_calls:
            type: integer
            minimum: 0
            description: "Observable tool-call count for this iteration."
          quality_per_1k_tokens:
            type: number
            description: "Quality score divided by thousands of tokens used."
          quality_per_minute:
            type: number
            description: "Quality score divided by wall-clock minutes."
          baseline_comparison:
            $ref: "#/$defs/BaselineComparison"
          delta:
            type: object
            properties:
              quality: { type: number }
              tests_passing: { type: number }
              code_quality: { type: number }
              coverage: { type: number }
              complexity: { type: number }
          cumulative:
            type: object
            properties:
              total_improvement: { type: number }
              plateau_count: { type: integer }
              regression_count: { type: integer }
              flat_cycle_count: { type: integer }

      quality_weights:
        type: object
        description: "Weights for aggregate quality calculation"
        properties:
          tests_passing: { type: number, default: 0.4 }
          code_quality: { type: number, default: 0.3 }
          coverage: { type: number, default: 0.2 }
          complexity: { type: number, default: 0.1 }

      scoring_methods:
        type: object
        properties:
          tests_passing:
            type: string
            default: "passed / total tests"
          code_quality:
            type: string
            default: "1 - (avg_severity / max_severity)"
          coverage:
            type: string
            default: "line_coverage / 100"
          complexity:
            type: string
            default: "max(0, 1 - (cyclomatic_complexity / threshold))"

  StoppingCriteria:
    type: object
    description: "Adaptive stopping rules"
    properties:
      quality_threshold:
        type: object
        properties:
          description: { type: string, default: "Stop if quality exceeds threshold" }
          value: { type: number, default: 0.95 }
          enabled: { type: boolean, default: true }

      improvement_threshold:
        type: object
        properties:
          description: { type: string, default: "Stop if improvement falls below threshold" }
          value: { type: number, default: 0.05 }
          enabled: { type: boolean, default: true }

      iteration_limits:
        type: object
        properties:
          min: { type: integer, default: 2, description: "Always run at least this many" }
          max: { type: integer, default: 10, description: "Hard limit" }

      plateau_detection:
        type: object
        properties:
          description: { type: string, default: "Stop if average improvement plateaus" }
          window: { type: integer, default: 3, description: "Iterations to check" }
          threshold: { type: number, default: 0.03, description: "Minimum avg improvement" }
          enabled: { type: boolean, default: true }

      regression_protection:
        type: object
        properties:
          allow_regression: { type: boolean, default: false }
          max_regressions: { type: integer, default: 2 }
          enabled: { type: boolean, default: true }

      budget_limits:
        type: object
        description: "Hard stop ceilings. Unobservable dimensions may be omitted, but declared dimensions stop the loop when exhausted."
        properties:
          wall_clock_minutes:
            type: integer
            minimum: 1
          output_tokens:
            type: integer
            minimum: 1
          total_tokens:
            type: integer
            minimum: 1
          spend_usd:
            type: number
            minimum: 0
          tool_calls:
            type: integer
            minimum: 1
          rate_caps_are_hard_stops:
            type: boolean
            default: false

      exploration_quota:
        type: object
        description: "Bounded entropy control for non-improving loops."
        properties:
          enabled: { type: boolean, default: true }
          k:
            type: integer
            minimum: 1
            default: 3
            description: "Maximum non-terminal cycles before a structural variant or stop is required."
          flat_cycle_action:
            type: string
            enum: ["structural_variant", "stop_with_best_output", "human_review"]
            default: "structural_variant"

      stopping_decision_flow:
        type: string
        default: |
          1. Check hard iteration limits
          2. Check declared budget limits → stop with best-output report if exhausted
          3. If min_iterations not met → continue
          4. If max_iterations reached → stop with best-output report
          5. Check quality threshold → stop if exceeded
          6. Check improvement threshold → require structural variant or stop
          7. Check plateau detection → require structural variant or stop
          8. Check regression protection → stop if exceeded
          9. Continue iterating

  ExperimentRecord:
    type: object
    description: "Falsifiable hypothesis captured before a material loop action."
    required:
      - hypothesis
      - expected_failure_mode
      - distinguishing_diagnostic
    properties:
      hypothesis:
        type: string
      expected_failure_mode:
        type: string
      distinguishing_diagnostic:
        type: string
      structural_variant:
        type: string
        description: "How this iteration differs from the last non-improving attempt."
      result:
        type: string
        description: "Observed outcome after validation."
      probe_or_generalization_signal:
        type: string
        description: "Probe, dev/holdout aggregate, or other generalization signal."

  BudgetStopReport:
    type: object
    description: "Report emitted when a hard loop budget stops execution."
    required:
      - stop_reason
      - budgets
    # NOTE: the authoritative RUNTIME shape is
    # agentic/code/addons/agent-loop/schemas/iteration-analytics-output.yaml
    # ($defs/BudgetStopReport), validated against the code in
    # tools/ralph-external/iteration-analytics.test.mjs (#1771). This
    # framework-level copy is kept in sync with it.
    properties:
      stop_reason:
        type: string
        # Emitted by getBudgetStopTrigger(): "<dimension>_exhausted".
        enum:
          - wall_clock_exhausted
          - output_tokens_exhausted
          - total_tokens_exhausted
          - input_tokens_exhausted
          - spend_exhausted
          - tool_calls_exhausted
      budgets:
        # Parallel maps, matching the code (not per-dimension {limit,observed}).
        type: object
        required: [limits, observed, exhausted, unobservable]
        properties:
          limits: { type: object }
          observed: { type: object }
          exhausted: { type: array }
          unobservable: { type: array, items: { type: string } }
      # Nullable when there is no selected/final iteration yet (#1771).
      selected_iteration:
        type: [integer, "null"]
      final_iteration:
        type: [integer, "null"]
      best_score:
        type: [number, "null"]
      final_score:
        type: [number, "null"]
      hypothesis_outcomes:
        type: array
        items:
          $ref: "#/$defs/ExperimentRecord"
      next_recommended_action:
        type: string

  BaselineComparison:
    type: object
    description: "Optional chance/random-walk baseline comparison for token-utilization and speed-of-accuracy reporting."
    required:
      - baseline_type
      - baseline_quality_score
      - quality_lift
    properties:
      baseline_type:
        type: string
        enum: ["random_walk", "chance", "tool_shortlist_random", "declared_control"]
        default: "random_walk"
      source:
        type: string
        description: "Harness, fixture, or issue/reference that produced the baseline."
      baseline_quality_score:
        type: number
      quality_lift:
        type: number
        description: "Iteration quality minus baseline quality."
      quality_lift_pct:
        type: number
        description: "Quality lift divided by absolute baseline quality, when defined."
      baseline_tokens_used:
        type: integer
        minimum: 0
      token_efficiency_lift:
        type: number
        description: "Quality-per-1K-token lift over the baseline, when token counts are observable."
      baseline_execution_time_ms:
        type: integer
        minimum: 0
      speed_efficiency_lift:
        type: number
        description: "Quality-per-minute lift over the baseline, when wall-clock time is observable."
      baseline_tool_calls:
        type: integer
        minimum: 0
      tool_call_savings:
        type: integer
        description: "Baseline tool calls minus iteration tool calls, when tool-call counts are observable."

  EvalHarnessContract:
    type: object
    # STATUS: IMPLEMENTED (#1776). The runtime lives at
    # tools/ralph-external/eval-harness.mjs (EvalHarness); the orchestrator runs
    # it per iteration when a loop declares `evalHarness`, folds VOID into the
    # iteration verification_status, writes eval-harness-result.json, and fences
    # VOID iterations out of best-output selection. Holdout isolation (VOID-safe
    # optimizer feedback + private diagnostics) is enforced by
    # buildOptimizerFeedback(). Opt-in: loops without a contract are unaffected.
    description: "LFD-style score/lint/probe/status harness contract for eval-driven loops (implemented — #1776)."
    required:
      - score
      - lint
      - probe
      - status
      - diagnostics_policy
    properties:
      manifest_path:
        type: string
        description: "Versioned harness manifest or config path."
      locked_harness:
        type: boolean
        default: true
        description: "Whether the harness implementation is fixed for comparable runs."
      score:
        type: object
        required: [command, optimizer_visible_output]
        properties:
          command: { type: string }
          accepts_dev_feedback: { type: boolean, default: true }
          accepts_holdout_feedback: { type: boolean, default: false }
          optimizer_visible_output:
            type: string
            enum: ["aggregate_only", "void_only", "none"]
            default: "aggregate_only"
      lint:
        type: object
        required: [command, void_on_violation]
        properties:
          command: { type: string }
          void_on_violation: { type: boolean, default: true }
          optimizer_visible_output:
            type: string
            enum: ["aggregate_only", "void_only", "none"]
            default: "void_only"
          private_diagnostics_path: { type: string }
      probe:
        type: object
        required: [command, purpose]
        properties:
          command: { type: string }
          purpose:
            type: string
            enum: ["memorization_gap", "generalization_gap", "harness_integrity", "policy_boundary"]
          optimizer_visible_output:
            type: string
            enum: ["aggregate_only", "void_only", "none"]
            default: "aggregate_only"
      status:
        type: object
        required: [command]
        properties:
          command: { type: string }
          reports:
            type: array
            items:
              type: string
              enum:
                - score_history
                - budget_usage
                - burn_rate
                - best_iteration
                - harness_version
                - leakage_audit
                - structural_variant
                - baseline_comparison
      diagnostics_policy:
        type: object
        required:
          - optimizer_visible
          - private_human
        properties:
          optimizer_visible:
            type: string
            enum: ["aggregate_only", "void_only", "none"]
          private_human:
            type: string
            description: "Path or channel for detailed diagnostics unavailable to the optimizing agent."
          forbidden_optimizer_fields:
            type: array
            items:
              type: string
            default:
              - holdout_case_ids
              - holdout_answers
              - oracle_traces
              - detailed_lint_findings
              - fixture_membership

  EvalHarnessResult:
    type: object
    description: "Result shape returned by an LFD-style eval harness instrument."
    required:
      - status
      - optimizer_feedback
    properties:
      status:
        type: string
        enum: ["pass", "fail", "void", "error"]
      optimizer_feedback:
        type: object
        description: "Aggregate-only or VOID-safe feedback visible to the optimizing agent."
        properties:
          score: { type: number }
          pass_count: { type: integer }
          total_count: { type: integer }
          status: { type: string }
          void_reason: { type: string }
      private_diagnostics_ref:
        type: string
        description: "Private diagnostics path or identifier, not optimizer-readable."
      leakage_audit:
        type: object
        properties:
          checked: { type: boolean }
          result:
            type: string
            enum: ["pass", "fail", "not_applicable"]

  RalphIntegration:
    type: object
    description: "Integration with agent loop"
    properties:
      enabled:
        type: boolean
        default: true

      hook_points:
        type: object
        properties:
          after_iteration:
            type: string
            default: "Score iteration and check stopping criteria"
          on_stop:
            type: string
            default: "Log final analytics and stopping reason"
          on_regression:
            type: string
            default: "Log warning and consider rollback"

      config_path:
        type: string
        default: ".aiwg/ralph/analytics-config.json"

      output_paths:
        type: object
        properties:
          iteration_history: { type: string, default: ".aiwg/ralph/tasks/{task_id}/analytics.json" }
          quality_chart: { type: string, default: ".aiwg/ralph/tasks/{task_id}/quality-chart.txt" }
          eval_harness_result: { type: string, default: ".aiwg/ralph/tasks/{task_id}/eval-harness-result.json" }

# Analytics result schema
analytics_result:
  type: object
  properties:
    task_id:
      type: string
    iterations:
      type: integer
    stopping_reason:
      type: string
    budget_stop_report:
      $ref: "#/$defs/BudgetStopReport"
    eval_harness:
      $ref: "#/$defs/EvalHarnessContract"
    eval_harness_result:
      $ref: "#/$defs/EvalHarnessResult"
    final_quality:
      type: number
    total_improvement:
      type: number
    baseline_comparison:
      type: object
      properties:
        count:
          type: integer
        best_quality_lift:
          type: number
        best_token_efficiency_lift:
          type: number
        best_speed_efficiency_lift:
          type: number
    history:
      type: array
      items:
        $ref: "#/$defs/QualityTracking/properties/metrics_schema"
    time_saved_estimate:
      type: string
      description: "Estimated time saved by adaptive stopping"

# CLI commands
cli_commands:
  ralph_analytics:
    command: "aiwg ralph-analytics <task-id>"
    description: "View iteration analytics for a task"
    options:
      - name: "--export"
        description: "Export analytics to file"
      - name: "--format"
        description: "Output format (text, json, csv)"
        default: "text"

  config_stopping:
    command: "aiwg config set ralph.<key> <value>"
    description: "Configure stopping criteria"
    examples:
      - "aiwg config set ralph.minIterations 2"
      - "aiwg config set ralph.maxIterations 10"
      - "aiwg config set ralph.improvementThreshold 0.05"

# Agent protocol
agent_protocol:
  score_iteration:
    description: "Score quality of current iteration"
    steps:
      - collect_test_results
      - collect_lint_results
      - collect_coverage_results
      - analyze_complexity
      - attach_pre_change_experiment_record
      - compute_weighted_quality
      - compute_delta_from_previous
      - update_cumulative_stats
      - return_iteration_metrics

  eval_harness_iteration:
    description: "Run optional LFD-style score/lint/probe/status instruments."
    steps:
      - load_eval_harness_contract
      - run_dev_score_or_visible_score
      - run_lint_with_void_semantics
      - run_probe_for_generalization_or_integrity_gap
      - run_status_for_budget_and_burn_rate
      - write_private_diagnostics_outside_optimizer_context
      - return_aggregate_or_void_feedback_only

  check_stopping:
    description: "Evaluate stopping criteria"
    steps:
      - check_iteration_limits
      - check_declared_budget_limits
      - if_min_not_met:
          - continue
      - if_max_reached:
          - stop_with_best_output_report
      - check_quality_threshold
      - check_improvement_threshold
      - check_plateau_detection
      - if_flat_cycle_quota_reached:
          - require_structural_variant_or_stop
      - check_regression_protection
      - return_decision

  generate_visualization:
    description: "Generate quality progression chart"
    steps:
      - load_iteration_history
      - calculate_axis_ranges
      - render_ascii_chart
      - include_stopping_annotations
      - return_visualization

# Storage
storage:
  analytics_data: ".aiwg/ralph/analytics/"
  quality_history: ".aiwg/ralph/tasks/{task_id}/quality-history.jsonl"
  stopping_log: ".aiwg/logs/ralph-stopping.jsonl"

# Success metrics
success_metrics:
  iteration_reduction: "30% fewer iterations on average"
  quality_maintained: "No regression in final output quality"
  time_savings: "40% reduction in total Ralph execution time"
  stopping_accuracy: "90% of stops occur within 1 iteration of human judgment"

# Example analytics output
example_analytics_output: |
  $ aiwg ralph-analytics task-001

  Iteration Analytics: task-001
  ═══════════════════════════════════

  Quality Over Iterations
  ───────────────────────
  1: ████████████████████████████████ 0.65 (+0.650)
  2: ███████████████████████████████████████ 0.78 (+0.130)
  3: █████████████████████████████████████████ 0.82 (+0.040)

  Stopping: Improvement below threshold (0.04 < 0.05)

  ───────────────────────
  Final Quality: 0.82
  Total Iterations: 3
  Total Improvement: +0.17
  Time Saved: ~7 iterations avoided

  Breakdown:
  - Tests Passing: 95% → 100% (+5%)
  - Code Quality: 0.72 → 0.85 (+13%)
  - Coverage: 78% → 85% (+7%)
  - Complexity: 0.60 → 0.65 (+5%)

# References
references:
  research:
    - "@.aiwg/research/findings/REF-015-self-refine.md"
  implementation:
    - "#250"
  related:
    - "@tools/ralph-external/loop-executor.ts"
    - "@agentic/code/frameworks/sdlc-complete/schemas/flows/reliability-patterns.yaml"