{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://github.com/p-yan-6908/pi-context-engineer/blob/main/bench/effectiveness-result.schema.json",
  "title": "Pi agent effectiveness paired CE-on/off result",
  "$comment": "This contract describes real Pi agent outcomes scored by hidden validators. It is not the deterministic marker-retention benchmark and does not assert non-inferiority.",
  "type": "object",
  "required": ["suite", "generatedAt", "note", "environment", "cases", "iterations", "rows", "totals"],
  "properties": {
    "suite": { "const": "pi-agent-effectiveness" },
    "generatedAt": { "type": "string", "format": "date-time" },
    "note": { "type": "string" },
    "environment": { "$ref": "#/$defs/environment" },
    "cases": { "type": "integer", "minimum": 0 },
    "iterations": { "type": "integer", "minimum": 1 },
    "rows": { "type": "array", "items": { "$ref": "#/$defs/row" } },
    "totals": {
      "type": "object",
      "required": ["ceOff", "ceOn", "paired"],
      "properties": {
        "ceOff": { "$ref": "#/$defs/aggregate" },
        "ceOn": { "$ref": "#/$defs/aggregate" },
        "paired": {
          "type": "object",
          "required": ["bothCorrect", "ceOnOnly", "ceOffOnly", "neitherCorrect"],
          "properties": {
            "bothCorrect": { "type": "integer", "minimum": 0 },
            "ceOnOnly": { "type": "integer", "minimum": 0 },
            "ceOffOnly": { "type": "integer", "minimum": 0 },
            "neitherCorrect": { "type": "integer", "minimum": 0 }
          }
        }
      }
    }
  },
  "$defs": {
    "environment": {
      "type": "object",
      "required": ["sourceCommit", "dirty", "nodeVersion", "piVersion", "model", "provider", "optIn", "realProviderCallsPossible"],
      "properties": {
        "sourceCommit": { "type": "string" },
        "dirty": { "type": "boolean" },
        "nodeVersion": { "type": "string" },
        "piVersion": { "type": ["string", "null"] },
        "model": { "type": ["string", "null"] },
        "provider": { "type": ["string", "null"] },
        "optIn": { "type": "boolean" },
        "realProviderCallsPossible": { "type": "boolean" }
      }
    },
    "usage": {
      "type": "object",
      "required": ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "totalTokens", "cost", "calls", "records", "availability", "missingMeasurements"],
      "properties": {
        "inputTokens": { "type": ["number", "null"] },
        "outputTokens": { "type": ["number", "null"] },
        "cacheReadTokens": { "type": ["number", "null"] },
        "cacheWriteTokens": { "type": ["number", "null"] },
        "totalTokens": { "type": ["number", "null"] },
        "cost": { "$ref": "#/$defs/cost" },
        "calls": { "type": ["integer", "null"] },
        "records": { "type": "integer", "minimum": 0 },
        "availability": { "enum": ["reported", "partial", "unavailable"] },
        "missingMeasurements": { "type": "array", "items": { "type": "string" } }
      }
    },
    "cost": {
      "type": "object",
      "required": ["inputUsd", "outputUsd", "cacheReadUsd", "cacheWriteUsd", "totalUsd"],
      "properties": {
        "inputUsd": { "type": ["number", "null"] },
        "outputUsd": { "type": ["number", "null"] },
        "cacheReadUsd": { "type": ["number", "null"] },
        "cacheWriteUsd": { "type": ["number", "null"] },
        "totalUsd": { "type": ["number", "null"] }
      }
    },
    "metrics": {
      "type": "object",
      "required": ["mode", "wallTimeMs", "modelCalls", "childModelCalls", "toolCalls", "recoveryCalls", "recoverySuccesses", "recoveryFailures", "recoveryUnknown", "retryCount", "telemetryPresent", "telemetryEvents", "telemetryParseErrors", "telemetryBytesRead", "childUsageOutsideParentCalls", "childUsageIncomplete", "finalAnswerCorrect", "taskCompleted", "answerBytes", "usage", "parseErrors", "timedOut"],
      "properties": {
        "mode": { "enum": ["ce-off", "ce-on"] },
        "wallTimeMs": { "type": "number", "minimum": 0 },
        "modelCalls": { "type": "integer", "minimum": 0 },
        "childModelCalls": { "type": "integer", "minimum": 0 },
        "toolCalls": { "type": "integer", "minimum": 0 },
        "recoveryCalls": { "type": "integer", "minimum": 0 },
        "recoverySuccesses": { "type": "integer", "minimum": 0 },
        "recoveryFailures": { "type": "integer", "minimum": 0 },
        "recoveryUnknown": { "type": "integer", "minimum": 0 },
        "retryCount": { "type": "integer", "minimum": 0 },
        "telemetryPresent": { "type": "boolean" },
        "telemetryEvents": { "type": "integer", "minimum": 0 },
        "telemetryParseErrors": { "type": "integer", "minimum": 0 },
        "telemetryBytesRead": { "type": "integer", "minimum": 0 },
        "childUsageOutsideParentCalls": { "type": "integer", "minimum": 0 },
        "childUsageIncomplete": { "type": "boolean" },
        "finalAnswerCorrect": { "type": "boolean" },
        "taskCompleted": { "type": "boolean" },
        "answerBytes": { "type": "integer", "minimum": 0 },
        "usage": { "$ref": "#/$defs/usage" },
        "parseErrors": { "type": "integer", "minimum": 0 },
        "timedOut": { "type": "boolean" },
        "error": { "type": "string" }
      }
    },
    "validation": {
      "type": "object",
      "required": ["finalAnswerCorrect", "taskCompleted", "score", "reason", "checks"],
      "properties": {
        "finalAnswerCorrect": { "type": "boolean" },
        "taskCompleted": { "type": "boolean" },
        "score": { "type": "number", "minimum": 0, "maximum": 1 },
        "reason": { "type": "string" },
        "checks": { "type": "object", "additionalProperties": { "type": "boolean" } }
      }
    },
    "sample": {
      "type": "object",
      "required": ["iteration", "answer", "validation", "metrics"],
      "properties": {
        "iteration": { "type": "integer", "minimum": 1 },
        "answer": { "type": "string" },
        "validation": { "$ref": "#/$defs/validation" },
        "metrics": { "$ref": "#/$defs/metrics" }
      }
    },
    "aggregate": {
      "type": "object",
      "required": ["runs", "finalAnswerCorrect", "finalAnswerCorrectRate", "taskCompleted", "taskCompletedRate", "wallTimeMs", "wallTimeP95Ms", "modelCalls", "childModelCalls", "toolCalls", "recoveryCalls", "recoverySuccesses", "recoveryFailures", "recoveryUnknown", "retryCount", "telemetryPresent", "telemetryEvents", "telemetryParseErrors", "telemetryBytesRead", "childUsageOutsideParentCalls", "childUsageIncomplete", "timedOut", "errors", "usage"],
      "properties": {
        "runs": { "type": "integer", "minimum": 0 },
        "finalAnswerCorrect": { "type": "integer", "minimum": 0 },
        "finalAnswerCorrectRate": { "type": "number", "minimum": 0, "maximum": 1 },
        "taskCompleted": { "type": "integer", "minimum": 0 },
        "taskCompletedRate": { "type": "number", "minimum": 0, "maximum": 1 },
        "wallTimeMs": { "type": "number", "minimum": 0 },
        "wallTimeP95Ms": { "type": "number", "minimum": 0 },
        "modelCalls": { "type": "integer", "minimum": 0 },
        "childModelCalls": { "type": "integer", "minimum": 0 },
        "toolCalls": { "type": "integer", "minimum": 0 },
        "recoveryCalls": { "type": "integer", "minimum": 0 },
        "recoverySuccesses": { "type": "integer", "minimum": 0 },
        "recoveryFailures": { "type": "integer", "minimum": 0 },
        "recoveryUnknown": { "type": "integer", "minimum": 0 },
        "retryCount": { "type": "integer", "minimum": 0 },
        "telemetryPresent": { "type": "boolean" },
        "telemetryEvents": { "type": "integer", "minimum": 0 },
        "telemetryParseErrors": { "type": "integer", "minimum": 0 },
        "telemetryBytesRead": { "type": "integer", "minimum": 0 },
        "childUsageOutsideParentCalls": { "type": "integer", "minimum": 0 },
        "childUsageIncomplete": { "type": "boolean" },
        "timedOut": { "type": "integer", "minimum": 0 },
        "errors": { "type": "integer", "minimum": 0 },
        "usage": { "$ref": "#/$defs/usage" }
      }
    },
    "row": {
      "type": "object",
      "required": ["id", "title", "task", "ceOff", "ceOn", "samples"],
      "properties": {
        "id": { "type": "string" },
        "title": { "type": "string" },
        "task": { "type": "string" },
        "ceOff": { "$ref": "#/$defs/aggregate" },
        "ceOn": { "$ref": "#/$defs/aggregate" },
        "samples": { "type": "array", "items": { "type": "object", "required": ["iteration", "ceOff", "ceOn"], "properties": { "iteration": { "type": "integer", "minimum": 1 }, "ceOff": { "$ref": "#/$defs/sample" }, "ceOn": { "$ref": "#/$defs/sample" } } } }
      }
    }
  }
}