{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://raw.githubusercontent.com/darks0l/remem/main/benchmarks/PUBLIC-RESULTS.schema.json",
  "title": "ReMEM public benchmark results manifest",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "$schema",
    "schemaVersion",
    "generatedAt",
    "generator",
    "benchmark",
    "claimBoundary",
    "harness",
    "historicalBaseline",
    "currentValidation",
    "artifactDigests"
  ],
  "properties": {
    "$schema": {
      "type": "string",
      "const": "./PUBLIC-RESULTS.schema.json"
    },
    "schemaVersion": {
      "type": "integer",
      "const": 1
    },
    "generatedAt": {
      "type": "string",
      "format": "date-time"
    },
    "generator": {
      "type": "string",
      "const": "benchmarks/generate-public-results.mjs"
    },
    "benchmark": {
      "type": "string",
      "const": "remem-context-window-suite-v1"
    },
    "claimBoundary": {
      "type": "object",
      "additionalProperties": false,
      "required": ["summary", "safeShortClaim", "notSafeToClaim"],
      "properties": {
        "summary": { "type": "string", "minLength": 1 },
        "safeShortClaim": { "type": "string", "minLength": 1 },
        "notSafeToClaim": {
          "type": "array",
          "minItems": 1,
          "items": { "type": "string", "minLength": 1 }
        }
      }
    },
    "harness": {
      "type": "object",
      "additionalProperties": false,
      "required": ["script", "storage", "seed", "metrics"],
      "properties": {
        "script": { "type": "string", "minLength": 1 },
        "storage": { "type": "string", "minLength": 1 },
        "seed": { "type": "integer" },
        "metrics": {
          "type": "array",
          "minItems": 1,
          "items": { "type": "string", "minLength": 1 }
        }
      }
    },
    "historicalBaseline": {
      "type": "object",
      "additionalProperties": false,
      "required": ["releasedAt", "runs"],
      "properties": {
        "releasedAt": {
          "type": "string",
          "pattern": "^\\d{4}-\\d{2}-\\d{2}$"
        },
        "runs": {
          "type": "object",
          "additionalProperties": false,
          "required": ["memories2000", "memories10000", "memories50000", "semantic80"],
          "properties": {
            "memories2000": { "$ref": "#/$defs/run" },
            "memories10000": { "$ref": "#/$defs/run" },
            "memories50000": { "$ref": "#/$defs/run" },
            "semantic80": { "$ref": "#/$defs/run" }
          }
        }
      }
    },
    "currentValidation": {
      "type": "object",
      "additionalProperties": false,
      "required": ["rerunDate", "latest50kEnvironment", "correctedTopicFilteredExactId", "deltasVsHistoricalBaseline"],
      "properties": {
        "rerunDate": {
          "type": "string",
          "pattern": "^\\d{4}-\\d{2}-\\d{2}$"
        },
        "latest50kEnvironment": {
          "$ref": "#/$defs/environment"
        },
        "correctedTopicFilteredExactId": {
          "type": "array",
          "minItems": 3,
          "items": { "$ref": "#/$defs/validationRow" }
        },
        "deltasVsHistoricalBaseline": {
          "type": "array",
          "minItems": 3,
          "items": { "$ref": "#/$defs/validationDeltaRow" }
        }
      }
    },
    "artifactDigests": {
      "type": "array",
      "minItems": 1,
      "items": { "$ref": "#/$defs/artifactDigest" }
    }
  },
  "$defs": {
    "sha256": {
      "type": "string",
      "pattern": "^[a-f0-9]{64}$"
    },
    "sourceFile": {
      "type": "string",
      "pattern": "^benchmarks/"
    },
    "environment": {
      "type": "object",
      "additionalProperties": true,
      "properties": {
        "node": { "type": "string" },
        "platform": { "type": "string" },
        "arch": { "type": "string" },
        "cwd": { "type": "string" },
        "argv": {
          "type": "array",
          "items": { "type": "string" }
        }
      }
    },
    "contextPressure": {
      "type": "object",
      "additionalProperties": true,
      "required": ["corpusApproxTokens", "fixedWindowApproxTokens", "effectiveCorpusToWindowMultiple"],
      "properties": {
        "corpusApproxTokens": { "type": "number" },
        "fixedWindowApproxTokens": { "type": "number" },
        "effectiveCorpusToWindowMultiple": { "type": "number" }
      }
    },
    "metrics": {
      "type": "object",
      "additionalProperties": true,
      "required": [
        "fixedContextRecallAt1",
        "rememRecallAt1",
        "rememRecallAtK",
        "rememMRR",
        "avgQueryMs",
        "p95QueryMs"
      ],
      "properties": {
        "fixedContextRecallAt1": { "type": "number" },
        "rememRecallAt1": { "type": "number" },
        "rememRecallAtK": { "type": "number" },
        "rememMRR": { "type": "number" },
        "avgQueryMs": { "type": "number" },
        "p95QueryMs": { "type": "number" },
        "storeMs": { "type": "number" }
      }
    },
    "scenario": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "name",
        "label",
        "sourceFile",
        "sourceSha256",
        "queryStyle",
        "queryTopics",
        "embeddings",
        "metrics"
      ],
      "properties": {
        "name": { "type": "string", "minLength": 1 },
        "label": { "type": "string", "minLength": 1 },
        "sourceFile": { "$ref": "#/$defs/sourceFile" },
        "sourceSha256": { "$ref": "#/$defs/sha256" },
        "queryStyle": { "type": "string" },
        "queryTopics": { "type": "boolean" },
        "embeddings": {
          "oneOf": [
            { "type": "null" },
            { "type": "boolean" },
            {
              "type": "object",
              "additionalProperties": true,
              "required": ["model", "baseUrl"],
              "properties": {
                "model": { "type": "string" },
                "baseUrl": { "type": "string" }
              }
            }
          ]
        },
        "metrics": { "$ref": "#/$defs/metrics" }
      }
    },
    "run": {
      "type": "object",
      "additionalProperties": false,
      "required": ["sourceFile", "sourceSha256", "contextPressure", "queryCount", "scenarios"],
      "properties": {
        "sourceFile": { "$ref": "#/$defs/sourceFile" },
        "sourceSha256": { "$ref": "#/$defs/sha256" },
        "contextPressure": { "$ref": "#/$defs/contextPressure" },
        "queryCount": { "type": "integer", "minimum": 1 },
        "scenarios": {
          "type": "array",
          "minItems": 1,
          "items": { "$ref": "#/$defs/scenario" }
        }
      }
    },
    "validationRow": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "corpusSize",
        "sourceFile",
        "sourceSha256",
        "fixedRecallAt1",
        "exactCodenameRecallAt1",
        "exactCodenameRecallAt5",
        "topicFilteredExactIdRecallAt1",
        "topicFilteredExactIdRecallAt5",
        "avgExactCodenameQueryMs",
        "avgTopicFilteredQueryMs"
      ],
      "properties": {
        "corpusSize": { "type": "integer", "minimum": 1 },
        "sourceFile": { "$ref": "#/$defs/sourceFile" },
        "sourceSha256": { "$ref": "#/$defs/sha256" },
        "fixedRecallAt1": { "type": "number" },
        "exactCodenameRecallAt1": { "type": "number" },
        "exactCodenameRecallAt5": { "type": "number" },
        "topicFilteredExactIdRecallAt1": { "type": "number" },
        "topicFilteredExactIdRecallAt5": { "type": "number" },
        "avgExactCodenameQueryMs": { "type": "number" },
        "avgTopicFilteredQueryMs": { "type": "number" }
      }
    },
    "validationDeltaRow": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "corpusSize",
        "baselineSourceFile",
        "baselineSourceSha256",
        "validationSourceFile",
        "validationSourceSha256",
        "exactCodenameRecallAt1Delta",
        "exactCodenameRecallAt5Delta",
        "topicFilteredExactIdRecallAt1Delta",
        "topicFilteredExactIdRecallAt5Delta",
        "avgExactCodenameQueryMsDelta",
        "avgTopicFilteredQueryMsDelta"
      ],
      "properties": {
        "corpusSize": { "type": "integer", "minimum": 1 },
        "baselineSourceFile": { "$ref": "#/$defs/sourceFile" },
        "baselineSourceSha256": { "$ref": "#/$defs/sha256" },
        "validationSourceFile": { "$ref": "#/$defs/sourceFile" },
        "validationSourceSha256": { "$ref": "#/$defs/sha256" },
        "exactCodenameRecallAt1Delta": { "type": "number" },
        "exactCodenameRecallAt5Delta": { "type": "number" },
        "topicFilteredExactIdRecallAt1Delta": { "type": "number" },
        "topicFilteredExactIdRecallAt5Delta": { "type": "number" },
        "avgExactCodenameQueryMsDelta": { "type": "number" },
        "avgTopicFilteredQueryMsDelta": { "type": "number" }
      }
    },
    "artifactDigest": {
      "type": "object",
      "additionalProperties": false,
      "required": ["sourceFile", "sha256", "timestamp", "totalMemories", "queryCount"],
      "properties": {
        "sourceFile": { "$ref": "#/$defs/sourceFile" },
        "sha256": { "$ref": "#/$defs/sha256" },
        "timestamp": { "type": "string", "minLength": 1 },
        "totalMemories": { "type": "integer", "minimum": 1 },
        "queryCount": { "type": "integer", "minimum": 1 }
      }
    }
  }
}