{
  "type": "study-task-suite",
  "schemaVersion": 1,
  "suiteVersion": "v1",
  "protocolVersion": "v1",
  "title": "Controlled documentation and knowledge discovery task suite",
  "population": ["consumer-01", "consumer-02", "consumer-03", "consumer-04", "consumer-05", "consumer-06"],
  "modelIds": ["low-cost-model", "reference-model"],
  "scenarioIds": ["repository-only", "deterministic-doc-bridge", "registry-assisted"],
  "maxTokensPerTask": 400000,
  "maxRuntimeMsPerTask": 180000,
  "maxRuns": 288,
  "replicatesPerTask": 2,
  "ordering": { "strategy": "balanced-counter-order", "seed": "phase-1-baseline" },
  "tasks": [
    {
      "id": "consumer-01-discovery", "repositoryId": "consumer-01", "category": "discovery", "difficulty": "easy",
      "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.",
      "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"],
      "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 },
      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } },
      "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-01-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }]
    },
    {
      "id": "consumer-01-architecture", "repositoryId": "consumer-01", "category": "architecture", "difficulty": "medium",
      "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.",
      "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"],
      "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 },
      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } },
      "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-01-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }]
    },
    {
      "id": "consumer-01-documentation", "repositoryId": "consumer-01", "category": "documentation", "difficulty": "hard",
      "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.",
      "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"],
      "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 },
      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } },
      "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-01-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }]
    },
    {
      "id": "consumer-01-implementation", "repositoryId": "consumer-01", "category": "implementation", "difficulty": "hard",
      "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.",
      "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"],
      "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 },
      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } },
      "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-01-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }]
    },

    {
      "id": "consumer-02-discovery", "repositoryId": "consumer-02", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-02-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
    {
      "id": "consumer-02-architecture", "repositoryId": "consumer-02", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-02-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
    {
      "id": "consumer-02-documentation", "repositoryId": "consumer-02", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-02-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
    {
      "id": "consumer-02-implementation", "repositoryId": "consumer-02", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-02-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },

    {
      "id": "consumer-03-discovery", "repositoryId": "consumer-03", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-03-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
    {
      "id": "consumer-03-architecture", "repositoryId": "consumer-03", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-03-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
    {
      "id": "consumer-03-documentation", "repositoryId": "consumer-03", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-03-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
    {
      "id": "consumer-03-implementation", "repositoryId": "consumer-03", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-03-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },

    {
      "id": "consumer-04-discovery", "repositoryId": "consumer-04", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-04-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
    {
      "id": "consumer-04-architecture", "repositoryId": "consumer-04", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-04-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
    {
      "id": "consumer-04-documentation", "repositoryId": "consumer-04", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-04-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
    {
      "id": "consumer-04-implementation", "repositoryId": "consumer-04", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-04-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },

    {
      "id": "consumer-05-discovery", "repositoryId": "consumer-05", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-05-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
    {
      "id": "consumer-05-architecture", "repositoryId": "consumer-05", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-05-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
    {
      "id": "consumer-05-documentation", "repositoryId": "consumer-05", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-05-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
    {
      "id": "consumer-05-implementation", "repositoryId": "consumer-05", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-05-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },

    {
      "id": "consumer-06-discovery", "repositoryId": "consumer-06", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-06-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
    {
      "id": "consumer-06-architecture", "repositoryId": "consumer-06", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-06-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
    {
      "id": "consumer-06-documentation", "repositoryId": "consumer-06", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-06-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
    {
      "id": "consumer-06-implementation", "repositoryId": "consumer-06", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-06-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] }
  ],
  "contentHashAlgo": "sha256-normalized-v1",
  "contentHash": "d7810828bce3c6616f00897fd7c92dfa4ecdf12e549923795cd95cc81a2b0029"
}