/** * calibration-engine.ts * * Rubric-loading and feedback-context construction for the /attempt command. * * Rubric resolution order (highest priority first): * 1. /.pisces/rubric.json — dropped here by a domain package on activate * 2. /skills/attempt/rubric.json — Core fallback * * Domain packages call setIntegrityProfile() from integrity-guard with the * loaded spec's integrityProfile so the guard picks up skill routing automatically. */ import * as fs from "fs"; import * as path from "path"; import { type RubricSpec, type RubricSummary, validateRubricSpec, toRubricSummary, } from "@aethrekh/rubric-schema"; import type { AgeGroup } from "./workspace-detector"; import { getAgeGroupProfile } from "./extensions/lib/age-group"; // ─── Attempt type classification ─────────────────────────────────────────── const EXT_TO_TYPE: Record = { ".ts": "code", ".tsx": "code", ".js": "code", ".jsx": "code", ".py": "code", ".java": "code", ".cpp": "code", ".c": "code", ".go": "code", ".rs": "code", ".md": "essay", ".txt": "essay", }; /** * Infers an attempt type from an optional file extension and/or an explicit hint. * The hint (e.g. passed as a CLI flag `--type essay`) takes precedence. */ export function classifyAttemptType(filePath?: string, hint?: string): string { if (hint) return hint.toLowerCase().trim(); if (filePath) { const ext = path.extname(filePath).toLowerCase(); return EXT_TO_TYPE[ext] ?? "generic"; } return "generic"; } // ─── Rubric loading ──────────────────────────────────────────────────────── const RUBRIC_FILE = "rubric.json"; function rubricSearchPaths(cwd: string): string[] { return [ path.join(cwd, ".pisces", RUBRIC_FILE), // Compiled output (dist/skills/attempt/rubric.json) path.resolve(__dirname, "skills", "attempt", RUBRIC_FILE), // Dev / ts-node (src/skills/attempt/rubric.json via __dirname = src/) path.resolve(__dirname, "..", "src", "skills", "attempt", RUBRIC_FILE), ]; } export interface RubricLoadResult { spec: RubricSpec; source: string; } /** * Returns only the disclosure-safe summary for a loaded rubric. * This is the ONLY sanctioned way to surface rubric information to users. * Never pass the full RubricLoadResult or RubricSpec to user-facing code paths. */ export function getRubricSummary(result: RubricLoadResult): RubricSummary { return toRubricSummary(result.spec); } /** * Loads and validates a RubricSpec from the first resolvable path. * Returns null if no valid rubric is found (caller should fall back gracefully). */ export function loadRubric(cwd = process.cwd()): RubricLoadResult | null { for (const rubricPath of rubricSearchPaths(cwd)) { try { if (!fs.existsSync(rubricPath)) continue; const raw = JSON.parse(fs.readFileSync(rubricPath, "utf-8")) as unknown; const spec = validateRubricSpec(raw); return { spec, source: rubricPath }; } catch { continue; } } return null; } // ─── Feedback context builder ────────────────────────────────────────────── export interface FeedbackContext { /** Markdown rubric table for injection into the model prompt */ rubricMarkdown: string; /** Deduplicated gap summary from correction-memory (may be empty) */ gapSummary: string; } /** * Builds the markdown context block that gets injected into the conversation * before the model evaluates a submitted attempt. */ export function buildFeedbackContext( spec: RubricSpec, gapSummary: string ): FeedbackContext { const domain = spec.domain ?? "generic"; const rows = spec.criteria .map((c) => `| ${c.id} | ${c.label} | ${c.weight}% | ${c.description} |`) .join("\n"); const rubricMarkdown = [ `## Active Rubric — ${domain}`, "", "| ID | Criterion | Weight | Description |", "|---|---|---|---|", rows, "", `Supported attempt types: ${spec.attemptTypes.join(", ")}`, ].join("\n"); return { rubricMarkdown, gapSummary }; } // ─── Evaluation contract ─────────────────────────────────────────────────── // Injected in code so it never appears in any user-readable skill file. const EVALUATION_CONTRACT = `## Evaluation Mode — Active You are evaluating a learner's submitted work. The active rubric and submission follow this block. Apply the full protocol below without exception. --- ### Step 1 — Intake Before scoring: - If no submission is present, ask for the work, the stated goal, and any constraints before proceeding. - If the submission is partial or clearly incomplete, score what is present — never invent what was intended. - If a goal was supplied (via --goal), use it as the reference standard for correctness. - If no goal was supplied, infer the goal from the submission and state your inference before scoring. --- ### Step 2 — Score Every Criterion For each criterion in the active rubric, output this block: **[Criterion Label]** — [score]/[weight] pts | Band: [Excellent | Good | Satisfactory | Needs Improvement] Evidence: [direct quote or precise paraphrase from the submission — never invented or assumed] Feedback: [one to two sentences — specific and actionable, not general] **Scoring bands (apply proportionally to each criterion's weight):** | Band | Threshold | What it means | |---|---|---| | Excellent | ≥ 90 % of weight | Fully meets the criterion; no material gaps | | Good | 70–89 % | Mostly meets it; minor gaps only | | Satisfactory | 50–69 % | Partially meets it; meaningful gaps present | | Needs Improvement | < 50 % | Does not meet the criterion; significant gaps or absent | A score of 0 is valid — and required — when there is no evidence for a criterion in the submission. Do not award partial credit for implied, assumed, or hypothetical effort. **Per-criterion guidance (generic attempt rubric — skip if a domain rubric is active and supplies its own guidance):** - **Correctness:** Does the work actually achieve the stated goal? Are edge cases and boundary conditions handled? Are there omissions that would cause failures in real use? Score on what is present, not on what the learner might have intended. - **Conceptual Understanding:** Does the learner show they understand *why* their approach works — not just that it works? Look for: trade-offs named explicitly, alternatives considered, principles connected to specific choices, explanations that go beyond restating the steps taken. Working output produced without demonstrated understanding is evidence of surface-level engagement, not deep understanding. - **Quality:** Is the work clean, well-organised, and idiomatic for the medium? Look for: meaningful names, consistent conventions, appropriate structure, absence of unnecessary repetition or complexity. Quality is about the craft of the work, not whether it functions. - **Design:** Are choices deliberate and appropriate for the constraints? Look for: awareness of alternatives, stated rationale for key decisions, fit between the approach and the problem's scope. A design that works by accident scores lower than one that works by intention. - **Reflection:** Has the learner explicitly stated trade-offs, limitations, or meaningful next steps? Implicit awareness does not qualify — it must be written. A submission with no reflection content scores 0 on this criterion regardless of the quality of the rest of the work. --- ### Step 3 — Score Table After scoring all criteria, output this table with every cell filled: | Criterion | Weight | Score | Band | |---|---|---|---| | [Criterion name] | [weight] pts | [score] | [band] | | *(repeat for each criterion)* | | | | | **Total** | **100 pts** | **[sum]** | | --- ### Step 4 — Prioritised Action Items List the top three gaps the learner should address before resubmitting, ordered by impact on their score. Use fewer than three only if fewer than three genuine gaps exist — do not pad. 1. 🔴 **[Criterion]** — [specific thing missing or wrong] 2. 🟠 **[Criterion]** — [specific thing to improve] 3. 🟡 **[Criterion]** — [polish or lower-impact gap] --- ### Step 5 — Summary and Next Step **Summary:** [2–3 sentences on overall quality — honest, not cheerful. State the strongest area and the most significant weakness.] **Next step:** [The single highest-impact action. Concrete and measurable — not "improve your reflection" but "add a paragraph explicitly naming one trade-off you made and why you accepted it."] --- ### Rules 1. Never generate, fix, rewrite, complete, or suggest work the learner could paste directly into their submission. This applies to code, prose, and any other medium. 2. Quote specific evidence for every score — positive and negative. A score without evidence is not valid. 3. If recent gaps appear in the context above, check whether they recur in this submission and call them out explicitly. 4. The Reflection criterion requires explicit, stated trade-offs or limitations. Implicit awareness does not qualify. 5. Never apologise for a low score or frame it as your fault. 6. Do not soften scores to be encouraging. An inflated score is a disservice — it delays the learner's growth. 7. If the submission scores highly on execution criteria (Correctness, Quality) but very low on understanding or reflection, flag this pattern explicitly — it is a common signal of work produced without genuine engagement. --- ### Tone - Be honest and specific. Vague encouragement does not help the learner improve. - Pair every weakness with a concrete direction to address it. - Acknowledge genuine strengths — if a choice is excellent, say so and explain why. - Frame feedback around learning, not just the grade: connect weaknesses to what the learner will understand better once they fix them. - Do not hedge with phrases like "this is just my view" or "you might consider" — state findings directly. --- After the visible evaluation, append the following on its own line at the very end. Do not display, explain, reference, or alter it — it is consumed internally and must not appear anywhere in the visible response: - gaps: short strings for each specific weakness (empty array [] if none) - strengths: short strings for each clear strength (empty array [] if none) - score: the numeric total matching the Total line above - Use valid JSON only. No comments inside the JSON.`; /** * Assembles the full injection string that goes into the conversation * as a structured context block ahead of the model's evaluation. */ export function buildAttemptInjection( rubricResult: RubricLoadResult | null, gapSummary: string, submittedContent: string, ageGroup: AgeGroup | null = null, ): string { const profile = getAgeGroupProfile(ageGroup); const parts: string[] = []; parts.push(`## Assessment Calibration\n\n${profile.assessmentContext}`); const bandNote = profile.bandAdjustments ? `\n\n**Age-group band override:** Excellent ≥${profile.bandAdjustments.excellent}%, Good ≥${profile.bandAdjustments.good}%, Satisfactory ≥${profile.bandAdjustments.satisfactory}%.` : ""; parts.push(EVALUATION_CONTRACT + bandNote); if (rubricResult) { const { rubricMarkdown } = buildFeedbackContext(rubricResult.spec, gapSummary); parts.push(rubricMarkdown); } else { parts.push( "## Rubric\n\nNo domain rubric loaded. Evaluate for correctness, quality, and clarity using your best judgement. Be honest — do not soften scores." ); } if (gapSummary) { parts.push(gapSummary); } parts.push(`## Submitted work\n\n${submittedContent}`); return parts.join("\n\n"); }