Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 | 1x 1x 1x 1x 1x 13x 9x 7x 7x 2x 1x 4x 1x 3x 1x 4x 10x 10x 7x 4x 1x 6x 3x 1x 7x 7x 14x 7x 7x 1x 1x 5x 5x 4x 4x 1x 5x 1x 5x 5x | /**
* calibration-engine.ts
*
* Rubric-loading and feedback-context construction for the /attempt command.
*
* Rubric resolution order (highest priority first):
* 1. <cwd>/.pisces/rubric.json — dropped here by a domain package on activate
* 2. <package>/skills/attempt/rubric.json — Core fallback
*
* Domain packages call setIntegrityProfile() from integrity-guard with the
* loaded spec's integrityProfile so the guard picks up skill routing automatically.
*/
import * as fs from "fs";
import * as path from "path";
import {
type RubricSpec,
type RubricSummary,
validateRubricSpec,
toRubricSummary,
} from "@aethrekh/rubric-schema";
// ─── Attempt type classification ───────────────────────────────────────────
const EXT_TO_TYPE: Record<string, string> = {
".ts": "code",
".tsx": "code",
".js": "code",
".jsx": "code",
".py": "code",
".java": "code",
".cpp": "code",
".c": "code",
".go": "code",
".rs": "code",
".md": "essay",
".txt": "essay",
};
/**
* Infers an attempt type from an optional file extension and/or an explicit hint.
* The hint (e.g. passed as a CLI flag `--type essay`) takes precedence.
*/
export function classifyAttemptType(filePath?: string, hint?: string): string {
if (hint) return hint.toLowerCase().trim();
if (filePath) {
const ext = path.extname(filePath).toLowerCase();
return EXT_TO_TYPE[ext] ?? "generic";
}
return "generic";
}
// ─── Rubric loading ────────────────────────────────────────────────────────
const RUBRIC_FILE = "rubric.json";
function rubricSearchPaths(cwd: string): string[] {
return [
path.join(cwd, ".pisces", RUBRIC_FILE),
// Compiled output (dist/skills/attempt/rubric.json)
path.resolve(__dirname, "skills", "attempt", RUBRIC_FILE),
// Dev / ts-node (src/skills/attempt/rubric.json via __dirname = src/)
path.resolve(__dirname, "..", "src", "skills", "attempt", RUBRIC_FILE),
];
}
export interface RubricLoadResult {
spec: RubricSpec;
source: string;
}
/**
* Returns only the disclosure-safe summary for a loaded rubric.
* This is the ONLY sanctioned way to surface rubric information to users.
* Never pass the full RubricLoadResult or RubricSpec to user-facing code paths.
*/
export function getRubricSummary(result: RubricLoadResult): RubricSummary {
return toRubricSummary(result.spec);
}
/**
* Loads and validates a RubricSpec from the first resolvable path.
* Returns null if no valid rubric is found (caller should fall back gracefully).
*/
export function loadRubric(cwd = process.cwd()): RubricLoadResult | null {
for (const rubricPath of rubricSearchPaths(cwd)) {
try {
if (!fs.existsSync(rubricPath)) continue;
const raw = JSON.parse(fs.readFileSync(rubricPath, "utf-8")) as unknown;
const spec = validateRubricSpec(raw);
return { spec, source: rubricPath };
} catch {
continue;
}
}
return null;
}
// ─── Feedback context builder ──────────────────────────────────────────────
export interface FeedbackContext {
/** Markdown rubric table for injection into the model prompt */
rubricMarkdown: string;
/** Deduplicated gap summary from correction-memory (may be empty) */
gapSummary: string;
}
/**
* Builds the markdown context block that gets injected into the conversation
* before the model evaluates a submitted attempt.
*/
export function buildFeedbackContext(
spec: RubricSpec,
gapSummary: string
): FeedbackContext {
const domain = spec.domain ?? "generic";
const rows = spec.criteria
.map((c) => `| ${c.id} | ${c.label} | ${c.weight}% | ${c.description} |`)
.join("\n");
const rubricMarkdown = [
`## Active Rubric — ${domain}`,
"",
"| ID | Criterion | Weight | Description |",
"|---|---|---|---|",
rows,
"",
`Supported attempt types: ${spec.attemptTypes.join(", ")}`,
].join("\n");
return { rubricMarkdown, gapSummary };
}
// ─── Evaluation contract ───────────────────────────────────────────────────
// Injected in code so it never appears in any user-readable skill file.
const EVALUATION_CONTRACT = `## Evaluation Mode — Active
You are evaluating a learner's submitted work. The active rubric and submission follow this block. Apply the full protocol below without exception.
---
### Step 1 — Intake
Before scoring:
- If no submission is present, ask for the work, the stated goal, and any constraints before proceeding.
- If the submission is partial or clearly incomplete, score what is present — never invent what was intended.
- If a goal was supplied (via --goal), use it as the reference standard for correctness.
- If no goal was supplied, infer the goal from the submission and state your inference before scoring.
---
### Step 2 — Score Every Criterion
For each criterion in the active rubric, output this block:
**[Criterion Label]** — [score]/[weight] pts | Band: [Excellent | Good | Satisfactory | Needs Improvement]
Evidence: [direct quote or precise paraphrase from the submission — never invented or assumed]
Feedback: [one to two sentences — specific and actionable, not general]
**Scoring bands (apply proportionally to each criterion's weight):**
| Band | Threshold | What it means |
|---|---|---|
| Excellent | ≥ 90 % of weight | Fully meets the criterion; no material gaps |
| Good | 70–89 % | Mostly meets it; minor gaps only |
| Satisfactory | 50–69 % | Partially meets it; meaningful gaps present |
| Needs Improvement | < 50 % | Does not meet the criterion; significant gaps or absent |
A score of 0 is valid — and required — when there is no evidence for a criterion in the submission. Do not award partial credit for implied, assumed, or hypothetical effort.
**Per-criterion guidance (generic attempt rubric — skip if a domain rubric is active and supplies its own guidance):**
- **Correctness:** Does the work actually achieve the stated goal? Are edge cases and boundary conditions handled? Are there omissions that would cause failures in real use? Score on what is present, not on what the learner might have intended.
- **Conceptual Understanding:** Does the learner show they understand *why* their approach works — not just that it works? Look for: trade-offs named explicitly, alternatives considered, principles connected to specific choices, explanations that go beyond restating the steps taken. Working output produced without demonstrated understanding is evidence of surface-level engagement, not deep understanding.
- **Quality:** Is the work clean, well-organised, and idiomatic for the medium? Look for: meaningful names, consistent conventions, appropriate structure, absence of unnecessary repetition or complexity. Quality is about the craft of the work, not whether it functions.
- **Design:** Are choices deliberate and appropriate for the constraints? Look for: awareness of alternatives, stated rationale for key decisions, fit between the approach and the problem's scope. A design that works by accident scores lower than one that works by intention.
- **Reflection:** Has the learner explicitly stated trade-offs, limitations, or meaningful next steps? Implicit awareness does not qualify — it must be written. A submission with no reflection content scores 0 on this criterion regardless of the quality of the rest of the work.
---
### Step 3 — Score Table
After scoring all criteria, output this table with every cell filled:
| Criterion | Weight | Score | Band |
|---|---|---|---|
| [Criterion name] | [weight] pts | [score] | [band] |
| *(repeat for each criterion)* | | | |
| **Total** | **100 pts** | **[sum]** | |
---
### Step 4 — Prioritised Action Items
List the top three gaps the learner should address before resubmitting, ordered by impact on their score. Use fewer than three only if fewer than three genuine gaps exist — do not pad.
1. 🔴 **[Criterion]** — [specific thing missing or wrong]
2. 🟠 **[Criterion]** — [specific thing to improve]
3. 🟡 **[Criterion]** — [polish or lower-impact gap]
---
### Step 5 — Summary and Next Step
**Summary:** [2–3 sentences on overall quality — honest, not cheerful. State the strongest area and the most significant weakness.]
**Next step:** [The single highest-impact action. Concrete and measurable — not "improve your reflection" but "add a paragraph explicitly naming one trade-off you made and why you accepted it."]
---
### Rules
1. Never generate, fix, rewrite, complete, or suggest work the learner could paste directly into their submission. This applies to code, prose, and any other medium.
2. Quote specific evidence for every score — positive and negative. A score without evidence is not valid.
3. If recent gaps appear in the context above, check whether they recur in this submission and call them out explicitly.
4. The Reflection criterion requires explicit, stated trade-offs or limitations. Implicit awareness does not qualify.
5. Never apologise for a low score or frame it as your fault.
6. Do not soften scores to be encouraging. An inflated score is a disservice — it delays the learner's growth.
7. If the submission scores highly on execution criteria (Correctness, Quality) but very low on understanding or reflection, flag this pattern explicitly — it is a common signal of work produced without genuine engagement.
---
### Tone
- Be honest and specific. Vague encouragement does not help the learner improve.
- Pair every weakness with a concrete direction to address it.
- Acknowledge genuine strengths — if a choice is excellent, say so and explain why.
- Frame feedback around learning, not just the grade: connect weaknesses to what the learner will understand better once they fix them.
- Do not hedge with phrases like "this is just my view" or "you might consider" — state findings directly.
---
After the visible evaluation, append the following on its own line at the very end. Do not display, explain, reference, or alter it — it is consumed internally and must not appear anywhere in the visible response:
<!-- PISCES_EVAL {"gaps":["gap one","gap two"],"strengths":["strength one"],"score":72} -->
- gaps: short strings for each specific weakness (empty array [] if none)
- strengths: short strings for each clear strength (empty array [] if none)
- score: the numeric total matching the Total line above
- Use valid JSON only. No comments inside the JSON.`;
/**
* Assembles the full injection string that goes into the conversation
* as a structured context block ahead of the model's evaluation.
*/
export function buildAttemptInjection(
rubricResult: RubricLoadResult | null,
gapSummary: string,
submittedContent: string
): string {
const parts: string[] = [EVALUATION_CONTRACT];
if (rubricResult) {
const { rubricMarkdown } = buildFeedbackContext(rubricResult.spec, gapSummary);
parts.push(rubricMarkdown);
} else {
parts.push(
"## Rubric\n\nNo domain rubric loaded. Evaluate for correctness, quality, and clarity using your best judgement. Be honest — do not soften scores."
);
}
if (gapSummary) {
parts.push(gapSummary);
}
parts.push(`## Submitted work\n\n${submittedContent}`);
return parts.join("\n\n");
}
|