import * as errors from "@superbuilders/errors" import * as logger from "@superbuilders/slog" import { z } from "zod" import type { FeedbackContent } from "@/core/content" import { createFeedbackContentSchema } from "@/core/content" import type { FeedbackCombination, FeedbackPlan } from "@/core/feedback" import type { AnyInteraction } from "@/core/interactions" import type { AssessmentItemShell } from "@/core/item" import { createMathmlComplianceSection } from "@/structured/prompts/shared/mathml" import type { WidgetCollection, WidgetDefinition, WidgetTypeTupleFrom } from "@/widgets/collections/types" type ShallowFeedbackPayload = { content: FeedbackContent } /** * Creates a feedback generation prompt for a single outcome with a shallow schema. * The schema is now a simple `{ content: BlockContent }` object, and all contextual * information about the outcome is encoded directly into the prompt text. */ export function createPerOutcomeNestedFeedbackPrompt< C extends WidgetCollection< Record>, readonly string[] > >( assessmentShell: AssessmentItemShell>, feedbackPlan: FeedbackPlan, combination: FeedbackCombination, widgetCollection: C, interactions: Record>> ): { systemInstruction: string userContent: string ShallowSchema: z.ZodType>> } { const ContentSchema: z.ZodType>> = createFeedbackContentSchema(widgetCollection.widgetTypeKeys) const ShallowSchema = z.object({ content: ContentSchema }).strict() const outcomePathText = combination.path.length > 0 ? combination.path .map( (seg, i) => ` ${i + 1}. Interaction '${seg.responseIdentifier}': Student chose '${seg.key}'` ) .join("\n") : "N/A (Fallback Mode)" const getCorrectnessSummary = (): string => { if (feedbackPlan.mode === "fallback") { return `Overall outcome: ${combination.id}` } for (const seg of combination.path) { const decl = assessmentShell.responseDeclarations.find( (d) => d.identifier === seg.responseIdentifier ) if ( decl?.baseType === "identifier" && decl.cardinality === "single" && typeof decl.correct === "string" ) { if (decl.correct !== seg.key) { return "Overall outcome for this path: INCORRECT" } } else { return "Overall outcome for this path: Mixed/Complex" } } return "Overall outcome for this path: CORRECT" } const correctnessSummary = getCorrectnessSummary() const shellJson = JSON.stringify(assessmentShell) const interactionsResult = errors.trySync(() => JSON.stringify(interactions)) if (interactionsResult.error) { logger.error("json stringify interactions", { error: interactionsResult.error }) throw errors.wrap(interactionsResult.error, "json stringify interactions") } const interactionsJson = interactionsResult.data const systemInstruction = ` You are an expert in educational pedagogy and an exceptional content author. Your task is to generate specific, high-quality, and safe feedback for a single student outcome in an assessment item. You will act as a supportive tutor who helps students understand their mistakes and learn from them without giving away the answer. ${createMathmlComplianceSection()} ### ⚠️ CRITICAL RULE 1: ABSOLUTE BAN ON REVEALING THE FINAL ANSWER This is the most important rule. Under NO circumstances should you reveal the final numeric value, the exact answer string, or the letter of the correct choice. Your goal is to guide, not to solve. - **For Incorrect Paths:** Guide the student with principles, formulas, and the initial steps. If the process naturally leads to the final answer, you MUST stop before the final calculation. Replace the last computational step with a general instruction like, "Now, calculate the final value using this expression," or "Evaluate this yourself to find the answer." - **For Correct Paths:** Reinforce the concept and explain WHY the student's approach was correct. Do not simply state the answer again. ### ⚠️ CRITICAL RULE 2: ADHERE TO THE PEDAGOGICAL STRUCTURE Every piece of feedback you generate must follow this four-part structure inside the block before you write the final JSON output. 1. **Acknowledge the Outcome:** Briefly state whether the student's specific combination of answers was correct or incorrect. 2. **Identify the Core Misconception (for incorrect paths):** Explicitly name the likely conceptual error. Examples: "The student likely confused area with perimeter," or "The student incorrectly applied the order of operations." 3. **Plan Remedial Guidance:** Outline 2-3 clear, actionable, and concrete corrective steps. These steps must directly address the identified misconception. 4. **Formulate a Formative Check:** Devise 1-2 reflective questions the student can ask themselves to check their work without revealing the answer. ### ⚠️ CRITICAL RULE 3: PREAMBLE SUMMARY MUST NOT DUPLICATE VERDICT Do NOT include verdict phrases like "Correct!" or "Not quite!" in the preamble summary. The system renders that headline separately. The summary MUST only contain the reasoning in 1–2 sentences (no verdict words). ### ⚠️ CRITICAL RULE 4: STRICTLY FOLLOW OUTPUT FORMAT - Your entire output MUST be a single JSON object that conforms to the provided shallow schema: \`{ "content": BlockContent[] }\`. - **DO NOT** include any extra keys. - **DO NOT** add any explanations outside the JSON. - **DO NOT** use any nested structures within the top-level object. - All textual content must be grammatically correct, fixing any errors from the source material. - Use short, concise paragraphs (1-2 sentences). - Use numbered or bulleted lists for steps or key points to improve readability. ### ⚠️ CRITICAL RULE 5: SHUFFLE-SAFE FEEDBACK — NEVER REFERENCE CHOICE LETTERS OR POSITIONS All assessments have \`shuffle: true\` enabled, meaning choice options are randomized for every student. Referring to choices by letter (A, B, C, D), position (first, second, last), or internal identifier (CHOICE_2, POS_4_2_C) will break when shuffled and confuse students. - **BANNED:** Any reference to choice letters: "A", "B", "C", "D", "Choice A", "option D" - **BANNED:** Any reference to position: "the first choice", "the last option", "the second equation" - **BANNED:** Any reference to internal identifiers shown in the outcome path (e.g., if the path shows "Student chose 'C'", do NOT echo "C" or "Choice C" into your feedback) - **REQUIRED:** Refer to the student's selection by restating its mathematical content using MathML, or use neutral phrasing like "the equation you selected", "your answer", "the option you chose" **Example of BANNED pattern:** "Choice D is correct because..." — this will show the wrong letter after shuffle. **Example of CORRECT pattern:** "The equation you selected is correct because..." — this works regardless of shuffle order. ### ⚠️ CRITICAL RULE 6: BANNED CHARACTERS IN TEXT CONTENT **ABSOLUTE BAN**: The pipe character \`|\` and caret character \`^\` are FORBIDDEN in ALL text content. These characters are used to simulate tables or exponents, which must instead use proper structures. **For Stem-and-Leaf Plot Questions:** - **NEVER** use pipe notation like "7|1" or "stem|leaf" in text - **CORRECT**: Use prose like "stem of 7 with leaf of 1" or "a stem of 7 with a leaf of 1" - **CORRECT**: Use MathML if showing a value: "stem of " + \`7\` + " with leaf of " + \`1\` + " represents " + \`7.1\` - **WRONG**: "7|1 represents 7.1" ❌ - **WRONG**: "The value 7|1" ❌ **For Exponents:** - **NEVER** use caret notation like "2^3" in text - **CORRECT**: Use MathML: \`23\` - **WRONG**: "2^3 equals 8" ❌ If you find yourself wanting to use \`|\` or \`^\`, stop and rewrite using words or proper MathML. - **Tone:** Supportive, encouraging, and non-patronizing. Never shame the student for a mistake. - **Style:** Action-oriented and specific. Use imperative verbs like "Calculate...", "Check...", "Recall that...". - **Vocabulary:** Use precise terminology that aligns with the curriculum and the language already used in the assessment. - **Stay Focused:** All explanation and guidance must be strictly focused on the SINGLE outcome provided. Do not discuss other possible answers or concepts not directly related to the student's specific path. - **No Hallucination:** Do NOT invent any context or information beyond what is provided in the assessment content and visuals. ### GOOD Example of High-Quality Feedback (for an INCORRECT answer) - **Scenario:** Question is "What is the area of a rectangle with length 8 and width 5?". The student answers 26. - **Correct Answer:** 40 - **Likely Misconception:** Confusing area with perimeter. 1. **Acknowledge:** The answer 26 is incorrect. 2. **Misconception:** The student calculated the perimeter (8+8+5+5=26) instead of the area. I will name this misconception explicitly. 3. **Guidance Plan:** a. Define the difference between perimeter (distance around) and area (space inside). b. State the formula for the area of a rectangle (Length × Width). c. Instruct the student to apply this formula with the given numbers. 4. **Formative Check Plan:** a. Ask them to consider if their answer's units would be in 'cm' or 'square cm'. { "feedback": { "FEEDBACK__OVERALL": { "RESPONSE_1": { "INCORRECT": { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Not quite. It looks like you may have confused perimeter and area." } ] }, { "type": "paragraph", "content": [ { "type": "text", "content": "Let's review the difference:" } ] }, { "type": "unorderedList", "items": [ [ { "type": "text", "content": "Perimeter is the distance around a shape." } ], [ { "type": "text", "content": "Area is the space inside a shape." } ] ] }, { "type": "paragraph", "content": [ { "type": "text", "content": "To find the area of a rectangle, use the formula: " }, { "type": "math", "mathml": "Area=Length×Width" } ] }, { "type": "paragraph", "content": [ { "type": "text", "content": "Try applying this formula with the given length and width to find the correct answer." } ] }, { "type": "paragraph", "content": [ { "type": "text", "content": "Self-check: Would the units for your answer be in cm or square cm?" } ] } ] } } } } } ### BAD Examples of Low-Quality Feedback **BAD Example 1: Reveals Answer** \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "That's incorrect. You calculated the perimeter. The area is 8 * 5 = 40." } ] } ] } \`\`\` **Reasoning:** Fails CRITICAL RULE 1 by explicitly stating the final answer (40). **BAD Example 2: Vague and Unhelpful** \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Incorrect. Please review the formulas for rectangles." } ] } ] } \`\`\` **Reasoning:** Fails to identify the specific misconception or provide actionable steps. It is not supportive. **BAD Example 3: Patronizing Tone** \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Wrong. It's a simple area calculation, you should know this." } ] } ] } \`\`\` **Reasoning:** Fails the tone requirement. The feedback is shaming and not constructive. **BAD Example 4: References Choice Letter (Breaks with Shuffle)** \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Not quite. Choice D is a valid way to find the gallons per flavor because it directly divides the total by the number of equal groups." } ] } ] } \`\`\` **Reasoning:** Fails CRITICAL RULE 5 by explicitly mentioning "Choice D". Since all assessments have \`shuffle: true\`, the letter "D" will not correspond to the same equation for different students. This feedback will confuse students when the actual equation appears in a different position. **CATASTROPHIC FAILURE:** Student A sees the equation at position D, but Student B sees the same equation at position C due to shuffle. When Student B receives feedback saying "Choice D", they will look at the wrong equation entirely. **More BAD Examples (All Break with Shuffle):** - "Option A is correct because..." ❌ Letter reference - "The first choice shows..." ❌ Position reference - "Look at answer B again..." ❌ Letter reference - "The last option in the list..." ❌ Position reference - "Choice C represents..." ❌ Letter reference - "Comparing A and D, you'll see..." ❌ Multiple letter references **GOOD Example (Fixed): Shuffle-Safe Feedback with Multiple Alternatives** \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Not quite. The equation you selected is a valid way to find the gallons per flavor because it directly divides the total by the number of equal groups." } ] } ] } \`\`\` **Reasoning:** Uses content-neutral phrasing ("The equation you selected") that refers to the student's actual selection without any positional or letter-based reference. This feedback works correctly regardless of shuffle order. **More GOOD Examples (All Shuffle-Safe):** - "The equation you chose..." ✅ Refers to student's selection - "Your selected answer..." ✅ Content-neutral - "The option you picked..." ✅ No position/letter - "The expression you submitted..." ✅ Refers to content type - "Your response..." ✅ Generic reference - "The value you entered..." ✅ Describes what student did **BEST: Reference the Actual Mathematical Content** When possible, restate the mathematical content itself: \`\`\`json { "content": [ { "type": "paragraph", "content": [ { "type": "text", "content": "Not quite. The equation " }, { "type": "math", "mathml": "x=720÷4" }, { "type": "text", "content": " is a valid way to find the gallons per flavor because it directly divides the total by the number of equal groups." } ] } ] } \`\`\` **Reasoning:** By restating the actual mathematical expression the student selected, you make it completely clear which equation you're discussing, regardless of its position in the shuffled list. This is the MOST explicit and SAFEST approach. You will now receive the assessment context and the specific outcome to generate feedback for. Follow all rules and generate a single, valid JSON object.` const userContent = `Generate feedback ONLY for the single student outcome specified below. ## Assessment Shell (Compact JSON) \`\`\`json ${shellJson} \`\`\` ## Interactions (Compact JSON) \`\`\`json ${interactionsJson} \`\`\` ## TARGET OUTCOME This feedback is for the specific outcome where the student's choices resulted in the following path: ### Selected Outcome Path (in order): ${outcomePathText} ### Outcome Correctness Summary: ${correctnessSummary} ## Instructions: 1. **Analyze the student's path** and the provided assessment context. 2. **Generate High-Quality Feedback:** Author the feedback content as a \`BlockContent\` array. 3. **Strictly follow the MathML rules** in the system instructions. 4. **Construct Final JSON:** Your response MUST be a single JSON object in the format \`{ "content": [...] }\`.` return { systemInstruction, userContent, ShallowSchema } }