import type * as logger from "@superbuilders/slog"
import he from "he"
/**
* Converts problematic HTML entities to their numeric equivalents or actual characters.
* This prevents validation errors from the QTI API which doesn't accept named HTML entities.
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with HTML entities converted
*/
export function convertHtmlEntities(
xml: string,
logger: logger.Logger
): string {
const exceptions = new Set([""", "'", "<", ">", "&"])
const entityRegex = /&(#?[a-zA-Z0-9]+);/g
let totalReplacements = 0
const convertedXml = xml.replace(entityRegex, (match) => {
if (exceptions.has(match)) {
return match
}
totalReplacements++
return he.decode(match)
})
if (totalReplacements > 0) {
logger.debug("converted html entities to unicode characters", {
totalReplacements,
originalLength: xml.length,
convertedLength: convertedXml.length
})
}
return convertedXml
}
/**
* Fixes unescaped angle brackets within MathML mo elements.
* This handles the common case where < and > symbols are not properly escaped
* within mathematical operator elements.
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with properly escaped angle brackets in mo elements
*/
export function fixMathMLOperators(xml: string, logger: logger.Logger): string {
let fixCount = 0
// Fix unescaped < in mo elements
// This specifically looks for < pattern
const moLessThanRegex = /]*)?>(<)<\/mo>/gi
let fixedXml = xml.replace(moLessThanRegex, (match) => {
fixCount++
return match.replace("<", "<")
})
// Fix unescaped > in mo elements
// This specifically looks for > pattern
const moGreaterThanRegex = /]*)?>><\/mo>/gi
fixedXml = fixedXml.replace(moGreaterThanRegex, (match) => {
fixCount++
return match.replace(">", ">")
})
// Fix unescaped <= in mo elements by converting to Unicode ≤
// This specifically looks for <= pattern
const moLessThanEqualRegex = /]*)?><=(\/mo>)/gi
fixedXml = fixedXml.replace(moLessThanEqualRegex, (match) => {
fixCount++
return match.replace("<=", "≤")
})
// Fix unescaped >= in mo elements by converting to Unicode ≥
// This specifically looks for >= pattern
const moGreaterThanEqualRegex = /]*)?>>=(<\/mo>)/gi
fixedXml = fixedXml.replace(moGreaterThanEqualRegex, (match) => {
fixCount++
return match.replace(">=", "≥")
})
if (fixCount > 0) {
logger.debug("fixed unescaped angle brackets in MathML mo elements", {
fixCount,
originalLength: xml.length,
fixedLength: fixedXml.length
})
}
return fixedXml
}
/**
* Strips all XML comments from the provided XML string.
* This prevents issues with malformed comments that could cause XML parsing errors.
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with all comments removed
*/
export function stripXmlComments(xml: string, logger: logger.Logger): string {
// EXTREMELY robust regex for matching XML comments
// This handles multi-line comments and ensures we don't accidentally match
// things that look like comments but aren't (e.g., within CDATA sections)
//
// The regex breakdown:
// ) : Negative lookahead - ensures we don't match -->
// [\s\S] : Matches any character including newlines
// )* : Zero or more of any character that isn't part of -->
// ) : End of content capture group
// --> : Matches the exact closing of an XML comment
const commentRegex = /)[\s\S])*)-->/g
// Count comments for logging
const commentMatches = xml.match(commentRegex)
const commentCount = commentMatches ? commentMatches.length : 0
if (commentCount > 0) {
logger.debug("stripping xml comments", {
commentCount,
firstComment: commentMatches?.[0]?.substring(0, 100),
originalLength: xml.length
})
}
// Replace all comments with empty string
const strippedXml = xml.replace(commentRegex, "")
if (commentCount > 0) {
logger.debug("xml comments stripped", {
commentCount,
originalLength: xml.length,
strippedLength: strippedXml.length,
bytesRemoved: xml.length - strippedXml.length
})
}
return strippedXml
}
/**
* Removes consecutive newlines from the XML string, leaving only single newlines.
* This handles all types of line endings (Unix \n, Windows \r\n, and mixed).
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with consecutive newlines reduced to single newlines
*/
export function removeDoubleNewlines(
xml: string,
logger: logger.Logger
): string {
// First normalize all line endings to \n for consistent processing
// This handles \r\n (Windows), \r (old Mac), and leaves \n (Unix) as is
const normalizedXml = xml.replace(/\r\n/g, "\n").replace(/\r/g, "\n")
// Count the number of double (or more) newline sequences before replacement
const doubleNewlineMatches = normalizedXml.match(/\n{2,}/g)
const doubleNewlineCount = doubleNewlineMatches
? doubleNewlineMatches.length
: 0
if (doubleNewlineCount > 0) {
// Calculate total excess newlines
const excessNewlines = doubleNewlineMatches?.reduce(
(sum, match) => sum + match.length - 1,
0
)
logger.debug("removing double newlines", {
doubleNewlineSequences: doubleNewlineCount,
totalExcessNewlines: excessNewlines,
originalLength: xml.length
})
}
// Replace any sequence of 2 or more newlines with a single newline
const cleanedXml = normalizedXml.replace(/\n{2,}/g, "\n")
if (doubleNewlineCount > 0) {
logger.debug("double newlines removed", {
sequencesRemoved: doubleNewlineCount,
originalLength: xml.length,
cleanedLength: cleanedXml.length,
bytesRemoved: xml.length - cleanedXml.length
})
}
return cleanedXml
}
/**
* Fixes Khan Academy graphie URLs by appending .svg extension if missing.
* This handles URLs in the format: https://cdn.kastatic.org/ka-perseus-graphie/[hash]
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with properly formatted Khan graphie URLs
*/
export function fixKhanGraphieUrls(xml: string, logger: logger.Logger): string {
// Match Khan graphie URLs that don't already have a file extension
// Named capture groups for clarity
const khanGraphieRegex =
/(?https:\/\/cdn\.kastatic\.org\/ka-perseus-graphie\/(?[a-zA-Z0-9]+))(?!\.(?:svg|png|jpg|jpeg|gif|webp))(?[\s"'<>&]|$)/g
let fixCount = 0
const fixedXml = xml.replace(
khanGraphieRegex,
(_match, url, _hash, after) => {
fixCount++
return `${url}.svg${after}`
}
)
if (fixCount > 0) {
logger.debug("fixed khan graphie urls by appending .svg", {
fixCount,
originalLength: xml.length,
fixedLength: fixedXml.length
})
}
return fixedXml
}
/**
* MISSION CRITICAL: Fixes unescaped inequality operators throughout the XML.
* - For qti-value elements: Escapes to HTML entities (<= and >=) for ASCII input
* - For all other content: Converts to Unicode symbols (≤ and ≥) for proper display
*
* This function uses EXTREMELY robust named capture groups and multiple safeguards
* to ensure it NEVER corrupts valid XML.
*
* @param xml The XML string to process
* @param logger The logger instance
* @returns The XML string with properly handled inequality operators
*/
export function fixInequalityOperators(
xml: string,
logger: logger.Logger
): string {
let totalFixCount = 0
// CRITICAL: Only process qti-value elements that contain mathematical inequalities
// This regex uses named capture groups for maximum clarity and robustness
const qtiValueRegex = new RegExp(
// Opening tag with exact match
"" +
// Capture the content - using lazy match to stop at first closing tag
"(?[\\s\\S]*?)" +
// Closing tag with exact match
"",
"gi"
)
let qtiValueFixCount = 0
const xmlAfterQtiValueFix = xml.replace(
qtiValueRegex,
(fullMatch, content) => {
// CRITICAL SAFEGUARDS:
// 1. If content already has ANY HTML entities, skip it entirely
if (/&[a-zA-Z]+;|\d+;|[0-9a-fA-F]+;/.test(content)) {
logger.debug("skipping qti-value with existing entities", { content })
return fullMatch
}
// 2. Process ANY ASCII inequality sequences in qti-value content
const hasInequalityOperator = /[<>]=?/.test(content)
if (!hasInequalityOperator) {
return fullMatch
}
// 3. Additional safeguard: content should be relatively short (typical answers)
if (content.length > 50) {
logger.warn("skipping suspiciously long qti-value content", {
contentLength: content.length,
content: `${content.substring(0, 50)}...`
})
return fullMatch
}
let modifiedContent = content
let localFixCount = 0
// Fix inequalities with EXPLICIT patterns
// CRITICAL: Don't use .test() with global regex - it maintains state!
// Pattern 1: <= (less than or equal)
modifiedContent = modifiedContent.replace(
/(?[a-zA-Z0-9\s\-+*/()^.]+)(?<=)(?[a-zA-Z0-9\s\-+*/()^.]+)/g,
(_match: string, before: string, _operator: string, after: string) => {
localFixCount++
return `${before}<=${after}`
}
)
// Pattern 2: >= (greater than or equal)
modifiedContent = modifiedContent.replace(
/(?[a-zA-Z0-9\s\-+*/()^.]+)(?>=)(?[a-zA-Z0-9\s\-+*/()^.]+)/g,
(_match: string, before: string, _operator: string, after: string) => {
localFixCount++
return `${before}>=${after}`
}
)
// Pattern 3: < (less than) - but NOT part of <=
modifiedContent = modifiedContent.replace(
/(?[a-zA-Z0-9\s\-+*/()^.]+)(?<)(?(?!=)[a-zA-Z0-9\s\-+*/()^.]+)/g,
(_match: string, before: string, _operator: string, after: string) => {
localFixCount++
return `${before}<${after}`
}
)
// Pattern 4: > (greater than) - but NOT part of >=
modifiedContent = modifiedContent.replace(
/(?[a-zA-Z0-9\s\-+*/()^.]+)(?>)(?(?!=)[a-zA-Z0-9\s\-+*/()^.]+)/g,
(_match: string, before: string, _operator: string, after: string) => {
localFixCount++
return `${before}>${after}`
}
)
if (localFixCount > 0) {
qtiValueFixCount++
logger.debug("fixed unescaped inequality in qti-value", {
original: content,
fixed: modifiedContent,
fixCount: localFixCount
})
}
return `${modifiedContent}`
}
)
// Handle inequalities in other text content - convert to Unicode
// CRITICAL: Use extremely specific patterns to avoid false positives
let generalFixCount = 0
// Only process text content that's clearly between XML tags
// Named groups for maximum clarity
const textContentRegex = new RegExp(
// Must start after a closing >
">(?" +
// Content that doesn't contain < or > (to ensure we're not in a tag)
"[^<>]+" +
// Must end before an opening <
")<",
"g"
)
const finalXml = xmlAfterQtiValueFix.replace(
textContentRegex,
(fullMatch, content) => {
// CRITICAL: Skip if this might be inside a qti-value element
// Check a wider context to be absolutely sure
const startIndex = xmlAfterQtiValueFix.lastIndexOf(
"<",
xmlAfterQtiValueFix.indexOf(fullMatch)
)
const contextBefore = xmlAfterQtiValueFix.substring(
Math.max(0, startIndex - 20),
startIndex + 20
)
if (contextBefore.includes("qti-value>")) {
return fullMatch
}
// Only process if content contains mathematical inequalities
if (!/[<>]=?/.test(content)) {
return fullMatch
}
// Check if this looks like mathematical content
// Must have numbers or common math variables
if (!/[0-9xyzabc]/.test(content)) {
return fullMatch
}
let modifiedContent = content
let hasChanges = false
// ONLY replace inequalities in clear mathematical contexts
// Use very specific patterns with named groups
// Pattern 1: Variable <= Number or Variable >= Number
const variableInequalityRegex =
/(?[a-zA-Z]+)\s*(?[<>]=?)\s*(?-?\d+(?:\.\d+)?)/g
modifiedContent = modifiedContent.replace(
variableInequalityRegex,
(match: string, variable: string, operator: string, number: string) => {
if (operator === "<=") {
hasChanges = true
return `${variable} ≤ ${number}`
}
if (operator === ">=") {
hasChanges = true
return `${variable} ≥ ${number}`
}
// Don't convert < or > to unicode in general content - could break things
return match
}
)
// Pattern 2: Number <= Variable or Number >= Variable
const numberInequalityRegex =
/(?-?\d+(?:\.\d+)?)\s*(?[<>]=?)\s*(?[a-zA-Z]+)/g
modifiedContent = modifiedContent.replace(
numberInequalityRegex,
(match: string, number: string, operator: string, variable: string) => {
if (operator === "<=") {
hasChanges = true
return `${number} ≤ ${variable}`
}
if (operator === ">=") {
hasChanges = true
return `${number} ≥ ${variable}`
}
return match
}
)
if (hasChanges) {
generalFixCount++
logger.debug("converted inequality to unicode in general content", {
original: content.substring(0, 100),
fixed: modifiedContent.substring(0, 100)
})
}
return `>${modifiedContent}<`
}
)
totalFixCount = qtiValueFixCount + generalFixCount
if (totalFixCount > 0) {
logger.info("MISSION CRITICAL: fixed unescaped inequalities", {
totalFixCount,
qtiValueFixCount,
generalContentFixCount: generalFixCount,
originalLength: xml.length,
fixedLength: finalXml.length
})
}
return finalXml
}