{"version":3,"file":"tokenize.cjs","names":[],"sources":["../../../src/components/CodeBlock/tokenize.ts"],"sourcesContent":["/** What a token is painted as. */\nexport type TokenKind =\n    | \"plain\"\n    | \"comment\"\n    | \"string\"\n    | \"number\"\n    | \"keyword\"\n    | \"literal\"\n    | \"function\"\n    | \"punctuation\"\n    | \"tag\"\n    | \"attribute\"\n    | \"property\";\n\nexport interface Token {\n    kind: TokenKind;\n    value: string;\n}\n\n/** Languages the tokenizer knows. Anything else renders as plain text. */\nexport type CodeLanguage =\n    | \"typescript\"\n    | \"javascript\"\n    | \"tsx\"\n    | \"jsx\"\n    | \"json\"\n    | \"css\"\n    | \"html\"\n    | \"bash\"\n    | \"python\"\n    | \"sql\"\n    | \"plain\";\n\n/** Aliases people actually type, mapped onto the grammars above. */\nconst ALIASES: Record<string, CodeLanguage> = {\n    ts: \"typescript\",\n    typescript: \"typescript\",\n    js: \"javascript\",\n    javascript: \"javascript\",\n    mjs: \"javascript\",\n    cjs: \"javascript\",\n    tsx: \"tsx\",\n    jsx: \"jsx\",\n    json: \"json\",\n    jsonc: \"json\",\n    css: \"css\",\n    scss: \"css\",\n    html: \"html\",\n    xml: \"html\",\n    svg: \"html\",\n    sh: \"bash\",\n    bash: \"bash\",\n    shell: \"bash\",\n    zsh: \"bash\",\n    console: \"bash\",\n    py: \"python\",\n    python: \"python\",\n    sql: \"sql\",\n};\n\n/**\n * Normalise a language name onto a known grammar.\n *\n * @param language - Whatever the caller passed, e.g. `\"ts\"` or `\"Shell\"`.\n * @returns The grammar to use; `\"plain\"` when there is no match.\n */\nexport function resolveLanguage(language: string | undefined): CodeLanguage {\n    if (!language) return \"plain\";\n    return ALIASES[language.toLowerCase()] ?? \"plain\";\n}\n\n/** One grammar rule: a sticky pattern and the kind it produces. */\ninterface Rule {\n    kind: TokenKind;\n    pattern: RegExp;\n}\n\nconst JS_KEYWORDS =\n    \"as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|keyof|let|new|of|private|protected|public|readonly|return|satisfies|set|static|super|switch|this|throw|try|type|typeof|var|void|while|with|yield\";\n\nconst JS_LITERALS = \"true|false|null|undefined|NaN|Infinity\";\n\n/**\n * Rules per grammar, tried in order at each position.\n *\n * Order carries meaning: comments and strings come first so a keyword inside a\n * string stays a string. Every pattern is sticky (`y`) and anchored by\n * `lastIndex`, so a rule can only match where the scanner currently stands.\n */\nconst GRAMMARS: Record<CodeLanguage, Rule[]> = {\n    typescript: jsRules(false),\n    javascript: jsRules(false),\n    tsx: jsRules(true),\n    jsx: jsRules(true),\n    json: [\n        { kind: \"property\", pattern: /\"(?:[^\"\\\\]|\\\\.)*\"(?=\\s*:)/y },\n        { kind: \"string\", pattern: /\"(?:[^\"\\\\]|\\\\.)*\"/y },\n        { kind: \"number\", pattern: /-?\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?/y },\n        { kind: \"literal\", pattern: /\\b(?:true|false|null)\\b/y },\n        { kind: \"punctuation\", pattern: /[{}[\\],:]/y },\n    ],\n    css: [\n        { kind: \"comment\", pattern: /\\/\\*[\\s\\S]*?\\*\\//y },\n        { kind: \"string\", pattern: /\"(?:[^\"\\\\]|\\\\.)*\"|'(?:[^'\\\\]|\\\\.)*'/y },\n        { kind: \"keyword\", pattern: /@[a-zA-Z-]+/y },\n        { kind: \"property\", pattern: /--?[a-zA-Z][\\w-]*(?=\\s*:)/y },\n        { kind: \"function\", pattern: /[a-zA-Z-]+(?=\\()/y },\n        { kind: \"number\", pattern: /-?\\d+(?:\\.\\d+)?(?:px|rem|em|%|s|ms|vh|vw|fr|deg)?/y },\n        { kind: \"punctuation\", pattern: /[{}();:,>+~]/y },\n    ],\n    html: [\n        { kind: \"comment\", pattern: /<!--[\\s\\S]*?-->/y },\n        { kind: \"string\", pattern: /\"(?:[^\"\\\\]|\\\\.)*\"|'(?:[^'\\\\]|\\\\.)*'/y },\n        { kind: \"tag\", pattern: /<\\/?[a-zA-Z][\\w:-]*/y },\n        { kind: \"attribute\", pattern: /[a-zA-Z][\\w:-]*(?==)/y },\n        { kind: \"punctuation\", pattern: /\\/?>|=/y },\n    ],\n    bash: [\n        { kind: \"comment\", pattern: /#[^\\n]*/y },\n        { kind: \"string\", pattern: /\"(?:[^\"\\\\]|\\\\.)*\"|'[^']*'/y },\n        {\n            kind: \"keyword\",\n            pattern:\n                /\\b(?:if|then|else|elif|fi|for|while|do|done|case|esac|function|return|export|local|source|set|cd|echo)\\b/y,\n        },\n        { kind: \"property\", pattern: /\\$\\{?[A-Za-z_][\\w]*\\}?/y },\n        { kind: \"attribute\", pattern: /(?<=\\s)--?[a-zA-Z][\\w-]*/y },\n        { kind: \"punctuation\", pattern: /[|&;()<>]/y },\n    ],\n    python: [\n        { kind: \"comment\", pattern: /#[^\\n]*/y },\n        {\n            kind: \"string\",\n            pattern: /\"\"\"[\\s\\S]*?\"\"\"|'''[\\s\\S]*?'''|\"(?:[^\"\\\\]|\\\\.)*\"|'(?:[^'\\\\]|\\\\.)*'/y,\n        },\n        {\n            kind: \"keyword\",\n            pattern:\n                /\\b(?:and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\\b/y,\n        },\n        { kind: \"literal\", pattern: /\\b(?:True|False|None|self)\\b/y },\n        { kind: \"number\", pattern: /\\b\\d+(?:\\.\\d+)?\\b/y },\n        { kind: \"function\", pattern: /[A-Za-z_]\\w*(?=\\()/y },\n        { kind: \"punctuation\", pattern: /[{}[\\]().,:;=+\\-*/<>!]/y },\n    ],\n    sql: [\n        { kind: \"comment\", pattern: /--[^\\n]*|\\/\\*[\\s\\S]*?\\*\\//y },\n        { kind: \"string\", pattern: /'(?:[^']|'')*'/y },\n        {\n            kind: \"keyword\",\n            // Case-insensitive: SQL is written both ways and both should paint.\n            pattern:\n                /\\b(?:select|from|where|insert|into|values|update|set|delete|create|table|alter|drop|index|join|inner|left|right|outer|on|group|by|order|having|limit|offset|as|and|or|not|in|is|null|distinct|union|all|with|returning|primary|key|foreign|references|default)\\b/iy,\n        },\n        { kind: \"number\", pattern: /\\b\\d+(?:\\.\\d+)?\\b/y },\n        { kind: \"punctuation\", pattern: /[(),;*=<>]/y },\n    ],\n    plain: [],\n};\n\n/** The shared JavaScript-family rules, with JSX tags added for tsx/jsx. */\nfunction jsRules(withJsx: boolean): Rule[] {\n    const rules: Rule[] = [\n        { kind: \"comment\", pattern: /\\/\\/[^\\n]*|\\/\\*[\\s\\S]*?\\*\\//y },\n        {\n            kind: \"string\",\n            pattern: /`(?:[^`\\\\]|\\\\.)*`|\"(?:[^\"\\\\]|\\\\.)*\"|'(?:[^'\\\\]|\\\\.)*'/y,\n        },\n        { kind: \"keyword\", pattern: new RegExp(`\\\\b(?:${JS_KEYWORDS})\\\\b`, \"y\") },\n        { kind: \"literal\", pattern: new RegExp(`\\\\b(?:${JS_LITERALS})\\\\b`, \"y\") },\n        {\n            kind: \"number\",\n            pattern: /\\b(?:0[xX][\\da-fA-F_]+|\\d[\\d_]*(?:\\.[\\d_]+)?(?:[eE][+-]?\\d+)?)\\b/y,\n        },\n        { kind: \"function\", pattern: /[A-Za-z_$][\\w$]*(?=\\s*\\()/y },\n        { kind: \"punctuation\", pattern: /[{}[\\]().,;:?!=<>+\\-*/%&|^~]/y },\n    ];\n    if (withJsx) {\n        // Before the keyword rule, so `<Button` reads as a tag and not as `<`.\n        rules.splice(2, 0, { kind: \"tag\", pattern: /<\\/?[A-Za-z][\\w.]*|\\/>/y });\n    }\n    return rules;\n}\n\n/**\n * Split source into coloured tokens.\n *\n * This is a **scanner, not a parser**: it recognises comments, strings, numbers,\n * keywords and punctuation by pattern, and knows nothing about scope, types or\n * grammar. That is a deliberate ceiling. A real parser per language is a\n * dependency the size of the rest of the SDK, and the payoff — being right about\n * the corner cases in a documentation snippet — is small. Where it is unsure it\n * emits `plain`, which renders as ordinary text rather than as something wrong.\n *\n * Unknown languages produce a single `plain` token, so an unhighlighted block is\n * a normal outcome and never an error.\n *\n * @param code - The source.\n * @param language - Grammar name or alias.\n * @returns Tokens covering the input exactly, in order.\n *\n * @example\n * tokenize(\"const x = 1;\", \"ts\");\n * // [{kind: \"keyword\", value: \"const\"}, {kind: \"plain\", value: \" x \"}, …]\n */\nexport function tokenize(code: string, language: string | undefined): Token[] {\n    const rules = GRAMMARS[resolveLanguage(language)];\n    if (rules.length === 0) return code === \"\" ? [] : [{ kind: \"plain\", value: code }];\n\n    const tokens: Token[] = [];\n    let plainFrom = 0;\n    let at = 0;\n\n    const flushPlain = (until: number) => {\n        if (until > plainFrom) tokens.push({ kind: \"plain\", value: code.slice(plainFrom, until) });\n    };\n\n    while (at < code.length) {\n        let matched: Token | null = null;\n        for (const rule of rules) {\n            rule.pattern.lastIndex = at;\n            const found = rule.pattern.exec(code);\n            if (found && found[0].length > 0) {\n                matched = { kind: rule.kind, value: found[0] };\n                break;\n            }\n        }\n        if (matched) {\n            flushPlain(at);\n            tokens.push(matched);\n            at += matched.value.length;\n            plainFrom = at;\n        } else {\n            at++;\n        }\n    }\n    flushPlain(code.length);\n    return tokens;\n}\n\n/**\n * The same tokens, split at newlines so each line can be rendered on its own.\n *\n * Line numbers and highlighted lines both need a per-line structure, and a token\n * is free to span a line break — a block comment usually does. Splitting here\n * keeps that out of the component.\n *\n * @param code - The source.\n * @param language - Grammar name or alias.\n * @returns One token array per line. Always at least one line.\n */\nexport function tokenizeLines(code: string, language: string | undefined): Token[][] {\n    const lines: Token[][] = [[]];\n    for (const token of tokenize(code, language)) {\n        const pieces = token.value.split(\"\\n\");\n        pieces.forEach((piece, index) => {\n            if (index > 0) lines.push([]);\n            if (piece !== \"\") lines[lines.length - 1].push({ kind: token.kind, value: piece });\n        });\n    }\n    return lines;\n}\n"],"mappings":"AAkCA,IAAM,EAAwC,CAC1C,GAAI,aACJ,WAAY,aACZ,GAAI,aACJ,WAAY,aACZ,IAAK,aACL,IAAK,aACL,IAAK,MACL,IAAK,MACL,KAAM,OACN,MAAO,OACP,IAAK,MACL,KAAM,MACN,KAAM,OACN,IAAK,OACL,IAAK,OACL,GAAI,OACJ,KAAM,OACN,MAAO,OACP,IAAK,OACL,QAAS,OACT,GAAI,SACJ,OAAQ,SACR,IAAK,KACT,EAQA,SAAgB,EAAgB,EAA4C,CAExE,OADK,EACE,EAAQ,EAAS,YAAY,IAAM,QADpB,OAE1B,CAQA,IAAM,EACF,uUAEE,EAAc,yCASd,EAAyC,CAC3C,WAAY,EAAQ,EAAK,EACzB,WAAY,EAAQ,EAAK,EACzB,IAAK,EAAQ,EAAI,EACjB,IAAK,EAAQ,EAAI,EACjB,KAAM,CACF,CAAE,KAAM,WAAY,QAAS,4BAA6B,EAC1D,CAAE,KAAM,SAAU,QAAS,oBAAqB,EAChD,CAAE,KAAM,SAAU,QAAS,mCAAoC,EAC/D,CAAE,KAAM,UAAW,QAAS,0BAA2B,EACvD,CAAE,KAAM,cAAe,QAAS,YAAa,CACjD,EACA,IAAK,CACD,CAAE,KAAM,UAAW,QAAS,mBAAoB,EAChD,CAAE,KAAM,SAAU,QAAS,sCAAuC,EAClE,CAAE,KAAM,UAAW,QAAS,cAAe,EAC3C,CAAE,KAAM,WAAY,QAAS,4BAA6B,EAC1D,CAAE,KAAM,WAAY,QAAS,mBAAoB,EACjD,CAAE,KAAM,SAAU,QAAS,oDAAqD,EAChF,CAAE,KAAM,cAAe,QAAS,eAAgB,CACpD,EACA,KAAM,CACF,CAAE,KAAM,UAAW,QAAS,kBAAmB,EAC/C,CAAE,KAAM,SAAU,QAAS,sCAAuC,EAClE,CAAE,KAAM,MAAO,QAAS,sBAAuB,EAC/C,CAAE,KAAM,YAAa,QAAS,uBAAwB,EACtD,CAAE,KAAM,cAAe,QAAS,SAAU,CAC9C,EACA,KAAM,CACF,CAAE,KAAM,UAAW,QAAS,UAAW,EACvC,CAAE,KAAM,SAAU,QAAS,4BAA6B,EACxD,CACI,KAAM,UACN,QACI,2GACR,EACA,CAAE,KAAM,WAAY,QAAS,yBAA0B,EACvD,CAAE,KAAM,YAAa,QAAS,2BAA4B,EAC1D,CAAE,KAAM,cAAe,QAAS,YAAa,CACjD,EACA,OAAQ,CACJ,CAAE,KAAM,UAAW,QAAS,UAAW,EACvC,CACI,KAAM,SACN,QAAS,oEACb,EACA,CACI,KAAM,UACN,QACI,wLACR,EACA,CAAE,KAAM,UAAW,QAAS,+BAAgC,EAC5D,CAAE,KAAM,SAAU,QAAS,oBAAqB,EAChD,CAAE,KAAM,WAAY,QAAS,qBAAsB,EACnD,CAAE,KAAM,cAAe,QAAS,yBAA0B,CAC9D,EACA,IAAK,CACD,CAAE,KAAM,UAAW,QAAS,4BAA6B,EACzD,CAAE,KAAM,SAAU,QAAS,iBAAkB,EAC7C,CACI,KAAM,UAEN,QACI,oQACR,EACA,CAAE,KAAM,SAAU,QAAS,oBAAqB,EAChD,CAAE,KAAM,cAAe,QAAS,aAAc,CAClD,EACA,MAAO,CAAC,CACZ,EAGA,SAAS,EAAQ,EAA0B,CACvC,IAAM,EAAgB,CAClB,CAAE,KAAM,UAAW,QAAS,8BAA+B,EAC3D,CACI,KAAM,SACN,QAAS,wDACb,EACA,CAAE,KAAM,UAAW,QAAa,OAAO,SAAS,EAAY,MAAO,GAAG,CAAE,EACxE,CAAE,KAAM,UAAW,QAAa,OAAO,SAAS,EAAY,MAAO,GAAG,CAAE,EACxE,CACI,KAAM,SACN,QAAS,mEACb,EACA,CAAE,KAAM,WAAY,QAAS,4BAA6B,EAC1D,CAAE,KAAM,cAAe,QAAS,+BAAgC,CACpE,EAKA,OAJI,GAEA,EAAM,OAAO,EAAG,EAAG,CAAE,KAAM,MAAO,QAAS,yBAA0B,CAAC,EAEnE,CACX,CAuBA,SAAgB,EAAS,EAAc,EAAuC,CAC1E,IAAM,EAAQ,EAAS,EAAgB,CAAQ,GAC/C,GAAI,EAAM,SAAW,EAAG,OAAO,IAAS,GAAK,CAAC,EAAI,CAAC,CAAE,KAAM,QAAS,MAAO,CAAK,CAAC,EAEjF,IAAM,EAAkB,CAAC,EACrB,EAAY,EACZ,EAAK,EAEH,EAAc,GAAkB,CAC9B,EAAQ,GAAW,EAAO,KAAK,CAAE,KAAM,QAAS,MAAO,EAAK,MAAM,EAAW,CAAK,CAAE,CAAC,CAC7F,EAEA,KAAO,EAAK,EAAK,QAAQ,CACrB,IAAI,EAAwB,KAC5B,IAAK,IAAM,KAAQ,EAAO,CACtB,EAAK,QAAQ,UAAY,EACzB,IAAM,EAAQ,EAAK,QAAQ,KAAK,CAAI,EACpC,GAAI,GAAS,EAAM,EAAE,CAAC,OAAS,EAAG,CAC9B,EAAU,CAAE,KAAM,EAAK,KAAM,MAAO,EAAM,EAAG,EAC7C,KACJ,CACJ,CACI,GACA,EAAW,CAAE,EACb,EAAO,KAAK,CAAO,EACnB,GAAM,EAAQ,MAAM,OACpB,EAAY,GAEZ,GAER,CAEA,OADA,EAAW,EAAK,MAAM,EACf,CACX,CAaA,SAAgB,EAAc,EAAc,EAAyC,CACjF,IAAM,EAAmB,CAAC,CAAC,CAAC,EAC5B,IAAK,IAAM,KAAS,EAAS,EAAM,CAAQ,EAEvC,EADqB,MAAM,MAAM;CACjC,CAAA,CAAO,SAAS,EAAO,IAAU,CACzB,EAAQ,GAAG,EAAM,KAAK,CAAC,CAAC,EACxB,IAAU,IAAI,EAAM,EAAM,OAAS,EAAE,CAAC,KAAK,CAAE,KAAM,EAAM,KAAM,MAAO,CAAM,CAAC,CACrF,CAAC,EAEL,OAAO,CACX"}