import type { DesignToken, TokenCategory, TokenParseError, TokenParseResult, } from "../token-types.js"; import { inferTokenCategory } from "./categories.js"; import { isDTCGFile, parseDtcgTokens } from "./dtcg.js"; import { parseScssTokens } from "./scss.js"; import { containsTailwindV4Theme, parseTailwindV4Theme } from "./tailwind-v4.js"; import type { DesignTokenParseOptions, ParsedToken, TokenParseOutput } from "./types.js"; // Token declaration value class excludes `;` (the normal terminator), and `{`/`}` // so the final declaration in a block (no trailing `;` — valid CSS, #34) and a // value that wraps across newlines (#35) are both captured without swallowing a // following block. Matched over the FULL content string (not per-line) so the // `[^;{}]+?` class can span `\n`. Terminates on `;`, the next `}`, or EOF. const TOKEN_DECLARATION_PATTERN = /--([a-zA-Z0-9_-]+)\s*:\s*([^;{}]+?)\s*(?:;|(?=\})|$)/g; const VAR_REFERENCE_PATTERN = /var\(\s*--([a-zA-Z0-9_-]+)(?:\s*,\s*([^)]+))?\s*\)/; export function parseDesignTokenContent( content: string, options: DesignTokenParseOptions = {} ): TokenParseResult { const startTime = performance.now(); const tokens: DesignToken[] = []; const errors: TokenParseError[] = []; const warnings: string[] = []; const filePath = options.filePath ?? "unknown"; const themeSelectors = options.themeSelectors ?? { ":root": "default" }; const includeTailwindV4 = options.includeTailwindV4 ?? true; // Format dispatch (single chokepoint for the CLI's buildContractVocabularyFacts). // DTCG/JSON and SCSS files produce NO `--name:` CSS declarations, so the CSS // line-loop below yields an empty vocabulary and FUI2015 silently disables — the // worst failure mode (#10 DTCG, #32 SCSS). Route those through the sibling // parsers (both already emit `--kebab` CSS names the rule consumes unchanged), // each returning early. A 0-token result surfaces a NON-SILENT warning rather // than a quietly-empty vocabulary. The CSS path (with the #34/#35 robustness // fixes) is the fallthrough default. const trimmed = content.trimStart(); const looksDtcg = isDTCGFile(filePath) || /\.jsonc?$|\.tokens$/i.test(filePath) || trimmed.startsWith("{"); if (looksDtcg) { try { // Fail-soft per the architecture spine: a malformed .json must never throw // or break the scan — fall through to the CSS path on any parse error. const out = parseDtcgTokens(content, filePath); const dtTokens = parsedOutputToDesignTokens(out, filePath); const dtcgWarnings = dtTokens.length === 0 ? [ `Token source file ${filePath}: DTCG/JSON file produced no token names; FUI2015 token-drift is inactive for it.`, ] : []; return { tokens: dtTokens, errors: [], warnings: dtcgWarnings, parseTimeMs: performance.now() - startTime, }; } catch { // Fall through to the CSS path; if nothing matches there, the 0-token // warning at the end of the CSS path still fires (non-silent). } } const isScss = /\.(scss|sass)$/i.test(filePath); const hasCssVar = /--[\w-]+\s*:/.test(content); const hasScssVar = /(^|\n)\s*\$[\w-]+\s*:/.test(content); if (isScss || (!hasCssVar && hasScssVar)) { // parseScssTokens emits BOTH `--css-var` and `$scss-var` names. Normalize // every `$name` to `--name`: the mandated dual-fallback convention // `var(--fui-x, $fui-x)` guarantees a 1:1 `$`↔`--` correspondence, and `--x` // is exactly the name the rule judges (it reads `var(--x)` references). const out = parseScssTokens(content, filePath); const scssTokens = scssOutputToDesignTokens(out, filePath); const scssWarnings = scssTokens.length === 0 ? [ `Token source file ${filePath}: SCSS file produced no token names; FUI2015 token-drift is inactive for it.`, ] : []; return { tokens: scssTokens, errors: [], warnings: scssWarnings, parseTimeMs: performance.now() - startTime, }; } if (includeTailwindV4 && containsTailwindV4Theme(content)) { const v4Result = parseTailwindV4Theme(content, filePath); tokens.push(...v4Result.tokens); warnings.push(...v4Result.warnings); for (const err of v4Result.errors) { errors.push({ message: err, file: filePath }); } } const tokensByName = new Map(); // Match over the FULL content so a value spanning newlines (#35) is captured. // The line number is recomputed from the match index because findSelectorForLine // and extractDescription still attribute by line. Internal whitespace (incl. // newlines) is collapsed so a multi-line value stores cleanly. for (const match of content.matchAll(TOKEN_DECLARATION_PATTERN)) { const [, name, rawValue] = match; const fullName = `--${name}`; const lineNumber = countNewlines(content, match.index ?? 0) + 1; tokensByName.set(fullName, { rawValue: rawValue.trim().replace(/\s+/g, " "), line: lineNumber, }); } for (const [name, { rawValue, line }] of tokensByName) { const selector = findSelectorForLine(content, line ?? 1); const theme = themeSelectors[selector] || "default"; const { resolvedValue, chain, hasCircular, unresolvedRef } = resolveDesignTokenValue( rawValue, tokensByName ); if (hasCircular) { warnings.push(`Circular reference detected for ${name} at line ${line}`); } if (unresolvedRef) { warnings.push(`Unresolved reference in ${name}: ${unresolvedRef}`); } tokens.push({ name, rawValue, resolvedValue, category: inferTokenCategory(name, resolvedValue || rawValue), level: inferTokenLevel(name, rawValue, chain), referenceChain: chain, sourceFile: filePath, lineNumber: line, theme, selector, description: extractDescription(content, line ?? 1), }); } // NON-SILENT invariant: a non-empty CSS file that yields no token names means // FUI2015 has no vocabulary from it — surface a warning instead of a quietly // empty result (mirrors the DTCG/SCSS 0-token warnings above). if (tokens.length === 0 && trimmed.length > 0) { warnings.push( `Token source file ${filePath}: CSS file produced no token names; FUI2015 token-drift is inactive for it.` ); } return { tokens, errors, warnings, parseTimeMs: performance.now() - startTime, }; } /** Count the `\n` characters in `content` up to (not including) `index`. */ function countNewlines(content: string, index: number): number { let count = 0; for (let i = 0; i < index && i < content.length; i++) { if (content.charCodeAt(i) === 10) count++; } return count; } /** * Flatten a DTCG/JSON `TokenParseOutput` into `DesignToken[]`. Names pass through * verbatim (parseDtcgTokens already emits `--kebab` CSS names via * tokenPathToCSSName). Determinism: walkTokenTree preserves JSON object key order. */ function parsedOutputToDesignTokens(output: TokenParseOutput, filePath: string): DesignToken[] { const tokens: DesignToken[] = []; for (const cat of Object.values(output.categories)) { for (const t of cat) { tokens.push(designTokenFromParsed(t.name, t, filePath)); } } return tokens; } /** * Flatten an SCSS `TokenParseOutput` into `DesignToken[]`, normalizing every * `$name` to `--name` (the mandated dual-fallback `var(--fui-x, $fui-x)` makes * this 1:1 and deterministic) and de-duping so a file declaring both `--fui-x` * and `$fui-x` yields one `--fui-x`. */ function scssOutputToDesignTokens(output: TokenParseOutput, filePath: string): DesignToken[] { const names = new Set(); const tokens: DesignToken[] = []; for (const cat of Object.values(output.categories)) { for (const t of cat) { const cssName = t.name.startsWith("$") ? `--${t.name.slice(1)}` : t.name; if (!cssName.startsWith("--") || names.has(cssName)) continue; names.add(cssName); tokens.push(designTokenFromParsed(cssName, t, filePath)); } } return tokens; } /** Build a `DesignToken` from a sibling-parser `ParsedToken` under a CSS name. */ function designTokenFromParsed(name: string, parsed: ParsedToken, filePath: string): DesignToken { return { name, rawValue: parsed.value ?? "", resolvedValue: parsed.resolvedValue ?? parsed.value ?? "", category: normalizeParsedTokenCategory( parsed.category, name, parsed.resolvedValue ?? parsed.value ?? "" ), level: 1, referenceChain: [], sourceFile: filePath, theme: "default", selector: ":root", description: parsed.description, }; } function normalizeParsedTokenCategory( category: string | undefined, name: string, value: string ): TokenCategory { switch (category?.toLowerCase()) { case "colors": case "color": case "surfaces": case "surface": case "text": return "color"; case "spacing": case "space": return "spacing"; case "typography": case "font": return "typography"; case "radius": case "radii": return "radius"; case "shadows": case "shadow": return "shadow"; case "borders": case "border": return "border"; case "animation": case "animations": case "transitions": case "transition": return "animation"; case "sizing": case "size": return "sizing"; case "z-index": case "zindex": return "z-index"; default: return inferTokenCategory(name, value); } } export function resolveDesignTokenValue( rawValue: string, tokensByName: Map, visited = new Set() ): { resolvedValue: string; chain: string[]; hasCircular: boolean; unresolvedRef?: string; } { const chain: string[] = []; let current = rawValue; let hasCircular = false; let unresolvedRef: string | undefined; const maxIterations = 20; let iterations = 0; while (iterations < maxIterations) { iterations++; const varMatch = current.match(VAR_REFERENCE_PATTERN); if (!varMatch) break; const [, refName, fallback] = varMatch; const fullRefName = `--${refName}`; if (visited.has(fullRefName)) { hasCircular = true; break; } visited.add(fullRefName); chain.push(fullRefName); const refToken = tokensByName.get(fullRefName); if (refToken) { current = current.replace(varMatch[0], refToken.rawValue); } else if (fallback) { current = current.replace(varMatch[0], fallback.trim()); } else { unresolvedRef = fullRefName; break; } } return { resolvedValue: normalizeTokenValue(current.trim()), chain, hasCircular, unresolvedRef, }; } export function normalizeTokenValue(value: string): string { value = value.replace(/#[0-9a-fA-F]+/g, (match) => match.toLowerCase()); value = value.replace(/\s+/g, " ").trim(); value = value.replace( /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*([\d.]+))?\s*\)/g, (_, r, g, b, a) => (a !== undefined ? `rgba(${r}, ${g}, ${b}, ${a})` : `rgb(${r}, ${g}, ${b})`) ); return value; } function inferTokenLevel(name: string, rawValue: string, referenceChain: string[]): 1 | 2 | 3 { const lowerName = name.toLowerCase(); if (/btn|button|input|card|modal|dialog|menu|nav|header|footer|table|form/i.test(lowerName)) { return 3; } if (referenceChain.length > 0) { return 2; } if (rawValue.match(/^#[0-9a-fA-F]+$/) || rawValue.match(/^\d+(\.\d+)?(px|rem|em|%|vh|vw)?$/)) { return 1; } return 2; } function findSelectorForLine(content: string, targetLine: number): string { const lines = content.split("\n"); let currentSelector = ":root"; let braceDepth = 0; for (let i = 0; i < Math.min(targetLine, lines.length); i++) { const line = lines[i]; const selectorMatch = line.match(/^\s*([^{]+)\s*\{/); if (selectorMatch) { const selector = selectorMatch[1].trim(); if ((line.match(/\{/g) || []).length > (line.match(/\}/g) || []).length) { currentSelector = selector; } } braceDepth += (line.match(/\{/g) || []).length; braceDepth -= (line.match(/\}/g) || []).length; if (braceDepth === 0) { currentSelector = ":root"; } } return currentSelector; } function extractDescription(content: string, line: number): string | undefined { const lines = content.split("\n"); if (line <= 1) return undefined; const prevLine = lines[line - 2]?.trim(); const singleLineMatch = prevLine?.match(/\/\/\s*(.+)$/); if (singleLineMatch) { return singleLineMatch[1].trim(); } const multiLineMatch = prevLine?.match(/\*\s*(.+)\s*\*\//); if (multiLineMatch) { return multiLineMatch[1].trim(); } const inlineMatch = prevLine?.match(/\/\*\s*(.+)\s*\*\//); if (inlineMatch) { return inlineMatch[1].trim(); } return undefined; }