/* Tier 1 typography rule engine (RFC-1068). Implements 14 rules across 5 families: PUNCT (4), SPACE (4), CASE (2), LOCALE (3), YAML (2). All regexes use the `u` flag and Unicode property escapes. Rules operate on TextSegment objects produced by the text-surface extractor, never on raw file bytes. TypographyRuleId and TypographyRuleFamily include Tier 2 families (HEAD, PAIR, MD per RFC-1069; NUM, ABBR, APOS per RFC-1070) and Tier 3 families (UNICODE, LINK, SENT per RFC-1071) so that Tier 2 and Tier 3 rule arrays can be typed as TypographyRule[]. Do not extract text — that is text-surface.ts. Do not handle file I/O or command registration — that is the command adapter. Do not auto-fix — autofix is RFC-1072. RFC-1069: extended TypographyRuleId and TypographyRuleFamily with HEAD, PAIR, MD families. RFC-1070: extended TypographyRuleId and TypographyRuleFamily with NUM, ABBR, APOS families; added localeDefaults to TypographyContext. RFC-1071: extended TypographyRuleId and TypographyRuleFamily with UNICODE, LINK, SENT families. RFC-1072: changed TypographyFinding.fix from null to FixAction | null; extended finding() helper with optional FixAction param; populated fix on PUNCT-01, SPACE-01, SPACE-02. RFC-1201: TYPO-LOCALE-01 fixHint branches on token shape — path-like tokens (`/` or `<` present) name the accepted domain-qualified URL-placeholder normalization form; plain mixed-script words keep the generic single-script hint. Detection unchanged. RFC-1068 */ import type { TextSegment } from "./text-surface.ts"; import type { FixAction } from "./fix.ts"; import { LOCALE_DEFAULTS, getLocaleDefaults, type LocaleTypographyDefaults, } from "./locale-defaults.ts"; // --------------------------------------------------------------------------- // Types (from RFC-1068) // --------------------------------------------------------------------------- export type TypographyRuleId = `TYPO-${ | "PUNCT" | "SPACE" | "CASE" | "LOCALE" | "YAML" | "HEAD" | "PAIR" | "MD" | "NUM" | "ABBR" | "APOS" | "UNICODE" | "LINK" | "SENT"}-${string}`; export type TypographyRuleFamily = | "PUNCT" | "SPACE" | "CASE" | "LOCALE" | "YAML" | "HEAD" | "PAIR" | "MD" | "NUM" | "ABBR" | "APOS" | "UNICODE" | "LINK" | "SENT"; export interface TypographyFinding { ruleId: TypographyRuleId; segment: TextSegment; /** 0-based column inside segment.text. */ column: number; /** Offending substring (for messages and fix). */ match: string; message: string; fixHint: string; /** Set by rules whitelisted in RFC-1072; null for non-fixable rules. */ fix: FixAction | null; } export interface TypographyRule { id: TypographyRuleId; family: TypographyRuleFamily; tier: 1 | 2 | 3; severity: "error" | "warning"; check(segment: TextSegment, ctx: TypographyContext): TypographyFinding[]; } export interface TypographyContext { locale: string; allowedTokens: ReadonlySet; abbreviations: ReadonlySet; /** Locale-specific typography defaults (number format, apostrophe policy). Added by RFC-1070. */ localeDefaults: LocaleTypographyDefaults; } // --------------------------------------------------------------------------- // Default allowed tokens seed (TYPO-CASE-01 and TYPO-LOCALE-03 exclusions) // --------------------------------------------------------------------------- export const DEFAULT_ALLOWED_TOKENS = new Set([ "iPhone", "iPad", "eBay", "GitHub", "YouTube", "LinkedIn", "WordPress", "JavaScript", "TypeScript", "PayPal", "macOS", "iOS", "OpenAI", ]); // --------------------------------------------------------------------------- // Helper: create a finding // --------------------------------------------------------------------------- export function finding( ruleId: TypographyRuleId, segment: TextSegment, match: string, column: number, message: string, fixHint: string, fix: FixAction | null = null, ): TypographyFinding { return { ruleId, segment, match, column, message, fixHint, fix }; } // --------------------------------------------------------------------------- // Helper: check if a token is in allowedTokens // --------------------------------------------------------------------------- function isAllowedToken(token: string, ctx: TypographyContext): boolean { if (ctx.allowedTokens.has(token)) return true; // Also check with leading/trailing punctuation stripped const stripped = token.replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, ""); if (stripped !== token && ctx.allowedTokens.has(stripped)) return true; // Also check hyphenated parts (e.g. LinkedIn-Profil → LinkedIn) for (const part of stripped.split("-")) { if (part.length > 0 && ctx.allowedTokens.has(part)) return true; } return false; } // --------------------------------------------------------------------------- // Helper: extract tokens (maximal non-whitespace sequences) from text // --------------------------------------------------------------------------- function tokenize(text: string): string[] { return text.split(/\s+/).filter(Boolean); } // --------------------------------------------------------------------------- // Placeholder characters — treated as letters for spacing, skipped for casing/locale // --------------------------------------------------------------------------- const FORMULA_PLACEHOLDER = "\uE000"; const URL_PLACEHOLDER = "\uE001"; const CMS_PLACEHOLDER = "\uE002"; // --------------------------------------------------------------------------- // TYPO-PUNCT-01: Doubled or clashing punctuation // Pattern: [,;:!?]\. · \.[,;] · :: · ;; · ,, · [!?]{3,} // Exclusions: ?! and !? are allowed // --------------------------------------------------------------------------- const PUNCT01_PATTERNS: RegExp[] = [/[,;:!?]\./u, /\.[,;]/u, /::/u, /;;/u, /,,/u, /[!?]{3,}/u]; /** * Deduplicate clashing punctuation for PUNCT-01 fix. * [,;:!?]. → . (keep the period) * .[,;] → . (keep the period) * :: → : * ;; → ; * ,, → , * [!?]{3,} → !! (keep first two) */ function dedupPunct(matched: string): string { // [,;:!?]. — closing mark followed by period → keep period if (/^[,;:!?]\.$/.test(matched)) return "."; // .[,;] — period followed by comma/semicolon → keep period if (/^\.[,;]$/.test(matched)) return "."; // :: → : if (matched === "::") return ":"; // ;; → ; if (matched === ";;") return ";"; // ,, → , if (matched === ",,") return ","; // [!?]{3,} → keep first two if (/^[!?]{3,}$/.test(matched)) return matched.slice(0, 2); return matched; } const PUNCT01_MESSAGES: Record = { "[,;:!?]\\.": { message: "Clashing punctuation — a closing mark followed by a period.", fixHint: "Keep exactly one closing mark.", }, "\\.[,;]": { message: "Clashing punctuation — a period followed by a comma or semicolon.", fixHint: "Keep exactly one closing mark.", }, "::": { message: "Doubled colon.", fixHint: "Use a single colon.", }, ";;": { message: "Doubled semicolon.", fixHint: "Use a single semicolon.", }, ",,": { message: "Doubled comma.", fixHint: "Use a single comma.", }, "[!?]{3,}": { message: "Three or more exclamation/question marks in a row.", fixHint: "Use at most two exclamation or question marks.", }, }; const punct01: TypographyRule = { id: "TYPO-PUNCT-01", family: "PUNCT", tier: 1, severity: "error", check(segment, ctx) { const findings: TypographyFinding[] = []; const text = segment.text; for (const pattern of PUNCT01_PATTERNS) { const match = pattern.exec(text); if (match) { // Check if it's ?! or !? — those are allowed const matched = match[0]; if (matched === "?!" || matched === "!?") continue; // Exclusion: period followed by comma/semicolon where the period // is the last char of a known abbreviation (e.g. "e.V.,", "Inc.,") if (pattern.source === "\\.[,;]") { const textUpToDot = text.slice(0, match.index + 1); let isAbbrev = false; for (const abbr of ctx.abbreviations) { if (textUpToDot.endsWith(abbr)) { isAbbrev = true; break; } } if (isAbbrev) continue; } const key = pattern.source; const msg = PUNCT01_MESSAGES[key] ?? { message: `Doubled or clashing punctuation: "${matched}".`, fixHint: "Keep exactly one closing mark.", }; findings.push( finding(this.id, segment, matched, match.index, msg.message, msg.fixHint, { type: "replace", old: matched, new: dedupPunct(matched), }), ); } } return findings; }, }; // --------------------------------------------------------------------------- // TYPO-PUNCT-02: Two dots or four-plus dots (not three-dot ellipsis) // Pattern: (? 0 && !/\s/.test(text[tokenStart - 1])) tokenStart--; let tokenEnd = matchEnd; while (tokenEnd < text.length && !/\s/.test(text[tokenEnd])) tokenEnd++; const token = text.slice(tokenStart, tokenEnd); // Skip if token contains placeholders (formula/URL) if ( token.includes(FORMULA_PLACEHOLDER) || token.includes(URL_PLACEHOLDER) || token.includes(CMS_PLACEHOLDER) ) return []; if (isAllowedToken(token, ctx)) return []; return [ finding( this.id, segment, m[0], m.index, `Uppercase letter between two lowercase letters inside a word: "${m[0]}" in "${token}".`, "Fix the casing — this is likely a corruption from a regex edit or translation tool.", ), ]; }, }; // --------------------------------------------------------------------------- // TYPO-CASE-02: Lowercase sentence start after sentence-ending punctuation inside a paragraph // Pattern: [.!?]\s+\p{Ll} // Exclusions: preceding token is a known abbreviation for the file locale; // preceding token is ...; preceding token is a digit sequence (\d+) // --------------------------------------------------------------------------- const CASE02 = /[.!?]\s+\p{Ll}/gu; const case02: TypographyRule = { id: "TYPO-CASE-02", family: "CASE", tier: 1, severity: "error", check(segment, ctx) { const text = segment.text; for (const m of text.matchAll(CASE02)) { const punctIndex = m.index; // Find the token before the punctuation // Scan backwards to find the token (maximal non-whitespace before punct) let tokenEnd = punctIndex; while (tokenEnd > 0 && /\s/.test(text[tokenEnd - 1])) tokenEnd--; let tokenStart = tokenEnd; while (tokenStart > 0 && !/\s/.test(text[tokenStart - 1])) tokenStart--; const token = text.slice(tokenStart, tokenEnd); // Exclusion: preceding token is a known abbreviation if (ctx.abbreviations.has(token)) continue; // Exclusion: text ending at the punctuation matches a multi-word abbreviation // (e.g. "z. B." — the period at punctIndex is part of the abbreviation) // Also check text including the matched characters, for abbreviations like // "u. a." where the first period triggers the match but the abbreviation // continues past the matched lowercase letter const textUpToPunct = text.slice(0, punctIndex + 1); const textUpToMatch = text.slice(0, m.index + m[0].length + 1); let isAbbrev = false; for (const abbr of ctx.abbreviations) { if (textUpToPunct.endsWith(abbr) || textUpToMatch.endsWith(abbr)) { isAbbrev = true; break; } } if (isAbbrev) continue; // Exclusion: preceding token is "..." if (token === "...") continue; // Exclusion: preceding token is a digit sequence if (/^\d+$/.test(token)) continue; // Exclusion: preceding token contains a placeholder (formula/URL/CMS) if ( token.includes(FORMULA_PLACEHOLDER) || token.includes(URL_PLACEHOLDER) || token.includes(CMS_PLACEHOLDER) ) continue; return [ finding( this.id, segment, m[0], m.index, "Lowercase sentence start after sentence-ending punctuation.", "Capitalize the first letter of the sentence.", ), ]; } return []; }, }; // --------------------------------------------------------------------------- // TYPO-LOCALE-01: Mixed-script word (Latin and Cyrillic letters in one token) // Pattern: token matches both \p{Script=Latin} and \p{Script=Cyrillic} // Exclusions: Latin parts of hyphenated tokens in typography.allowedTokens // --------------------------------------------------------------------------- const LATIN_SCRIPT = /\p{Script=Latin}/u; const CYRILLIC_SCRIPT = /\p{Script=Cyrillic}/u; const LATIN_ONLY = /^[\p{Script=Latin}0-9.]+$/u; const HAS_LATIN_LETTER = /\p{Script=Latin}/u; // RFC-1201: path-like flagged tokens get the domain-qualified URL-placeholder // hint — the same normalization the validator already accepts for // `domain.tld/path/` forms. function isPathLikeToken(token: string): boolean { return token.includes("/") || token.includes("<"); } function extractLatinParts(token: string): string[] { const parts = token.split("-"); return parts .map((p) => p.replace(/[^\p{Script=Latin}0-9.]/gu, "")) .filter((p) => p.length > 0 && LATIN_ONLY.test(p) && HAS_LATIN_LETTER.test(p)); } const locale01: TypographyRule = { id: "TYPO-LOCALE-01", family: "LOCALE", tier: 1, severity: "error", check(segment, ctx) { const tokens = tokenize(segment.text); for (const token of tokens) { // Skip tokens with placeholders if ( token.includes(FORMULA_PLACEHOLDER) || token.includes(URL_PLACEHOLDER) || token.includes(CMS_PLACEHOLDER) ) continue; if (LATIN_SCRIPT.test(token) && CYRILLIC_SCRIPT.test(token)) { // Check if all Latin parts are in allowedTokens const latinParts = extractLatinParts(token); if (latinParts.length > 0 && latinParts.every((p) => ctx.allowedTokens.has(p))) { continue; } return [ finding( this.id, segment, token, segment.text.indexOf(token), `Mixed-script word (Latin and Cyrillic letters in one token): "${token}".`, isPathLikeToken(token) ? `If "${token}" is a URL/path placeholder, use the domain-qualified form (e.g. "domain.tld/path/") — URL-like tokens are normalized to placeholders. Otherwise rewrite the token in a single script.` : "Separate scripts — this is likely a homoglyph or translation artifact.", ), ]; } } return []; }, }; // --------------------------------------------------------------------------- // TYPO-LOCALE-02: Foreign-alphabet letters for a Cyrillic locale // Pattern: file locale `uk`: [ыэъёЫЭЪЁ] // Exclusions: tokens in typography.allowedTokens // --------------------------------------------------------------------------- const UK_FOREIGN_LETTERS = /[ыэъёЫЭЪЁ]/u; const locale02: TypographyRule = { id: "TYPO-LOCALE-02", family: "LOCALE", tier: 1, severity: "error", check(segment, ctx) { if (segment.locale !== "uk") return []; const text = segment.text; const m = UK_FOREIGN_LETTERS.exec(text); if (!m) return []; // Check if the containing token is in allowedTokens const matchIndex = m.index; let tokenStart = matchIndex; while (tokenStart > 0 && !/\s/.test(text[tokenStart - 1])) tokenStart--; let tokenEnd = matchIndex + 1; while (tokenEnd < text.length && !/\s/.test(text[tokenEnd])) tokenEnd++; const token = text.slice(tokenStart, tokenEnd); if (isAllowedToken(token, ctx)) return []; return [ finding( this.id, segment, m[0], m.index, `Foreign-alphabet letter "${m[0]}" in a Ukrainian locale (these letters are Russian, not Ukrainian).`, "Replace with the Ukrainian equivalent.", ), ]; }, }; // --------------------------------------------------------------------------- // TYPO-LOCALE-03: Cyrillic letters in a non-Cyrillic locale // Pattern: file locale outside CYRILLIC_SCRIPT_LOCALES: \p{Script=Cyrillic} // Exclusions: tokens in typography.allowedTokens; markdown blockquote lines (lines starting with >) // --------------------------------------------------------------------------- /** Locales (primary subtag) whose default script is Cyrillic — TYPO-LOCALE-03 never fires there. */ export const CYRILLIC_SCRIPT_LOCALES: ReadonlySet = new Set([ "be", "bg", "kk", "ky", "mk", "mn", "ru", "sr", "tg", "uk", ]); const locale03: TypographyRule = { id: "TYPO-LOCALE-03", family: "LOCALE", tier: 1, severity: "error", check(segment, ctx) { if (CYRILLIC_SCRIPT_LOCALES.has(segment.locale.split("-")[0]!)) return []; const text = segment.text; const m = CYRILLIC_SCRIPT.exec(text); if (!m) return []; // Check if the containing token is in allowedTokens const matchIndex = m.index; let tokenStart = matchIndex; while (tokenStart > 0 && !/\s/.test(text[tokenStart - 1])) tokenStart--; let tokenEnd = matchIndex + 1; while (tokenEnd < text.length && !/\s/.test(text[tokenEnd])) tokenEnd++; const token = text.slice(tokenStart, tokenEnd); if (isAllowedToken(token, ctx)) return []; // Check if this is a blockquote line (raw starts with >) if (segment.source === "body" && /^\s*>/.test(segment.raw)) return []; return [ finding( this.id, segment, m[0], m.index, `Cyrillic letter "${m[0]}" in a non-Cyrillic locale (${segment.locale}).`, "Replace with the Latin equivalent or wrap in a blockquote/fenced block.", ), ]; }, }; // --------------------------------------------------------------------------- // TYPO-YAML-02: Odd number of straight double quotes in a text value // Pattern: count of " in value is odd // Exclusions: none // --------------------------------------------------------------------------- const yaml02: TypographyRule = { id: "TYPO-YAML-02", family: "YAML", tier: 1, severity: "error", check(segment, _ctx) { if (segment.source !== "frontmatter") return []; const quoteCount = (segment.raw.match(/"/g) ?? []).length; if (quoteCount % 2 !== 0) { return [ finding( this.id, segment, '"', 0, `Odd number of straight double quotes (${quoteCount}) in a text value.`, "Balance the quotes — add or remove a double quote.", ), ]; } return []; }, }; // --------------------------------------------------------------------------- // Export all Tier 1 rules // --------------------------------------------------------------------------- export const TIER1_RULES: readonly TypographyRule[] = [ punct01, punct02, punct03, punct04, space01, space02, space03, space04, case01, case02, locale01, locale02, locale03, yaml02, ]; // Note: TYPO-YAML-01 is not a rule function — it is emitted by the command // adapter when the text-surface extractor returns a non-null yamlError. export function createTypographyContext( locale: string, allowedTokens: ReadonlySet, abbreviationsOverride?: ReadonlySet, ): TypographyContext { const defaults = getLocaleDefaults(locale); return { locale, allowedTokens: new Set([...DEFAULT_ALLOWED_TOKENS, ...allowedTokens]), abbreviations: abbreviationsOverride ?? defaults.abbreviations, localeDefaults: defaults, }; } export { LOCALE_DEFAULTS, getLocaleDefaults };