/* Maintains packages/share/src/semantic/extract.ts as an authored share authored module so agents can evolve it without rediscovering local boundaries. Do not introduce app-specific runtime composition or deployment behavior into this reusable package source file. RFC-0133: backfilled MODULE_MAP and CHANGE_SUMMARY markers for compass.validate compliance. RFC-0915: removed custom slugify() — replaced by slugId from @warpgogol/werkstatt-shared/slug. */ import type { SemanticPerson, SemanticPostalAddress } from "./models.ts"; export function normalizeWhitespace(value: string): string { return value.replace(/\s+/g, " ").trim(); } export type MarkdownSection = { heading: string; body: string; }; export function splitMarkdownSections(markdown: string, level: 2 | 3 = 2): MarkdownSection[] { const normalizedMarkdown = markdown.trim(); if (!normalizedMarkdown) { return []; } const marker = "#".repeat(level); const splitPattern = new RegExp(`\\n(?=${marker}\\s+)`); const headingPattern = new RegExp(`^${marker}\\s+`); return normalizedMarkdown .split(splitPattern) .map((section) => section.trim()) .filter((section) => headingPattern.test(section)) .map((section) => { const [headingLine = "", ...restLines] = section.split("\n"); return { heading: headingLine.replace(headingPattern, "").trim(), body: restLines.join("\n").trim(), }; }); } export function extractParagraphs(markdown: string): string[] { return markdown .split(/\n\s*\n/) .map((part) => normalizeWhitespace(part)) .filter(Boolean); } const SENTENCE_ABBREVIATIONS: Record = { de: [ "z.B.", "z. B.", "z.", "etc.", "Nr.", "Abs.", "§", "S.", "ca.", "u.a.", "u. a.", "u.", "vgl.", "bspw.", "zzgl.", "inkl.", "exkl.", "bzw.", "ggf.", "ggfls.", "sog.", "o.Ä.", ], uk: [ "т.д.", "т.п.", "п.", "ст.", "див.", "пор.", "напр.", "ім.", "о.", "вкл.", "тис.", "млн.", "млрд.", "грн.", "№", ], en: ["e.g.", "i.e.", "etc.", "vs.", "Mr.", "Mrs.", "Dr.", "Inc.", "Ltd."], }; function escapeRegex(text: string): string { return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } function buildAbbreviationPattern(abbreviations: string[]): RegExp | undefined { if (abbreviations.length === 0) return undefined; const sorted = [...abbreviations].sort((a, b) => b.length - a.length); return new RegExp(`(?:^|[^a-zA-Zа-яА-ЯёЁїЇіІєЄäöüÄÖÜß])(${sorted.map(escapeRegex).join("|")})$`); } export function splitSentences(text: string, locale: string = "en"): string[] { const normalized = text.trim(); if (!normalized) return []; const abbreviations = SENTENCE_ABBREVIATIONS[locale] ?? SENTENCE_ABBREVIATIONS.en; const abbrevPattern = buildAbbreviationPattern(abbreviations); const sentences: string[] = []; let current = ""; const chars = [...normalized]; for (let i = 0; i < chars.length; i++) { const char = chars[i]; current += char; if (char !== "." && char !== "!" && char !== "?") continue; const nextChar = chars[i + 1]; if (nextChar === undefined) { sentences.push(current.trim()); current = ""; continue; } if (nextChar !== " " && nextChar !== "\n" && nextChar !== "\t") continue; const beforeAbbrCheck = current.trimEnd(); if (abbrevPattern && abbrevPattern.test(beforeAbbrCheck)) continue; // Skip numbered list markers (e.g., "1. ", "2. ") — a period after // digits is a list marker, not a sentence boundary. After normalizeWhitespace // newlines are spaces, so we check the token before the period. const beforePeriod = current.slice(0, -1); const lastSpaceIdx = Math.max(beforePeriod.lastIndexOf(" "), beforePeriod.lastIndexOf("\n")); const tokenBeforePeriod = beforePeriod.slice(lastSpaceIdx + 1).trim(); if (/^\d+$/.test(tokenBeforePeriod)) continue; const afterWhitespace = chars .slice(i + 1) .join("") .match(/^\s*([A-ZÄÖÜА-ЯЁЇІЄ])/); if (!afterWhitespace) continue; sentences.push(current.trim()); current = ""; } if (current.trim()) sentences.push(current.trim()); return sentences.filter(Boolean); } export type MarkdownList = { marker: "-" | "*" | "+" | "digit"; items: string[]; }; const LIST_MARKER_PATTERNS: Array<{ marker: MarkdownList["marker"]; regex: RegExp }> = [ { marker: "-", regex: /^-\s+/ }, { marker: "*", regex: /^\*\s+/ }, { marker: "+", regex: /^\+\s+/ }, { marker: "digit", regex: /^\d+\.\s+/ }, ]; function matchListMarker( line: string, ): { marker: MarkdownList["marker"]; text: string } | undefined { for (const { marker, regex } of LIST_MARKER_PATTERNS) { const match = line.match(regex); if (match) { return { marker, text: line.slice(match[0].length).trim() }; } } return undefined; } function isTopLevelListLine( line: string, ): { marker: MarkdownList["marker"]; text: string } | undefined { // Top-level: no leading whitespace before the marker if (/^\s/.test(line)) return undefined; return matchListMarker(line); } function isIndentedListLine(line: string): boolean { // Indented list line: starts with whitespace + list marker if (!/^\s/.test(line)) return false; const trimmed = line.trimStart(); return LIST_MARKER_PATTERNS.some(({ regex }) => regex.test(trimmed)); } export function extractLists(markdown: string): MarkdownList[] { const normalized = markdown.trim(); if (!normalized) return []; const lines = normalized.split("\n"); const lists: MarkdownList[] = []; let currentList: MarkdownList | null = null; let inCodeFence = false; for (const line of lines) { // Track fenced code blocks if (line.trimStart().startsWith("```")) { inCodeFence = !inCodeFence; if (currentList) { lists.push(currentList); currentList = null; } continue; } if (inCodeFence) { continue; } const topMatch = isTopLevelListLine(line); if (topMatch) { // Continue current list if same marker type if (currentList && currentList.marker === topMatch.marker) { currentList.items.push(topMatch.text); } else { // Different marker or first list — flush previous, start new if (currentList) { lists.push(currentList); } currentList = { marker: topMatch.marker, items: [topMatch.text] }; } continue; } // Indented list line — part of parent item, not a new item if (isIndentedListLine(line)) { continue; } // Non-list line (blank, prose, heading, etc.) — terminate current list if (currentList) { lists.push(currentList); currentList = null; } } // Flush trailing list if (currentList) { lists.push(currentList); } return lists; } export function extractListFacts(markdown: string): string[] { return markdown .split("\n") .map((line) => line.trim()) .filter((line) => line.startsWith("- ") || /^\d+\.\s+/.test(line)) .map((line) => line .replace(/^-\s+/, "") .replace(/^\d+\.\s+/, "") .trim(), ) .filter(Boolean); } export function extractEmail(markdown: string): string | undefined { const match = markdown.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i); return match?.[0]; } export function extractPostalAddress( markdown: string, country: string = "DE", ): SemanticPostalAddress | undefined { const match = markdown.match(/([^\n,]+),\s*(\d{5})\s+([^\n]+)/); if (!match) { return undefined; } return { streetAddress: match[1].trim(), postalCode: match[2].trim(), addressLocality: match[3].trim(), addressCountry: country, }; } export function mergePeople(...groups: Array): SemanticPerson[] { const merged = new Map(); for (const group of groups) { for (const person of group ?? []) { const existing = merged.get(person.name); merged.set(person.name, { name: person.name, role: existing?.role ?? person.role, description: existing?.description ?? person.description, isDeceased: existing?.isDeceased || person.isDeceased, }); } } return Array.from(merged.values()); }