/** * obsidian-parser — pure functions for parsing an Obsidian vault on disk. * * No DB access, no MCP, no I/O side effects other than fs reads. Composed by * the `obsidian-vault-import` MCP tool in the memory plugin. * * Doctrine: every regex is linear-time (no catastrophic backtracking), every * function is deterministic. The agent never calls this code; the server does. */ import { createHash } from "node:crypto"; import { promises as fs } from "node:fs"; import * as path from "node:path"; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface ParsedPage { /** Path relative to the vault root, with forward slashes. The MERGE natural key. */ obsidianPath: string; /** Filename without `.md` — the default wikilink target form. */ title: string; /** Aliases declared in frontmatter (`aliases:` array). */ aliases: string[]; /** Tag tokens (without `#`), unioned from frontmatter `tags:` and inline `#tag` body matches. */ tags: string[]; /** Body text (frontmatter stripped). Stored verbatim. */ body: string; /** SHA-256 of the on-disk file bytes — idempotency key. */ contentHash: string; /** Detected daily-note date (YYYY-MM-DD) or null. */ noteDate: string | null; /** Wikilinks emitted by this page. */ wikilinks: WikilinkOccurrence[]; /** Embedded attachments declared via `![[file.png]]` or `![alt](file.png)`. */ attachments: AttachmentOccurrence[]; /** Last-modified timestamp from fs stat — used for modified-since filtering. */ mtimeMs: number; } export interface WikilinkOccurrence { /** The exact text inside `[[…]]` before any alias `|`. May contain `#anchor`. */ rawTarget: string; /** Target after stripping `#anchor`. */ target: string; /** Anchor portion (after `#`) or null. */ anchor: string | null; /** Alias text after `|`, or null when absent. The display text. */ alias: string | null; } export interface AttachmentOccurrence { /** Path as written in the markdown. May be relative to the page or to the vault root. */ rawPath: string; /** Resolved path relative to vault root, normalised. May not exist on disk yet. */ resolvedPath: string; /** Display alt text (from `![alt](path)` form) or null for embed-syntax `![[file]]`. */ alt: string | null; } export interface VaultWalkOptions { /** Filter: only pages whose path starts with one of these folder prefixes. */ folderPrefixes?: string[]; /** Filter: only pages whose tags intersect with this set. */ tagFilter?: string[]; /** Filter: only pages modified at or after this epoch-ms. */ modifiedSinceMs?: number; } // --------------------------------------------------------------------------- // Vault walk // --------------------------------------------------------------------------- /** * Walk the vault directory and return one ParsedPage per `.md` file that * survives the optional filter set. * * Path-traversal defence: paths containing `..` segments are rejected at * entry. The `vaultRoot` passed in is assumed to have already been verified * by the caller as living under the account's archive directory. */ export async function walkVault( vaultRoot: string, opts: VaultWalkOptions = {}, ): Promise { const pages: ParsedPage[] = []; const queue: string[] = [vaultRoot]; while (queue.length > 0) { const dir = queue.shift()!; let entries; try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch (err) { throw new Error( `obsidian-parser: cannot read directory ${dir}: ${ err instanceof Error ? err.message : String(err) }`, ); } for (const entry of entries) { if (entry.name.startsWith(".")) continue; const abs = path.join(dir, entry.name); if (entry.isDirectory()) { queue.push(abs); continue; } if (!entry.isFile()) continue; if (!entry.name.toLowerCase().endsWith(".md")) continue; const rel = toPosixRelative(vaultRoot, abs); if (rel.split("/").includes("..")) continue; if (opts.folderPrefixes && opts.folderPrefixes.length > 0) { const matchesPrefix = opts.folderPrefixes.some((p) => rel === p || rel.startsWith(p.replace(/\/?$/, "/")), ); if (!matchesPrefix) continue; } let stat; let bytes; try { stat = await fs.stat(abs); bytes = await fs.readFile(abs); } catch (err) { // Per-file isolation — log and skip, don't abort the walk. process.stderr.write( `[obsidian-parser] event=read-failed file=${rel} reason=${ err instanceof Error ? err.message : String(err) }\n`, ); continue; } if (opts.modifiedSinceMs && stat.mtimeMs < opts.modifiedSinceMs) continue; let parsed: ParsedPage; try { parsed = parsePage(rel, bytes, stat.mtimeMs); } catch (err) { process.stderr.write( `[obsidian-parser] event=parse-failed file=${rel} reason=${ err instanceof Error ? err.message : String(err) }\n`, ); continue; } if (opts.tagFilter && opts.tagFilter.length > 0) { const wanted = new Set(opts.tagFilter.map(normalizeTag)); const hit = parsed.tags.some((t) => wanted.has(normalizeTag(t))); if (!hit) continue; } pages.push(parsed); } } pages.sort((a, b) => a.obsidianPath.localeCompare(b.obsidianPath)); return pages; } function toPosixRelative(root: string, abs: string): string { const rel = path.relative(root, abs); return rel.split(path.sep).join("/"); } // --------------------------------------------------------------------------- // Page parser // --------------------------------------------------------------------------- const DAILY_NOTE_FILENAME_RE = /^(\d{4})-(\d{2})-(\d{2})(?:\.md)?$/; const DAILY_NOTE_FOLDER_NAMES = new Set([ "daily", "daily-notes", "daily notes", "journal", "journals", ]); export function parsePage( obsidianPath: string, bytes: Buffer, mtimeMs: number, ): ParsedPage { const contentHash = createHash("sha256").update(bytes).digest("hex"); const text = bytes.toString("utf8"); const { frontmatter, body } = splitFrontmatter(text); const title = path .basename(obsidianPath) .replace(/\.md$/i, "") .normalize("NFC"); const aliases = extractAliases(frontmatter); const frontmatterTags = extractFrontmatterTags(frontmatter); const { bodyTags, wikilinks, attachments } = scanBody(body, obsidianPath); const tags = dedupeNormalized([...frontmatterTags, ...bodyTags]); const noteDate = detectDailyNote(obsidianPath, frontmatter); return { obsidianPath, title, aliases: aliases.map((a) => a.normalize("NFC")), tags, body, contentHash, noteDate, wikilinks, attachments, mtimeMs, }; } // --------------------------------------------------------------------------- // Frontmatter // --------------------------------------------------------------------------- interface ParsedFrontmatter { raw: string; /** Map of top-level key → raw value string. Arrays are returned as the * block following the key (caller parses per-key). */ fields: Map; } const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/; export function splitFrontmatter( text: string, ): { frontmatter: ParsedFrontmatter; body: string } { const match = text.match(FRONTMATTER_RE); if (!match) { return { frontmatter: { raw: "", fields: new Map() }, body: text }; } const raw = match[1]; const body = text.slice(match[0].length); // Minimal YAML-subset parser: top-level `key: value` and `key:` followed by // ` - item` lines. Not a full YAML — we explicitly limit the surface so a // malformed frontmatter degrades to "no fields found" rather than throwing. const fields = new Map(); const lines = raw.split(/\r?\n/); let currentKey: string | null = null; let currentBlock: string[] = []; for (const line of lines) { if (/^[#]/.test(line.trim())) continue; // comment const flush = () => { if (currentKey !== null) { const existing = fields.get(currentKey); const next = currentBlock.join("\n"); fields.set(currentKey, existing ? existing + "\n" + next : next); } currentKey = null; currentBlock = []; }; const topMatch = line.match(/^([A-Za-z_][\w-]*)\s*:\s*(.*)$/); if (topMatch && !line.startsWith(" ") && !line.startsWith("\t")) { flush(); currentKey = topMatch[1]; const value = topMatch[2]; if (value.trim() === "") { currentBlock = []; } else { currentBlock = [value]; flush(); } continue; } if (currentKey !== null) { currentBlock.push(line); } } if (currentKey !== null) { const existing = fields.get(currentKey); const next = currentBlock.join("\n"); fields.set(currentKey, existing ? existing + "\n" + next : next); } return { frontmatter: { raw, fields }, body }; } function extractAliases(fm: ParsedFrontmatter): string[] { return parseYamlList(fm.fields.get("aliases")); } function extractFrontmatterTags(fm: ParsedFrontmatter): string[] { return parseYamlList(fm.fields.get("tags")).map(normalizeTag); } function parseYamlList(value: string | undefined): string[] { if (!value) return []; const trimmed = value.trim(); if (!trimmed) return []; // Flow-style: [a, b, c] if (trimmed.startsWith("[") && trimmed.endsWith("]")) { return trimmed .slice(1, -1) .split(",") .map((s) => s.trim().replace(/^["']|["']$/g, "")) .filter(Boolean); } // Block-style: - item per line, or single scalar const lines = trimmed.split(/\r?\n/); if (lines.length === 1 && !lines[0].startsWith("-")) { return [lines[0].replace(/^["']|["']$/g, "").trim()].filter(Boolean); } const out: string[] = []; for (const line of lines) { const m = line.match(/^\s*-\s*(.*?)\s*$/); if (m) { const v = m[1].replace(/^["']|["']$/g, "").trim(); if (v) out.push(v); } } return out; } // --------------------------------------------------------------------------- // Body scan: wikilinks, tags, attachments — all code-fence-aware // --------------------------------------------------------------------------- const WIKILINK_RE = /\[\[([^\]\r\n]+)\]\]/g; const EMBED_RE = /!\[\[([^\]\r\n]+)\]\]/g; const IMAGE_RE = /!\[([^\]\r\n]*)\]\(([^)\r\n]+)\)/g; const INLINE_TAG_RE = /(^|[\s(])#([A-Za-z][\w/-]*)/g; const ATTACHMENT_EXT_RE = /\.(png|jpe?g|gif|webp|svg|pdf|mp3|mp4|mov|wav|m4a|webm)$/i; interface BodyScan { bodyTags: string[]; wikilinks: WikilinkOccurrence[]; attachments: AttachmentOccurrence[]; } function scanBody(body: string, obsidianPath: string): BodyScan { const bodyTags: string[] = []; const wikilinks: WikilinkOccurrence[] = []; const attachments: AttachmentOccurrence[] = []; for (const segment of splitOnCodeFences(body)) { if (segment.isCode) continue; const text = segment.text; // Embeds first — they share `[[…]]` with wikilinks but the `!` prefix // routes them to attachments. const embedSpans: Array<[number, number]> = []; for (const m of text.matchAll(EMBED_RE)) { const inner = m[1].trim(); const resolved = resolveAttachmentPath(inner, obsidianPath); if (resolved && ATTACHMENT_EXT_RE.test(resolved)) { attachments.push({ rawPath: inner, resolvedPath: resolved, alt: null }); } else { // Non-attachment embed (e.g. another note transclusion). Treat as a // wikilink so the cross-reference is preserved. wikilinks.push(parseWikilinkInner(inner)); } embedSpans.push([m.index ?? 0, (m.index ?? 0) + m[0].length]); } for (const m of text.matchAll(WIKILINK_RE)) { const start = m.index ?? 0; if (embedSpans.some(([s, e]) => start >= s - 1 && start < e)) continue; wikilinks.push(parseWikilinkInner(m[1])); } for (const m of text.matchAll(IMAGE_RE)) { const rawPath = m[2].trim(); const alt = m[1].trim() || null; if (/^https?:\/\//i.test(rawPath)) continue; // remote const resolved = resolveAttachmentPath(rawPath, obsidianPath); if (resolved && ATTACHMENT_EXT_RE.test(resolved)) { attachments.push({ rawPath, resolvedPath: resolved, alt }); } } for (const m of text.matchAll(INLINE_TAG_RE)) { bodyTags.push(normalizeTag(m[2])); } } return { bodyTags: dedupeNormalized(bodyTags), wikilinks, attachments, }; } function parseWikilinkInner(inner: string): WikilinkOccurrence { const trimmed = inner.trim(); const aliasIdx = trimmed.indexOf("|"); let raw = trimmed; let alias: string | null = null; if (aliasIdx >= 0) { raw = trimmed.slice(0, aliasIdx).trim(); alias = trimmed.slice(aliasIdx + 1).trim() || null; } const anchorIdx = raw.indexOf("#"); let target = raw; let anchor: string | null = null; if (anchorIdx >= 0) { target = raw.slice(0, anchorIdx).trim(); anchor = raw.slice(anchorIdx + 1).trim() || null; } return { rawTarget: raw, target: target.normalize("NFC"), anchor, alias }; } function resolveAttachmentPath( rawPath: string, obsidianPath: string, ): string | null { if (!rawPath) return null; if (rawPath.startsWith("/") || /^[A-Za-z]:[\\/]/.test(rawPath)) return null; if (rawPath.split("/").includes("..")) return null; // Obsidian's default: vault-root relative. We also accept page-relative. // For the resolved form we prefer vault-root-relative (more deterministic // for MERGE), normalising path separators. const norm = rawPath.replace(/\\/g, "/"); if (norm.includes("/")) return norm; // Bare filename — could be in any attachments folder. Caller's index does // the lookup; we record the bare form. return norm; } function splitOnCodeFences(text: string): Array<{ text: string; isCode: boolean }> { const out: Array<{ text: string; isCode: boolean }> = []; const fenceRe = /(^|\n)(```|~~~)[^\n]*\n([\s\S]*?)\n(```|~~~)/g; let lastIdx = 0; for (const m of text.matchAll(fenceRe)) { const start = (m.index ?? 0) + (m[1] ? m[1].length : 0); out.push({ text: text.slice(lastIdx, start), isCode: false }); out.push({ text: m[0], isCode: true }); lastIdx = start + m[0].length - (m[1] ? m[1].length : 0); } out.push({ text: text.slice(lastIdx), isCode: false }); return out; } // --------------------------------------------------------------------------- // Tag normalization // --------------------------------------------------------------------------- export function normalizeTag(tag: string): string { return tag .replace(/^#/, "") .normalize("NFC") .trim() .toLowerCase(); } function dedupeNormalized(values: string[]): string[] { const seen = new Set(); const out: string[] = []; for (const v of values) { const k = v.normalize("NFC").trim().toLowerCase(); if (!k) continue; if (seen.has(k)) continue; seen.add(k); out.push(v); } return out; } // --------------------------------------------------------------------------- // Daily-notes detection // --------------------------------------------------------------------------- export function detectDailyNote( obsidianPath: string, fm: ParsedFrontmatter, ): string | null { // 1. Frontmatter `date:` or `created:` wins if it's an ISO date. for (const key of ["date", "created"]) { const v = fm.fields.get(key); if (v) { const m = v.trim().match(/^(\d{4}-\d{2}-\d{2})/); if (m) return m[1]; } } // 2. Filename matches YYYY-MM-DD.md. const basename = path.basename(obsidianPath); const fileMatch = basename.match(DAILY_NOTE_FILENAME_RE); if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`; // 3. Filename is YYYY-MM-DD inside a known daily-notes folder. const parts = obsidianPath.split("/"); const inDailyFolder = parts .slice(0, -1) .some((p) => DAILY_NOTE_FOLDER_NAMES.has(p.toLowerCase())); if (inDailyFolder) { const stem = basename.replace(/\.md$/i, ""); const m = stem.match(/^(\d{4})-(\d{2})-(\d{2})$/); if (m) return `${m[1]}-${m[2]}-${m[3]}`; } return null; } // --------------------------------------------------------------------------- // Wikilink resolution helpers // --------------------------------------------------------------------------- export interface VaultIndex { /** Lowercased title (without `.md`) → set of `obsidianPath` candidates. */ byTitle: Map; /** Lowercased alias → set of `obsidianPath` candidates. */ byAlias: Map; } export function buildVaultIndex(pages: ParsedPage[]): VaultIndex { const byTitle = new Map(); const byAlias = new Map(); for (const p of pages) { pushKey(byTitle, p.title.toLowerCase(), p.obsidianPath); for (const a of p.aliases) { pushKey(byAlias, a.toLowerCase(), p.obsidianPath); } } return { byTitle, byAlias }; } function pushKey(m: Map, k: string, v: string): void { const arr = m.get(k); if (arr) { if (!arr.includes(v)) arr.push(v); } else { m.set(k, [v]); } } /** * Look up a wikilink target inside the vault, returning candidate page paths. * * The target is matched against (a) page titles and (b) declared aliases. * Path-form targets (`Folder/Page`) match directly. Returns an empty array * when no intra-vault candidate exists — the caller falls through to entity * resolution. */ export function resolveWikilinkInVault( target: string, index: VaultIndex, knownPaths: Set, ): string[] { const t = target.normalize("NFC").trim(); if (!t) return []; // Path-form: contains a slash or ends with `.md`. if (t.includes("/") || t.toLowerCase().endsWith(".md")) { const norm = t.replace(/\\/g, "/").replace(/\.md$/i, "") + ".md"; if (knownPaths.has(norm)) return [norm]; return []; } const titleHits = index.byTitle.get(t.toLowerCase()) ?? []; const aliasHits = index.byAlias.get(t.toLowerCase()) ?? []; const seen = new Set(); const out: string[] = []; for (const p of [...titleHits, ...aliasHits]) { if (seen.has(p)) continue; seen.add(p); out.push(p); } return out; } // --------------------------------------------------------------------------- // Import-id helper // --------------------------------------------------------------------------- export function makeImportId(): string { return `obs-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; }