/** * THE OFFSET-PRESERVING SOURCE TOKENIZER (rich-content PLAN decision 6). * * Cuts stored markdown into blocks that cover the text EXACTLY: the * concatenation of every block's `raw` is the input, always, byte for byte. * Nothing is trimmed, normalized, or re-serialized here — the tokenizer only * draws boundaries. Every custom construct the platform stores (kinds and * `__kind` JSON, typed JSON blocks, special fences, XML sections including * attribute-bearing, mid-line and unknown containers, `{{variables}}`, * ``, math, raw HTML, code fences, HTML comments and pinned * anchors ``) is an ISLAND: an atomic span no editor may parse * into its own model. Islands that sit inside running prose (a variable, a * citation, inline math) stay inside their paragraph as `inlines`, so a * visual editor keeps the paragraph whole and holds the island as an atom. * * Block kinds: * - `prose` — a markdown paragraph run (headings, lists, tables…): the only * thing a visual editor may re-serialize, and only when edited. * - `island` — a protected construct; `islandType` says which. * - `gap` — the line terminator(s) and blank lines between blocks. Never * edited; they keep the document's exact spacing. * * Streaming-tolerant: never throws on partial input. A construct whose * closer has not arrived yet becomes an island running to the end of the text * with `complete: false` — the same ownership rule the renderer's splitter * applies, so a half-streamed fence or XML section is never reflowed. * * Boundaries follow matrx-frontend's renderer splitter (`content-splitter-v2.ts`) * wherever it decides where a construct ends, so the editor and the renderer * agree on what is locked. Offsets are UTF-16 (`start`/`end`) and code points * (`startCp`/`endCp`). */ type SourceBlockKind = "prose" | "island" | "gap"; /** Island types that own whole blocks. */ type BlockIslandType = /** ``` / ~~~ fenced code of any language (meta.lang, meta.special). */ "fence" /** A bare JSON object block — typed JSON, `__kind` instances, directives. */ | "json" /** A known attribute-less XML region (``, ``, …). */ | "xml_region" /** An attribute-bearing XML element (``, ``, …). */ | "xml_attr" /** Any other line-leading, balanced XML element. */ | "xml_container" /** Allow-listed raw HTML block. */ | "html_block" /** `` comment. */ | "html_comment" /** A pinned annotation anchor ``. */ | "anchor" /** `$$ … $$` display math on its own lines. */ | "math_block" /** YAML (`---`) or TOML (`+++`) front matter at the start of the text (meta.format). */ | "front_matter" /** A box-drawing tree diagram. */ | "tree" /** * A generic-directive container `:::name[label]{attrs}` … `:::` (tabs, * columns, details, figures, callouts, TOC — meta.name), a leaf directive * `::name` on its own line (meta.leaf), or a MkDocs admonition * `!!! type "Title"` / `??? type` with its indented body (meta.syntax "mkdocs"). */ | "directive" /** A GitHub alert / Obsidian callout: a blockquote opening `> [!TYPE]` (meta.name). */ | "callout" /** A footnote definition `[^id]: …` with its indented continuation lines (meta.id). */ | "footnote_def"; /** Island types that live inside a prose block. */ type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref" /** `[[Page]]`, `[[Page|alias]]` (meta.target) — or `![[Page]]` (meta.embed). */ | "wikilink"; type IslandType = BlockIslandType | InlineIslandType; type IslandMeta = Readonly>; interface SourceSpan { /** UTF-16 offset of the first code unit. */ readonly start: number; /** UTF-16 offset just past the last code unit. */ readonly end: number; /** Code-point offset of `start`. */ readonly startCp: number; /** Code-point offset of `end`. */ readonly endCp: number; /** Exactly `text.slice(start, end)`. */ readonly raw: string; } interface SourceIsland extends SourceSpan { readonly islandType: IslandType; /** True when the island sits inside a prose block. */ readonly inline: boolean; /** False when the construct's closer never arrived (streaming / malformed). */ readonly complete: boolean; readonly meta: IslandMeta; } interface SourceBlock extends SourceSpan { readonly kind: SourceBlockKind; /** Set only on `island` blocks. */ readonly islandType: BlockIslandType | null; readonly complete: boolean; readonly meta: IslandMeta; /** Inline islands of a `prose` block, in document order. Empty otherwise. */ readonly inlines: readonly SourceIsland[]; } /** * Every callout spelling the renderer resolves (matrx-frontend * components/markdown-core/syntax/callout-types.ts ALIASES — keep in step). */ declare const CALLOUT_NAMES: ReadonlySet; /** * Cut `text` into blocks whose raw concatenation is exactly `text`. * Never throws; unclosed constructs are islands with `complete: false`. */ interface TokenizeOptions { /** * The text is a prefix of a message still arriving. The final line (no * terminator yet) can still grow — a bare ``` there may become ```bash — so * it closes no fence until its newline arrives (verify-RC-B3 residual R2). * Omit for stored text: a complete document's last line is final. */ readonly streaming?: boolean; /** * DIAGNOSTIC — complexity tests only. When given, every loop that walks the * text itself adds the characters it visits to `meter.work`, so a test can * assert the tokenizer's work grows linearly with its input (deterministic, * unlike wall-clock timing). Loops over a slice are not charged here: the * slice's own cost covers them. Omit in production. */ readonly meter?: { work: number; }; } declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[]; /** The exact source the blocks were cut from. */ declare function joinSource(blocks: readonly SourceBlock[]): string; /** Every island — block-level and inline — in document order. */ declare function listIslands(blocks: readonly SourceBlock[]): SourceIsland[]; /** The block containing UTF-16 offset `offset` (the later block at a boundary). */ declare function blockAt(blocks: readonly SourceBlock[], offset: number): SourceBlock | null; /** * SAVE = SPLICE (rich-content PLAN decision 6). * * An editor never hands back a whole re-serialized document. It hands back * per-block replacements for exactly the blocks the person changed; every * other byte of the stored text is written back as it was. This module * applies those replacements and returns: * * - the new text, * - a change set (old range → new range, in UTF-16 AND code points, * narrowed to the bytes that actually differ) that the annotation layer * uses to carry every live anchor through the save * (STORE-DESIGN §3.19 — the ProseMirror / Google Docs position-mapping * model), and * - an integrity guarantee: every island not explicitly replaced must still * be the same island, at its mapped position, in the new text. An edit * that opens an unclosed fence and swallows a kind below it is REFUSED. * * ISLANDS CHANGE ONLY THROUGH `islandEdit`. A prose/range edit must hand back * every island inside its range byte-identical and in order (a visual editor * holds them as atoms); if one is altered or dropped the save throws * `SourceSpliceError("island_edit")` naming the island. A kind editor or code * view changes an island with `islandEdit(island, newRaw)`. * * A no-edit save (or edits whose text equals the original block) returns the * original string itself — byte-identical, zero changes. */ /** * Replace `[start, end)` (UTF-16) of the original with `text`. A plain edit * starts and ends on block boundaries; an island edit (`island: true`, made by * `islandEdit`) names exactly one island's span. */ interface SourceEdit { readonly start: number; readonly end: number; readonly text: string; readonly island?: boolean; } interface SourceChange { /** The block-aligned range the edit named, in the ORIGINAL text. */ readonly blockStart: number; readonly blockEnd: number; /** The narrowed range that actually differs: old text … */ readonly oldStart: number; readonly oldEnd: number; /** … and where its replacement sits in the NEW text. */ readonly newStart: number; readonly newEnd: number; readonly oldStartCp: number; readonly oldEndCp: number; readonly newStartCp: number; readonly newEndCp: number; } interface DisturbedIsland { readonly islandType: IslandType; readonly inline: boolean; /** Where the island was in the original (UTF-16). */ readonly start: number; readonly end: number; /** Where it should have been in the new text. */ readonly mappedStart: number; } interface SpliceIntegrity { /** True when every untouched island survived intact and outside bytes are identical. */ readonly ok: boolean; /** Always true by construction; checked anyway because it is THE guarantee. */ readonly bytesOutsideEditsIdentical: boolean; readonly disturbed: readonly DisturbedIsland[]; } interface SpliceResult { readonly text: string; readonly changed: boolean; readonly changes: readonly SourceChange[]; readonly integrity: SpliceIntegrity; /** Tokenization of the new text. */ readonly blocks: readonly SourceBlock[]; } type SourceSpliceErrorCode = "out_of_range" | "misaligned" | "overlap" | "stale_blocks" | "island_edit" | "integrity"; declare class SourceSpliceError extends Error { readonly code: SourceSpliceErrorCode; constructor(code: SourceSpliceErrorCode, message: string); } interface SpliceOptions { /** The tokenization of `original`, when the caller already holds it. */ readonly blocks?: readonly SourceBlock[]; /** * Default `true`: a save that disturbs any island it did not explicitly * replace throws `SourceSpliceError("integrity")`. `false` is for TOOLING * ONLY (diagnostic scripts that want the report instead of the refusal) — * no editor or persistence path may pass it. */ readonly requireIntegrity?: boolean; } /** An edit that replaces one block's raw text. */ declare function blockEdit(block: SourceBlock, text: string): SourceEdit; /** The one door that changes an island: replace exactly its span with `text`. */ declare function islandEdit(island: { readonly start: number; readonly end: number; }, text: string): SourceEdit; /** An edit that replaces a contiguous run of blocks `[first, last]`. */ declare function blockRangeEdit(first: SourceBlock, last: SourceBlock, text: string): SourceEdit; /** * Apply block-aligned replacements to `original`. Bytes outside the edits are * copied, never rebuilt; an edit equal to its block changes nothing. */ declare function spliceSave(original: string, edits: readonly SourceEdit[], options?: SpliceOptions): SpliceResult; interface MapOptions { /** -1 keeps a position at an insertion point before the inserted text; 1 (default) moves it after. */ readonly assoc?: -1 | 1; /** Offset unit of `pos` and of the result. Default `utf16`. */ readonly unit?: "utf16" | "codepoint"; } interface MappedPosition { readonly pos: number; /** True when the position sat strictly inside replaced text. */ readonly deleted: boolean; } /** * Carry a position in the old text through a change set (ProseMirror StepMap * semantics): positions before a change stay, positions after shift by the * size difference, a position strictly inside replaced text is `deleted` and * lands on the replacement's edge chosen by `assoc`. */ declare function mapPosition(changes: readonly SourceChange[], pos: number, options?: MapOptions): MappedPosition; interface MappedRange { readonly start: number; readonly end: number; /** True when a change overlapped the range's interior. */ readonly touched: boolean; /** True when the whole range was replaced away. */ readonly collapsed: boolean; } /** Carry an anchor range through a change set; it stays inside its text when edits happen around it. */ declare function mapRange(changes: readonly SourceChange[], start: number, end: number, unit?: "utf16" | "codepoint"): MappedRange; /** * THE `{{variable}}` detector — one rule for every place that looks for variables in prompt text. * * The author's escape (Arman, 2026-10-10): `{{!name}}` means "this is an EXAMPLE, not a variable". * The server turns `{{!` into `{{` once, on the first turn, so the model sees `{{name}}`; the * server does nothing else with braces. So in every editor, `{{!name}}` is plain text — never * colored as a variable, never reported undeclared, never counted as used. Real `{{name}}` * tokens behave exactly as before. Pure TS. */ /** The character right after `{{` that makes a token an example, not a variable. */ declare const EXAMPLE_VARIABLE_MARK = "!"; /** True for `{{!name}}` — the author's "this is an example" token (raw includes the braces). */ declare const isExampleVariableToken: (raw: string) => boolean; /** * SECTIONS (Mustache): `{{#name}}…{{/name}}` stays only when the declared variable `name` is filled, * `{{^name}}…{{/name}}` only when it is empty. A section marker is NOT a variable called `#name`: it * is a reference to `name`, so `name` counts as used and `#name` / `/name` are never reported * undeclared. The server only acts on markers of declared names (matrx_ai template_substitution). */ type SectionMark = "open" | "inverse" | "close"; /** The section marker `{{#name}}` / `{{^name}}` / `{{/name}}` (exact, no spaces) as written, or null. */ declare function sectionMarkerOf(raw: string): { mark: SectionMark; name: string; } | null; /** * PROMPT BLOCKS (owner ruling, 2026-10-10): `{{}}` places the library block `name` the agent * DECLARES (`agent.definition.prompt_blocks`); the server replaces it with `…` on the * first turn. It is never a variable. `{{!}}` is the author's literal example. */ declare const PROMPT_BLOCK_NAME: RegExp; /** True for `{{}}` (raw includes the braces) with a valid block name. */ declare const isPromptBlockToken: (raw: string) => boolean; interface PromptBlockToken { raw: string; name: string; start: number; end: number; } /** Every `{{}}` prompt-block token in a text, in order (escaped `{{!}}` skipped). */ declare function findPromptBlockTokens(text: string): PromptBlockToken[]; /** Unique prompt-block names placed in a text, in order of first appearance. */ declare function promptBlockReferencesIn(text: string): string[]; /** The exact token that places block `name`. */ declare const promptBlockToken: (name: string) => string; interface VariableToken { /** The token as written, braces included. */ raw: string; /** The name inside the braces, trimmed. For a section marker: the variable it is about. */ name: string; /** Set when the token is a section marker, not a plain `{{name}}`. */ section?: SectionMark; /** UTF-16 offsets of the token in the text. */ start: number; end: number; } /** Every real `{{variable}}` token in a text, in order (repeats kept). `{{!example}}` is skipped. */ declare function findVariableTokens(text: string): VariableToken[]; /** Unique real variable names in a text, in order of first appearance, trimmed. */ declare function variableReferencesIn(text: string): string[]; type VariableSegment = { kind: "text"; text: string; } | { kind: "variable"; text: string; name: string; section?: SectionMark; }; /** The text split into plain runs and real variable tokens (`{{!example}}` stays inside the plain text). */ declare function splitVariableText(text: string): VariableSegment[]; interface VariableUsage { /** Declared names the text uses. */ used: string[]; /** Names the text uses that nobody declared. */ undeclared: string[]; } /** Which declared variables a text uses, and which names it uses that are not declared. */ declare function analyzeVariableUsage(text: string, declared: readonly string[]): VariableUsage; /** One matched `{{#name}}…{{/name}}` (or `{{^name}}…{{/name}}`) pair. */ interface VariableSection { name: string; /** `open` keeps the text when filled, `inverse` when empty. */ mode: "open" | "inverse"; openToken: VariableToken; closeToken: VariableToken; } /** * The matched sections of a text, outer first, the way the server pairs them: each close pairs with * the nearest open of the same name; an unclosed or unmatched marker pairs with nothing (it renders as * written). `isDeclared` limits sections to declared names (the server's rule); omit it to take all. */ declare function findVariableSections(text: string, isDeclared?: (name: string) => boolean): VariableSection[]; /** * Wrap a selection so it is kept only when `name` is filled (`when: "filled"`) or empty. A selection * spanning lines (or ending in a newline) gets the markers on their own lines, so removing the section * leaves no blank line; a one-line selection is wrapped inline. */ declare function wrapInSection(selection: string, name: string, when: "filled" | "empty"): string; /** * `wrapInSection` for a plain-text field: a selection that covers whole lines (it starts at a line * start and ends at a line end) gets the markers on their own lines even when it is one line long. */ declare function wrapSelectionInText(text: string, start: number, end: number, name: string, when: "filled" | "empty"): { text: string; selectionStart: number; selectionEnd: number; }; /** * THE variable-name rule for `{{name}}`: what a declarable variable name looks like. Pure TS. * The agent variable editors, the context-policy key, and the rich editor's `{{token}}` states * all ask it (the token grammar itself is `source/variable-tokens.ts`). * * - Trim whitespace, lowercase * - Spaces and hyphens → underscores * - Remove non-alphanumeric/underscore characters * - Collapse consecutive underscores to one * - Strip leading/trailing underscores */ declare const sanitizeVariableName: (input: string) => string; /** * True when a `{{token}}` could ever name a real variable. * * Declared names always survive `sanitizeVariableName` unchanged, and the runtime substitutes by * exact name match against the declared set — so a token that isn't in that grammar * (`{{step_N.output.field}}`, `{{a b}}`) can NEVER be substituted and is literal text the model * receives verbatim. Use this to avoid flagging documentation of the syntax as an undeclared * variable. Names that ARE declared bypass this check entirely. */ declare const isDeclarableVariableName: (name: string) => boolean; /** * THE code-language list: each language a person can pick for a code file or snippet (Monaco * language ids, so a pick maps 1:1 to the code editor's `language` prop) with its file * extensions, plus the two lookups every door asks — the extension a language saves under and * the language a file name implies. Pure TS. One list for the Save to Code window, a code * block's download/copy-as-file, and the host app (fence-language aliases live beside it in * `source/constants.ts`). */ interface LanguageOption { value: string; label: string; /** Common file extensions (no leading dot). */ exts: string[]; } declare const LANGUAGE_OPTIONS: LanguageOption[]; /** The extension (no dot) a language saves under; `txt` for a language not on the list. */ declare function extensionForLanguage(language: string): string; /** The language a file name's extension implies; `plaintext` when it implies none. */ declare function languageFromName(name: string): string; /** The thinking family — the only XML regions recognized mid-line and as orphan closers. */ declare const THINKING_FAMILY: readonly string[]; /** * THE raw-HTML tag allow-list: tags a model may emit as raw HTML that the renderer honors * (rich-content rehypeSafeRawHtml, the content splitter, the GFM table vectors and prose-prepare * all read this one set). Kept in sync with the pre-escape gate in BasicMarkdownContent — a tag * left un-escaped there must be allowed here, or it renders as literal text. Deliberately excludes * `div` (layout container) and any scripting or embedding tags (`script`, `style`, `iframe`, * `object`, `embed`, `form`…). */ declare const ALLOWED_RAW_HTML_TAGS: ReadonlySet; /** * `SPECIAL_CODE_LANGUAGES` — code-fence languages that promote to a first-class block type (the * fence language becomes the block type). Mirrors SPECIAL_CODE_LANGUAGES in aidream * block_detector.py — keep the two in lockstep. `html` and `react` are deliverables (a webpage, a * live component) that materialize into persistent artifacts; a ```matrx fence carries one Matrx * Envelope. */ declare const SPECIAL_CODE_LANGUAGES: ReadonlySet; /** `CODE_LANGUAGE_ALIASES` — fence-language aliases that normalize to a canonical special language (```mmd → mermaid; jsx/tsx render as a live React component). Mirrors block_detector.py. */ declare const CODE_LANGUAGE_ALIASES: Readonly>; /** * THE HTML element list — the one place that says "this tag is real HTML". * * Why it exists (2026-10-21 production regression): the splitter, the source tokenizer and the * Python block detector claimed every line-leading tag that was not a Matrx tag and not on the * sanitizer's small raw-HTML allow-list (`ALLOWED_RAW_HTML_TAGS`) as a CUSTOM XML container. So * `
`, `
`, `