/**
* THE OFFSET-PRESERVING SOURCE TOKENIZER (rich-content PLAN decision 6).
*
* Cuts stored markdown into blocks that cover the text EXACTLY: the
* concatenation of every block's `raw` is the input, always, byte for byte.
* Nothing is trimmed, normalized, or re-serialized here — the tokenizer only
* draws boundaries. Every custom construct the platform stores (kinds and
* `__kind` JSON, typed JSON blocks, special fences, XML sections including
* attribute-bearing, mid-line and unknown containers, `{{variables}}`,
* ``, math, raw HTML, code fences, HTML comments and pinned
* anchors ``) is an ISLAND: an atomic span no editor may parse
* into its own model. Islands that sit inside running prose (a variable, a
* citation, inline math) stay inside their paragraph as `inlines`, so a
* visual editor keeps the paragraph whole and holds the island as an atom.
*
* Block kinds:
* - `prose` — a markdown paragraph run (headings, lists, tables…): the only
* thing a visual editor may re-serialize, and only when edited.
* - `island` — a protected construct; `islandType` says which.
* - `gap` — the line terminator(s) and blank lines between blocks. Never
* edited; they keep the document's exact spacing.
*
* Streaming-tolerant: never throws on partial input. A construct whose
* closer has not arrived yet becomes an island running to the end of the text
* with `complete: false` — the same ownership rule the renderer's splitter
* applies, so a half-streamed fence or XML section is never reflowed.
*
* Boundaries follow matrx-frontend's renderer splitter (`content-splitter-v2.ts`)
* wherever it decides where a construct ends, so the editor and the renderer
* agree on what is locked. Offsets are UTF-16 (`start`/`end`) and code points
* (`startCp`/`endCp`).
*/
type SourceBlockKind = "prose" | "island" | "gap";
/** Island types that own whole blocks. */
type BlockIslandType =
/** ``` / ~~~ fenced code of any language (meta.lang, meta.special). */
"fence"
/** A bare JSON object block — typed JSON, `__kind` instances, directives. */
| "json"
/** A known attribute-less XML region (``, ``, …). */
| "xml_region"
/** An attribute-bearing XML element (``, ``, …). */
| "xml_attr"
/** Any other line-leading, balanced XML element. */
| "xml_container"
/** Allow-listed raw HTML block. */
| "html_block"
/** `` comment. */
| "html_comment"
/** A pinned annotation anchor ``. */
| "anchor"
/** `$$ … $$` display math on its own lines. */
| "math_block"
/** YAML (`---`) or TOML (`+++`) front matter at the start of the text (meta.format). */
| "front_matter"
/** A box-drawing tree diagram. */
| "tree"
/**
* A generic-directive container `:::name[label]{attrs}` … `:::` (tabs,
* columns, details, figures, callouts, TOC — meta.name), a leaf directive
* `::name` on its own line (meta.leaf), or a MkDocs admonition
* `!!! type "Title"` / `??? type` with its indented body (meta.syntax "mkdocs").
*/
| "directive"
/** A GitHub alert / Obsidian callout: a blockquote opening `> [!TYPE]` (meta.name). */
| "callout"
/** A footnote definition `[^id]: …` with its indented continuation lines (meta.id). */
| "footnote_def";
/** Island types that live inside a prose block. */
type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref"
/** `[[Page]]`, `[[Page|alias]]` (meta.target) — or `![[Page]]` (meta.embed). */
| "wikilink";
type IslandType = BlockIslandType | InlineIslandType;
type IslandMeta = Readonly>;
interface SourceSpan {
/** UTF-16 offset of the first code unit. */
readonly start: number;
/** UTF-16 offset just past the last code unit. */
readonly end: number;
/** Code-point offset of `start`. */
readonly startCp: number;
/** Code-point offset of `end`. */
readonly endCp: number;
/** Exactly `text.slice(start, end)`. */
readonly raw: string;
}
interface SourceIsland extends SourceSpan {
readonly islandType: IslandType;
/** True when the island sits inside a prose block. */
readonly inline: boolean;
/** False when the construct's closer never arrived (streaming / malformed). */
readonly complete: boolean;
readonly meta: IslandMeta;
}
interface SourceBlock extends SourceSpan {
readonly kind: SourceBlockKind;
/** Set only on `island` blocks. */
readonly islandType: BlockIslandType | null;
readonly complete: boolean;
readonly meta: IslandMeta;
/** Inline islands of a `prose` block, in document order. Empty otherwise. */
readonly inlines: readonly SourceIsland[];
}
/**
* Every callout spelling the renderer resolves (matrx-frontend
* components/markdown-core/syntax/callout-types.ts ALIASES — keep in step).
*/
declare const CALLOUT_NAMES: ReadonlySet;
/**
* Cut `text` into blocks whose raw concatenation is exactly `text`.
* Never throws; unclosed constructs are islands with `complete: false`.
*/
interface TokenizeOptions {
/**
* The text is a prefix of a message still arriving. The final line (no
* terminator yet) can still grow — a bare ``` there may become ```bash — so
* it closes no fence until its newline arrives (verify-RC-B3 residual R2).
* Omit for stored text: a complete document's last line is final.
*/
readonly streaming?: boolean;
/**
* DIAGNOSTIC — complexity tests only. When given, every loop that walks the
* text itself adds the characters it visits to `meter.work`, so a test can
* assert the tokenizer's work grows linearly with its input (deterministic,
* unlike wall-clock timing). Loops over a slice are not charged here: the
* slice's own cost covers them. Omit in production.
*/
readonly meter?: {
work: number;
};
}
declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[];
/** The exact source the blocks were cut from. */
declare function joinSource(blocks: readonly SourceBlock[]): string;
/** Every island — block-level and inline — in document order. */
declare function listIslands(blocks: readonly SourceBlock[]): SourceIsland[];
/** The block containing UTF-16 offset `offset` (the later block at a boundary). */
declare function blockAt(blocks: readonly SourceBlock[], offset: number): SourceBlock | null;
/**
* SAVE = SPLICE (rich-content PLAN decision 6).
*
* An editor never hands back a whole re-serialized document. It hands back
* per-block replacements for exactly the blocks the person changed; every
* other byte of the stored text is written back as it was. This module
* applies those replacements and returns:
*
* - the new text,
* - a change set (old range → new range, in UTF-16 AND code points,
* narrowed to the bytes that actually differ) that the annotation layer
* uses to carry every live anchor through the save
* (STORE-DESIGN §3.19 — the ProseMirror / Google Docs position-mapping
* model), and
* - an integrity guarantee: every island not explicitly replaced must still
* be the same island, at its mapped position, in the new text. An edit
* that opens an unclosed fence and swallows a kind below it is REFUSED.
*
* ISLANDS CHANGE ONLY THROUGH `islandEdit`. A prose/range edit must hand back
* every island inside its range byte-identical and in order (a visual editor
* holds them as atoms); if one is altered or dropped the save throws
* `SourceSpliceError("island_edit")` naming the island. A kind editor or code
* view changes an island with `islandEdit(island, newRaw)`.
*
* A no-edit save (or edits whose text equals the original block) returns the
* original string itself — byte-identical, zero changes.
*/
/**
* Replace `[start, end)` (UTF-16) of the original with `text`. A plain edit
* starts and ends on block boundaries; an island edit (`island: true`, made by
* `islandEdit`) names exactly one island's span.
*/
interface SourceEdit {
readonly start: number;
readonly end: number;
readonly text: string;
readonly island?: boolean;
}
interface SourceChange {
/** The block-aligned range the edit named, in the ORIGINAL text. */
readonly blockStart: number;
readonly blockEnd: number;
/** The narrowed range that actually differs: old text … */
readonly oldStart: number;
readonly oldEnd: number;
/** … and where its replacement sits in the NEW text. */
readonly newStart: number;
readonly newEnd: number;
readonly oldStartCp: number;
readonly oldEndCp: number;
readonly newStartCp: number;
readonly newEndCp: number;
}
interface DisturbedIsland {
readonly islandType: IslandType;
readonly inline: boolean;
/** Where the island was in the original (UTF-16). */
readonly start: number;
readonly end: number;
/** Where it should have been in the new text. */
readonly mappedStart: number;
}
interface SpliceIntegrity {
/** True when every untouched island survived intact and outside bytes are identical. */
readonly ok: boolean;
/** Always true by construction; checked anyway because it is THE guarantee. */
readonly bytesOutsideEditsIdentical: boolean;
readonly disturbed: readonly DisturbedIsland[];
}
interface SpliceResult {
readonly text: string;
readonly changed: boolean;
readonly changes: readonly SourceChange[];
readonly integrity: SpliceIntegrity;
/** Tokenization of the new text. */
readonly blocks: readonly SourceBlock[];
}
type SourceSpliceErrorCode = "out_of_range" | "misaligned" | "overlap" | "stale_blocks" | "island_edit" | "integrity";
declare class SourceSpliceError extends Error {
readonly code: SourceSpliceErrorCode;
constructor(code: SourceSpliceErrorCode, message: string);
}
interface SpliceOptions {
/** The tokenization of `original`, when the caller already holds it. */
readonly blocks?: readonly SourceBlock[];
/**
* Default `true`: a save that disturbs any island it did not explicitly
* replace throws `SourceSpliceError("integrity")`. `false` is for TOOLING
* ONLY (diagnostic scripts that want the report instead of the refusal) —
* no editor or persistence path may pass it.
*/
readonly requireIntegrity?: boolean;
}
/** An edit that replaces one block's raw text. */
declare function blockEdit(block: SourceBlock, text: string): SourceEdit;
/** The one door that changes an island: replace exactly its span with `text`. */
declare function islandEdit(island: {
readonly start: number;
readonly end: number;
}, text: string): SourceEdit;
/** An edit that replaces a contiguous run of blocks `[first, last]`. */
declare function blockRangeEdit(first: SourceBlock, last: SourceBlock, text: string): SourceEdit;
/**
* Apply block-aligned replacements to `original`. Bytes outside the edits are
* copied, never rebuilt; an edit equal to its block changes nothing.
*/
declare function spliceSave(original: string, edits: readonly SourceEdit[], options?: SpliceOptions): SpliceResult;
interface MapOptions {
/** -1 keeps a position at an insertion point before the inserted text; 1 (default) moves it after. */
readonly assoc?: -1 | 1;
/** Offset unit of `pos` and of the result. Default `utf16`. */
readonly unit?: "utf16" | "codepoint";
}
interface MappedPosition {
readonly pos: number;
/** True when the position sat strictly inside replaced text. */
readonly deleted: boolean;
}
/**
* Carry a position in the old text through a change set (ProseMirror StepMap
* semantics): positions before a change stay, positions after shift by the
* size difference, a position strictly inside replaced text is `deleted` and
* lands on the replacement's edge chosen by `assoc`.
*/
declare function mapPosition(changes: readonly SourceChange[], pos: number, options?: MapOptions): MappedPosition;
interface MappedRange {
readonly start: number;
readonly end: number;
/** True when a change overlapped the range's interior. */
readonly touched: boolean;
/** True when the whole range was replaced away. */
readonly collapsed: boolean;
}
/** Carry an anchor range through a change set; it stays inside its text when edits happen around it. */
declare function mapRange(changes: readonly SourceChange[], start: number, end: number, unit?: "utf16" | "codepoint"): MappedRange;
/**
* THE `{{variable}}` detector — one rule for every place that looks for variables in prompt text.
*
* The author's escape (Arman, 2026-10-10): `{{!name}}` means "this is an EXAMPLE, not a variable".
* The server turns `{{!` into `{{` once, on the first turn, so the model sees `{{name}}`; the
* server does nothing else with braces. So in every editor, `{{!name}}` is plain text — never
* colored as a variable, never reported undeclared, never counted as used. Real `{{name}}`
* tokens behave exactly as before. Pure TS.
*/
/** The character right after `{{` that makes a token an example, not a variable. */
declare const EXAMPLE_VARIABLE_MARK = "!";
/** True for `{{!name}}` — the author's "this is an example" token (raw includes the braces). */
declare const isExampleVariableToken: (raw: string) => boolean;
/**
* SECTIONS (Mustache): `{{#name}}…{{/name}}` stays only when the declared variable `name` is filled,
* `{{^name}}…{{/name}}` only when it is empty. A section marker is NOT a variable called `#name`: it
* is a reference to `name`, so `name` counts as used and `#name` / `/name` are never reported
* undeclared. The server only acts on markers of declared names (matrx_ai template_substitution).
*/
type SectionMark = "open" | "inverse" | "close";
/** The section marker `{{#name}}` / `{{^name}}` / `{{/name}}` (exact, no spaces) as written, or null. */
declare function sectionMarkerOf(raw: string): {
mark: SectionMark;
name: string;
} | null;
/**
* PROMPT BLOCKS (owner ruling, 2026-10-10): `{{}}` places the library block `name` the agent
* DECLARES (`agent.definition.prompt_blocks`); the server replaces it with `…` on the
* first turn. It is never a variable. `{{!}}` is the author's literal example.
*/
declare const PROMPT_BLOCK_NAME: RegExp;
/** True for `{{}}` (raw includes the braces) with a valid block name. */
declare const isPromptBlockToken: (raw: string) => boolean;
interface PromptBlockToken {
raw: string;
name: string;
start: number;
end: number;
}
/** Every `{{}}` prompt-block token in a text, in order (escaped `{{!}}` skipped). */
declare function findPromptBlockTokens(text: string): PromptBlockToken[];
/** Unique prompt-block names placed in a text, in order of first appearance. */
declare function promptBlockReferencesIn(text: string): string[];
/** The exact token that places block `name`. */
declare const promptBlockToken: (name: string) => string;
interface VariableToken {
/** The token as written, braces included. */
raw: string;
/** The name inside the braces, trimmed. For a section marker: the variable it is about. */
name: string;
/** Set when the token is a section marker, not a plain `{{name}}`. */
section?: SectionMark;
/** UTF-16 offsets of the token in the text. */
start: number;
end: number;
}
/** Every real `{{variable}}` token in a text, in order (repeats kept). `{{!example}}` is skipped. */
declare function findVariableTokens(text: string): VariableToken[];
/** Unique real variable names in a text, in order of first appearance, trimmed. */
declare function variableReferencesIn(text: string): string[];
type VariableSegment = {
kind: "text";
text: string;
} | {
kind: "variable";
text: string;
name: string;
section?: SectionMark;
};
/** The text split into plain runs and real variable tokens (`{{!example}}` stays inside the plain text). */
declare function splitVariableText(text: string): VariableSegment[];
interface VariableUsage {
/** Declared names the text uses. */
used: string[];
/** Names the text uses that nobody declared. */
undeclared: string[];
}
/** Which declared variables a text uses, and which names it uses that are not declared. */
declare function analyzeVariableUsage(text: string, declared: readonly string[]): VariableUsage;
/** One matched `{{#name}}…{{/name}}` (or `{{^name}}…{{/name}}`) pair. */
interface VariableSection {
name: string;
/** `open` keeps the text when filled, `inverse` when empty. */
mode: "open" | "inverse";
openToken: VariableToken;
closeToken: VariableToken;
}
/**
* The matched sections of a text, outer first, the way the server pairs them: each close pairs with
* the nearest open of the same name; an unclosed or unmatched marker pairs with nothing (it renders as
* written). `isDeclared` limits sections to declared names (the server's rule); omit it to take all.
*/
declare function findVariableSections(text: string, isDeclared?: (name: string) => boolean): VariableSection[];
/**
* Wrap a selection so it is kept only when `name` is filled (`when: "filled"`) or empty. A selection
* spanning lines (or ending in a newline) gets the markers on their own lines, so removing the section
* leaves no blank line; a one-line selection is wrapped inline.
*/
declare function wrapInSection(selection: string, name: string, when: "filled" | "empty"): string;
/**
* `wrapInSection` for a plain-text field: a selection that covers whole lines (it starts at a line
* start and ends at a line end) gets the markers on their own lines even when it is one line long.
*/
declare function wrapSelectionInText(text: string, start: number, end: number, name: string, when: "filled" | "empty"): {
text: string;
selectionStart: number;
selectionEnd: number;
};
/**
* THE variable-name rule for `{{name}}`: what a declarable variable name looks like. Pure TS.
* The agent variable editors, the context-policy key, and the rich editor's `{{token}}` states
* all ask it (the token grammar itself is `source/variable-tokens.ts`).
*
* - Trim whitespace, lowercase
* - Spaces and hyphens → underscores
* - Remove non-alphanumeric/underscore characters
* - Collapse consecutive underscores to one
* - Strip leading/trailing underscores
*/
declare const sanitizeVariableName: (input: string) => string;
/**
* True when a `{{token}}` could ever name a real variable.
*
* Declared names always survive `sanitizeVariableName` unchanged, and the runtime substitutes by
* exact name match against the declared set — so a token that isn't in that grammar
* (`{{step_N.output.field}}`, `{{a b}}`) can NEVER be substituted and is literal text the model
* receives verbatim. Use this to avoid flagging documentation of the syntax as an undeclared
* variable. Names that ARE declared bypass this check entirely.
*/
declare const isDeclarableVariableName: (name: string) => boolean;
/**
* THE code-language list: each language a person can pick for a code file or snippet (Monaco
* language ids, so a pick maps 1:1 to the code editor's `language` prop) with its file
* extensions, plus the two lookups every door asks — the extension a language saves under and
* the language a file name implies. Pure TS. One list for the Save to Code window, a code
* block's download/copy-as-file, and the host app (fence-language aliases live beside it in
* `source/constants.ts`).
*/
interface LanguageOption {
value: string;
label: string;
/** Common file extensions (no leading dot). */
exts: string[];
}
declare const LANGUAGE_OPTIONS: LanguageOption[];
/** The extension (no dot) a language saves under; `txt` for a language not on the list. */
declare function extensionForLanguage(language: string): string;
/** The language a file name's extension implies; `plaintext` when it implies none. */
declare function languageFromName(name: string): string;
/** The thinking family — the only XML regions recognized mid-line and as orphan closers. */
declare const THINKING_FAMILY: readonly string[];
/**
* THE raw-HTML tag allow-list: tags a model may emit as raw HTML that the renderer honors
* (rich-content rehypeSafeRawHtml, the content splitter, the GFM table vectors and prose-prepare
* all read this one set). Kept in sync with the pre-escape gate in BasicMarkdownContent — a tag
* left un-escaped there must be allowed here, or it renders as literal text. Deliberately excludes
* `div` (layout container) and any scripting or embedding tags (`script`, `style`, `iframe`,
* `object`, `embed`, `form`…).
*/
declare const ALLOWED_RAW_HTML_TAGS: ReadonlySet;
/**
* `SPECIAL_CODE_LANGUAGES` — code-fence languages that promote to a first-class block type (the
* fence language becomes the block type). Mirrors SPECIAL_CODE_LANGUAGES in aidream
* block_detector.py — keep the two in lockstep. `html` and `react` are deliverables (a webpage, a
* live component) that materialize into persistent artifacts; a ```matrx fence carries one Matrx
* Envelope.
*/
declare const SPECIAL_CODE_LANGUAGES: ReadonlySet;
/** `CODE_LANGUAGE_ALIASES` — fence-language aliases that normalize to a canonical special language (```mmd → mermaid; jsx/tsx render as a live React component). Mirrors block_detector.py. */
declare const CODE_LANGUAGE_ALIASES: Readonly>;
/**
* THE HTML element list — the one place that says "this tag is real HTML".
*
* Why it exists (2026-10-21 production regression): the splitter, the source tokenizer and the
* Python block detector claimed every line-leading tag that was not a Matrx tag and not on the
* sanitizer's small raw-HTML allow-list (`ALLOWED_RAW_HTML_TAGS`) as a CUSTOM XML container. So
* ``, `