/** * Search indexes, derived from `manifest.searchables` (#827). * * The declaration has been in the manifest since the beginning and nothing read * it — `kernel-design.md` §"Search backends" deferred the decision "to first * search consumer". This is that consumer, and it takes the FTS half only: * `vec0` is rejected by Durable Object SQLite today, so semantic search stays * deferred rather than half-built. * * ## Why the kernel owns the index rather than the module * * Three reasons, in order of how much they'd hurt to get wrong. * * 1. **Rule 4 makes it the only legal home for a cross-entity search.** A module * may not reference another module's tables, so a palette searching customers * *and* orders cannot be written by either module. A kernel-owned index over * declared entities can answer it without anyone reading anyone's tables. * 2. **Triggers keep the `ctx.sql`-only rule intact.** The index is maintained by * SQL triggers generated from the declaration, so it stays correct no matter * who writes the row — no new write path, no module opting in per `INSERT`, * and nothing to forget. A module that never hears about the index cannot * desynchronise it. * 3. **It stays read-after-write correct.** Indexing from the event spine instead * would be eventually consistent, which is why Stripe has to warn "don't use * search for read-after-write flows". Create a customer, then find it in the * picker in the same breath: with triggers that simply works, and a picker * that cannot see what the user just created is the bug report. * * ## What is derived, and from where * * Everything. `searchables` names an entity and its fields; `manifestEntities()` * enriches that with the entity's `table` and single-column key from the same * registry the fields are checked against. So there is no second description of * where a customer lives — the model already said it. */ import type { DerivedObject } from './derived-object.js'; import type { SqlMigration } from './scope-host.js'; import type { EntityStateName } from '@substrat-run/contracts'; /** How a declared searchable is matched. `prefix` unless it says otherwise. */ export type SearchTokenizer = 'prefix' | 'substring'; /** * One `manifest.searchables` entry, as the kernel needs it. * * `table` and `idColumn` are not authored — `manifestEntities()` fills them in * from the entity registry. A hand-written manifest that declares a searchable * without a table is refused rather than guessed at. */ export interface SearchableDeclaration { readonly entityType: string; readonly fields: readonly string[]; readonly table?: string; readonly idColumn?: string; readonly tokenizer?: SearchTokenizer; } /** A resolved searchable: everything the DDL and the query need, nothing optional. */ export interface SearchIndexPlan { readonly moduleId: string; readonly entityType: string; readonly table: string; readonly idColumn: string; readonly fields: readonly string[]; readonly tokenizer: SearchTokenizer; /** The FTS5 table. Kernel-owned: `_substrat_*`, which module code may read and never write. */ readonly indexTable: string; } /** One hit. Ids only — the caller hydrates through its own read path. */ export interface SearchHit { readonly entityType: string; readonly id: string; /** FTS5 bm25: lower is better. Carried so a caller can merge two entity types into one list. */ readonly rank: number; } export interface SearchOptions { /** Defaults to `DEFAULT_SEARCH_LIMIT`, capped at `MAX_SEARCH_LIMIT`. */ readonly limit?: number; /** * Which rows (#119): `active` when unset, so an archived or trashed row never surfaces in a * picker nobody asked to show it in. `archived` asks for the archive. The bin is not here — * `ctx.searchTrashed` reads it, checking the declared trash key per hit. */ readonly view?: Exclude; } /** * A typeahead wants ten rows, not ten thousand. The cap is the design: a ranked * result set has no stable sort key, so it has no honest cursor either — Stripe * documents that paginating a ranked search "can reorder some records, causing * them to be missing or duplicated on a page". Capping sidesteps that instead of * shipping a cursor that lies. Deep paging over search is #811's `Page` over a * declared sort, which is a different question with a different answer. */ export declare const DEFAULT_SEARCH_LIMIT = 20; export declare const MAX_SEARCH_LIMIT = 100; /** Raised for a term the index cannot answer. Distinguishable from a crash at the seam. */ export declare class SearchTermTooShort extends Error { readonly minimum: number; constructor(minimum: number); } /** Raised for an entity type no registered module declares searchable. */ export declare class NotSearchable extends Error { readonly entityType: string; constructor(entityType: string); } /** * Is this table part of a derived search index? * * True for the FTS5 table AND its shadow tables (`…_data`, `…_idx`, `…_config`, * `…_docsize`), which SQLite names by prefixing the virtual table's own name. * * **Used to keep the index OUT of a scope dump.** A dump replays DDL and then * inserts rows, and a shadow table refuses both — workerd answers "object name * reserved for internal use", and D1's exporter refuses a whole database that * merely contains an fts5 table (cloudflare/workers-sdk#9519). Excluding it is * also just correct: an index is derived data, and the import rebuilds it from * the content tables it just loaded. Nothing is lost that was not recomputable. * * **The attachment text tables share the prefix on purpose (#1575).** * `_substrat_search__attachment_text` holds text extracted from attachment BYTES, and a * dump carries an attachment's metadata row and never its bytes. Keeping the text out of * the dump keeps it wherever the bytes are; a load re-queues extraction for the rows it * brought back (`reconcileAttachmentText`) instead of carrying a plaintext copy of every * file into every exported dump and backup. */ export declare function isSearchIndexTable(name: string): boolean; /** * Resolve one module's declarations into plans. * * Refuses rather than skips. A `searchables` entry that cannot become an index is * a declaration the author believes is live, and the failure mode of skipping is * a search box that returns nothing with no error anywhere — the exact silence * this feature exists to end. */ export declare function searchIndexPlans(moduleId: string, searchables: readonly SearchableDeclaration[] | undefined): SearchIndexPlan[]; /** * The DDL for one plan: an external-content FTS5 table, the three triggers that * keep it in step, and a rebuild. * * **External content, not a copy.** The index stores terms and points at the * source table's `rowid`; the text itself is not duplicated. That also makes * `rebuild` the whole backfill story — it reads the content table. * * **Drop-then-create, every time the declaration changes.** The version below is * the declaration itself, so adding a field re-runs this, and re-running has to * produce an index matching the NEW declaration rather than silently keeping the * old columns. Nothing is lost by dropping: an index is derived data, and the * rebuild at the end reconstructs it from rows that never moved. */ export declare function searchIndexDdl(plan: SearchIndexPlan): string; /** * What `searchIndexDdl` creates, each with its CREATE statement: the FTS table, then the three * triggers that keep it in step (insert, delete, update). */ export declare function searchIndexObjects(plan: SearchIndexPlan): DerivedObject[]; /** * The migrations that provision a module's declared indexes, journaled like any * other so a scope applies them once and a changed declaration re-applies. * * **The version IS the declaration.** `search/customer:prefix:name+number` says * everything that decides the DDL, so adding a field or switching a tokenizer * produces a new version and re-runs; changing nothing produces the same version * and does not. A hash would say the same thing less legibly in the one place an * operator reads when a scope is stuck (`_substrat_migrations`). * * Appended AFTER the module's own migrations by the adapter, which is what makes * the content table exist by the time the trigger references it. */ export declare function searchIndexMigrations(moduleId: string, searchables: readonly SearchableDeclaration[] | undefined): SqlMigration[]; /** One plan's migration version — the declaration itself, as above. */ export declare const searchIndexVersion: (plan: SearchIndexPlan) => string; /** * Turn what a person typed into an FTS5 MATCH expression. * * **Everything is quoted, always.** FTS5's query syntax has operators (`AND`, * `OR`, `NOT`, `*`, `^`, `:`, `"`) and a picker's input is a name, not a query — * someone searching for `AND Sons` or `O"Brien` must get rows, not a syntax * error, and must not be able to steer the match. So the input is split into * bare terms and each is re-emitted as a quoted string. * * **In prefix mode every term is a prefix, not just the last.** The obvious rule * — only the final token is still being typed — was measured against a real index * and is wrong: `unicode61` tokenizes "Andersson Fastigheter AB" into whole * words, so someone typing "anders fast" matched NOTHING while "anders" alone * matched. In a picker every token is plausibly half-typed. Prefixing all of them * can only widen the result set, and the set is ranked and capped anyway. * * **In substring mode nothing is appended, and short terms are dropped.** Trigram * matching is already inside-the-word, and appending `*` there is a documented * trap that changes the results rather than widening them. A term below the * trigram floor is silently ignored BY THE INDEX (measured: `"AND" "x" "OR"` * matched "Andersson" — the `x` constrained nothing), so it is dropped here * instead, where the behaviour is written down rather than inferred from a match * that looks wrong. */ export declare function searchMatchExpression(term: string, tokenizer: SearchTokenizer): string; /** * The read: ids and ranks, joined back to the content table for the declared id. * * Ids ONLY, deliberately. The row shape a caller wants is its own — its * projections, its joins, its permission narrowing — and returning rows here * would be a second read path with a second answer to "what is a customer". * Hydrating through the module's own query keeps that one code path, at the cost * of one extra `WHERE id IN (…)` the caller was going to write anyway. * * **The FTS table is not aliased, and cannot be.** `MATCH` and `bm25()` both take * the table's own name — that name IS the hidden column being matched — so * aliasing it to `idx` and writing `WHERE idx MATCH ?` fails at prepare time with * "no such column: idx". Found by running the emitted SQL rather than reading it. */ export declare function searchQuery(plan: SearchIndexPlan, match: string, limit: number, /** * A further predicate over the source row, aliased `src` — the archive/trash view (#119). * Applied BEFORE the limit, so hiding rows never shortens a ranked top-N. */ where?: string): { sql: string; params: [string, number]; }; /** Clamp a caller's `limit` into the range the index will answer. */ export declare function searchLimit(requested: number | undefined): number; //# sourceMappingURL=search-index.d.ts.map