/** * Backlog dedup EMBED FORM + lexical corroboration (the 2026-08-11 register-inflation fix). * * Zero-dep PURE module: imported by BOTH `backlog.ts` (query/mirror/harmonize) and * `agentdb-index.ts` (the reindex/index write path), so every dz-backlog vector — fresh mirror, * reindex, or query — is built from the SAME text form. A second definition of this form is exactly * the query-vs-row space split this module exists to prevent. * * WHY the bounded excerpt (all numbers MEASURED 2026-08-11 on the real 105-idea store, reproducer: * scratch pairwise-cosine harness over `EmbeddingService` / Xenova-paraphrase-multilingual-MiniLM-L12-v2, * the same model + `${taskType}: ${text}` form the tool uses): * * - Full-length embeddings INVERT the duplicate signal on long texts. Genuine reworded paraphrases * of 1073–1509-char ideas scored 0.35–0.61 against their own sources, while topically DISJOINT * long-RU pairs scored up to 0.9195 (genai-tweets x dz-cost) — i.e. above every genuine long * paraphrase. No threshold can separate classes that are ordered the wrong way round. * - The inflation is a register/length effect, not topic: long-RU x long-RU pairs mean cosine * 0.6467 vs short-EN x short-EN 0.3929 (+0.25) over the same store; the 2026-08-05 incident * (an agentdb zombie-process idea absorbed as DUPLICATE of a dz-guard publish-gate rule at * cosine 0.941) shared only the register — правило/проверка/гейт/уровень/измерено. * - Bounded to the first 400 chars, the same pairs separate: paraphrase-vs-source 0.73–0.95 with * the source as top-1 in 8/8 probes, store-wide max non-duplicate pair 0.9181, and the * patient-values long-form scored 0.80 against its own short reword (was 0.26 at full length, * with an unrelated record at 0.89 on top). * * WHY the lexical corroboration: cosine alone still cannot tell a same-register pair from a true * paraphrase near the threshold. A TRUE re-capture shares the idea's DISTINCTIVE vocabulary * (subject nouns, identifiers); a register-only pair shares function words. Measured containment on * the labeled set: true duplicates 0.33–0.97 (>= 0.538 for every pair that also cleared the cosine * threshold); register-only false-positive pairs <= 0.171. */ /** The vector-store namespace that isolates ideas from lessons (ADR-001/005). */ export declare const BACKLOG_TASK_TYPE = "dz-backlog"; /** * Version of the dedup embed FORM (not the model). v1 = `dz-backlog: `; v2 = bounded * excerpt (this module). Stored vectors written under an older form are in a DIFFERENT space than * v2 queries for texts longer than the cap — `ensureBacklogEmbedForm` re-mirrors them once. */ export declare const DEDUP_EMBED_FORM_VERSION = 2; /** * Embed-input cap in UTF-16 units. NOT config: the cap is part of the embed form — two stores (or a * query and a row) built with different caps silently live in different spaces. 400 chosen over 300 * by measurement: same store-wide max (0.9181) with higher paraphrase-vs-source cosines (0.88 vs * 0.84 on the hardest long-RU probe). */ export declare const DEDUP_EMBED_CAP = 400; /** * The DISTINCTIVE excerpt of an idea for embedding: whitespace-collapsed, capped at * {@link DEDUP_EMBED_CAP}. Short texts pass through untouched (byte-identical semantics to v1 for * them). The cut never splits a surrogate pair. */ export declare function dedupExcerpt(text: string): string; /** The ONE dedup embed form (v2) — used by the query, the mirror write, and the reindex write. */ export declare function dedupEmbedText(text: string): string; /** * The distinctive-token set of an idea text: lowercased word-ish tokens (letters/digits, allowing * inner `_ . - /` so identifiers and paths survive whole), length >= 4, register stopwords removed, * Cyrillic words longer than 6 chars folded to their 6-char prefix (a crude but MEASURED-adequate * stem that matches RU inflections: подкрепление/подкрепления → подкре). */ export declare function distinctiveTokens(text: string): Set; /** * Lexical containment in [0,1]: |A ∩ B| / min(|A|, |B|) over distinctive tokens. `min` (containment, * not Jaccard) so a SHORT re-capture of a LONG idea — whose vocabulary is a subset — still scores * high; that asymmetry is exactly the length-moved-the-verdict incident. Empty token sets score 0 * (no evidence is not corroboration). */ export declare function lexicalContainment(a: string, b: string): number; /** The corroboration knobs of the two-signal decision (defaults in `readBacklogConfig`). */ export interface DedupBandConfig { readonly duplicateThreshold: number; readonly relatednessFloor: number; /** DUPLICATE-by-cosine additionally requires containment >= this (undefined containment waives). */ readonly corroborationFloor: number; /** Containment >= this + cosine >= subsetCosineFloor promotes to duplicate BELOW the threshold. */ readonly subsetContainment: number; readonly subsetCosineFloor: number; } export type DedupPairBand = 'duplicate' | 'demoted' | 'subset-duplicate' | 'below'; /** * THE two-signal pair decision — shared by `classifyDedup` (capture) and `harmonizeBacklog` (batch) * so the two paths can never disagree about what a duplicate is. * * duplicate cosine >= duplicateThreshold AND (containment unknown OR >= corroborationFloor) * demoted cosine >= duplicateThreshold but containment < corroborationFloor — the * register-only false positive (zombie x publish-gate @ 0.941, containment 0.077): * high cosine with DISJOINT subject vocabulary is not a duplicate. * subset-duplicate cosine in [subsetCosineFloor, duplicateThreshold) AND containment >= * subsetContainment — the same idea re-captured at a different LENGTH (measured * long-vs-short patient-values: cosine 0.8019, containment 0.971). * below everything else (related/new banding is the caller's job). */ export declare function dedupPairBand(cosine: number, containment: number | undefined, cfg: DedupBandConfig): DedupPairBand; //# sourceMappingURL=backlog-embed.d.ts.map