import { type LLMResponse, type Message, type Topic, type Person, type Quote, } from "../types.js"; import type { PersonIdentifier } from "../types/data-items.js"; import type { StateManager } from "../state-manager.js"; import type { ItemMatchResult, ExposureImpact, TopicUpdateResult, PersonUpdateResult } from "../../prompts/human/types.js"; import { queueTopicUpdate, queueTopicValidate, type ExtractionContext } from "../orchestrators/index.js"; import { getEmbeddingService, getTopicEmbeddingText, getPersonEmbeddingText } from "../embedding-service.js"; import { calculateExposureCurrent } from "../utils/exposure.js"; import { mergeOverlappingQuotes, unionIds } from "../corrections.js"; import { resolveMessageWindow, getMessageText } from "./utils.js"; import { sanitizeEiPersonaIdentifiers, normalizeIdentifierType, isEiPersonaIdentifierType } from "../utils/identifier-utils.js"; export function handleTopicMatch(response: LLMResponse, state: StateManager): void { const result = response.parsed as ItemMatchResult | undefined; if (!result) { throw new Error("[handleTopicMatch] No parsed result"); } const personaId = response.request.data.personaId as string; const personaDisplayName = response.request.data.personaDisplayName as string; const roomId = response.request.data.roomId as string | undefined; const { messages_context, messages_analyze } = resolveMessageWindow(response, state); let matched_guid = result.matched_guid; let resolvedTopic: import('../types/data-items.js').Topic | null = null; if (matched_guid === "new") { matched_guid = null; } else if (matched_guid) { const human = state.getHuman(); resolvedTopic = human.topics.find(t => t.id === matched_guid) ?? null; if (!resolvedTopic) { console.warn(`[handleTopicMatch] matched_guid "${matched_guid}" not found in topics — treating as new`); matched_guid = null; } } result.matched_guid = matched_guid; const context: ExtractionContext & { candidateName: string; candidateDescription: string; candidateCategory: string; extraction_model?: string; } = { personaId, channelDisplayName: personaDisplayName, roomId, messages_context, messages_analyze, sources: response.request.data.sources as string[] | undefined, candidateName: response.request.data.candidateName as string, candidateDescription: response.request.data.candidateDescription as string, candidateCategory: response.request.data.candidateCategory as string, extraction_model: response.request.data.extraction_model as string | undefined, }; queueTopicUpdate(result, context, state, resolvedTopic); const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new topic)"; console.log(`[handleTopicMatch] topic "${context.candidateName}": ${matched}`); } export async function handleTopicUpdate(response: LLMResponse, state: StateManager): Promise { const result = response.parsed as (TopicUpdateResult & { quotes?: Array<{ text: string; reason: string }> }) | undefined; if (!result || Object.keys(result).length === 0) { console.log("[handleTopicUpdate] No changes needed (empty result)"); return; } const isNewItem = response.request.data.isNewItem as boolean; const existingItemId = response.request.data.existingItemId as string | undefined; const personaId = response.request.data.personaId as string; const personaDisplayName = response.request.data.personaDisplayName as string; const candidateCategory = response.request.data.candidateCategory as string | undefined; const candidateName = response.request.data.candidateName as string | undefined; const candidateDescription = response.request.data.candidateDescription as string | undefined; const personaIds = personaId.split("|").filter(Boolean); const primaryId = personaIds[0] ?? personaId; const now = new Date().toISOString(); const { messages_analyze } = resolveMessageWindow(response, state); const earliestMessageTimestamp = messages_analyze.length > 0 ? messages_analyze.reduce((a, b) => a.timestamp < b.timestamp ? a : b).timestamp : now; const human = state.getHuman(); const resolveItemId = (): string => { if (isNewItem || !existingItemId) return crypto.randomUUID(); return human.topics.find(t => t.id === existingItemId) ? existingItemId : crypto.randomUUID(); }; const itemId = resolveItemId(); const allPersonaGroups = personaIds .map(id => state.persona_getById(id)?.group_primary) .filter((g): g is string => g != null); const existingTopic = isNewItem ? undefined : human.topics.find(t => t.id === existingItemId); const resolvedName = result.name || existingTopic?.name || candidateName; const resolvedDescription = typeof result.description === 'string' ? result.description : existingTopic?.description ?? candidateDescription; const resolvedSentiment = result.sentiment !== undefined ? result.sentiment : existingTopic?.sentiment ?? 0; if (!resolvedName || !resolvedDescription) { if (isNewItem) { throw new Error(`[handleTopicUpdate] Cannot create new topic — missing required fields: name=${resolvedName}, description=${!!resolvedDescription}`); } console.log(`[handleTopicUpdate] Skipping update for "${resolvedName ?? existingItemId}" — no description available and existing record preserved`); return; } let embedding: number[] | undefined; try { const embeddingService = getEmbeddingService(); const category = result.category ?? candidateCategory ?? existingTopic?.category; const text = getTopicEmbeddingText({ name: resolvedName, category, description: resolvedDescription }); embedding = await embeddingService.embed(text); } catch (err) { console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${resolvedName}":`, err); } const exposureImpact = result.exposure_impact as ExposureImpact | undefined; const interestedPersonas = isNewItem ? personaIds : [...new Set([...(existingTopic?.interested_personas ?? []), ...personaIds])]; const personaGroupsMerged = isNewItem ? (allPersonaGroups.length > 0 ? allPersonaGroups : existingTopic?.persona_groups) : [...new Set([...(existingTopic?.persona_groups ?? []), ...allPersonaGroups])]; const incomingSources = (response.request.data.sources ?? []) as string[]; const sources = isNewItem ? incomingSources : [...new Set([...(existingTopic?.sources ?? []), ...incomingSources])]; const newDescLen = resolvedDescription?.length ?? 0; const existingFloor = existingTopic?.rewrite_length_floor; // No stored floor to grow past (new record, or an existing record that's // already floorless) -- omit the override so the choke point's own // default computes a fresh floor from the new description (ADR-032 // amendment). Only an existing floor the new description actually // reaches/exceeds gets the explicit clear. const floorOverride = existingFloor === undefined ? undefined : newDescLen < existingFloor ? existingFloor : null; const topic: Topic = { id: itemId, name: resolvedName, description: resolvedDescription, sentiment: resolvedSentiment, category: result.category ?? candidateCategory ?? existingTopic?.category, exposure_current: calculateExposureCurrent(exposureImpact, existingTopic?.exposure_current ?? 0), exposure_desired: result.exposure_desired ?? 0.5, last_updated: now, learned_on: isNewItem ? earliestMessageTimestamp : existingTopic?.learned_on, last_mentioned: now, learned_by: isNewItem ? primaryId : existingTopic?.learned_by, last_changed_by: primaryId, interested_personas: interestedPersonas, sources: sources.length > 0 ? sources : undefined, persona_groups: personaGroupsMerged, embedding, }; state.human_topic_upsert(topic, floorOverride); await validateAndStoreQuotes(result.quotes, messages_analyze, itemId, personaDisplayName, allPersonaGroups, state); if (isNewItem && embedding) { const extractionModel = (response.request.data as Record).extraction_model as string | undefined; await queueTopicValidate(topic, state, extractionModel); } console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${resolvedName}"`); } function ensureEiPersonaHasNickname(identifiers: PersonIdentifier[], state: StateManager): PersonIdentifier[] { const eiPersonaId = identifiers.find(i => isEiPersonaIdentifierType(i.type))?.value; if (!eiPersonaId) return identifiers; const persona = state.persona_getById(eiPersonaId); if (!persona) return identifiers; const hasNickname = identifiers.some(i => i.type === 'Nickname' && i.value === persona.display_name); if (hasNickname) return identifiers; const withoutPrimary = identifiers.map(i => isEiPersonaIdentifierType(i.type) ? { ...i, is_primary: undefined } : i ).map(({ is_primary, ...rest }) => is_primary ? { ...rest, is_primary } : rest); return [ { type: 'Nickname', value: persona.display_name, is_primary: true as const }, ...withoutPrimary.map(i => isEiPersonaIdentifierType(i.type) ? { type: i.type, value: i.value } : i), ]; } export async function handlePersonUpdate(response: LLMResponse, state: StateManager): Promise { const result = response.parsed as (PersonUpdateResult & { identifiers?: PersonIdentifier[]; identifiers_to_add?: PersonIdentifier[]; quotes?: Array<{ text: string; reason: string }>; }) | undefined; if (!result || Object.keys(result).length === 0) { console.log("[handlePersonUpdate] No changes needed (empty result)"); return; } const isNewItem = response.request.data.isNewItem as boolean; const existingItemId = response.request.data.existingItemId as string | undefined; const personaId = response.request.data.personaId as string; const personaDisplayName = response.request.data.personaDisplayName as string; const candidateRelationship = response.request.data.candidateRelationship as string | undefined; const candidateIdentifiers = (response.request.data.candidateIdentifiers ?? []) as PersonIdentifier[]; const candidateName = response.request.data.candidateName as string; const candidateDescription = response.request.data.candidateDescription as string | undefined; const personaIds = personaId.split("|").filter(Boolean); const primaryId = personaIds[0] ?? personaId; const now = new Date().toISOString(); const { messages_analyze } = resolveMessageWindow(response, state); const earliestMessageTimestamp = messages_analyze.length > 0 ? messages_analyze.reduce((a, b) => a.timestamp < b.timestamp ? a : b).timestamp : now; const human = state.getHuman(); const resolveItemId = (): string => { if (isNewItem || !existingItemId) return crypto.randomUUID(); return human.people.find(p => p.id === existingItemId) ? existingItemId : crypto.randomUUID(); }; const itemId = resolveItemId(); const allPersonaGroups = personaIds .map(id => state.persona_getById(id)?.group_primary) .filter((g): g is string => g != null); const existingPerson = isNewItem ? undefined : human.people.find(p => p.id === existingItemId); const resolvedDescription = typeof result.description === 'string' ? result.description : existingPerson?.description ?? candidateDescription; const resolvedSentiment = result.sentiment !== undefined ? result.sentiment : existingPerson?.sentiment ?? 0; if (!resolvedDescription) { if (isNewItem) { throw new Error(`[handlePersonUpdate] Cannot create new person "${candidateName}" — no description available`); } console.log(`[handlePersonUpdate] Skipping update for "${candidateName}" — no description available and existing record preserved`); return; } let embedding: number[] | undefined; try { const embeddingService = getEmbeddingService(); const relationship = result.relationship ?? candidateRelationship ?? existingPerson?.relationship; const text = getPersonEmbeddingText({ name: candidateName, relationship, description: resolvedDescription }); embedding = await embeddingService.embed(text); } catch (err) { console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${candidateName}":`, err); } const exposureImpact = result.exposure_impact as ExposureImpact | undefined; const interestedPersonas = isNewItem ? personaIds : [...new Set([...(existingPerson?.interested_personas ?? []), ...personaIds])]; const personaGroupsMerged = isNewItem ? (allPersonaGroups.length > 0 ? allPersonaGroups : existingPerson?.persona_groups) : [...new Set([...(existingPerson?.persona_groups ?? []), ...allPersonaGroups])]; const incomingPersonSources = (response.request.data.sources ?? []) as string[]; const personSources = isNewItem ? incomingPersonSources : [...new Set([...(existingPerson?.sources ?? []), ...incomingPersonSources])]; let resolvedIdentifiers: PersonIdentifier[]; if (isNewItem) { const llmIdentifiers: PersonIdentifier[] = sanitizeEiPersonaIdentifiers( (result.identifiers ?? []).map(i => ({ type: normalizeIdentifierType(i.type, state), value: i.value, ...(i.is_primary ? { is_primary: i.is_primary } : {}), })), state.persona_getAll() ); const allCandidateIds = [...llmIdentifiers, ...candidateIdentifiers]; if (allCandidateIds.length === 0) { const hasSpace = candidateName.includes(' '); allCandidateIds.push({ type: hasSpace ? "Full Name" : "Nickname", value: candidateName, is_primary: true }); } const deduped: PersonIdentifier[] = []; for (const id of allCandidateIds) { if (!deduped.some(e => e.value === id.value)) { deduped.push(id); } } resolvedIdentifiers = ensureEiPersonaHasNickname(deduped, state); } else { const base = [...(existingPerson?.identifiers ?? [])]; const sanitizedToAdd = sanitizeEiPersonaIdentifiers( (result.identifiers_to_add ?? []).map(i => ({ ...i, type: normalizeIdentifierType(i.type, state), })), state.persona_getAll() ); for (const id of sanitizedToAdd) { if (!base.some(e => e.value === id.value)) { base.push({ type: id.type, value: id.value, ...(id.is_primary ? { is_primary: id.is_primary } : {}) }); } } resolvedIdentifiers = ensureEiPersonaHasNickname(base, state); } const personName = resolvedIdentifiers.find(i => i.is_primary && i.type !== 'Ei Persona')?.value ?? resolvedIdentifiers.find(i => i.type !== 'Ei Persona')?.value ?? candidateName; const personExistingFloor = existingPerson?.rewrite_length_floor; const personNewLen = resolvedDescription?.length ?? 0; // Same reasoning as handleTopicUpdate's floorOverride above: no stored // floor to grow past omits the override (fresh compute); only reaching // or exceeding an EXISTING floor gets the explicit clear. const floorOverride = personExistingFloor === undefined ? undefined : personNewLen < personExistingFloor ? personExistingFloor : null; const person: Person = { id: itemId, name: personName, description: resolvedDescription, sentiment: resolvedSentiment, relationship: result.relationship ?? candidateRelationship ?? existingPerson?.relationship ?? "Unknown", exposure_current: calculateExposureCurrent(exposureImpact, existingPerson?.exposure_current ?? 0), exposure_desired: result.exposure_desired ?? 0.5, identifiers: resolvedIdentifiers, validated_date: isNewItem ? '' : (existingPerson?.validated_date ?? ''), last_updated: now, learned_on: isNewItem ? earliestMessageTimestamp : existingPerson?.learned_on, last_mentioned: now, learned_by: isNewItem ? primaryId : existingPerson?.learned_by, last_changed_by: primaryId, interested_personas: interestedPersonas, sources: personSources.length > 0 ? personSources : undefined, persona_groups: personaGroupsMerged, embedding, }; state.human_person_upsert(person, undefined, floorOverride); await validateAndStoreQuotes(result.quotes, messages_analyze, itemId, personaDisplayName, allPersonaGroups, state); const primaryValue = resolvedIdentifiers.find(i => i.is_primary)?.value ?? candidateName; const resolvedName = (!primaryValue || primaryValue.toLowerCase() === 'unknown') ? (result.relationship ?? candidateRelationship ?? '(unknown)') : primaryValue; console.log(`[handlePersonUpdate] ${isNewItem ? "Created" : "Updated"} person "${resolvedName}"`); } function normalizeWithMap(text: string): { normalized: string; map: number[] } { const chars: string[] = []; const map: number[] = []; for (let i = 0; i < text.length; i++) { const ch = text[i]; let mapped: string; if (/[\u201C\u201D]/.test(ch)) { mapped = '"'; // curly double quotes } else if (/[\u2018\u2019\u0060\u00B4]/.test(ch)) { mapped = "'"; // curly single, backtick, acute accent } else if (/[\u2014\u2013\u2012]/.test(ch)) { mapped = '-'; // em-dash, en-dash, figure dash } else if (ch === '\u00A0') { mapped = ' '; // non-breaking space } else if (/[\u2000-\u200F]/.test(ch)) { mapped = ' '; // unicode space variants } else if (/[*_`~]/.test(ch)) { continue; // Markdown emphasis/code chars — deleted, no output, no map entry } else { mapped = ch; } chars.push(mapped); map.push(i); } map.push(text.length); // sentinel — makes the end-of-string normalized index addressable return { normalized: chars.join(''), map }; } function stripPunctuation(text: string): string { return text .replace(/[*_`~]/g, ' ') // Markdown chars (kept by \w, must strip explicitly) .replace(/[^\w\s]/gu, ' ') // replace non-word, non-space with space .replace(/\s+/g, ' ') // collapse multiple spaces .trim() .toLowerCase(); } export interface WordBoundaryMatch { start: number; end: number; text: string; } export function expandToWordBoundaries(text: string, start: number, end: number): WordBoundaryMatch { // Only walk backward if start is mid-word (not already at a word boundary) if (start > 0 && !/\s/.test(text[start])) while (start > 0 && !/\s/.test(text[start - 1])) start--; // Only walk forward if end is mid-word if (end > 0 && !/\s/.test(text[end - 1])) while (end < text.length && !/\s/.test(text[end])) end++; return { start, end, text: text.slice(start, end) }; } export function findQuoteByWords(quoteText: string, msgText: string): WordBoundaryMatch | null { const strippedQuote = stripPunctuation(quoteText); const quoteWords = strippedQuote.split(' ').filter(w => w.length > 0); if (quoteWords.length < 2) return null; // Too short to trust — require at least 2 words // Build word token list from original message with original positions. // Each \S+ token is re-split into sub-tokens (sharing the parent's start/end) // so that contractions stripped by stripPunctuation (e.g. don't → "don t") // align correctly with quoteWords which is also split on spaces. const wordTokens: Array<{ word: string; start: number; end: number }> = []; const wordRegex = /\S+/g; let match: RegExpExecArray | null; while ((match = wordRegex.exec(msgText)) !== null) { const tokenStart = match.index; const tokenEnd = match.index + match[0].length; const stripped = stripPunctuation(match[0]); const subWords = stripped.split(' ').filter(w => w.length > 0); for (const sub of subWords) { wordTokens.push({ word: sub, start: tokenStart, end: tokenEnd }); } } // Find contiguous sequence of word tokens matching the quote words for (let i = 0; i <= wordTokens.length - quoteWords.length; i++) { let allMatch = true; for (let j = 0; j < quoteWords.length; j++) { if (wordTokens[i + j].word !== quoteWords[j]) { allMatch = false; break; } } if (allMatch) { const startToken = wordTokens[i]; const endToken = wordTokens[i + quoteWords.length - 1]; return expandToWordBoundaries(msgText, startToken.start, endToken.end); } } return null; } export interface QuoteMatch extends WordBoundaryMatch { level: "exact" | "word-boundary"; } /** * Matches candidate quote text against a single message's text, trying a * normalized-exact match first (Level 1) and falling back to a word-boundary * scan (Level 2, ≥2-word threshold) if that fails. Extracted from * `validateAndStoreQuotes`'s inline per-message matching so the same logic * can be reused outside the extraction pipeline (e.g. `ei create`/`ei fix * quote`'s server-side verification). */ export function matchQuoteInMessage(candidateText: string, msgText: string): QuoteMatch | null { // Level 1: normalized exact match const { normalized: normalizedMsg, map } = normalizeWithMap(msgText); const normalizedQuote = normalizeWithMap(candidateText).normalized; const start = normalizedQuote.length > 0 ? normalizedMsg.indexOf(normalizedQuote) : -1; if (start !== -1) { const rawStart = map[start]; const rawEnd = map[start + normalizedQuote.length - 1] + 1; const expanded = expandToWordBoundaries(msgText, rawStart, rawEnd); return { start: expanded.start, end: expanded.end, text: expanded.text, level: "exact" }; } // Level 2: word-boundary fallback const wordMatch = findQuoteByWords(candidateText, msgText); if (!wordMatch) return null; return { start: wordMatch.start, end: wordMatch.end, text: wordMatch.text, level: "word-boundary" }; } async function validateAndStoreQuotes( candidates: Array<{ text: string; reason: string }> | undefined, messages: Message[], dataItemId: string, channelDisplayName: string, personaGroups: string[], state: StateManager ): Promise { if (!candidates || candidates.length === 0) return; for (const candidate of candidates) { if (!candidate.text) { console.warn('[extraction] Skipping quote candidate with missing text field'); continue; } let found = false; for (const message of messages) { const msgText = getMessageText(message); const match = matchQuoteInMessage(candidate.text, msgText); if (!match) continue; const matchStart = match.start; const matchEnd = match.end; const matchText = match.text; const matchLevel = match.level; const existing = state.human_quote_getForMessage(message.id); const merge = mergeOverlappingQuotes(existing, { message_id: message.id, start: matchStart, end: matchEnd, text: matchText }); if (merge) { const survivor = merge.absorbed[0]; const others = merge.absorbed.slice(1); const groups = personaGroups.length > 0 ? personaGroups : ["General"]; const dataItemIds = unionIds(survivor.data_item_ids, [dataItemId], ...others.map((q) => q.data_item_ids)); const mergedPersonaGroups = unionIds(survivor.persona_groups, groups, ...others.map((q) => q.persona_groups)); let embedding = survivor.embedding; if (merge.text !== survivor.text) { try { const embeddingService = getEmbeddingService(); embedding = await embeddingService.embed(merge.text); } catch (err) { console.warn(`[extraction] Failed to recompute embedding for merged quote: "${merge.text.slice(0, 30)}..."`, err); } } state.human_quote_update(survivor.id, { start: merge.start, end: merge.end, text: merge.text, data_item_ids: dataItemIds, persona_groups: mergedPersonaGroups, embedding, }); for (const absorbed of others) { state.human_quote_remove(absorbed.id); } console.log(`[extraction] Merged ${1 + others.length} overlapping quote(s): "${merge.text.slice(0, 50)}..." (${merge.start}-${merge.end})`); found = true; break; } let embedding: number[] | undefined; try { const embeddingService = getEmbeddingService(); embedding = await embeddingService.embed(matchText); } catch (err) { console.warn(`[extraction] Failed to compute embedding for quote: "${matchText.slice(0, 30)}..."`, err); } const quote: Quote = { id: crypto.randomUUID(), message_id: message.id, data_item_ids: [dataItemId], persona_groups: personaGroups.length > 0 ? personaGroups : ["General"], text: matchText, speaker: message.role === "human" ? "human" : (message.speaker_name ?? channelDisplayName), channel: channelDisplayName, timestamp: message.timestamp, start: matchStart, end: matchEnd, created_at: new Date().toISOString(), created_by: "extraction", embedding, }; state.human_quote_add(quote); if (matchLevel === "word-boundary") { console.log(`[extraction] Captured quote (word-boundary match): "${matchText.slice(0, 50)}..."`); } else { console.log(`[extraction] Captured quote: "${matchText.slice(0, 50)}..."`); } found = true; break; } if (!found) { console.warn(`[extraction] Quote not found in messages (both levels), skipping: "${candidate.text}"`); } } }