/** * Turning what the user said about people and companies into knowledge-store * writes. A clear match or a new name is written; a name that could be more than * one known entity waits in the review queue instead of being guessed. */ import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import type { CompanyInput, IngestInput, PersonInput, } from "../../tools/knowledge/ingest"; import { findMentions, foldName, indexedEntities, isExcluded, lookupName, type NamedEntity, type NameIndex, proseOnly, } from "./entity-names"; import { paths } from "./paths"; import { isSystemText, stripInjectedTags } from "./prompt-text"; import { type Message, messagesAfter } from "./transcript"; type Cursor = string | number; export interface ExtractedEntity { kind: "person" | "company"; name: string; existing: string; aliases: string[]; role: string; organization: string; relation: string; fact: string; } export interface ResearchProfile { match: "sure" | "unsure" | "none"; summary: string; role: string; organization: string; website: string; linkedin: string; socials: { platform: string; url: string }[]; registry: { name: string; number: string; taxNumber: string; seat: string; status: string; activity: string; managers: string[]; url: string; }; news: { date: string; title: string; url: string }[]; sources: string[]; } export interface ReviewItem { ts: string; source: string; id: string; reason: "ambiguous" | "unknown-existing" | "first-name-only" | "web-profile"; entity: ExtractedEntity; candidates: string[]; profile?: ResearchProfile; } export interface ExtractionPlan { ingest: IngestInput; review: ReviewItem[]; } export function closedObject

>(properties: P) { return { type: "object" as const, additionalProperties: false, properties, required: Object.keys(properties), }; } export function extractionSchema() { const text = { type: "string" as const }; const entity = closedObject({ kind: { type: "string" as const, enum: ["person", "company"] }, name: text, existing: text, aliases: { type: "array" as const, items: text }, role: text, organization: text, relation: text, fact: text, }); return closedObject({ entities: { type: "array" as const, items: entity } }); } const MAX_TEXT = 4000; const MIN_TEXT = 20; const MAX_KNOWN = 120; const CURSOR_KEEP = 200; function userTexts(messages: Message[]): string[] { return messages .filter((m) => m.role === "user" && typeof m.content === "string") .map((m) => stripInjectedTags(m.content as string)) .filter((t) => t.length > 0 && !isSystemText(t)); } /** * A count cursor predates id cursors: its session counts as read up to now. A * transcript without ids can hold no cursor, so only its latest user turn is new. */ function unseenMessages(messages: Message[], cursor: Cursor | undefined): Message[] { if (!messages.some((m) => m.id)) return messages.slice(messages.findLastIndex((m) => m.role === "user")); if (typeof cursor === "number") return []; return messagesAfter(messages, cursor); } /** The user's words since the last extraction, and the id to resume after. */ export function unseenUserText( messages: Message[], cursor: Cursor | undefined ): { text: string; count: number; lastId?: string } { const fresh = userTexts(unseenMessages(messages, cursor)); const text = fresh.map((t, i) => `${i + 1}. ${proseOnly(t).trim()}`).join("\n"); const lastId = messages.findLast((m) => m.id)?.id; return { text: text.slice(-MAX_TEXT), count: fresh.length, lastId }; } function hasUnknownProperName(text: string, ignored: Set): boolean { const sentences = proseOnly(text).split(/[.!?\n]+/); return sentences.some((s) => s .trim() .split(/\s+/) .slice(1) .some((w) => /^\p{Lu}\p{Ll}/u.test(w) && !ignored.has(foldName(w))) ); } /** Cheap gate before a model call: a known name, or a capitalised word mid-sentence. */ export function worthExtracting( text: string, index: NameIndex, excluded: string[] ): boolean { if (proseOnly(text).trim().length < MIN_TEXT) return false; if (findMentions(text, index).length > 0) return true; return hasUnknownProperName(text, new Set(excluded.map(foldName))); } function knownLine(entity: NamedEntity): string { const aliases = entity.aliases.length > 0 ? ` (also: ${entity.aliases.join(", ")})` : ""; const kind = entity.domain === "People" ? "person" : "company"; return `${entity.slug} | ${kind} | ${entity.title}${aliases}`; } export function extractionSystem(principal: string, assistant: string): string { return [ `You keep a contact book for ${principal}. From ${principal}'s own messages to an AI assistant, list the people and companies ${principal} talks about as part of their work and life.`, `- Never list ${principal} or the assistant (${assistant}).`, `- People: individuals ${principal} deals with (colleagues, clients, partners, friends, family, contacts). Skip public figures cited only for their work.`, `- Companies: only organisations ${principal} has a relationship with: employer, own company, client, prospect, partner, contractor, school. Skip tools, products, platforms, languages and vendors that are merely used or discussed.`, "- existing: the slug of the matching known entry below, or empty when it is someone new.", "- aliases: other names the messages use for it (nickname, short form), not the main name.", `- role, organization (a person's company), relation (how ${principal} relates to it: client, partner, colleague, friend, employer, prospect, own company): only what the messages state, else empty.`, "- fact: one sentence of what the messages state about it, else empty. Never guess.", "- Ignore names that appear only inside pasted logs, code or email headers.", "Return an empty list when nobody qualifies.", ].join("\n"); } export function extractionUser(text: string, index: NameIndex): string { const known = indexedEntities(index) .filter((e) => !isExcluded(e, index)) .slice(0, MAX_KNOWN) .map(knownLine); return `Known entries (slug | kind | name):\n${known.join("\n") || "(none)"}\n\nMessages:\n${text}`; } function domainOf(kind: ExtractedEntity["kind"]): NamedEntity["domain"] { return kind === "person" ? "People" : "Companies"; } function namesUser(e: ExtractedEntity, excluded: string[]): boolean { const own = new Set(excluded.filter(Boolean).map(foldName)); return [e.name, ...e.aliases].some((n) => own.has(foldName(n))); } type Resolution = | { kind: "new" } | { kind: "known"; entity: NamedEntity } | { kind: "review"; reason: ReviewItem["reason"]; candidates: NamedEntity[] }; function resolveExisting(e: ExtractedEntity, sameDomain: NamedEntity[]): Resolution { const entity = sameDomain.find((c) => c.slug === e.existing); return entity ? { kind: "known", entity } : { kind: "review", reason: "unknown-existing", candidates: [] }; } /** "Dan" alone may be a known Daniel under a nickname the store hasn't seen yet. */ function newcomer(e: ExtractedEntity): Resolution { const firstNameOnly = e.kind === "person" && !/\s/.test(e.name.trim()); return firstNameOnly ? { kind: "review", reason: "first-name-only", candidates: [] } : { kind: "new" }; } function resolveEntity(e: ExtractedEntity, index: NameIndex): Resolution { const domain = domainOf(e.kind); const sameDomain = indexedEntities(index).filter((c) => c.domain === domain); if (e.existing) return resolveExisting(e, sameDomain); const { entities, exact } = lookupName(e.name, index); const candidates = entities.filter((c) => c.domain === domain); if (candidates.length === 0) return newcomer(e); if (exact && candidates.length === 1) return { kind: "known", entity: candidates[0] }; return { kind: "review", reason: "ambiguous", candidates }; } function otherNames(e: ExtractedEntity, storedName: string): string[] { const names = [e.name, ...e.aliases].map((n) => n.trim()).filter(Boolean); return [...new Set(names)].filter((n) => foldName(n) !== foldName(storedName)); } /** "Acme" for a stored "Acme Kft." links the person to that file, not a new stub. */ function storedOrganization(organization: string, index: NameIndex): string | null { if (!organization.trim()) return null; const { entities, exact } = lookupName(organization, index); const companies = entities.filter((c) => c.domain === "Companies"); return exact && companies.length === 1 ? companies[0].title : organization; } function personInput( e: ExtractedEntity, name: string, index: NameIndex, slug: string | null ): PersonInput { return { name, slug, role: e.role || null, company: storedOrganization(e.organization, index), context: e.fact || null, aliases: otherNames(e, name), relation: e.relation || null, }; } function companyInput( e: ExtractedEntity, name: string, slug: string | null ): CompanyInput { return { name, slug, context: e.fact || null, aliases: otherNames(e, name), relation: e.relation || null, }; } /** Ingest input for one extracted entity: the known file it updates, or a new one. */ export function ingestInput( e: ExtractedEntity, index: NameIndex, known: NamedEntity | null, newName?: string ): IngestInput { const name = known?.title ?? (newName?.trim() || e.name.trim()); const slug = known?.slug ?? null; return e.kind === "person" ? { people: [personInput(e, name, index, slug)] } : { companies: [companyInput(e, name, slug)] }; } export function planExtraction( extracted: ExtractedEntity[], index: NameIndex, excluded: string[], source: string ): ExtractionPlan { const plan: ExtractionPlan = { ingest: { people: [], companies: [] }, review: [] }; const queue = ( entity: ExtractedEntity, reason: ReviewItem["reason"], candidates: NamedEntity[] ) => plan.review.push({ id: crypto.randomUUID().slice(0, 8), ts: new Date().toISOString(), source, reason, entity, candidates: candidates.map((c) => c.slug).sort(), }); for (const e of extracted) { if (!e.name.trim() || namesUser(e, excluded)) continue; const resolution = resolveEntity(e, index); if (resolution.kind === "review") { queue(e, resolution.reason, resolution.candidates); continue; } if (resolution.kind === "known" && isExcluded(resolution.entity, index)) continue; const known = resolution.kind === "known" ? resolution.entity : null; const input = ingestInput(e, index, known); plan.ingest.people?.push(...(input.people ?? [])); plan.ingest.companies?.push(...(input.companies ?? [])); } return plan; } export function parseExtraction(output: string): ExtractedEntity[] { try { const parsed = JSON.parse(output) as { entities?: ExtractedEntity[] }; return Array.isArray(parsed.entities) ? parsed.entities : []; } catch { return []; } } function cursorFile(): string { return resolve(paths.state(), "entity-extraction.json"); } function readCursors(): Record { const file = cursorFile(); if (!existsSync(file)) return {}; try { return JSON.parse(readFileSync(file, "utf-8")) as Record; } catch { return {}; } } export function seenCursor(sessionId: string): Cursor | undefined { return readCursors()[sessionId]; } export function markSeen(sessionId: string, lastId: string): void { const cursors = Object.entries({ ...readCursors(), [sessionId]: lastId }).slice( -CURSOR_KEEP ); writeFileSync( cursorFile(), `${JSON.stringify(Object.fromEntries(cursors), null, 2)}\n` ); } export function reviewQueueFile(): string { return resolve(paths.knowledge(), "_review.jsonl"); } export function queueForReview(items: ReviewItem[]): void { if (items.length === 0) return; appendFileSync(reviewQueueFile(), items.map((i) => `${JSON.stringify(i)}\n`).join("")); }