/** Portable JSON budgeting for already-authorized record evidence. Kept in the * CLI package so installed CLI reads and Convex/model reads use the same policy. */ type Row = Record; const object = (value: unknown): Row | undefined => value !== null && typeof value === "object" && !Array.isArray(value) ? (value as Row) : undefined; const encoder = new TextEncoder(); export const evidenceJsonBytes = (value: unknown) => encoder.encode(JSON.stringify(value)).length; /** Include JSON escaping and preserve Unicode code points. */ export function clipEvidenceText(value: string, limit: number): string { if (evidenceJsonBytes(value) <= limit) return value; let result = ""; let used = 5; // quotes and ellipsis for (const point of value) { used += evidenceJsonBytes(point) - 2; if (used > limit) break; result += point; } return `${result}…`; } const identityKeys = new Set([ "id", "entryId", "activityId", "objectName", "threadKey", "conversationId", "permalink", "href", "url", "at", "currentDate", "kind", "association", ]); const collections = [ "timeline", "relatedRecords", "humanCommentsAndNotes", ] as const; type Collection = (typeof collections)[number]; function eventTime(value: unknown): number { const row = object(value); const meeting = object(row?.meeting); const valueAt = meeting?.["Start At"] ?? meeting?.["Started At"] ?? row?.at; return typeof valueAt === "number" ? valueAt : Date.parse(String(valueAt)); } function category(collection: Collection, value: unknown, now: number): string { const row = object(value); if (collection === "relatedRecords") return String(row?.objectName ?? "record"); const kind = String(row?.kind ?? "unknown"); if (collection === "humanCommentsAndNotes") return kind; if (kind.startsWith("email")) return "email"; if (kind.startsWith("slack")) return "slack"; if (kind.startsWith("linkedin")) return "linkedin"; if (kind === "meeting" || kind.startsWith("calendar")) { const status = String(object(row?.meeting)?.Status ?? ""); if (/cancel/i.test(status)) return "meeting:cancelled"; if (/past|complet|ended/i.test(status)) return "meeting:past"; const at = eventTime(row); if (Number.isFinite(at) ? at > now : /^(scheduled|upcoming)$/i.test(status)) return "meeting:upcoming"; return kind === "calendar_notice" ? "meeting:notice" : "meeting:past"; } return kind; } /** Preserve representative source channels, past/upcoming meetings and related * object types before spending the remaining budget on repeated older detail. * Omission counts describe this projection, not the completeness of the CRM. */ export function boundRecordEvidence(input: Row, limit: number) { if (!Number.isFinite(limit) || limit < 512) throw new Error("Record evidence budget must be at least 512 bytes."); const data = JSON.parse(JSON.stringify(input)) as Row; if (evidenceJsonBytes(data) <= limit) return { data, truncated: data.truncated === true }; const previous = object(data.evidenceOmissions); const omissions = { timeline: Number(previous?.timeline) || 0, relatedRecords: Number(previous?.relatedRecords) || 0, humanCommentsAndNotes: Number(previous?.humanCommentsAndNotes) || 0, values: Number(previous?.values) || 0, }; data.truncated = true; data.evidenceOmissions = omissions; const fits = () => evidenceJsonBytes(data) <= limit; const now = Date.parse(String(data.currentDate)) || 0; const rows = (key: Collection) => Array.isArray(data[key]) ? (data[key] as unknown[]) : []; const shorten = ( value: unknown, key: string, textLimit: number, depth = 0, ): unknown => { if (typeof value === "string") { if (identityKeys.has(key)) return value; const clipped = clipEvidenceText(value, textLimit); if (clipped !== value) omissions.values++; return clipped; } if (depth > 16) { omissions.values++; return null; } if (Array.isArray(value)) { const keep = depth === 1 && collections.includes(key as Collection) ? value.length : 8; omissions.values += Math.max(0, value.length - keep); const selected = key === "recentMessages" ? value.slice(-keep) : value.slice(0, keep); return selected.map((item) => shorten(item, "", textLimit, depth + 1)); } const row = object(value); if (!row || key === "evidenceOmissions") return value; return Object.fromEntries( Object.entries(row).map(([name, child]) => [ name, shorten(child, name, textLimit, depth + 1), ]), ); }; const compress = (textLimit: number) => { for (const [key, value] of Object.entries(data)) if (key !== "evidenceOmissions") data[key] = shorten(value, key, textLimit, 1); }; // First shorten oversized prose, retaining every evidence row where possible. compress(2_000); const removable = (key: Collection, preserveCategories: boolean) => { const list = rows(key); if (!preserveCategories) return list.length - 1; const protectedRows = new Map(); list.forEach((row, index) => { const group = category(key, row, now); const old = protectedRows.get(group); // The nearest scheduled meeting matters before a more distant booking. if ( old === undefined || (group === "meeting:upcoming" && eventTime(row) < eventTime(list[old])) ) protectedRows.set(group, index); }); const protectedIndexes = new Set(protectedRows.values()); for (let index = list.length - 1; index >= 0; index--) if (!protectedIndexes.has(index)) return index; return -1; }; const removeOne = (preserveCategories: boolean) => { const candidates = collections .flatMap((key) => { const index = removable(key, preserveCategories); return index < 0 ? [] : [{ key, index, bytes: evidenceJsonBytes(rows(key)[index]) }]; }) .sort((a, b) => b.bytes - a.bytes); const next = candidates[0]; if (!next) return false; rows(next.key).splice(next.index, 1); omissions[next.key]++; return true; }; while (!fits() && removeOne(true)) { /* retain one source per category */ } for (const textLimit of [800, 320, 120]) { if (fits()) break; compress(textLimit); } // Research and an old generated Brief are lower priority than source facts. for (const key of ["dossier", "previousBrief"]) { if (fits()) break; if (key in data) { delete data[key]; omissions.values++; } } while (!fits() && removeOne(false)) { /* atomic source handles are never clipped */ } if (!fits()) { // Malformed/oversized metadata can itself exceed the budget. Keep a valid, // explicitly incomplete packet rather than returning broken IDs or JSON. omissions.values += Object.keys(data).filter( (key) => key !== "truncated" && key !== "evidenceOmissions" && !collections.includes(key as Collection), ).length; return { data: { truncated: true, evidenceOmissions: omissions, notice: "Evidence exceeds this response budget. Read the source records.", } as Row, truncated: true, }; } return { data, truncated: true }; }