/** * Assemble fetched/extracted pages into the markdown blob handed to the * summarizer (or the main agent), plus the canonical Sources list. * * Pure (no I/O): given the already-fetched pages it formats text and enforces * the total character budget. The per-page origin is computed once. */ import { ageDays, ageLabel, dateSuffix } from "./recency.ts"; import { BLOCKED_MARKER, DO_NOT_RETRY_MARKER } from "../session/block-ledger.ts"; import type { LlmstxtResult } from "../fetch/llms-txt.ts"; import type { SearchResult } from "../search/search.ts"; export interface AssembledPage extends SearchResult { markdown?: string; truncated?: boolean; error?: string; llmsTxt?: LlmstxtResult; // Set when the fetch was a terminal bot-block (all escalation tiers exhausted). blocked?: true; status?: number; blockReason?: string; tiersTried?: string; } function originOf(url: string): string { try { return new URL(url).origin; } catch { return url; } } export interface AssembleResult { rawText: string; sourcesList: string; } /** * Build the full assembled markdown and the canonical Sources list. * `results` is the ordered result set used for the deterministic Sources list * (so reference URLs survive even if the summarizer omits one). */ /** * The ready-made standard citation for one result: publish date + age as plain * text (when known), then the URL as a markdown link whose link text is the URL * itself. Assembled deterministically in code so the main agent only relays it * verbatim — it never composes (and thus never mangles) the format. */ export function citationFor(r: SearchResult): string { const d = ageDays(r.published); const date = r.published && d !== undefined ? `${r.published.slice(0, 10)} (${ageLabel(d)}) ` : ""; return `${date}[${r.url}](${r.url})`; } /** * Build the canonical, deterministic "Sources" list from an ordered result set, * so reference URLs survive even if the summarizer omits one. Exported so callers * (e.g. multi-hop, which discovers extra sources) can build the final list from a * combined array. Each entry ends with its ready-made citation (see citationFor) * so relaying a source means copying, not formatting. */ export function sourcesListFor(results: SearchResult[]): string { return [ "## 🔗 Sources (for further reading)", ...results.map((r, i) => `- [${i + 1}] ${r.title} — ${citationFor(r)}`), ].join("\n"); } export function assembleResults( heading: string, pages: AssembledPage[], results: SearchResult[], totalMaxChars: number, ): AssembleResult { const parts: string[] = [heading, ""]; let used = 0; for (let i = 0; i < pages.length; i++) { const p = pages[i]; const num = i + 1; const origin = originOf(p.url); parts.push(`### 📄 Source ${num}: ${p.title}${dateSuffix(p)}`); parts.push(`**URL:** <${p.url}>`); if (p.llmsTxt) { const llmsBody = p.llmsTxt.content; // already structure-aware truncated + self-annotated if (used + llmsBody.length <= totalMaxChars) { parts.push("", `> **📋 [llms.txt](https://llmstxt.org/) from ${origin}:**`); parts.push(llmsBody); used += llmsBody.length; } else { parts.push( "", `> **📋 [llms.txt](https://llmstxt.org/) from ${origin}:** _[skipped: total content budget reached]_`, ); } } if (p.markdown) { if (used >= totalMaxChars) { // Budget already exhausted: say so explicitly rather than silently // dropping the page down to its snippet. parts.push("_[content skipped: total budget reached]_"); } else if (used + p.markdown.length > totalMaxChars) { const body = p.markdown.slice(0, Math.max(0, totalMaxChars - used)); parts.push(body, "", "_[content truncated: total budget reached]_"); used += body.length; } else { parts.push(p.markdown); if (p.truncated) parts.push("", "_[content truncated: per-page limit]_"); used += p.markdown.length; } } else if (p.error) { const blockTag = p.blocked ? ` ${BLOCKED_MARKER} ${DO_NOT_RETRY_MARKER} (${origin} blocks automated access)` : ""; parts.push(`_[could not fetch content: ${p.error}]_${blockTag}${p.snippet ? `\n\n${p.snippet}` : ""}`); } else if (p.snippet) { parts.push(p.snippet); } parts.push("", "---", ""); } // Source index for cross-referencing during summarization. parts.push("", "---", "", `### 🔗 Source Index`); for (let i = 0; i < results.length; i++) { parts.push(`[${i + 1}] ${results[i].title} — <${results[i].url}>`); } const rawText = parts.join("\n").trim(); return { rawText, sourcesList: sourcesListFor(results) }; }