import type { NormalizedFetchResponse, NormalizedSearchResponse } from "./normalize.js"; export interface FormatOptions { maxOutputChars: number; fullContentRef?: string; recommendationThreshold?: number; } export interface SearchFormatOptions extends FormatOptions { fullContentRef?: string; onTruncate?: () => void; } export type RecommendedRetrievalMode = "inline" | "file"; export interface TargetVisibilityMetadata { totalChars: number; visibleChars: number; remainingChars: number; recommendedRetrievalMode: RecommendedRetrievalMode; } export interface FetchTruncationMetadata { truncated: boolean; maxOutputChars: number; targets: { title: TargetVisibilityMetadata; content: TargetVisibilityMetadata; links: TargetVisibilityMetadata; }; } export interface SearchResultTruncationMetadata { targets: { title: TargetVisibilityMetadata; url: TargetVisibilityMetadata; content: TargetVisibilityMetadata; }; } export interface SearchTruncationMetadata { truncated: boolean; maxOutputChars: number; omittedResultCount: number; results: SearchResultTruncationMetadata[]; } export interface FormattedSearchResult { text: string; truncation: SearchTruncationMetadata; } export interface FormattedFetchResult { text: string; truncation: FetchTruncationMetadata; } const RETRIEVAL_INLINE_THRESHOLD = 12_000; function applySafetyCap(text: string, maxOutputChars: number, truncationNotice?: string): string { if (text.length <= maxOutputChars) return text; const notice = truncationNotice ?? `\n\n[Output truncated to ${maxOutputChars} characters to protect pi context.]`; if (notice.length >= maxOutputChars) { return notice.slice(0, maxOutputChars); } const available = maxOutputChars - notice.length; return text.slice(0, available).trimEnd() + notice; } function buildTargetVisibilityMetadata( totalChars: number, visibleChars: number, recommendationThreshold: number, ): TargetVisibilityMetadata { const clampedVisibleChars = Math.max(0, Math.min(totalChars, visibleChars)); const remainingChars = Math.max(0, totalChars - clampedVisibleChars); return { totalChars, visibleChars: clampedVisibleChars, remainingChars, recommendedRetrievalMode: remainingChars <= recommendationThreshold ? "inline" : "file", }; } function buildSearchResultChunk(index: number, title: string, url: string, content: string): { separator: string; prefix: string; chunk: string } { const separator = index === 0 ? "" : "\n\n---\n\n"; const prefix = `[${index + 1}] ${title}\nURL: ${url}\nContent:\n`; return { separator, prefix, chunk: separator + prefix + content }; } function buildSearchResultVisibility( index: number, title: string, url: string, content: string, visibleChunkChars: number, recommendationThreshold: number, ): SearchResultTruncationMetadata { const { separator } = buildSearchResultChunk(index, title, url, content); let remaining = Math.max(0, visibleChunkChars - separator.length); const leading = `[${index + 1}] `; const between = "\nURL: "; const beforeContent = "\nContent:\n"; const consume = (size: number): number => { const consumed = Math.max(0, Math.min(size, remaining)); remaining -= consumed; return consumed; }; consume(leading.length); const titleVisible = consume(title.length); consume(between.length); const urlVisible = consume(url.length); consume(beforeContent.length); const contentVisible = consume(content.length); return { targets: { title: buildTargetVisibilityMetadata(title.length, titleVisible, recommendationThreshold), url: buildTargetVisibilityMetadata(url.length, urlVisible, recommendationThreshold), content: buildTargetVisibilityMetadata(content.length, contentVisible, recommendationThreshold), }, }; } function getSearchTruncationNotice(maxOutputChars: number, fullContentRef?: string): string { if (fullContentRef) { return `[Output truncated to ${maxOutputChars} characters to protect pi context. Use ollama_web_read_full with ref ${fullContentRef}.]`; } return `[Output truncated to ${maxOutputChars} characters to protect pi context. See details for retrieval metadata.]`; } function getSearchOmissionNotice(fullContentRef?: string): string { if (fullContentRef) { return `Additional search results were omitted from visible output. Use resultIndex with ref ${fullContentRef} to retrieve them.`; } return "Additional search results were omitted from visible output. See details for omitted result targets."; } function getFetchTruncationNotice(maxOutputChars: number, fullContentRef?: string): string { if (fullContentRef) { return `[Output truncated to ${maxOutputChars} characters to protect pi context. Use ollama_web_read_full with ref ${fullContentRef}.]`; } return `[Output truncated to ${maxOutputChars} characters to protect pi context. See details for retrieval metadata.]`; } function buildSearchTrailingNotices(maxOutputChars: number, omittedResultCount: number, fullContentRef?: string): string { const truncationNotice = getSearchTruncationNotice(maxOutputChars, fullContentRef); const omissionNotice = getSearchOmissionNotice(fullContentRef); const withBoth = `\n\n${truncationNotice}${omittedResultCount > 0 ? `\n\n${omissionNotice}` : ""}`; if (withBoth.length <= maxOutputChars) { return withBoth; } if (omittedResultCount > 0) { const omissionOnly = `\n\n${omissionNotice}`; if (omissionOnly.length <= maxOutputChars) { return omissionOnly; } return applySafetyCap(omissionNotice, maxOutputChars); } const truncationOnly = `\n\n${truncationNotice}`; if (truncationOnly.length <= maxOutputChars) { return truncationOnly; } return applySafetyCap(truncationNotice, maxOutputChars); } export function formatSearchResultsWithMetadata(response: NormalizedSearchResponse, options: SearchFormatOptions): FormattedSearchResult { const recommendationThreshold = options.recommendationThreshold ?? RETRIEVAL_INLINE_THRESHOLD; if (response.results.length === 0) { return { text: "No results found.", truncation: { truncated: false, maxOutputChars: options.maxOutputChars, omittedResultCount: 0, results: [], }, }; } const header = "Search results:\n\n"; const chunks = response.results.map((result, index) => buildSearchResultChunk(index, result.title, result.url, result.content)); const fullText = header + chunks.map((chunk) => chunk.chunk).join(""); if (fullText.length <= options.maxOutputChars) { return { text: fullText, truncation: { truncated: false, maxOutputChars: options.maxOutputChars, omittedResultCount: 0, results: response.results.map((result) => ({ targets: { title: buildTargetVisibilityMetadata(result.title.length, result.title.length, recommendationThreshold), url: buildTargetVisibilityMetadata(result.url.length, result.url.length, recommendationThreshold), content: buildTargetVisibilityMetadata(result.content.length, result.content.length, recommendationThreshold), }, })), }, }; } options.onTruncate?.(); const metadata: SearchResultTruncationMetadata[] = response.results.map((result) => ({ targets: { title: buildTargetVisibilityMetadata(result.title.length, 0, recommendationThreshold), url: buildTargetVisibilityMetadata(result.url.length, 0, recommendationThreshold), content: buildTargetVisibilityMetadata(result.content.length, 0, recommendationThreshold), }, })); let visibleChunkText = ""; let partialChunk = ""; let omittedResultCount = response.results.length; for (let index = 0; index < response.results.length; index += 1) { const result = response.results[index]; const chunk = chunks[index]; const omittedIfStopHere = Math.max(0, response.results.length - (index + 1)); const trailingNotices = buildSearchTrailingNotices(options.maxOutputChars, omittedIfStopHere, options.fullContentRef); const fullChunkFits = header.length + visibleChunkText.length + chunk.chunk.length + trailingNotices.length <= options.maxOutputChars; if (fullChunkFits) { visibleChunkText += chunk.chunk; metadata[index] = { targets: { title: buildTargetVisibilityMetadata(result.title.length, result.title.length, recommendationThreshold), url: buildTargetVisibilityMetadata(result.url.length, result.url.length, recommendationThreshold), content: buildTargetVisibilityMetadata(result.content.length, result.content.length, recommendationThreshold), }, }; continue; } const availableForChunk = Math.max(0, options.maxOutputChars - header.length - visibleChunkText.length - trailingNotices.length); const prefixLength = chunk.separator.length + chunk.prefix.length; if (availableForChunk >= prefixLength) { const visibleContentChars = Math.min(result.content.length, availableForChunk - prefixLength); partialChunk = chunk.separator + chunk.prefix + result.content.slice(0, visibleContentChars); metadata[index] = buildSearchResultVisibility( index, result.title, result.url, result.content, partialChunk.length, recommendationThreshold, ); omittedResultCount = omittedIfStopHere; } else { partialChunk = ""; omittedResultCount = response.results.length - index; } break; } const trailingNotices = buildSearchTrailingNotices(options.maxOutputChars, omittedResultCount, options.fullContentRef); const leadingText = header + visibleChunkText + partialChunk; const availableForLeadingText = Math.max(0, options.maxOutputChars - trailingNotices.length); const leadingVisibleText = leadingText.slice(0, availableForLeadingText); const text = leadingVisibleText + trailingNotices; return { text, truncation: { truncated: true, maxOutputChars: options.maxOutputChars, omittedResultCount, results: metadata, }, }; } export function formatSearchResults(response: NormalizedSearchResponse, options: SearchFormatOptions): string { return formatSearchResultsWithMetadata(response, options).text; } function buildLinksSection(links: string[]): string { return links.length === 0 ? "No links found." : links.map((link, i) => `[${i + 1}] ${link}`).join("\n"); } export function formatFetchResultWithMetadata(response: NormalizedFetchResponse, options: FormatOptions): FormattedFetchResult { const recommendationThreshold = options.recommendationThreshold ?? RETRIEVAL_INLINE_THRESHOLD; const linksSection = buildLinksSection(response.links); const headerPrefix = "Fetched page:\n\nTitle: "; const betweenTitleAndContent = "\n\nContent:\n"; const linksHeader = "\nLinks:\n"; const fullText = `${headerPrefix}${response.title}${betweenTitleAndContent}${response.content}${linksHeader}${linksSection}`; if (fullText.length <= options.maxOutputChars) { return { text: fullText, truncation: { truncated: false, maxOutputChars: options.maxOutputChars, targets: { title: buildTargetVisibilityMetadata(response.title.length, response.title.length, recommendationThreshold), content: buildTargetVisibilityMetadata(response.content.length, response.content.length, recommendationThreshold), links: buildTargetVisibilityMetadata(linksSection.length, linksSection.length, recommendationThreshold), }, }, }; } const notice = getFetchTruncationNotice(options.maxOutputChars, options.fullContentRef); const noticeBlock = `\n\n${notice}`; let visibleTitleChars = response.title.length; let visibleContentChars = response.content.length; let visibleLinksChars = linksSection.length; const measure = () => headerPrefix.length + visibleTitleChars + betweenTitleAndContent.length + visibleContentChars + noticeBlock.length + linksHeader.length + visibleLinksChars; let overflow = measure() - options.maxOutputChars; if (overflow > 0) { const contentReduction = Math.min(overflow, visibleContentChars); visibleContentChars -= contentReduction; overflow -= contentReduction; } if (overflow > 0) { const linksReduction = Math.min(overflow, visibleLinksChars); visibleLinksChars -= linksReduction; overflow -= linksReduction; } if (overflow > 0) { const titleReduction = Math.min(overflow, visibleTitleChars); visibleTitleChars -= titleReduction; overflow -= titleReduction; } const titleVisible = response.title.slice(0, visibleTitleChars); const contentVisible = response.content.slice(0, visibleContentChars); const linksVisible = linksSection.slice(0, visibleLinksChars); const truncatedText = `${headerPrefix}${titleVisible}${betweenTitleAndContent}${contentVisible}${noticeBlock}${linksHeader}${linksVisible}`; if (overflow > 0) { return { text: applySafetyCap( truncatedText, options.maxOutputChars, `\n\n${getFetchTruncationNotice(options.maxOutputChars, options.fullContentRef)}`, ), truncation: { truncated: true, maxOutputChars: options.maxOutputChars, targets: { title: buildTargetVisibilityMetadata(response.title.length, 0, recommendationThreshold), content: buildTargetVisibilityMetadata(response.content.length, 0, recommendationThreshold), links: buildTargetVisibilityMetadata(linksSection.length, 0, recommendationThreshold), }, }, }; } return { text: truncatedText, truncation: { truncated: true, maxOutputChars: options.maxOutputChars, targets: { title: buildTargetVisibilityMetadata(response.title.length, visibleTitleChars, recommendationThreshold), content: buildTargetVisibilityMetadata(response.content.length, visibleContentChars, recommendationThreshold), links: buildTargetVisibilityMetadata(linksSection.length, visibleLinksChars, recommendationThreshold), }, }, }; } export function formatFetchResult(response: NormalizedFetchResponse, options: FormatOptions): string { return formatFetchResultWithMetadata(response, options).text; }