import { load } from "cheerio"; import { Type } from "typebox"; import { chunkAndSerializeResponse, serializeResponseWithMeta } from "../shared/serialize"; import { chunkingManager } from "../shared/instances"; import { HttpError, HttpStatusError } from "../shared/http"; import { isFetchEnabled } from "../shared/env"; import { BaseProvider, type HttpClientFactory, type ProviderMetadata, type ProviderResult, type ToolDefinition } from "./base"; function normalizePackageName(packageName: string): string { if (packageName.startsWith("https://godocs.io/")) { return packageName.slice("https://godocs.io/".length); } return packageName; } function byteTruncateUtf8(text: string, maxBytes: number): { content: string; truncated: boolean } { const bytes = new TextEncoder().encode(text); if (bytes.length <= maxBytes) { return { content: text, truncated: false }; } let cut = Math.max(0, maxBytes); if (cut === 0) { return { content: "", truncated: true }; } let lead = cut - 1; while (lead > 0 && (bytes[lead] & 0b1100_0000) === 0b1000_0000) { lead -= 1; } const first = bytes[lead]; let expected = 1; if ((first & 0b1000_0000) === 0) { expected = 1; } else if ((first & 0b1110_0000) === 0b1100_0000) { expected = 2; } else if ((first & 0b1111_0000) === 0b1110_0000) { expected = 3; } else if ((first & 0b1111_1000) === 0b1111_0000) { expected = 4; } const available = cut - lead; if (available < expected) { cut = lead; } return { content: new TextDecoder().decode(bytes.slice(0, cut)), truncated: true }; } export class GoDocsProvider extends BaseProvider { private readonly chunkingManager = chunkingManager; public constructor(httpClientFactory: HttpClientFactory) { super(httpClientFactory); } getMetadata(): ProviderMetadata { const toolNames = ["godocs_metadata"]; if (isFetchEnabled()) { toolNames.push("fetch_godocs_docs"); } return { name: "godocs", description: "GoDocs package documentation metadata", exposeAsTool: true, toolNames, supportsLibrarySearch: true, requiredEnvVars: [], optionalEnvVars: [], toolTiers: { godocs_metadata: { tier: 4, deferRecommended: true, category: "metadata", }, fetch_godocs_docs: { tier: 4, deferRecommended: true, category: "fetch", }, }, }; } async searchLibrary(library: string, _limit = 5): Promise { try { const data = await this._fetchMetadata(library); return { success: true, data, providerName: "godocs" }; } catch (error) { if (error instanceof HttpStatusError) { if (error.response.status === 404) { return { success: false, error: null, providerName: "godocs" }; } return { success: false, error: `GoDocs returned ${error.response.status}`, providerName: "godocs" }; } if (error instanceof HttpError) { return { success: false, error: `GoDocs request failed: ${error}`, providerName: "godocs" }; } return { success: false, error: null, providerName: "godocs" }; } } protected async _fetchMetadata(packageName: string): Promise> { const packagePath = normalizePackageName(packageName); const url = `https://godocs.io/${packagePath}`; const client = await this.httpClient(); const response = (await client.get(url, { headers: { "User-Agent": "curl/7.68.0" } })) as { text: string; raiseForStatus(): void; }; response.raiseForStatus(); const $ = load(response.text); let description = ""; const metaDescription = $('meta[name=description]').attr("content") ?? ""; if (metaDescription) { description = metaDescription; } if (!description || description.includes("godocs.io")) { const overviewHeader = $('h2#pkg-overview, h3#pkg-overview').first(); if (overviewHeader.length > 0) { for (const sibling of overviewHeader.nextAll().toArray()) { const tagName = typeof (sibling as { tagName?: string }).tagName === "string" ? (sibling as { tagName?: string }).tagName!.toLowerCase() : ""; if (tagName === "h2" || tagName === "h3") { break; } if (tagName === "p") { const text = $(sibling).text().trim(); if (text.startsWith('import "')) { continue; } description = text; break; } } } } return { name: packagePath, summary: description, url, source_url: `https://pkg.go.dev/${packagePath}`, }; } protected async _fetchGodocsDocs(packageName: string, maxBytes = 20480): Promise> { try { const packagePath = normalizePackageName(packageName); const url = `https://godocs.io/${packagePath}`; const client = await this.httpClient(); const response = (await client.get(url, { headers: { "User-Agent": "curl/7.68.0" } })) as { text: string; raiseForStatus(): void; }; response.raiseForStatus(); const $ = load(response.text); const contentParts: string[] = []; const overviewHeader = $('h2#pkg-overview, h3#pkg-overview').first(); if (overviewHeader.length > 0) { for (const sibling of overviewHeader.nextAll().toArray()) { const tagName = typeof (sibling as { tagName?: string }).tagName === "string" ? (sibling as { tagName?: string }).tagName!.toLowerCase() : ""; if (tagName === "h2" || tagName === "h3") { break; } if (tagName === "p" || tagName === "pre") { const text = $(sibling).text().trim(); if (text && !text.startsWith('import "')) { contentParts.push(text); } } } } let mainContent = $('div.container.main').first(); if (mainContent.length === 0) { mainContent = $('div#main').first(); } if (mainContent.length > 0) { const mainLines = mainContent .find("p, pre") .toArray() .map((node) => $(node).text().trim()) .filter((line) => line.length > 0) .slice(0, 50); contentParts.push(...mainLines); } const fullContent = contentParts.join("\n"); const { content, truncated } = byteTruncateUtf8(fullContent, maxBytes); return { package: packagePath, content, size_bytes: new TextEncoder().encode(content).length, source: "godocs", truncated, }; } catch (error) { if (error instanceof HttpStatusError) { if (error.response.status === 404) { return { package: normalizePackageName(packageName), content: "", error: "Package not found on GoDocs", size_bytes: 0, source: null, }; } return { package: normalizePackageName(packageName), content: "", error: `GoDocs returned ${error.response.status}`, size_bytes: 0, source: null, }; } if (error instanceof HttpError) { return { package: normalizePackageName(packageName), content: "", error: `GoDocs request failed: ${error}`, size_bytes: 0, source: null, }; } return { package: normalizePackageName(packageName), content: "", error: `Failed to fetch docs: ${error instanceof Error ? error.message : String(error)}`, size_bytes: 0, source: null, }; } } getTools(): Record { const godocsMetadata: ToolDefinition = { name: "godocs_metadata", label: "GoDocs Metadata", description: "Get Go package metadata (name, summary, URLs). For docs content, use fetch_godocs_docs.", promptSnippet: "Get Go package metadata for a package path or GoDocs URL.", parameters: Type.Object({ package: Type.String({ description: "Go package path or GoDocs URL" }), }), execute: async (_id: string, { package: packageName }: { package: string }) => { const result = await this._fetchMetadata(packageName); return serializeResponseWithMeta(result); }, }; if (!isFetchEnabled()) { return { godocs_metadata: godocsMetadata }; } const fetchGodocsDocs: ToolDefinition = { name: "fetch_godocs_docs", label: "Fetch GoDocs Docs", description: "Fetch Go package docs from godocs.io. Overview, functions, types, readable format.", promptSnippet: "Fetch Go package documentation content from GoDocs.", parameters: Type.Object({ package: Type.String({ description: "Go package path or GoDocs URL" }), max_bytes: Type.Optional(Type.Integer({ description: "Maximum content size in bytes" })), }), execute: async (_id: string, { package: packageName, max_bytes }: { package: string; max_bytes?: number }) => { const result = await this._fetchGodocsDocs(packageName, max_bytes ?? 20480); return chunkAndSerializeResponse(result, "content", this.chunkingManager); }, }; return { godocs_metadata: godocsMetadata, fetch_godocs_docs: fetchGodocsDocs, }; } }