import { Type } from "typebox"; import { HttpError, HttpResponseLike, HttpStatusError } from "../shared/http"; import { isFetchEnabled } from "../shared/env"; import { convertRstToMarkdown, extractSections, prioritizeSections } from "../shared/content"; import { chunkAndSerializeResponse, serializeResponseWithMeta } from "../shared/serialize"; import { chunkingManager } from "../shared/instances"; import type { HttpClientFactory, Provider, ProviderMetadata, ProviderResult, ToolDefinition } from "./base"; import { BaseProvider } from "./base"; function toUtf8Bytes(value: string): number { return Buffer.byteLength(value, "utf8"); } function safeJsonLoads(text: string): any { return JSON.parse(text); } export class PyPiProvider extends BaseProvider implements Provider { constructor(httpClientFactory: HttpClientFactory) { super(httpClientFactory); } getMetadata(): ProviderMetadata { const toolNames = ["pypi_metadata"]; if (isFetchEnabled()) { toolNames.push("fetch_pypi_docs"); } return { name: "pypi", description: "PyPI package metadata and documentation", exposeAsTool: true, toolNames, supportsLibrarySearch: true, requiredEnvVars: [], optionalEnvVars: ["VERIFIED_BY_PYPI"], toolTiers: { pypi_metadata: { tier: 2, deferRecommended: true, category: "metadata" }, fetch_pypi_docs: { tier: 3, deferRecommended: true, category: "fetch" }, }, }; } async searchLibrary(library: string, limit = 5): Promise { void limit; try { const data = await this._fetchMetadata(library); return { success: true, data, providerName: "pypi" }; } catch (error) { if (error instanceof HttpStatusError) { return { success: false, error: `PyPI returned ${error.response.status}`, providerName: "pypi", }; } if (error instanceof HttpError) { return { success: false, error: `PyPI request failed: ${error}`, providerName: "pypi", }; } return { success: false, error: `PyPI request failed: ${error instanceof Error ? error.message : String(error)}`, providerName: "pypi", }; } } async _checkVerification(packageName: string): Promise { const url = `https://pypi.org/project/${packageName}/`; try { const client = await this.httpClient(); const response = (await client.get(url)) as HttpResponseLike; response.raiseForStatus(); return response.text.includes('class="sidebar-section verified"'); } catch { return false; } } async _fetchMetadata(packageName: string, ignoreVerification = false): Promise> { if ((process.env.VERIFIED_BY_PYPI ?? "").toLowerCase() === "true" && !ignoreVerification) { const isVerified = await this._checkVerification(packageName); if (!isVerified) { return { name: packageName, error: `Project '${packageName}' is not verified by PyPI. Please ask the user if they want to trust this project.`, is_unverified: true, }; } } const url = `https://pypi.org/pypi/${packageName}/json`; const client = await this.httpClient(); const response = (await client.get(url)) as HttpResponseLike; response.raiseForStatus(); const payload = safeJsonLoads(response.text) as Record; const info = (payload.info && typeof payload.info === "object" ? payload.info : {}) as Record; const projectUrlsRaw = info.project_urls; const projectUrls = projectUrlsRaw && typeof projectUrlsRaw === "object" && !Array.isArray(projectUrlsRaw) ? (projectUrlsRaw as Record) : {}; const docsUrl = typeof projectUrls.Documentation === "string" ? projectUrls.Documentation : null; return { name: info.name, summary: (info.summary ?? "") as string, version: info.version, home_page: info.home_page, docs_url: docsUrl ?? null, project_urls: projectUrls, description: (info.description ?? "") as string, }; } _extractGithubUrl(projectUrls: Record): string | null { if (!projectUrls || typeof projectUrls !== "object") { return null; } for (const key of ["Source", "Repository", "Homepage", "Code"]) { const url = projectUrls[key]; if (typeof url === "string" && url.includes("github.com")) { return url; } } return null; } async _fetchPypiDocs( packageName: string, maxBytes = 20480, ignoreVerification = false, ): Promise> { try { const metadata = await this._fetchMetadata(packageName, ignoreVerification); if (metadata.error) { return { package: packageName, content: "", error: metadata.error, size_bytes: 0, source: null, }; } let content = typeof metadata.description === "string" ? metadata.description : ""; const source = "pypi"; if (content && (content.includes(".. ") || content.slice(0, 200).includes("::"))) { content = convertRstToMarkdown(content); } if (toUtf8Bytes(content) < 500) { const repoUrl = this._extractGithubUrl( (metadata.project_urls && typeof metadata.project_urls === "object" && !Array.isArray(metadata.project_urls) ? (metadata.project_urls as Record) : {}) as Record, ); if (repoUrl) { void repoUrl; } } const sections = extractSections(content); const finalContent = sections.length ? prioritizeSections(sections, maxBytes) : content.slice(0, maxBytes); return { package: packageName, content: finalContent, size_bytes: toUtf8Bytes(finalContent), source, truncated: toUtf8Bytes(content) > maxBytes, }; } catch (error) { if (error instanceof HttpStatusError) { return { package: packageName, content: "", error: `PyPI returned ${error.response.status}`, size_bytes: 0, source: null, }; } return { package: packageName, content: "", error: `Failed to fetch docs: ${error instanceof Error ? error.message : String(error)}`, size_bytes: 0, source: null, }; } } getTools(): Record { const pypiMetadataDescription = [ "Get PyPI package metadata (name, version, URLs). For docs content, use fetch_pypi_docs.", "", "When: Need package info or external doc links", 'Args: package="requests", ignore_verification=False', 'Ex: pypi_metadata("flask") → {name, version, docs_url, homepage}', ].join("\n"); const fetchPypiDocsDescription = [ "Fetch Python package docs from PyPI README. Extracts relevant sections, converts RST to MD.", "", "When: Need installation, usage examples, or API quickstart", "Not for: External doc sites (use pypi_metadata docs_url + WebFetch)", 'Args: package="requests", max_bytes=20480, ignore_verification=False', 'Ex: fetch_pypi_docs("numpy") → formatted README content', ].join("\n"); const pypiMetadata: ToolDefinition = { name: "pypi_metadata", label: "PyPI metadata", description: pypiMetadataDescription, promptSnippet: 'pypi_metadata("requests")', parameters: Type.Object({ package: Type.String({ description: "Package name" }), ignore_verification: Type.Optional(Type.Boolean()), }), execute: async (_id: string, { package: packageName, ignore_verification = false }: { package: string; ignore_verification?: boolean }) => { const result = await this._fetchMetadata(packageName, ignore_verification); return serializeResponseWithMeta(result); }, }; const tools: Record = { pypi_metadata: pypiMetadata, }; if (isFetchEnabled()) { tools.fetch_pypi_docs = { name: "fetch_pypi_docs", label: "Fetch PyPI docs", description: fetchPypiDocsDescription, promptSnippet: 'fetch_pypi_docs("numpy")', parameters: Type.Object({ package: Type.String({ description: "Package name" }), max_bytes: Type.Optional(Type.Integer({ description: "Max bytes" })), ignore_verification: Type.Optional(Type.Boolean()), }), execute: async (_id: string, { package: packageName, max_bytes = 20480, ignore_verification = false, }: { package: string; max_bytes?: number; ignore_verification?: boolean }) => { const result = await this._fetchPypiDocs(packageName, max_bytes, ignore_verification); return chunkAndSerializeResponse(result as Record, "content", chunkingManager); }, }; } return tools; } } export { PyPiProvider as PyPIProvider }; declare module "./crates" { interface CratesProvider { _searchCrates(query: string, perPage?: number): Promise>; _getCrateMetadata(crateName: string): Promise>; } }