import { Type } from "typebox"; import { HttpError, HttpResponseLike, HttpStatusError, USER_AGENT } from "../shared/http"; import { convertRelativeUrls } from "../shared/content"; import { chunkingManager } from "../shared/instances"; import { chunkAndSerializeResponse, serializeResponseWithMeta } from "../shared/serialize"; import { getGithubToken, isFetchEnabled } from "../shared/env"; import type { HttpClientFactory, Provider, ProviderMetadata, ProviderResult, ToolDefinition, } from "./base"; import { BaseProvider } from "./base"; function safeJsonLoads(text: string): any { return JSON.parse(text); } function toUtf8Bytes(value: string): number { return Buffer.byteLength(value, "utf8"); } function decodeUtf8FromBase64(content: string): string { const bytes = Buffer.from(content, "base64"); return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } function truncateUtf8(text: string, maxBytes: number): { text: string; truncated: boolean } { if (maxBytes <= 0) { return { text: "", truncated: toUtf8Bytes(text) > 0 }; } const encoded = Buffer.from(text, "utf8"); if (encoded.length <= maxBytes) { return { text, truncated: false }; } let slice = encoded.subarray(0, maxBytes); const decoder = new TextDecoder("utf-8", { fatal: true }); while (slice.length > 0) { try { return { text: decoder.decode(slice), truncated: true }; } catch { slice = slice.subarray(0, slice.length - 1); } } return { text: "", truncated: true }; } function encodePath(path: string): string { if (!path) return ""; return path .split("/") .filter((segment, index) => segment.length > 0 || index === 0) .map((segment) => encodeURIComponent(segment)) .join("/"); } function encodeRepoPath(owner: string, repo: string): string { return `${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`; } async function getResponse( client: { get(url: string, opts?: { headers?: Record; params?: Record }): Promise }, url: string, opts?: { headers?: Record; params?: Record }, ): Promise { return (await client.get(url, opts)) as HttpResponseLike; } function splitRepo(repo: string): { owner: string; repo: string } | null { const parts = repo.split("/", 2); if (parts.length !== 2 || !parts[0] || !parts[1]) { return null; } return { owner: parts[0], repo: parts[1] }; } export class GitHubProvider extends BaseProvider implements Provider { constructor(httpClientFactory: HttpClientFactory) { super(httpClientFactory); } getMetadata(): ProviderMetadata { const toolNames = ["github_repo_search", "github_code_search"]; if (isFetchEnabled()) { toolNames.push( "fetch_github_readme", "list_repo_contents", "get_file_content", "get_repo_tree", "get_commit_diff", "list_github_packages", "get_package_versions", ); } return { name: "github", description: "GitHub repository and code search, file browsing, and content fetching", exposeAsTool: true, toolNames, supportsLibrarySearch: true, requiredEnvVars: [], optionalEnvVars: ["GITHUB_TOKEN", "GITHUB_AUTH"], toolTiers: { github_repo_search: { tier: 1, deferRecommended: false, category: "search" }, github_code_search: { tier: 2, deferRecommended: true, category: "search" }, fetch_github_readme: { tier: 3, deferRecommended: true, category: "fetch" }, list_repo_contents: { tier: 3, deferRecommended: true, category: "fetch" }, get_file_content: { tier: 3, deferRecommended: true, category: "fetch" }, get_repo_tree: { tier: 3, deferRecommended: true, category: "fetch" }, get_commit_diff: { tier: 4, deferRecommended: true, category: "fetch" }, list_github_packages: { tier: 5, deferRecommended: true, category: "fetch" }, get_package_versions: { tier: 5, deferRecommended: true, category: "fetch" }, }, }; } async searchLibrary(library: string, limit = 5): Promise { try { const data = await this._searchRepos(library, limit); return { success: true, data, providerName: "github" }; } catch (error) { if (error instanceof HttpStatusError) { if (error.response.status === 404) { return { success: false, error: null, providerName: "github" }; } return { success: false, error: `GitHub API returned ${error.response.status}`, providerName: "github", }; } if (error instanceof HttpError) { return { success: false, error: `GitHub API request failed: ${error}`, providerName: "github", }; } return { success: false, error: `GitHub API request failed: ${error}`, providerName: "github", }; } } private _getHeaders(): Record { const headers: Record = { "User-Agent": USER_AGENT, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", }; const token = getGithubToken(); if (token) { headers.Authorization = `token ${token}`; } return headers; } async _searchRepos(query: string, limit = 5): Promise> { const client = await this.httpClient(); const response = await getResponse(client, "https://api.github.com/search/repositories", { headers: this._getHeaders(), params: { q: query, per_page: limit, }, }); response.raiseForStatus(); const payload = safeJsonLoads(response.text) as Record; const items = Array.isArray(payload.items) ? payload.items : []; const results = items.slice(0, limit).map((item) => ({ name: (item as Record).full_name ?? null, description: ((item as Record).description ?? "") as string, stars: (item as Record).stargazers_count ?? 0, url: (item as Record).html_url ?? null, default_branch: (item as Record).default_branch ?? null, })); return { query, results, total: payload.total_count ?? 0, source: "github", }; } async _searchCode(query: string, limit = 5): Promise> { const client = await this.httpClient(); const response = await getResponse(client, "https://api.github.com/search/code", { headers: this._getHeaders(), params: { q: query, per_page: limit, }, }); response.raiseForStatus(); const payload = safeJsonLoads(response.text) as Record; const items = Array.isArray(payload.items) ? payload.items : []; const results = items.slice(0, limit).map((item) => ({ name: (item as Record).name ?? null, path: (item as Record).path ?? null, repository: ((item as Record).repository as Record | undefined)?.full_name ?? null, url: (item as Record).html_url ?? null, })); return { query, results, total: payload.total_count ?? 0, source: "github", }; } async _fetchGithubReadme(owner: string, repo: string, maxBytes = 20480): Promise> { try { const client = await this.httpClient(); const response = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}/readme`, { headers: this._getHeaders(), }); response.raiseForStatus(); const data = safeJsonLoads(response.text) as Record; let content = decodeUtf8FromBase64(String(data.content ?? "")); const readmePath = typeof data.path === "string" ? data.path : ""; let baseUrl = `https://github.com/${owner}/${repo}/blob/main`; if (readmePath && readmePath.includes("/")) { const dirPath = readmePath.split("/").slice(0, -1).join("/"); baseUrl = `${baseUrl}/${dirPath}`; } content = convertRelativeUrls(content, baseUrl); const truncated = toUtf8Bytes(content) > maxBytes; if (truncated) { content = truncateUtf8(content, maxBytes).text; } return { repository: `${owner}/${repo}`, content, size_bytes: toUtf8Bytes(content), source: "github_readme", readme_path: readmePath, truncated, }; } catch (error) { if (error instanceof HttpStatusError) { return { repository: `${owner}/${repo}`, content: "", error: `GitHub returned ${error.response.status}`, size_bytes: 0, source: null, }; } return { repository: `${owner}/${repo}`, content: "", error: `Failed to fetch README: ${error}`, size_bytes: 0, source: null, }; } } async _listRepoContents(owner: string, repo: string, path = ""): Promise> { try { const client = await this.httpClient(); const response = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}/contents/${encodePath(path)}`, { headers: this._getHeaders(), }); response.raiseForStatus(); const data = safeJsonLoads(response.text) as Record | Record[]; const items = Array.isArray(data) ? data : [data]; const contents = items.map((item) => ({ name: (item as Record).name ?? null, path: (item as Record).path ?? null, type: (item as Record).type ?? null, size: (item as Record).size ?? null, sha: (item as Record).sha ?? null, url: (item as Record).html_url ?? null, download_url: (item as Record).download_url ?? null, })); return { repository: `${owner}/${repo}`, path: path || "/", contents, count: contents.length, }; } catch (error) { if (error instanceof HttpStatusError) { return { repository: `${owner}/${repo}`, path, contents: [], error: `GitHub returned ${error.response.status}`, }; } return { repository: `${owner}/${repo}`, path, contents: [], error: `Failed to list contents: ${error}`, }; } } async _getFileContent(owner: string, repo: string, path: string, maxBytes = 102400): Promise> { try { const client = await this.httpClient(); const response = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}/contents/${encodePath(path)}`, { headers: this._getHeaders(), }); response.raiseForStatus(); const data = safeJsonLoads(response.text) as Record; if (data.type !== "file") { return { repository: `${owner}/${repo}`, path, content: "", error: `Path is a ${data.type}, not a file`, }; } let content: string; try { content = decodeUtf8FromBase64(String(data.content ?? "")); } catch { return { repository: `${owner}/${repo}`, path, content: "", error: "File appears to be binary", size_bytes: data.size ?? 0, encoding: data.encoding ?? null, }; } let truncated = false; if (toUtf8Bytes(content) > maxBytes) { content = truncateUtf8(content, maxBytes).text; truncated = true; } return { repository: `${owner}/${repo}`, path, content, size_bytes: toUtf8Bytes(content), truncated, sha: data.sha ?? null, url: data.html_url ?? null, }; } catch (error) { if (error instanceof HttpStatusError) { return { repository: `${owner}/${repo}`, path, content: "", error: `GitHub returned ${error.response.status}`, }; } return { repository: `${owner}/${repo}`, path, content: "", error: `Failed to get file content: ${error}`, }; } } async _getRepoTree( owner: string, repo: string, recursive = false, maxItems = 1000, ): Promise> { try { const headers = this._getHeaders(); const client = await this.httpClient(); const repoResponse = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}`, { headers, }); repoResponse.raiseForStatus(); const repoData = safeJsonLoads(repoResponse.text) as Record; const defaultBranch = typeof repoData.default_branch === "string" && repoData.default_branch ? repoData.default_branch : "main"; const treeResponse = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}/git/trees/${encodeURIComponent(defaultBranch)}`, { headers, params: recursive ? { recursive: 1 } : undefined, }, ); treeResponse.raiseForStatus(); const data = safeJsonLoads(treeResponse.text) as Record; const treeItems = (Array.isArray(data.tree) ? data.tree : []).slice(0, maxItems); const tree = treeItems.map((item) => ({ path: (item as Record).path ?? null, type: (item as Record).type ?? null, size: (item as Record).size ?? null, sha: (item as Record).sha ?? null, url: (item as Record).url ?? null, })); return { repository: `${owner}/${repo}`, branch: defaultBranch, tree, count: tree.length, truncated: Boolean(data.truncated ?? false) || treeItems.length >= maxItems, }; } catch (error) { if (error instanceof HttpStatusError) { return { repository: `${owner}/${repo}`, tree: [], error: `GitHub returned ${error.response.status}`, }; } return { repository: `${owner}/${repo}`, tree: [], error: `Failed to get repository tree: ${error}`, }; } } async _getCommitDiff(owner: string, repo: string, base: string, head: string): Promise> { try { const headers = this._getHeaders(); headers.Accept = "application/vnd.github.diff"; const client = await this.httpClient(); const response = await getResponse(client, `https://api.github.com/repos/${encodeRepoPath(owner, repo)}/compare/${encodeURIComponent(base)}...${encodeURIComponent(head)}`, { headers, }); response.raiseForStatus(); const diffContent = response.text; return { repository: `${owner}/${repo}`, base, head, diff: diffContent, size_bytes: toUtf8Bytes(diffContent), }; } catch (error) { if (error instanceof HttpStatusError) { return { repository: `${owner}/${repo}`, base, head, diff: "", error: `GitHub returned ${error.response.status}`, }; } return { repository: `${owner}/${repo}`, base, head, diff: "", error: `Failed to get diff: ${error}`, }; } } async _listGithubPackages(owner: string, packageType = "container"): Promise[]> { const headers = this._getHeaders(); const client = await this.httpClient(); const endpoints = [ `https://api.github.com/users/${encodeURIComponent(owner)}/packages?package_type=${encodeURIComponent(packageType)}`, `https://api.github.com/orgs/${encodeURIComponent(owner)}/packages?package_type=${encodeURIComponent(packageType)}`, ]; let data: Record[] = []; let error: unknown = null; for (const url of endpoints) { try { const response = await getResponse(client, url, { headers }); if (response.status === 200) { data = safeJsonLoads(response.text) as Record[]; error = null; break; } if (response.status === 404) { continue; } response.raiseForStatus(); } catch (caught) { if (caught instanceof HttpError) { error = caught; continue; } error = caught; continue; } } if (!data && error) { throw error instanceof Error ? error : new Error(`Could not find packages for ${owner}`); } return data.map((item) => ({ name: (item as Record).name ?? null, package_type: (item as Record).package_type ?? null, owner: ((item as Record).owner as Record | undefined)?.login ?? null, repository: ((item as Record).repository as Record | undefined)?.full_name ?? null, url: (item as Record).html_url ?? null, version_count: (item as Record).version_count ?? 0, visibility: (item as Record).visibility ?? null, })); } async _getPackageVersions(owner: string, packageType: string, packageName: string): Promise[]> { const headers = this._getHeaders(); const client = await this.httpClient(); const endpoints = [ `https://api.github.com/users/${encodeURIComponent(owner)}/packages/${encodeURIComponent(packageType)}/${encodeURIComponent(packageName)}/versions`, `https://api.github.com/orgs/${encodeURIComponent(owner)}/packages/${encodeURIComponent(packageType)}/${encodeURIComponent(packageName)}/versions`, ]; let data: Record[] = []; let error: unknown = null; for (const url of endpoints) { try { const response = await getResponse(client, url, { headers }); if (response.status === 200) { data = safeJsonLoads(response.text) as Record[]; error = null; break; } if (response.status === 404) { continue; } response.raiseForStatus(); } catch (caught) { if (caught instanceof HttpError) { error = caught; continue; } error = caught; continue; } } if (!data && error) { throw error instanceof Error ? error : new Error(`Could not find versions for ${packageName}`); } return data.map((item) => { const metadata = ((item as Record).metadata as Record | undefined) ?? {}; const containerMetadata = (metadata.container as Record | undefined) ?? {}; const tags = Array.isArray(containerMetadata.tags) ? containerMetadata.tags : []; return { id: (item as Record).id ?? null, name: (item as Record).name ?? null, url: (item as Record).html_url ?? null, created_at: (item as Record).created_at ?? null, updated_at: (item as Record).updated_at ?? null, tags, }; }); } getTools(): Record { const tools: Record = { github_repo_search: { name: "github_repo_search", label: "GitHub repo search", description: "Search GitHub repositories by keyword. Returns names, descriptions, stars, URLs, and default branches.", promptSnippet: 'github_repo_search("http framework")', parameters: Type.Object({ query: Type.String({ description: "Search query" }), limit: Type.Optional(Type.Integer({ description: "Maximum results to return" })), }), execute: async (_id: string, { query, limit = 5 }: { query: string; limit?: number }) => { const result = await this._searchRepos(query, limit); return serializeResponseWithMeta(result); }, }, github_code_search: { name: "github_code_search", label: "GitHub code search", description: "Search GitHub code by keyword. Returns file paths and repository names.", promptSnippet: 'github_code_search("async def fetch")', parameters: Type.Object({ query: Type.String({ description: "Search query" }), limit: Type.Optional(Type.Integer({ description: "Maximum results to return" })), }), execute: async (_id: string, { query, limit = 5 }: { query: string; limit?: number }) => { const result = await this._searchCode(query, limit); return serializeResponseWithMeta(result); }, }, }; if (isFetchEnabled()) { tools.fetch_github_readme = { name: "fetch_github_readme", label: "Fetch GitHub README", description: "Fetch a README file from a GitHub repository and convert relative links to absolute URLs.", promptSnippet: 'fetch_github_readme("psf/requests")', parameters: Type.Object({ repo: Type.String({ description: "Repository in owner/repo form" }), max_bytes: Type.Optional(Type.Integer({ description: "Maximum bytes to return" })), }), execute: async (_id: string, { repo, max_bytes = 20480 }: { repo: string; max_bytes?: number }) => { const parsed = splitRepo(repo); if (!parsed) { return chunkAndSerializeResponse( { repository: repo, content: "", error: "Invalid repo format. Use 'owner/repo'", size_bytes: 0, source: null, }, "content", chunkingManager, ); } const result = await this._fetchGithubReadme(parsed.owner, parsed.repo, max_bytes); return chunkAndSerializeResponse(result, "content", chunkingManager); }, }; tools.list_repo_contents = { name: "list_repo_contents", label: "List GitHub repo contents", description: "List files and directories in a GitHub repository path.", promptSnippet: 'list_repo_contents("psf/requests", "requests")', parameters: Type.Object({ repo: Type.String({ description: "Repository in owner/repo form" }), path: Type.Optional(Type.String({ description: "Path inside repository" })), }), execute: async (_id: string, { repo, path = "" }: { repo: string; path?: string }) => { const parsed = splitRepo(repo); if (!parsed) { return serializeResponseWithMeta({ repository: repo, path, contents: [], error: "Invalid repo format. Use 'owner/repo'", }); } const result = await this._listRepoContents(parsed.owner, parsed.repo, path); return serializeResponseWithMeta(result); }, }; tools.get_file_content = { name: "get_file_content", label: "Get GitHub file content", description: "Read a UTF-8 file from a GitHub repository.", promptSnippet: 'get_file_content("psf/requests", "requests/api.py")', parameters: Type.Object({ repo: Type.String({ description: "Repository in owner/repo form" }), path: Type.String({ description: "File path inside repository" }), max_bytes: Type.Optional(Type.Integer({ description: "Maximum bytes to return" })), }), execute: async (_id: string, { repo, path, max_bytes = 102400 }: { repo: string; path: string; max_bytes?: number }) => { const parsed = splitRepo(repo); if (!parsed) { return serializeResponseWithMeta({ repository: repo, path, content: "", error: "Invalid repo format. Use 'owner/repo'", }); } const result = await this._getFileContent(parsed.owner, parsed.repo, path, max_bytes); return serializeResponseWithMeta(result); }, }; tools.get_repo_tree = { name: "get_repo_tree", label: "Get GitHub repo tree", description: "Get the file tree for a GitHub repository.", promptSnippet: 'get_repo_tree("psf/requests", recursive=true)', parameters: Type.Object({ repo: Type.String({ description: "Repository in owner/repo form" }), recursive: Type.Optional(Type.Boolean({ description: "Fetch the tree recursively" })), max_items: Type.Optional(Type.Integer({ description: "Maximum tree items to return" })), }), execute: async (_id: string, { repo, recursive = false, max_items = 1000 }: { repo: string; recursive?: boolean; max_items?: number }) => { const parsed = splitRepo(repo); if (!parsed) { return serializeResponseWithMeta({ repository: repo, tree: [], error: "Invalid repo format. Use 'owner/repo'", }); } const result = await this._getRepoTree(parsed.owner, parsed.repo, recursive, max_items); return serializeResponseWithMeta(result); }, }; tools.get_commit_diff = { name: "get_commit_diff", label: "Get GitHub commit diff", description: "Get the diff between two commits, branches, or tags.", promptSnippet: 'get_commit_diff("psf/requests", "v2.28.0", "v2.28.1")', parameters: Type.Object({ repo: Type.String({ description: "Repository in owner/repo form" }), base: Type.String({ description: "Base commit, branch, or tag" }), head: Type.String({ description: "Head commit, branch, or tag" }), }), execute: async (_id: string, { repo, base, head }: { repo: string; base: string; head: string }) => { const parsed = splitRepo(repo); if (!parsed) { return serializeResponseWithMeta({ repository: repo, base, head, diff: "", error: "Invalid repo format. Use 'owner/repo'", }); } const result = await this._getCommitDiff(parsed.owner, parsed.repo, base, head); return serializeResponseWithMeta(result); }, }; tools.list_github_packages = { name: "list_github_packages", label: "List GitHub packages", description: "List packages published by a user or organization on GitHub.", promptSnippet: 'list_github_packages("github")', parameters: Type.Object({ owner: Type.String({ description: "GitHub username or organization" }), package_type: Type.Optional(Type.String({ description: "Package type" })), }), execute: async (_id: string, { owner, package_type = "container" }: { owner: string; package_type?: string }) => { try { const packages = await this._listGithubPackages(owner, package_type); return serializeResponseWithMeta({ owner, package_type, packages, count: packages.length, }); } catch (error) { return serializeResponseWithMeta({ owner, error: `Failed to list packages: ${error}`, }); } }, }; tools.get_package_versions = { name: "get_package_versions", label: "Get GitHub package versions", description: "Get versions for a GitHub package.", promptSnippet: 'get_package_versions("github", "container", "rtfd")', parameters: Type.Object({ owner: Type.String({ description: "GitHub username or organization" }), package_type: Type.String({ description: "Package type" }), package_name: Type.String({ description: "Package name" }), }), execute: async (_id: string, { owner, package_type, package_name, }: { owner: string; package_type: string; package_name: string; }) => { try { const versions = await this._getPackageVersions(owner, package_type, package_name); return serializeResponseWithMeta({ owner, package_name, versions, count: versions.length, }); } catch (error) { return serializeResponseWithMeta({ owner, package_name, error: `Failed to get versions: ${error}`, }); } }, }; } return tools; } }