import { load } from "cheerio"; import { Type } from "typebox"; import { BaseProvider, type HttpClientFactory, type ProviderMetadata, type ProviderResult, type ToolDefinition } from "./base"; import { USER_AGENT, HttpError, HttpStatusError } from "../shared/http"; import { getGithubToken, isFetchEnabled } from "../shared/env"; import { extractSections, htmlToMarkdown, prioritizeSections } from "../shared/content"; import { chunkAndSerializeResponse, serializeResponseWithMeta } from "../shared/serialize"; import { chunkingManager } from "../shared/instances"; interface GcpServiceInfo { name: string; url: string; api: string; description: string; } interface SearchResult { name: string; description: string; api: string; docs_url: string; source: string; } interface SearchServicesState { results: SearchResult[]; githubStatus: number | null; githubDetail: string; githubHttpError: Error | null; otherError: unknown; } const GCP_SERVICE_DOCS: Record = { storage: { name: "Cloud Storage", url: "https://cloud.google.com/storage/docs", api: "storage.googleapis.com", description: "Object storage for companies of all sizes", }, compute: { name: "Compute Engine", url: "https://cloud.google.com/compute/docs", api: "compute.googleapis.com", description: "Virtual machines running in Google's data centers", }, bigquery: { name: "BigQuery", url: "https://cloud.google.com/bigquery/docs", api: "bigquery.googleapis.com", description: "Serverless, highly scalable, and cost-effective multicloud data warehouse", }, cloudfunctions: { name: "Cloud Functions", url: "https://cloud.google.com/functions/docs", api: "cloudfunctions.googleapis.com", description: "Event-driven serverless compute platform", }, run: { name: "Cloud Run", url: "https://cloud.google.com/run/docs", api: "run.googleapis.com", description: "Fully managed compute platform for deploying and scaling containerized applications", }, pubsub: { name: "Pub/Sub", url: "https://cloud.google.com/pubsub/docs", api: "pubsub.googleapis.com", description: "Asynchronous and scalable messaging service", }, firestore: { name: "Cloud Firestore", url: "https://cloud.google.com/firestore/docs", api: "firestore.googleapis.com", description: "NoSQL document database for mobile, web, and server development", }, datastore: { name: "Cloud Datastore", url: "https://cloud.google.com/datastore/docs", api: "datastore.googleapis.com", description: "Highly scalable NoSQL database for web and mobile applications", }, bigtable: { name: "Cloud Bigtable", url: "https://cloud.google.com/bigtable/docs", api: "bigtable.googleapis.com", description: "Fully managed, scalable NoSQL database service for large analytical and operational workloads", }, spanner: { name: "Cloud Spanner", url: "https://cloud.google.com/spanner/docs", api: "spanner.googleapis.com", description: "Fully managed, mission-critical, relational database service with transactional consistency", }, sql: { name: "Cloud SQL", url: "https://cloud.google.com/sql/docs", api: "sqladmin.googleapis.com", description: "Fully managed relational database service for MySQL, PostgreSQL, and SQL Server", }, gke: { name: "Google Kubernetes Engine", url: "https://cloud.google.com/kubernetes-engine/docs", api: "container.googleapis.com", description: "Managed Kubernetes service for running containerized applications", }, appengine: { name: "App Engine", url: "https://cloud.google.com/appengine/docs", api: "appengine.googleapis.com", description: "Platform for building scalable web applications and mobile backends", }, vision: { name: "Cloud Vision API", url: "https://cloud.google.com/vision/docs", api: "vision.googleapis.com", description: "Image analysis powered by machine learning", }, speech: { name: "Cloud Speech-to-Text", url: "https://cloud.google.com/speech-to-text/docs", api: "speech.googleapis.com", description: "Speech to text conversion powered by machine learning", }, translate: { name: "Cloud Translation API", url: "https://cloud.google.com/translate/docs", api: "translate.googleapis.com", description: "Dynamically translate between languages", }, monitoring: { name: "Cloud Monitoring", url: "https://cloud.google.com/monitoring/docs", api: "monitoring.googleapis.com", description: "Visibility into the performance, availability, and health of your applications", }, logging: { name: "Cloud Logging", url: "https://cloud.google.com/logging/docs", api: "logging.googleapis.com", description: "Store, search, analyze, monitor, and alert on logging data and events", }, iam: { name: "Identity and Access Management", url: "https://cloud.google.com/iam/docs", api: "iam.googleapis.com", description: "Manage access control by defining who (identity) has what access (role) for which resource", }, secretmanager: { name: "Secret Manager", url: "https://cloud.google.com/secret-manager/docs", api: "secretmanager.googleapis.com", description: "Store and manage access to secrets", }, }; function toUtf8Bytes(value: string): number { return Buffer.byteLength(value, "utf8"); } function byteTruncateUtf8(text: string, maxBytes: number): string { const bytes = new TextEncoder().encode(text); if (bytes.length <= maxBytes) { return text; } if (maxBytes <= 0) { return ""; } let cut = maxBytes; while (cut > 0) { try { return new TextDecoder().decode(bytes.slice(0, cut)); } catch { cut -= 1; } } return ""; } function safeJsonLoads(text: string): any { return JSON.parse(text); } export class GcpProvider extends BaseProvider { private readonly chunkingManager = chunkingManager; constructor(httpClientFactory: HttpClientFactory) { super(httpClientFactory); } getMetadata(): ProviderMetadata { const toolNames = ["search_gcp_services"]; if (isFetchEnabled()) { toolNames.push("fetch_gcp_service_docs"); } return { name: "gcp", description: "Google Cloud Platform API and service documentation", exposeAsTool: true, toolNames, supportsLibrarySearch: true, requiredEnvVars: [], optionalEnvVars: ["GITHUB_TOKEN", "GITHUB_AUTH"], toolTiers: { search_gcp_services: { tier: 4, deferRecommended: true, category: "search" }, fetch_gcp_service_docs: { tier: 4, deferRecommended: true, category: "fetch" }, }, }; } async searchLibrary(library: string, limit = 5): Promise { const state = await this._searchServicesDetailed(library, limit); if (state.results.length > 0) { return { success: true, data: state.results, providerName: "gcp" }; } if (state.githubStatus !== null) { if (state.githubStatus === 404) { return { success: false, error: null, providerName: "gcp" }; } return { success: false, error: `GitHub API returned ${state.githubStatus}: ${state.githubDetail}`, providerName: "gcp", }; } if (state.githubHttpError) { return { success: false, error: `GitHub API request failed: ${state.githubHttpError}`, providerName: "gcp", }; } if (state.otherError) { return { success: false, error: null, providerName: "gcp" }; } return { success: false, error: null, providerName: "gcp" }; } private _normalizeServiceName(query: string): string | null { const queryLower = query.toLowerCase().trim(); if (queryLower in GCP_SERVICE_DOCS) { return queryLower; } let normalized = queryLower; for (const prefix of ["google cloud ", "cloud ", "gcp ", "google "]) { if (normalized.startsWith(prefix)) { normalized = normalized.slice(prefix.length); } } if (normalized in GCP_SERVICE_DOCS) { return normalized; } const aliases: Record = { kubernetes: "gke", k8s: "gke", functions: "cloudfunctions", cloudrun: "run", "pub/sub": "pubsub", "cloud functions": "cloudfunctions", "cloud run": "run", "cloud storage": "storage", "compute engine": "compute", "big query": "bigquery", "app engine": "appengine", "secret manager": "secretmanager", }; return aliases[normalized] ?? null; } private async _searchServicesDetailed(query: string, limit = 5): Promise { const results: SearchResult[] = []; let githubStatus: number | null = null; let githubDetail = ""; let githubHttpError: Error | null = null; let otherError: unknown = null; const normalized = this._normalizeServiceName(query); if (normalized && normalized in GCP_SERVICE_DOCS) { const serviceInfo = GCP_SERVICE_DOCS[normalized]; results.push({ name: serviceInfo.name, description: serviceInfo.description, api: serviceInfo.api, docs_url: serviceInfo.url, source: "gcp_mapping", }); } const queryLower = query.toLowerCase(); const queryWords = queryLower.split(/\s+/).filter((word) => word.length > 0); if (queryWords.length > 1 && results.length === 0) { const firstWordNormalized = this._normalizeServiceName(queryWords[0]); if (firstWordNormalized && firstWordNormalized in GCP_SERVICE_DOCS) { const serviceInfo = GCP_SERVICE_DOCS[firstWordNormalized]; const topic = queryWords.slice(1).join(" "); results.push({ name: serviceInfo.name, description: `${serviceInfo.description} (searching for: ${topic})`, api: serviceInfo.api, docs_url: serviceInfo.url, source: "gcp_mapping_contextual", }); } } for (const [key, serviceInfo] of Object.entries(GCP_SERVICE_DOCS)) { if (normalized === key) { continue; } if (queryWords.length > 1 && this._normalizeServiceName(queryWords[0]) === key) { continue; } let queryMatches = false; for (const word of queryWords) { if (word.length > 2) { const wordLower = word.toLowerCase(); if ( serviceInfo.name.toLowerCase().includes(wordLower) || key.includes(wordLower) || serviceInfo.description.toLowerCase().includes(wordLower) ) { queryMatches = true; break; } } } if (queryMatches) { results.push({ name: serviceInfo.name, description: serviceInfo.description, api: serviceInfo.api, docs_url: serviceInfo.url, source: "gcp_mapping", }); } if (results.length >= limit) { break; } } if (results.length > 0) { return { results: results.slice(0, limit), githubStatus, githubDetail, githubHttpError, otherError }; } try { const cloudResults = await this._searchCloudGoogleCom(query, limit); if (cloudResults.length > 0) { results.push(...cloudResults); } } catch (error) { otherError = error; } if (results.length >= limit) { return { results: results.slice(0, limit), githubStatus, githubDetail, githubHttpError, otherError }; } try { const githubResults = await this._searchGithubGoogleapis(query, limit); results.push(...githubResults); } catch (error) { if (error instanceof HttpStatusError) { githubStatus = error.response.status; githubDetail = error.response.text.slice(0, 200); } else if (error instanceof HttpError) { githubHttpError = error; } else { otherError = error; } } return { results: results.slice(0, limit), githubStatus, githubDetail, githubHttpError, otherError }; } private async _searchGithubGoogleapis(query: string, limit = 5): Promise { const headers = this._getGithubHeaders(); const searchQuery = `${query} repo:googleapis/googleapis path:google/cloud`; const params = { q: searchQuery, per_page: String(limit) }; const client = await this.httpClient(); const response = (await client.get("https://api.github.com/search/code", { params, headers })) as { text: string; raiseForStatus(): void; }; response.raiseForStatus(); const payload = safeJsonLoads(response.text) as Record; const items = Array.isArray(payload.items) ? payload.items : []; const results: SearchResult[] = []; for (const item of items) { if (!item || typeof item !== "object") { continue; } const itemRecord = item as Record; const path = typeof itemRecord.path === "string" ? itemRecord.path : ""; const parts = path.split("/"); if (parts.length >= 3 && parts[0] === "google" && parts[1] === "cloud") { const serviceName = parts[2]; const serviceInfo = GCP_SERVICE_DOCS[serviceName]; if (serviceInfo) { results.push({ name: serviceInfo.name, description: serviceInfo.description, api: serviceInfo.api, docs_url: serviceInfo.url, source: "github_googleapis", }); } else { results.push({ name: serviceName.charAt(0).toUpperCase() + serviceName.slice(1), description: `Google Cloud ${serviceName} service`, api: `${serviceName}.googleapis.com`, docs_url: `https://cloud.google.com/${serviceName}/docs`, source: "github_googleapis", }); } } if (results.length >= limit) { break; } } return results; } private _getGithubHeaders(): Record { const headers: Record = { "User-Agent": USER_AGENT, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", }; const token = getGithubToken(); if (token) { headers.Authorization = `token ${token}`; } return headers; } private async _searchCloudGoogleCom(query: string, limit = 5): Promise { const url = `https://cloud.google.com/search?q=${query}`; const client = await this.httpClient(); const response = (await client.get(url, { headers: { "User-Agent": USER_AGENT } })) as { text: string; raiseForStatus(): void; }; response.raiseForStatus(); const $ = load(response.text); const results: SearchResult[] = []; const searchLinks = $('a[track-type="search-result"]').toArray(); for (const link of searchLinks) { const $link = $(link); const title = $link.text().trim(); let href = $link.attr("href") ?? ""; if (!href || !title) { continue; } if (href.startsWith("/")) { href = `https://cloud.google.com${href}`; } let description = `Search result for ${query}`; try { const container = $link.parent().parent(); const fullText = container.text().replace(/\s+/g, " ").trim(); const descText = fullText.replace(title, "").trim(); if (descText) { description = descText.length > 200 ? `${descText.slice(0, 200)}...` : descText; } } catch { // keep default description } results.push({ name: title, description, api: "", docs_url: href, source: "cloud_google_com", }); if (results.length >= limit) { break; } } return results; } private async _fetchServiceDocs(service: string, maxBytes = 20480): Promise> { try { const normalized = this._normalizeServiceName(service); let docsUrl: string; let serviceName: string; if (normalized && normalized in GCP_SERVICE_DOCS) { const serviceInfo = GCP_SERVICE_DOCS[normalized]; docsUrl = serviceInfo.url; serviceName = serviceInfo.name; } else { const searchResults = await this._searchServicesDetailed(service, 1); if (searchResults.results.length > 0) { const bestMatch = searchResults.results[0]; docsUrl = bestMatch.docs_url; serviceName = bestMatch.name; } else { const serviceSlug = service.toLowerCase().replace(/\s+/g, "-").replace(/_/g, "-"); docsUrl = `https://cloud.google.com/${serviceSlug}/docs`; serviceName = service; } } const client = await this.httpClient(); const response = (await client.get(docsUrl, { headers: { "User-Agent": USER_AGENT } })) as { text: string; raiseForStatus(): void; }; response.raiseForStatus(); const $ = load(response.text); let finalContent = ""; let mainContent = $("main").first(); if (!mainContent.length) { mainContent = $("div.devsite-article-body").first(); } if (!mainContent.length) { mainContent = $("article").first(); } if (mainContent.length > 0) { mainContent.find("nav, aside, footer, header, script, style").remove(); const htmlContent = $.html(mainContent) ?? mainContent.toString(); const markdownContent = htmlToMarkdown(htmlContent, docsUrl); const sections = extractSections(markdownContent); if (sections.length > 0) { finalContent = prioritizeSections(sections, maxBytes); } else if (toUtf8Bytes(markdownContent) > maxBytes) { finalContent = byteTruncateUtf8(markdownContent, maxBytes); } else { finalContent = markdownContent; } } else { finalContent = `Documentation for ${serviceName} is available at ${docsUrl}`; } return { service: serviceName, content: finalContent, size_bytes: toUtf8Bytes(finalContent), source: "gcp_docs", docs_url: docsUrl, truncated: toUtf8Bytes(finalContent) >= maxBytes, }; } catch (error) { if (error instanceof HttpStatusError) { if (error.response.status === 404) { return { service, content: "", error: "Service documentation not found", size_bytes: 0, source: null, }; } return { service, content: "", error: `GCP docs returned ${error.response.status}`, size_bytes: 0, source: null, }; } if (error instanceof HttpError) { return { service, content: "", error: `Failed to fetch docs: ${error}`, size_bytes: 0, source: null, }; } return { service, content: "", error: `Failed to process docs: ${String(error)}`, size_bytes: 0, source: null, }; } } getTools(): Record { const searchGcpServicesDescription = [ "Search GCP services by name/keywords. Searches local mapping, cloud.google.com, googleapis GitHub.", "", "When: Finding Google Cloud services or APIs", "See also: fetch_gcp_service_docs", "Note: GitHub fallback limited to 60 req/hour without token", 'Args: query="vertex ai", limit=5', 'Ex: search_gcp_services("bigquery") → services with docs links', ].join("\n"); const fetchGcpServiceDocsDescription = [ "Fetch GCP service docs from cloud.google.com. Extracts content, converts HTML to MD.", "", "When: Need detailed docs, guides, tutorials, API reference", "Works with: Exact names, abbreviations, multi-word queries", 'Args: service="vertex ai", max_bytes=20480', 'Ex: fetch_gcp_service_docs("Cloud Storage") → formatted docs', ].join("\n"); const tools: Record = { search_gcp_services: { name: "search_gcp_services", label: "Search GCP services", description: searchGcpServicesDescription, promptSnippet: 'search_gcp_services("bigquery")', parameters: Type.Object({ query: Type.String({ description: "Search query" }), limit: Type.Optional(Type.Integer({ description: "Maximum number of results" })), }), execute: async (_id: string, { query, limit = 5 }: { query: string; limit?: number }) => { const result = await this._searchServicesDetailed(query, limit); return serializeResponseWithMeta(result.results); }, }, }; if (isFetchEnabled()) { tools.fetch_gcp_service_docs = { name: "fetch_gcp_service_docs", label: "Fetch GCP service docs", description: fetchGcpServiceDocsDescription, promptSnippet: 'fetch_gcp_service_docs("Cloud Storage")', parameters: Type.Object({ service: Type.String({ description: "Service name" }), max_bytes: Type.Optional(Type.Integer({ description: "Maximum content size in bytes" })), }), execute: async (_id: string, { service, max_bytes = 20480 }: { service: string; max_bytes?: number }) => { const result = await this._fetchServiceDocs(service, max_bytes); return chunkAndSerializeResponse(result, "content", this.chunkingManager); }, }; } return tools; } }