import type { ToolDefinition } from '../../shared/index.ts' import { validateUrl } from '../../security/url' // ── In-memory cache (15-min TTL per URL) ── interface CacheEntry { content: string timestamp: number } const CACHE_TTL = 15 * 60 * 1000 // 15 minutes const cache = new Map() function getCached(url: string): string | undefined { const entry = cache.get(url) if (!entry) return undefined if (Date.now() - entry.timestamp > CACHE_TTL) { cache.delete(url) return undefined } return entry.content } function setCache(url: string, content: string): void { // Evict oldest entries if cache grows too large (max 200 URLs) if (cache.size >= 200) { const oldest = [...cache.entries()].sort((a, b) => a[1].timestamp - b[1].timestamp) for (let i = 0; i < 20 && oldest[i]; i++) { cache.delete(oldest[i]![0]) } } cache.set(url, { content, timestamp: Date.now() }) } // ── HTTP→HTTPS upgrade ── function upgradeToHttps(url: string): string { if (url.startsWith('http://')) { return url.replace('http://', 'https://') } return url } // ── HTML → Markdown conversion ── function htmlToMarkdown(html: string, baseUrl: string): string { let text = html // Remove scripts, styles, nav, header, footer text = text.replace(/]*>[\s\S]*?<\/script>/gi, '') text = text.replace(/]*>[\s\S]*?<\/style>/gi, '') text = text.replace(/]*>[\s\S]*?<\/nav>/gi, '') text = text.replace(/]*>[\s\S]*?<\/header>/gi, '') text = text.replace(/]*>[\s\S]*?<\/footer>/gi, '') // Convert headings text = text.replace(/]*>([\s\S]*?)<\/h1>/gi, (_, c) => `\n# ${stripTags(c).trim()}\n`) text = text.replace(/]*>([\s\S]*?)<\/h2>/gi, (_, c) => `\n## ${stripTags(c).trim()}\n`) text = text.replace(/]*>([\s\S]*?)<\/h3>/gi, (_, c) => `\n### ${stripTags(c).trim()}\n`) text = text.replace(/]*>([\s\S]*?)<\/h4>/gi, (_, c) => `\n#### ${stripTags(c).trim()}\n`) text = text.replace(/]*>([\s\S]*?)<\/h5>/gi, (_, c) => `\n##### ${stripTags(c).trim()}\n`) text = text.replace(/]*>([\s\S]*?)<\/h6>/gi, (_, c) => `\n###### ${stripTags(c).trim()}\n`) // Convert links: text → [text](url) text = text.replace(/]*href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, content) => { const resolved = resolveUrl(href, baseUrl) return `[${stripTags(content).trim()}](${resolved})` }) // Convert images: → ![alt](url) text = text.replace( /]*src=["']([^"']*)["'][^>]*alt=["']([^"']*)["'][^>]*\/?>/gi, (_, src, alt) => { const resolved = resolveUrl(src, baseUrl) return `![${alt || ''}](${resolved})` }, ) text = text.replace(/]*src=["']([^"']*)["'][^>]*\/?>/gi, (_, src) => { const resolved = resolveUrl(src, baseUrl) return `![](${resolved})` }) // Convert lists text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_, c) => `- ${stripTags(c).trim()}\n`) text = text.replace(/<\/ul>/gi, '\n') text = text.replace(/<\/ol>/gi, '\n') // Convert code blocks text = text.replace(/]*>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, c) => { const decoded = c .replace(/</g, '<') .replace(/>/g, '>') .replace(/&/g, '&') .replace(/"/g, '"') return `\n\`\`\`\n${decoded.trim()}\n\`\`\`\n` }) text = text.replace(/]*>([\s\S]*?)<\/code>/gi, (_, c) => `\`${c.trim()}\``) // Convert inline formatting text = text.replace(/]*>([\s\S]*?)<\/strong>/gi, '**$1**') text = text.replace(/]*>([\s\S]*?)<\/b>/gi, '**$1**') text = text.replace(/]*>([\s\S]*?)<\/em>/gi, '*$1*') text = text.replace(/]*>([\s\S]*?)<\/i>/gi, '*$1*') // Convert paragraph and line break tags text = text.replace(//gi, '\n') text = text.replace(/<\/p>/gi, '\n\n') text = text.replace(/]*>/gi, '') // Strip remaining HTML tags text = text.replace(/<[^>]*>/g, '') // Decode HTML entities text = text.replace(/</g, '<') text = text.replace(/>/g, '>') text = text.replace(/&/g, '&') text = text.replace(/"/g, '"') text = text.replace(/'/g, "'") text = text.replace(/'/g, "'") text = text.replace(/ /g, ' ') // Collapse whitespace (preserve intentional line breaks) text = text .split('\n') .map((l) => l.replace(/\s+/g, ' ').trim()) .join('\n') text = text.replace(/\n{3,}/g, '\n\n') return text.trim() } function stripTags(html: string): string { return html.replace(/<[^>]*>/g, '') } function resolveUrl(href: string, baseUrl: string): string { try { return new URL(href, baseUrl).toString() } catch { return href } } // ── Tool Definition ── export const webFetchTool: ToolDefinition = { name: 'WebFetch', description: 'Fetches a URL, converts the page to markdown. HTTP is upgraded to HTTPS. Cross-host redirects are returned to the caller. Responses are cached for 15 minutes per URL.', category: 'network', permission: 'self', parameters: { type: 'object', properties: { url: { type: 'string', format: 'uri', description: 'URL to fetch' }, prompt: { type: 'string', description: 'What to extract from the page (e.g., "find the API docs for authentication"). The tool returns the full page; the prompt helps focus extraction.', }, }, required: ['url'], }, async execute(params, _ctx) { const rawUrl = params.url as string const prompt = (params.prompt as string) || '' const url = upgradeToHttps(rawUrl) // Check cache const cached = getCached(url) if (cached) { return { success: true, content: cached, metadata: { cached: true, url }, } } // SSRF protection: validate URL before fetching const validationError = await validateUrl(url) if (validationError) { return { success: false, content: '', error: validationError } } try { const controller = new AbortController() const timer = setTimeout(() => controller.abort(), 30_000) // 30s timeout // Manual redirect handling: follow same-host redirects one hop at a time, // validating the target BEFORE each request (closes the SSRF-via-redirect gap). let currentUrl = url const originalHost = new URL(url).hostname let response!: Response for (let hop = 0; hop <= 5; hop++) { response = await fetch(currentUrl, { headers: { 'User-Agent': 'Mipham-Code/0.24.0', Accept: 'text/html,application/xhtml+xml,*/*', }, redirect: 'manual', signal: controller.signal, }) // Not a redirect (or no Location) → proceed with this response if (response.status < 300 || response.status >= 400) break const location = response.headers.get('location') if (!location || hop === 5) break const nextUrl = new URL(location, currentUrl).toString() // SSRF defense: validate the redirect target BEFORE following const redirectError = await validateUrl(nextUrl) if (redirectError) { clearTimeout(timer) return { success: false, content: '', error: `Redirect blocked: ${redirectError}`, } } // Cross-host redirects are returned to the caller (tool contract) if (new URL(nextUrl).hostname !== originalHost) { clearTimeout(timer) return { success: true, content: `Redirected to: ${nextUrl}\n\nFetch from this URL directly to retrieve content.`, metadata: { redirected: true, originalUrl: url, finalUrl: nextUrl }, } } currentUrl = nextUrl } clearTimeout(timer) // Determine content type; only convert HTML to markdown const contentType = response.headers.get('content-type') || '' const isHtml = contentType.includes('text/html') || contentType.includes('application/xhtml') if (!response.ok) { return { success: false, content: '', error: `HTTP ${response.status}: ${response.statusText}`, } } let content: string if (isHtml) { const html = await response.text() const baseUrl = response.url || url content = htmlToMarkdown(html, baseUrl) } else { // Plain text / JSON / etc. — return as-is content = await response.text() } // Truncate to 100K characters if (content.length > 100_000) { content = content.slice(0, 100_000) + '\n\n... (truncated)' } // Cache the result setCache(url, content) // Include prompt context if provided const header = prompt ? `── WebFetch: ${url} ──\nPrompt: ${prompt}\n\n` : `── WebFetch: ${url} ──\n\n` return { success: true, content: header + content, metadata: { url, size: content.length } } } catch (err) { const message = err instanceof Error && err.name === 'AbortError' ? 'Request timed out (30s)' : `Fetch failed: ${String(err)}. If direct network is blocked, retry via the web-access skill (CDP through the user's logged-in Chrome).` return { success: false, content: '', error: message } } }, }