import { dedent, extractContentFromHtml } from "./utils.js";
import type { ToolDefinition } from "./types.js";
import { JSDOM } from "jsdom";
const rawFetch = async (url: string) => {
try {
const parsedUrl = new URL(
['http://', 'https://'].some(p => url.startsWith(p)) ? url : `https://${url}`
);
// Use the same fetch logic as httpGet but optimized for analysis
const controller = new AbortController();
const timeoutId = setTimeout(() => {
console.log(`Fetch timeout reached for: ${url}`);
controller.abort();
}, 20000); // 20 second timeout for analysis
console.log(`Fetching content from: ${url}`);
const request = await fetch(parsedUrl, {
signal: controller.signal,
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WireJS-Analyzer/1.0)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
}
});
clearTimeout(timeoutId);
if (!request.ok) {
throw new Error(`HTTP ${request.status}: ${request.statusText}`);
}
const body = await request.text();
console.log(`Fetched ${body.length} characters from: ${url}`);
return body;
} catch (error) {
console.error(`Error fetching ${url}:`, error);
if (error instanceof Error && error.name === 'AbortError') {
throw new Error(`Fetch timeout after 20 seconds for: ${url}`);
}
throw error;
}
}
/**
* Standard tool definitions include searching and fetching web content.
*/
export const standard: ToolDefinition[] = [
{
name: 'fetch',
description: dedent`
Fetches raw content from an HTTP(S) URL via a GET request.
Use fetch ONLY when the URL is known and expected to contain
"raw data" like CSV, JSON, XML, YML, or plain text.
`,
parameters: {
type: 'object',
properties: {
url: {
type: 'string',
description: "Fully qualified URL string to fetch."
}
}
},
async execute({ url } : { url: string }) {
console.log(`[fetch] Received request for: ${url}`);
return rawFetch(url);
}
},
{
name: 'fetch_html_content_text',
description: dedent`
Extracts the text content from HTML at the given URL.
Use fetch_html_content_text ONLY when the URL is known and expected
to be an HTML page with meaningful content to extract.
`,
parameters: {
type: 'object',
properties: {
url: {
type: 'string',
description: "Fully qualified URL string to fetch."
}
}
},
async execute({ url } : { url: string }) {
console.log(`[natural language fetch] Received request for: ${url}`);
const content = extractContentFromHtml(await rawFetch(url));
console.log(`Extracted content:\n${content}`);
return content;
}
},
{
name: 'web_search',
description: dedent`
Searches the web using DuckDuckGo.
Use web_search ONLY when user intent indicates a need for information beyond
your training or if the subject matter is temporally sensitive AND when a
web search is likely to provide the URL you need for a future fetch or
fetch_html_content_text call.
`,
parameters: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'Search text to use for searching the web. Supports DuckDuckGo search syntax.'
}
}
},
async execute({ query } : { query: string }) {
console.log(`[searching] query: ${query}`);
const rawHtml = await rawFetch(`https://html.duckduckgo.com/html/?q=${query}`);
return parseDuckDuckGoResults(rawHtml);
}
}
];
const parseDuckDuckGoResults = (html: string) : string => {
const results: Array<{ url: string; title: string; description: string }> = [];
const dom = new JSDOM(html);
const doc = dom.window.document;
const resultElements = doc.querySelectorAll('.result');
resultElements.forEach((element) => {
const titleEl = element.querySelector('.result__a');
const snippetEl = element.querySelector('.result__snippet');
const linkEl = element.querySelector('a.result__a') as HTMLAnchorElement;
if (titleEl && snippetEl && linkEl) {
const url = new URL(linkEl.getAttribute('href') || '', 'https://html.duckduckgo.com');
const uddgParam = url.searchParams.get('uddg');
const decodedUrl = uddgParam ? decodeURIComponent(uddgParam) : '';
results.push({
url: decodedUrl,
title: titleEl.textContent?.trim() || '',
description: snippetEl.textContent?.trim() || ''
});
}
});
return JSON.stringify(results, null, 2);
};