/** * `dench web` — brand, scrape, sitemap, crawl, screenshot, and extract * through the Dench Cloud Gateway (`POST /v1/context/{op}`). * * Agents and humans use this instead of writing their own scrapers. * Auth matches `dench search`: org DENCH_API_KEY / paid-workspace gateway key. */ import { assertNoUnknownFlags, getFlag, hasFlag } from "./lib/cli-args"; import { normalizeGatewayBaseUrl } from "./lib/gateway-url"; type WebCliContext = { args: string[]; jsonOutput: boolean; bearerToken?: string; gatewayBaseUrl?: string; }; class WebCliError extends Error {} const CRAWL_DEFAULT_PAGES = 15; const CRAWL_MAX_PAGES = 50; function parsePositiveInt( name: string, raw: string | undefined, ): number | undefined { if (raw === undefined) return undefined; const parsed = Number(raw); if (!Number.isFinite(parsed) || parsed <= 0) { throw new WebCliError(`Invalid ${name} value: ${raw}`); } return Math.floor(parsed); } function resolveGatewayBaseUrl( ctx?: Pick, ): string { if (ctx?.gatewayBaseUrl?.trim()) { return normalizeGatewayBaseUrl(ctx.gatewayBaseUrl.trim()); } const explicit = process.env.DENCH_GATEWAY_URL?.trim() || process.env.GATEWAY_URL?.trim(); if (explicit) return normalizeGatewayBaseUrl(explicit); const host = process.env.DENCH_HOST?.trim(); if (host && /localhost|127\.0\.0\.1/.test(host)) { return "http://localhost:8787"; } return "https://gateway.merseoriginals.com"; } function requireBearerToken(ctx?: Pick): string { const bearerToken = ctx?.bearerToken?.trim() || process.env.DENCH_API_KEY?.trim(); if (!bearerToken) { throw new WebCliError( "No Dench gateway credential found. Set DENCH_API_KEY or run `dench signin` for a paid workspace.", ); } return bearerToken; } async function callWebOp( ctx: WebCliContext, op: string, input: Record, ): Promise { const bearerToken = requireBearerToken(ctx); const baseUrl = resolveGatewayBaseUrl(ctx); const response = await fetch(`${baseUrl}/v1/context/${op}`, { method: "POST", headers: { "content-type": "application/json", Authorization: `Bearer ${bearerToken}`, }, body: JSON.stringify({ input }), }); if (!response.ok) { let detail = ""; try { detail = await response.text(); } catch { // ignore } const truncated = detail.length > 500 ? `${detail.slice(0, 500)}…` : detail; throw new WebCliError( `Gateway /v1/context/${op} returned ${response.status}: ${truncated || response.statusText}`, ); } return response.json(); } function asString(value: unknown): string | undefined { return typeof value === "string" ? value : undefined; } function asRecord(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : undefined; } function out(ctx: WebCliContext, payload: unknown, formatted: string): void { if (ctx.jsonOutput) { console.log(JSON.stringify(payload, null, 2)); return; } console.log(formatted); } function formatBrand(data: unknown): string { const record = asRecord(data); if (!record) return "(empty brand)"; const lines = [ asString(record.title) ?? asString(record.name) ?? "(untitled brand)", asString(record.domain) ? ` ${asString(record.domain)}` : "", asString(record.description) ? ` ${asString(record.description)}` : "", ].filter((line) => line.length > 0); return lines.join("\n"); } function formatMarkdown(data: unknown): string { const record = asRecord(data); return asString(record?.markdown) ?? JSON.stringify(data, null, 2); } export function webHelp(): void { console.log(webHelpText()); } export function webHelpText(): string { return `Dench web Read pages, brand kits, sitemaps, screenshots, and structured extracts through the Dench gateway. Prefer these over writing your own scraper. dench web scrape [--json] dench web brand [--json] dench web sitemap [--limit N] [--json] dench web crawl [--max-pages N] [--json] dench web screenshot [--full-page] [--json] dench web extract --schema '' [--json] dench web product [--json] dench web competitors [--json] dench web styleguide [--json] dench web fonts [--json] dench web classify [--scheme naics|sic] [--json] dench web brand-search [--json] Crawl defaults to 15 pages and is hard-capped at 50. Search stays on \`dench search\`. \`dench context\` is workspace orientation, not this. `; } async function runWebSubcommand(ctx: WebCliContext): Promise { const sub = ctx.args[0]; const rest = ctx.args.slice(1); if (!sub || sub === "help" || sub === "--help" || sub === "-h") { webHelp(); return; } if (sub === "scrape") { assertNoUnknownFlags(rest, "web scrape"); const url = rest[0]; if (!url) throw new WebCliError("Usage: dench web scrape [--json]"); const data = await callWebOp(ctx, "web-scrape-markdown", { url, useMainContentOnly: true, }); out(ctx, data, formatMarkdown(data)); return; } if (sub === "brand") { assertNoUnknownFlags(rest, "web brand"); const identifier = rest[0]; if (!identifier) { throw new WebCliError("Usage: dench web brand [--json]"); } const input = identifier.includes("@") ? { email: identifier } : { domain: identifier }; const data = await callWebOp(ctx, "brand-retrieve", input); out(ctx, data, formatBrand(data)); return; } if (sub === "brand-search") { assertNoUnknownFlags(rest, "web brand-search"); const query = rest.join(" ").trim(); if (!query) { throw new WebCliError("Usage: dench web brand-search [--json]"); } const data = await callWebOp(ctx, "brand-search", { query }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "sitemap") { const limit = parsePositiveInt("--limit", getFlag(rest, "--limit")); assertNoUnknownFlags(rest, "web sitemap"); const domain = rest[0]; if (!domain) { throw new WebCliError( "Usage: dench web sitemap [--limit N] [--json]", ); } const data = await callWebOp(ctx, "web-sitemap", { domain, ...(limit != null ? { maxLinks: limit } : {}), }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "crawl") { const maxPages = parsePositiveInt("--max-pages", getFlag(rest, "--max-pages")) ?? CRAWL_DEFAULT_PAGES; if (maxPages > CRAWL_MAX_PAGES) { throw new WebCliError(`--max-pages cannot exceed ${CRAWL_MAX_PAGES}.`); } assertNoUnknownFlags(rest, "web crawl"); const url = rest[0]; if (!url) { throw new WebCliError( "Usage: dench web crawl [--max-pages N] [--json]", ); } const data = await callWebOp(ctx, "web-crawl", { url, maxPages, useMainContentOnly: true, }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "screenshot") { const fullPage = hasFlag(rest, "--full-page"); assertNoUnknownFlags(rest, "web screenshot"); const url = rest[0]; if (!url) { throw new WebCliError( "Usage: dench web screenshot [--full-page] [--json]", ); } const data = await callWebOp(ctx, "web-screenshot", { directUrl: url, fullScreenshot: fullPage, }); const record = asRecord(data); out( ctx, data, asString(record?.screenshot) ?? JSON.stringify(data, null, 2), ); return; } if (sub === "extract") { const schemaRaw = getFlag(rest, "--schema"); assertNoUnknownFlags(rest, "web extract"); const url = rest[0]; if (!url || !schemaRaw) { throw new WebCliError( "Usage: dench web extract --schema '' [--json]", ); } let schema: unknown; try { schema = JSON.parse(schemaRaw); } catch { throw new WebCliError("--schema must be valid JSON."); } if (!schema || typeof schema !== "object" || Array.isArray(schema)) { throw new WebCliError("--schema must be a JSON object."); } const data = await callWebOp(ctx, "web-extract", { url, schema }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "product") { assertNoUnknownFlags(rest, "web product"); const url = rest[0]; if (!url) throw new WebCliError("Usage: dench web product [--json]"); const data = await callWebOp(ctx, "web-product", { url }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "competitors") { assertNoUnknownFlags(rest, "web competitors"); const domain = rest[0]; if (!domain) { throw new WebCliError("Usage: dench web competitors [--json]"); } const data = await callWebOp(ctx, "web-competitors", { domain, numCompetitors: 5, }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "styleguide") { assertNoUnknownFlags(rest, "web styleguide"); const domain = rest[0]; if (!domain) { throw new WebCliError("Usage: dench web styleguide [--json]"); } const data = await callWebOp(ctx, "brand-styleguide", { domain }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "fonts") { assertNoUnknownFlags(rest, "web fonts"); const domain = rest[0]; if (!domain) throw new WebCliError("Usage: dench web fonts [--json]"); const data = await callWebOp(ctx, "brand-fonts", { domain }); out(ctx, data, JSON.stringify(data, null, 2)); return; } if (sub === "classify") { const scheme = getFlag(rest, "--scheme") ?? "naics"; if (scheme !== "naics" && scheme !== "sic") { throw new WebCliError("Invalid --scheme. Allowed: naics, sic"); } assertNoUnknownFlags(rest, "web classify"); const domain = rest[0]; if (!domain) { throw new WebCliError( "Usage: dench web classify [--scheme naics|sic] [--json]", ); } const data = await callWebOp( ctx, scheme === "sic" ? "classify-sic" : "classify-naics", { input: domain }, ); out(ctx, data, JSON.stringify(data, null, 2)); return; } throw new WebCliError( `Unknown dench web command "${sub}". Run \`dench web --help\`.`, ); } export async function runWebCommand(ctx: WebCliContext): Promise { try { await runWebSubcommand(ctx); } catch (error) { if (error instanceof WebCliError) { console.error(error.message); process.exitCode = 1; return; } throw error; } }