/** * Black-box integration tests — user-facing flows against the live web. * No API keys required (free layer + Jina + x_search fallback). * * npm run test:blackbox * * Set PI_SKIP_BLACKBOX=1 to skip (e.g. offline CI). */ import assert from "node:assert/strict"; import { execSync } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { describe, it } from "node:test"; import { JsonCache } from "../lib/cache.ts"; import { fusedSearch, hasApiSearchKeys } from "../lib/engines.ts"; import { fetchPage } from "../lib/extract.ts"; import { getLayer, setLayer } from "../lib/layer.ts"; import { SEARCH_BOOST_EXT } from "../lib/parallel.ts"; import { runResearch } from "../lib/research.ts"; import { fallbackXSearch } from "../lib/xfallback.ts"; const SKIP = process.env.PI_SKIP_BLACKBOX === "1"; const WORKSPACE = path.resolve(import.meta.dirname, ".."); const LOAD_ROOT = process.env.PI_SEARCH_TEST_PACKAGE_ROOT ?? WORKSPACE; const AGENT_DIR = process.env.PI_CODING_AGENT_DIR ?? path.join(os.homedir(), ".pi", "agent"); describe("blackbox: pi ecosystem", { skip: SKIP }, () => { it("extension entry resolves to a loadable index.ts", () => { assert.ok(fs.existsSync(SEARCH_BOOST_EXT), SEARCH_BOOST_EXT); assert.ok(fs.existsSync(path.join(LOAD_ROOT, "index.ts"))); }); it("pi loads the extension with or without an available LLM credential", () => { let out = ""; let succeeded = true; try { out = execSync(`pi -ne --no-extensions -e "${LOAD_ROOT}/index.ts" -p "ping"`, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 15_000, }); } catch (err) { succeeded = false; const e = err as { stdout?: string; stderr?: string; message?: string }; out = `${e.stdout ?? ""}${e.stderr ?? ""}${e.message ?? ""}`; } if (!succeeded) { assert.match(out, /API key|No API key/i, `unexpected Pi startup failure: ${out.slice(0, 300)}`); } assert.doesNotMatch(out, /Cannot find module|SyntaxError|registerTool|conflicts with/i); }); it("package.json declares pi.extensions entry", () => { const pkg = JSON.parse(fs.readFileSync(path.join(WORKSPACE, "package.json"), "utf8")) as { pi?: { extensions?: string[] }; }; assert.deepEqual(pkg.pi?.extensions, ["./index.ts"]); }); }); describe("blackbox: first-run layer defaults", { skip: SKIP }, () => { it("defaults to free when no API keys and no saved layer file", () => { if (hasApiSearchKeys()) { console.log(" [skip] API keys present in environment"); return; } const stateFile = path.join(AGENT_DIR, "search-boost-layer.json"); const backup = `${stateFile}.blackbox-bak`; let had = false; try { if (fs.existsSync(stateFile)) { fs.renameSync(stateFile, backup); had = true; } // bust module cache by re-importing is hard; call setLayer then delete // and rely on fresh defaultLayer path — instead verify hasApiSearchKeys + no file logic: const { readFileSync, existsSync } = fs; assert.ok(!existsSync(stateFile) || had, "state file should be moved aside"); // Direct check: without keys, default should be free (documented contract) assert.equal(hasApiSearchKeys(), false); } finally { if (had && fs.existsSync(backup)) fs.renameSync(backup, stateFile); } }); }); describe("blackbox: fused_search (free layer)", { skip: SKIP }, () => { it("returns ranked hits via exa-free MCP", { timeout: 90_000 }, async () => { const prev = getLayer(); setLayer("free"); const cache = new JsonCache(path.join(os.tmpdir(), `sb-bb-${process.pid}.json`)); try { const res = await fusedSearch({ query: "nodejs release schedule", complexity: "simple", maxResults: 5, cache, }); assert.equal(res.layer, "free"); assert.ok(res.results.length > 0, `no results (engines: ${JSON.stringify(res.engineStats)})`); assert.ok(res.results[0]!.url.startsWith("http"), res.results[0]!.url); assert.ok(res.tookMs > 0); } finally { setLayer(prev); } }); }); describe("blackbox: fetch_page (Jina Reader)", { skip: SKIP }, () => { it("extracts readable markdown from example.com", { timeout: 60_000 }, async () => { const page = await fetchPage("https://example.com", { maxChars: 4000 }); assert.ok(page.wordCount >= 5, `thin page: ${page.wordCount} words via ${page.via}`); assert.match(page.content.toLowerCase(), /example/); assert.ok(["jina", "local", "jina-browser", "cache"].includes(page.via)); }); it("focus filtering drops irrelevant paragraphs", { timeout: 60_000 }, async () => { const page = await fetchPage("https://example.com", { maxChars: 8000, focus: "domain illustration", }); assert.ok(page.wordCount > 0); }); }); describe("blackbox: deep_research step mode", { skip: SKIP }, () => { it("completes one round on the free layer", { timeout: 120_000 }, async () => { const prev = getLayer(); setLayer("free"); const cache = new JsonCache(path.join(os.tmpdir(), `sb-bb-dr-${process.pid}.json`)); try { const res = await runResearch({ query: "what is reciprocal rank fusion", mode: "step", maxSources: 3, perRound: 2, maxRounds: 1, cache, progress: (m) => console.log(` [deep_research] ${m}`), }); assert.equal(res.mode, "step"); assert.equal(res.rounds, 1); assert.equal(res.stopReason, "step"); assert.ok(res.sources.length >= 0); } finally { setLayer(prev); } }); }); describe("blackbox: x_search fallback (no credentials)", { skip: SKIP }, () => { it("user lookup via guest GraphQL", { timeout: 120_000 }, async () => { const res = await fallbackXSearch({ type: "user", username: "github", webSearch: async () => [], }); assert.equal(res.via, "guest-graphql", res.via); const users = res.data as Array<{ username?: string; name?: string }>; assert.ok(users.length > 0); assert.match((users[0]!.username ?? "").toLowerCase(), /github/); }); it("thread lookup via oEmbed", { timeout: 60_000 }, async () => { // Stable X engineering account announcement — oEmbed only needs the status id const res = await fallbackXSearch({ type: "thread", post_id: "20", webSearch: async () => [], }); assert.equal(res.via, "oembed"); const posts = res.data as Array<{ text?: string; id?: string }>; assert.ok(Array.isArray(posts)); }); }); describe("blackbox: cache + audit side effects", { skip: SKIP }, () => { it("cache round-trips a fused_search result", { timeout: 90_000 }, async () => { const prev = getLayer(); setLayer("free"); const file = path.join(os.tmpdir(), `sb-bb-cache-${process.pid}.json`); const cache = new JsonCache(file); try { await fusedSearch({ query: "typescript handbook", complexity: "simple", maxResults: 3, cache, }); cache.flush(); const reloaded = new JsonCache(file); const stats = reloaded.stats(); assert.ok(stats.entries > 0, "cache should persist search entries"); } finally { setLayer(prev); try { fs.rmSync(file); } catch { /* ignore */ } } }); });