/** * Unit tests for scripts/probe-creative-source.mjs logic. * * We test the script's internal functions (shape validation, redaction * assertion, exit-code semantics) by importing a thin harness module that * re-exports them. The real upstream is never hit — that is the probe-live * CI job's job, not ours. * * Strategy: use vi.stubGlobal / vi.mock to replace dynamic imports and * spawn the script as a child process so we can capture exit codes and * stdio without touching real networks. */ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; import { z } from "zod"; const execFileAsync = promisify(execFile); // --------------------------------------------------------------------------- // Inline helpers that mirror the script's logic — tested in isolation // --------------------------------------------------------------------------- const LICENSE_VALUES = [ "CC0", "PublicDomain", "CC-BY", "CC-BY-SA", "Unknown", "Restricted", ] as const; const RAW_SOURCE_ITEM_SCHEMA = z.object({ sourceUrl: z.string().url(), sourceName: z.string().min(1), title: z.string().min(1), license: z.enum(LICENSE_VALUES), artist: z.string().optional(), year: z.number().optional(), medium: z.string().optional(), type: z.enum(["project", "artist", "artwork", "technique", "cue_reference"]).optional(), tags: z.array(z.string()).optional(), rightsNotes: z.string().optional(), imageUrl: z.string().url().optional(), palette: z.array(z.string()).optional(), visualLanguage: z.string().optional(), }); type RawItem = z.infer; function assertNoSecretLeaked( json: string, secrets: string[], ): { ok: true } | { ok: false; fieldPath: string; hashPrefix: string } { for (const secret of secrets) { if (json.includes(secret)) { const idx = json.indexOf(secret); const snippet = json.slice(Math.max(0, idx - 60), idx); const fieldMatch = /"([^"]+)"\s*:\s*"[^"]*$/.exec(snippet); const fieldPath = fieldMatch?.[1] ?? "(unknown field)"; const hashPrefix = createHash("sha256").update(secret).digest("hex").slice(0, 8); return { ok: false, fieldPath, hashPrefix }; } } return { ok: true }; } function validateItems(items: unknown[]): Array<{ index: number; path: string; message: string }> { const fails: Array<{ index: number; path: string; message: string }> = []; for (let i = 0; i < items.length; i++) { const result = RAW_SOURCE_ITEM_SCHEMA.safeParse(items[i]); if (!result.success) { for (const issue of result.error.issues) { fails.push({ index: i, path: issue.path.join("."), message: issue.message, }); } } } return fails; } // --------------------------------------------------------------------------- // Well-shaped artic item fixture // --------------------------------------------------------------------------- const WELL_SHAPED_ITEM: RawItem = { sourceUrl: "https://www.artic.edu/artworks/12345", sourceName: "Art Institute of Chicago", title: "Landscape with Trees", license: "CC0", artist: "Unknown Artist", year: 1890, medium: "Oil on canvas", type: "artwork", tags: ["landscape", "oil"], }; // --------------------------------------------------------------------------- // 1. Happy path — well-shaped item passes schema + redaction // --------------------------------------------------------------------------- describe("shape validation", () => { it("passes a well-shaped item", () => { const fails = validateItems([WELL_SHAPED_ITEM]); expect(fails).toHaveLength(0); }); // 2. Shape drift — missing license field → validation fail it("fails an item missing 'license'", () => { const bad = { ...WELL_SHAPED_ITEM }; // @ts-expect-error intentional shape drift test delete bad.license; const fails = validateItems([bad]); expect(fails.length).toBeGreaterThan(0); const licenseFail = fails.find((f) => f.path === "license"); expect(licenseFail).toBeDefined(); }); it("fails an item with an invalid sourceUrl", () => { const bad = { ...WELL_SHAPED_ITEM, sourceUrl: "not-a-url" }; const fails = validateItems([bad]); expect(fails.length).toBeGreaterThan(0); const urlFail = fails.find((f) => f.path === "sourceUrl"); expect(urlFail).toBeDefined(); }); it("fails an item with an unknown license value", () => { const bad = { ...WELL_SHAPED_ITEM, license: "WTFPL" as unknown as RawItem["license"] }; const fails = validateItems([bad]); expect(fails.length).toBeGreaterThan(0); expect(fails[0]?.path).toBe("license"); }); }); // --------------------------------------------------------------------------- // 3. Redaction assertion // --------------------------------------------------------------------------- describe("redaction assertion", () => { it("passes when no secrets present in output", () => { const result = assertNoSecretLeaked(JSON.stringify([WELL_SHAPED_ITEM]), ["topsecret123"]); expect(result.ok).toBe(true); }); it("catches a secret leaked in sourceUrl", () => { const secret = "topsecret123"; const leakyItem = { ...WELL_SHAPED_ITEM, sourceUrl: `https://api.example.com/search?wskey=${secret}`, }; const json = JSON.stringify([leakyItem]); const result = assertNoSecretLeaked(json, [secret]); expect(result.ok).toBe(false); if (!result.ok) { // Must name the field path but NEVER echo the secret expect(result.fieldPath).toBeTruthy(); expect(result.hashPrefix).toBeTruthy(); // The secret value must not appear in the returned diagnostic strings expect(result.fieldPath).not.toContain(secret); expect(result.hashPrefix).not.toContain(secret); } }); it("does not leak the secret value in any diagnostic output", () => { const secret = "mysupersecretapikey"; const leakyItem = { ...WELL_SHAPED_ITEM, rightsNotes: `Provided by API key=${secret}`, }; const json = JSON.stringify([leakyItem]); const result = assertNoSecretLeaked(json, [secret]); expect(result.ok).toBe(false); if (!result.ok) { // Serialize the entire result object — confirm secret is absent const diagnosticStr = JSON.stringify(result); expect(diagnosticStr).not.toContain(secret); } }); it("passes when secrets list is empty", () => { const result = assertNoSecretLeaked(JSON.stringify([WELL_SHAPED_ITEM]), []); expect(result.ok).toBe(true); }); }); // --------------------------------------------------------------------------- // 4–6. Exit code behavior via child process // // We spawn the actual script with a mocked sources registry by patching env // vars and using a synthetic test harness. These tests verify the script's // exit codes, which are part of its public contract. // --------------------------------------------------------------------------- const SCRIPT_PATH = resolve( fileURLToPath(import.meta.url), "../../..", "scripts/probe-creative-source.ts", ); async function runScript( args: string[], env?: Record, ): Promise<{ exitCode: number; stdout: string; stderr: string }> { try { const { stdout, stderr } = await execFileAsync("npx", ["tsx", SCRIPT_PATH, ...args], { env: { ...process.env, ...env }, timeout: 30_000, }); return { exitCode: 0, stdout, stderr }; } catch (err: unknown) { const e = err as { code?: number; stdout?: string; stderr?: string }; return { exitCode: e.code ?? 1, stdout: e.stdout ?? "", stderr: e.stderr ?? "", }; } } describe("script exit codes", () => { // Missing source id argument → exit 2 (checked before registry import) it("exits 2 when no source id is provided", async () => { const { exitCode, stderr } = await runScript([]); expect(exitCode).toBe(2); expect(stderr).toMatch(/usage/i); }); }); // --------------------------------------------------------------------------- // 6. Unknown source id — tested via the resolveSources helper directly // --------------------------------------------------------------------------- describe("unknown source id (logic test)", () => { it("resolveSources returns empty array for unknown id", async () => { // Dynamic import of the compiled/ts sources index via tsx loader (vitest handles this) const { resolveSources } = await import("../../src/creativeRag/sources/index.js"); const result = resolveSources(["nonexistent-source-xyz"]); expect(result).toHaveLength(0); }); }); // --------------------------------------------------------------------------- // Redaction: integration-style check that secret not printed to stderr // --------------------------------------------------------------------------- describe("redaction in full script run", () => { // Use an unknown source id so the script exits at the registry-resolve step // (exit 2) without touching the real Europeana upstream — the prior test // exercised a live network dependency just to assert "secret is not echoed". it("does not echo a secret-shaped env var value to any output", async () => { const secret = "fake_probe_unit_test_key_abc123"; const { stdout, stderr } = await runScript(["nonexistent-source-xyz"], { // Use BOTH naming conventions so we cover the adapter env (TDMCP_RAG_*_KEY) // and the legacy alias (*_API_KEY) — the script's secret collector now // matches both, and neither must ever appear in output. TDMCP_RAG_EUROPEANA_KEY: secret, EUROPEANA_API_KEY: secret, }); expect(stdout).not.toContain(secret); expect(stderr).not.toContain(secret); }); });