import { ObjectId } from "mongodb"; import { getScanResourcesCollection, getScanKpiCounts, getScanPages, getScanResourceById, getScanResourceCountsByScanIds, deleteScanResourcesByScanId, listPageSummariesByScanIds, } from "../scanResources.getters"; import { URL_TASK_STATUSES } from "../scanResources.constants"; import { boardColumn } from "../scanResources.utils"; import { createScanResourceFixture } from "../../../test-utils/factories"; describe("db.websitalk.scanResources", () => { describe("boardColumn()", () => { it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => { expect(boardColumn("URL_FAILED")).toBe("FAILED"); }); it("folds URL_PERMANENT_FAILURE into FAILED", () => { expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED"); }); it("folds duplicate content into COMPLETED", () => { expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED"); }); }); describe("getScanKpiCounts()", () => { it("sums to the scan's total task count and omits zero-count statuses", async () => { const scanId = new ObjectId().toString(); await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId, status: "URL_COMPLETED" }), createScanResourceFixture({ scanId, status: "URL_COMPLETED" }), createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }), ]); const counts = await getScanKpiCounts(scanId); expect(counts.URL_COMPLETED).toBe(2); expect(counts.URL_PERMANENT_FAILURE).toBe(1); expect(counts.URL_PENDING).toBeUndefined(); const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0); expect(total).toBe(3); }); }); describe("getScanPages()", () => { it("never returns context or stageLogs, and paginates within one column", async () => { const scanId = new ObjectId().toString(); await getScanResourcesCollection().insertMany( Array.from({ length: 3 }, () => createScanResourceFixture({ scanId, status: "URL_COMPLETED", context: "heavy markdown content", stageLogs: [ { from: null, to: "URL_COMPLETED", occurredAt: new Date(), details: {}, }, ], }), ), ); const page1 = await getScanPages({ scanId, column: "COMPLETED", limit: 2, }); expect(page1.items).toHaveLength(2); expect(page1.nextCursor).not.toBeNull(); page1.items.forEach((item) => { expect(item).not.toHaveProperty("context"); expect(item).not.toHaveProperty("stageLogs"); }); const page2 = await getScanPages({ scanId, column: "COMPLETED", limit: 2, cursor: page1.nextCursor, }); expect(page2.items).toHaveLength(1); expect(page2.nextCursor).toBeNull(); }); it("folds URL_FAILED into the FAILED column query", async () => { const scanId = new ObjectId().toString(); await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }), createScanResourceFixture({ scanId, status: "URL_FAILED" }), createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }), ]); const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" }); expect(ready.items).toHaveLength(1); const failed = await getScanPages({ scanId, column: "FAILED" }); expect(failed.items).toHaveLength(2); }); it("finds a URL containing regex-special characters when searched verbatim", async () => { const scanId = new ObjectId().toString(); const url = "https://www.industry.org.il/index.php?dir=site&page=expert&op=item&cs=3022"; await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId, status: "URL_COMPLETED", url }), createScanResourceFixture({ scanId, status: "URL_COMPLETED", url: "https://example.com/other", }), ]); const result = await getScanPages({ scanId, column: "COMPLETED", search: url, }); expect(result.items).toHaveLength(1); expect(result.items[0].url).toBe(url); }); }); describe("getScanResourceCountsByScanIds()", () => { it("returns per-scan totals and omits scans with no resources", async () => { const scanA = new ObjectId().toString(); const scanB = new ObjectId().toString(); await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }), createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }), createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }), ]); const counts = await getScanResourceCountsByScanIds([ scanA, scanB, "missing", ]); expect(counts[scanA]).toBe(2); expect(counts[scanB]).toBe(1); expect(counts.missing).toBeUndefined(); }); }); describe("listPageSummariesByScanIds()", () => { it("returns completed pages for the given scans", async () => { const scanA = new ObjectId().toString(); const scanB = new ObjectId().toString(); await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId: scanA, status: URL_TASK_STATUSES.URL_COMPLETED, url: "https://a.example/page", title: "A", context: "summary a", summaryTokens: 12, unsummarized: false, }), createScanResourceFixture({ scanId: scanA, status: URL_TASK_STATUSES.URL_DUPLICATE_CONTENT, context: "dup", }), createScanResourceFixture({ scanId: scanB, status: URL_TASK_STATUSES.URL_COMPLETED, url: "https://b.example/page", context: "summary b", }), createScanResourceFixture({ scanId: new ObjectId().toString(), status: URL_TASK_STATUSES.URL_COMPLETED, context: "other scan", }), ]); const pages = await listPageSummariesByScanIds([scanA, scanB]); expect(pages).toHaveLength(2); expect(pages.map((page) => page.scanId).sort()).toEqual( [scanA, scanB].sort(), ); const pageA = pages.find((page) => page.scanId === scanA); expect(pageA).toMatchObject({ url: "https://a.example/page", title: "A", context: "summary a", summaryTokens: 12, unsummarized: false, status: URL_TASK_STATUSES.URL_COMPLETED, }); }); it("returns an empty array when scanIds is empty", async () => { expect(await listPageSummariesByScanIds([])).toEqual([]); }); }); describe("getScanResourceById()", () => { it("returns the full document including stageLogs in insertion order", async () => { const scanId = new ObjectId().toString(); const stageLogs = [ { from: null, to: "URL_PENDING" as const, occurredAt: new Date("2024-01-01"), details: {}, }, { from: "URL_PENDING" as const, to: "URL_SCANNING" as const, occurredAt: new Date("2024-01-02"), details: {}, }, ]; const fixture = createScanResourceFixture({ scanId, stageLogs, context: "full content", }); const { insertedId } = await getScanResourcesCollection().insertOne(fixture); const result = await getScanResourceById(scanId, insertedId.toString()); expect(result?.context).toBe("full content"); expect(result?.stageLogs).toEqual(stageLogs); }); it("returns null when the scan does not match", async () => { const fixture = createScanResourceFixture(); const { insertedId } = await getScanResourcesCollection().insertOne(fixture); const result = await getScanResourceById( new ObjectId().toString(), insertedId.toString(), ); expect(result).toBeNull(); }); }); describe("deleteScanResourcesByScanId()", () => { it("should delete all resources for the scan and return the count", async () => { const scanId = new ObjectId().toString(); await getScanResourcesCollection().insertMany([ createScanResourceFixture({ scanId }), createScanResourceFixture({ scanId }), createScanResourceFixture({ scanId: new ObjectId().toString() }), ]); const deleted = await deleteScanResourcesByScanId(scanId); expect(deleted).toBe(2); expect( await getScanResourcesCollection().countDocuments({ scanId }), ).toBe(0); }); }); });