import test from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import os from 'node:os'; import { XMLParser } from 'fast-xml-parser'; import { extractUrlsFromParsedSitemap, fetchSitemapUrls, discoverSitemapUrls } from './sitemap.js'; const parser = new XMLParser(); test('extractUrlsFromParsedSitemap: parses a plain sitemap', () => { const xml = ` https://example.com/ https://example.com/pricing https://example.com/about `; const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com'); assert.deepEqual(pageUrls.sort(), ['https://example.com/', 'https://example.com/about', 'https://example.com/pricing'].sort()); assert.deepEqual(subSitemapUrls, []); }); test('extractUrlsFromParsedSitemap: parses a into sub-sitemap URLs', () => { const xml = ` https://example.com/sitemap-pages.xml https://example.com/sitemap-blog.xml `; const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com'); assert.deepEqual(pageUrls, []); assert.deepEqual(subSitemapUrls, ['https://example.com/sitemap-pages.xml', 'https://example.com/sitemap-blog.xml']); }); test('extractUrlsFromParsedSitemap: filters out cross-origin URLs', () => { const xml = ` https://example.com/page https://attacker.example/evil `; const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com'); assert.deepEqual(pageUrls, ['https://example.com/page']); }); test('extractUrlsFromParsedSitemap: dedupes repeated URLs', () => { const xml = ` https://example.com/a https://example.com/a `; const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com'); assert.deepEqual(pageUrls, ['https://example.com/a']); }); test('extractUrlsFromParsedSitemap: a single entry is not an array (fast-xml-parser quirk) — still handled', () => { const xml = `https://example.com/only`; const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com'); assert.deepEqual(pageUrls, ['https://example.com/only']); }); test('extractUrlsFromParsedSitemap: handles empty/malformed input without throwing', () => { assert.deepEqual(extractUrlsFromParsedSitemap({}, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] }); assert.deepEqual(extractUrlsFromParsedSitemap(null, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] }); assert.deepEqual(extractUrlsFromParsedSitemap({ urlset: { url: [{ notLoc: 'x' }] } }, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] }); }); // --- Live end-to-end: a real local HTTP server, real fetch, real XML parse --- // Bound to the sandbox's actual (non-loopback) address — assertCrawlUrlSafe // deliberately blocks literal localhost/127.0.0.1 (meaningless as a crawl // target, would just hit the crawler container itself) but allows other // addresses, matching how a real deployment reaches a tenant's internal app. function getNonLoopbackAddress(): string { for (const ifaces of Object.values(os.networkInterfaces())) { for (const iface of ifaces as any[]) { if (iface.family === 'IPv4' && !iface.internal) return iface.address; } } throw new Error('No non-loopback IPv4 address found on this host — cannot run the live sitemap test'); } test('fetchSitemapUrls: live end-to-end against a real HTTP server serving a sitemap index', async () => { const host = getNonLoopbackAddress(); const indexXml = ` http://${host}:PORT/sub1.xml http://${host}:PORT/sub2.xml `; const sub1Xml = `http://${host}:PORT/page1http://${host}:PORT/page2`; const sub2Xml = `http://${host}:PORT/page3`; const server = http.createServer((req, res) => { res.writeHead(200, { 'Content-Type': 'application/xml' }); if (req.url === '/sitemap.xml') res.end(indexXml.replaceAll('PORT', String(port))); else if (req.url === '/sub1.xml') res.end(sub1Xml.replaceAll('PORT', String(port))); else if (req.url === '/sub2.xml') res.end(sub2Xml.replaceAll('PORT', String(port))); else { res.writeHead(404); res.end(); } }); await new Promise((resolve) => server.listen(0, host, resolve)); const port = (server.address() as any).port; try { const origin = `http://${host}:${port}`; const urls = await fetchSitemapUrls(`${origin}/sitemap.xml`, origin); assert.deepEqual( urls.sort(), [`${origin}/page1`, `${origin}/page2`, `${origin}/page3`].sort(), ); } finally { server.close(); } }); test('discoverSitemapUrls: defaults to /sitemap.xml at the app origin and returns [] gracefully on 404', async () => { const host = getNonLoopbackAddress(); const server = http.createServer((_req, res) => { res.writeHead(404); res.end(); }); await new Promise((resolve) => server.listen(0, host, resolve)); const port = (server.address() as any).port; try { const urls = await discoverSitemapUrls(`http://${host}:${port}/app`); assert.deepEqual(urls, []); } finally { server.close(); } }); test('discoverSitemapUrls: never throws for an invalid appUrl', async () => { const urls = await discoverSitemapUrls('not-a-valid-url'); assert.deepEqual(urls, []); }); test('discoverSitemapUrls: is blocked (returns []) for a disallowed target like localhost — assertCrawlUrlSafe still applies', async () => { const urls = await discoverSitemapUrls('http://localhost:1/app'); assert.deepEqual(urls, []); });