import http from 'http'; import { AddressInfo } from 'net'; import zlib from 'zlib'; import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from '@jest/globals'; import { createImageDownloader } from '../../../adapters/imageDownloader.js'; import type { ImageDownloader, ImageDownloaderOptions } from '../../../adapters/imageDownloader.js'; import type { LookupFunction } from '../../../../../lib/ssrf/guardedLookup.js'; import type { ImageError } from '../../../services/images/types.js'; const LIMIT = 1000; const PNG = Buffer.concat([ Buffer.from('89504e470d0a1a0a', 'hex'), Buffer.alloc(92, 7) ]); type Seen = { method?: string; url?: string; headers: http.IncomingHttpHeaders; }; /** `img.test` is what a user writes in a URL; the fake DNS sends it to the local server. */ const lookup: LookupFunction = (_hostname, _options, callback) => { callback(null, [{ address: '127.0.0.1', family: 4 }]); }; describe('createImageDownloader', () => { let server: http.Server; let port: number; let seen: Seen[]; let privateAllowed: boolean; let serverClosed: string[]; const open: ImageDownloader[] = []; const url = (path: string) => `http://img.test:${port}${path}`; const png = (res: http.ServerResponse, body: Buffer = PNG) => { res.writeHead(200, { 'content-type': 'image/png' }); res.end(body); }; const redirect = (res: http.ServerResponse, to: string, status = 302) => { res.writeHead(status, { location: to }); res.end(); }; const routes: Record< string, (req: http.IncomingMessage, res: http.ServerResponse) => void > = { '/ok.png': (_req, res) => png(res), '/params.png': (_req, res) => { res.writeHead(200, { 'content-type': 'Image/PNG; charset=binary' }); res.end(PNG); }, '/missing': (_req, res) => { res.writeHead(404); res.end('no'); }, '/boom': (_req, res) => { res.writeHead(500); res.end('no'); }, '/page': (_req, res) => { res.writeHead(200, { 'content-type': 'text/html' }); res.end(''); }, '/no-type': (_req, res) => { res.writeHead(200); res.end(PNG); }, '/svg': (_req, res) => { res.writeHead(200, { 'content-type': 'image/svg+xml' }); res.end(''); }, '/bmp': (_req, res) => { res.writeHead(200, { 'content-type': 'image/bmp' }); res.end(PNG); }, '/empty': (_req, res) => { res.writeHead(200, { 'content-type': 'image/png' }); res.end(); }, // redirects '/r3': (_req, res) => redirect(res, '/r2'), '/r2': (_req, res) => redirect(res, '/r1'), '/r1': (_req, res) => redirect(res, '/ok.png', 301), '/r4': (_req, res) => redirect(res, '/r3'), '/relative': (_req, res) => redirect(res, 'ok.png', 307), '/loop': (_req, res) => redirect(res, '/loop'), '/no-location': (_req, res) => { res.writeHead(302); res.end(); }, '/to-ftp': (_req, res) => redirect(res, 'ftp://example.com/a.png'), '/to-credentials': (_req, res) => redirect(res, `http://user:pw@img.test:${port}/ok.png`), // The first hop was allowed; from now on private addresses are not. '/to-private-name': (_req, res) => { privateAllowed = false; redirect(res, '/ok.png'); }, '/to-private-literal': (_req, res) => { privateAllowed = false; redirect(res, `http://127.0.0.1:${port}/ok.png`); }, '/to-metadata': (_req, res) => { privateAllowed = false; redirect(res, 'http://169.254.169.254/latest/meta-data/'); }, // sizes '/exact': (_req, res) => png(res, Buffer.alloc(LIMIT, 1)), '/over-by-one': (_req, res) => png(res, Buffer.alloc(LIMIT + 1, 1)), '/declared-huge': (req, res) => { res.writeHead(200, { 'content-type': 'image/png', 'content-length': String(LIMIT * 1000) }); // The headers go out now; no body follows, and the answer must not wait // for one. res.flushHeaders(); req.socket.on('close', () => serverClosed.push('declared-huge')); }, '/stream-huge': (_req, res) => { res.writeHead(200, { 'content-type': 'image/png' }); const timer = setInterval(() => res.write(Buffer.alloc(300, 2)), 5); res.on('close', () => { clearInterval(timer); serverClosed.push('stream-huge'); }); }, '/jpeg-bigger-limit': (_req, res) => { res.writeHead(200, { 'content-type': 'image/jpeg' }); res.end(Buffer.alloc(LIMIT * 2, 3)); }, '/jpeg-too-big': (_req, res) => { res.writeHead(200, { 'content-type': 'image/jpeg' }); res.end(Buffer.alloc(LIMIT * 2 + 1, 3)); }, '/gzip-bomb': (_req, res) => { res.writeHead(200, { 'content-type': 'image/png', 'content-encoding': 'gzip' }); res.end(zlib.gzipSync(Buffer.alloc(LIMIT * 500, 0))); }, // time '/hang': () => undefined, '/drip': (_req, res) => { res.writeHead(200, { 'content-type': 'image/png' }); const timer = setInterval(() => res.write('x'), 20); res.on('close', () => clearInterval(timer)); }, // HEAD '/head-405': (req, res) => { if (req.method === 'HEAD') { res.writeHead(405); res.end(); } else { res.writeHead(206, { 'content-type': 'image/png' }); res.end(PNG.subarray(0, 1)); } }, '/head-501': (req, res) => { if (req.method === 'HEAD') { res.writeHead(501); res.end(); } else { png(res); } }, '/head-405-endless': (req, res) => { if (req.method === 'HEAD') { res.writeHead(405); res.end(); } else { res.writeHead(206, { 'content-type': 'image/png' }); const timer = setInterval(() => res.write(Buffer.alloc(100, 1)), 5); res.on('close', () => clearInterval(timer)); } }, '/declared-exact': (_req, res) => { res.writeHead(200, { 'content-type': 'image/png', 'content-length': String(LIMIT) }); res.end(Buffer.alloc(LIMIT, 1)); }, '/declared-over-by-one': (req, res) => { res.writeHead(200, { 'content-type': 'image/png', 'content-length': String(LIMIT + 1) }); res.flushHeaders(); req.socket.on('close', () => serverClosed.push('declared-over-by-one')); }, '/head-404': (_req, res) => { res.writeHead(404); res.end(); }, '/head-403': (req, res) => { res.writeHead(req.method === 'HEAD' ? 403 : 200, { 'content-type': 'image/png' }); res.end(); } }; beforeAll(async () => { server = http.createServer((req, res) => { seen.push({ method: req.method, url: req.url, headers: req.headers }); const route = routes[req.url ?? '']; if (route) { route(req, res); } else { res.writeHead(404); res.end(); } }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve) ); port = (server.address() as AddressInfo).port; }); afterAll(async () => { server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); }); beforeEach(() => { seen = []; serverClosed = []; privateAllowed = true; }); afterEach(async () => { while (open.length > 0) { await open.pop()?.close(); } }); const make = (overrides: Partial = {}) => { const downloader = createImageDownloader({ allowPrivate: () => privateAllowed, lookup, timeoutMs: 5000, allowedTypes: () => [ 'image/jpeg', 'image/png', 'image/gif', 'image/webp' ], maxBytesForType: (type) => (type === 'image/jpeg' ? LIMIT * 2 : LIMIT), ...overrides }); open.push(downloader); return downloader; }; const reasonOf = async (work: Promise) => { try { await work; } catch (e) { return e as ImageError; } throw new Error('Expected a failure'); }; describe('download: the answer', () => { it('reads an image', async () => { const image = await make().download(url('/ok.png')); expect(image.bytes.equals(PNG)).toBe(true); expect(image.contentType).toBe('image/png'); expect(image.finalUrl).toBe(url('/ok.png')); }); it('reads the type without its parameters and in lower case', async () => { expect((await make().download(url('/params.png'))).contentType).toBe( 'image/png' ); }); it('asks for an image, in the identity encoding, and says who is asking', async () => { await make().download(url('/ok.png')); expect(seen).toHaveLength(1); expect(seen[0].method).toBe('GET'); expect(seen[0].headers['accept-encoding']).toBe('identity'); expect(seen[0].headers.accept).toBe('image/*'); expect(seen[0].headers['user-agent']).toBe('EverShop-Import'); expect(seen[0].headers.host).toBe(`img.test:${port}`); }); it.each([ ['/missing', 404], ['/boom', 500] ])('fails with http_status for %s', async (path, status) => { const error = await reasonOf(make().download(url(path))); expect(error.reason).toBe('http_status'); expect(error.params).toEqual({ status }); }); it('fails with not_an_image for a page, and for an answer that does not say what it is', async () => { const page = await reasonOf(make().download(url('/page'))); expect([page.reason, page.params]).toEqual([ 'not_an_image', { contentType: 'text/html' } ]); const none = await reasonOf(make().download(url('/no-type'))); expect([none.reason, none.params]).toEqual([ 'not_an_image', { contentType: null } ]); }); it('refuses SVG, even when the store accepts it', async () => { const downloader = make({ allowedTypes: () => ['image/png', 'image/svg+xml'] }); const error = await reasonOf(downloader.download(url('/svg'))); expect([error.reason, error.params]).toEqual([ 'type_not_allowed', { contentType: 'image/svg+xml' } ]); }); it('refuses a type of image the store does not accept', async () => { const error = await reasonOf(make().download(url('/bmp'))); expect([error.reason, error.params]).toEqual([ 'type_not_allowed', { contentType: 'image/bmp' } ]); }); it('fails with empty for an image with no bytes', async () => { expect((await reasonOf(make().download(url('/empty')))).reason).toBe( 'empty' ); }); }); describe('download: redirects (ยง 10)', () => { it('follows a redirect, also to a relative address', async () => { const image = await make().download(url('/relative')); expect(image.finalUrl).toBe(url('/ok.png')); expect(seen.map((s) => s.url)).toEqual(['/relative', '/ok.png']); }); it('follows three redirects and no more', async () => { const three = await make().download(url('/r3')); expect(three.finalUrl).toBe(url('/ok.png')); expect(seen).toHaveLength(4); seen = []; const error = await reasonOf(make().download(url('/r4'))); expect([error.reason, error.params]).toEqual([ 'too_many_redirects', { max: 3 } ]); expect(seen).toHaveLength(4); }); it('stops a redirect loop', async () => { expect((await reasonOf(make().download(url('/loop')))).reason).toBe( 'too_many_redirects' ); }); it('fails with http_status when a redirect has no Location', async () => { const error = await reasonOf(make().download(url('/no-location'))); expect([error.reason, error.params]).toEqual([ 'http_status', { status: 302 } ]); }); it.each([['/to-ftp'], ['/to-credentials']])( 'fails with invalid_url when %s redirects to an address that cannot be read', async (path) => { expect((await reasonOf(make().download(url(path)))).reason).toBe( 'invalid_url' ); expect(seen).toHaveLength(1); } ); it.each([ ['a name that now resolves to a private address', '/to-private-name'], ['an IP literal', '/to-private-literal'], ['the cloud metadata address', '/to-metadata'] ])( 'checks every hop: %s is refused after a first hop that was allowed', async (_name, path) => { const error = await reasonOf(make().download(url(path))); expect(error.reason).toBe('blocked'); // The first request was made; the redirected one was not. expect(seen.map((s) => s.url)).toEqual([path]); } ); }); describe('download: the size', () => { it('accepts exactly the limit and refuses one byte more', async () => { expect((await make().download(url('/exact'))).bytes.length).toBe(LIMIT); const error = await reasonOf(make().download(url('/over-by-one'))); expect([error.reason, error.params]).toEqual([ 'too_large', { contentType: 'image/png', maxMb: LIMIT / 1_048_576 } ]); }); it('judges the declared size on the same line as the body: exactly the limit passes, one byte more does not', async () => { expect((await make().download(url('/declared-exact'))).bytes.length).toBe( LIMIT ); const error = await reasonOf( make().download(url('/declared-over-by-one')) ); expect(error.reason).toBe('too_large'); }); it('uses the limit of the type', async () => { const image = await make().download(url('/jpeg-bigger-limit')); expect(image.bytes.length).toBe(LIMIT * 2); expect( (await reasonOf(make().download(url('/jpeg-too-big')))).reason ).toBe('too_large'); }); it('refuses an image whose declared size is over the limit without waiting for the body', async () => { const started = Date.now(); const error = await reasonOf(make().download(url('/declared-huge'))); expect(error.reason).toBe('too_large'); expect(Date.now() - started).toBeLessThan(2000); }); it('stops reading and closes the connection when a body without a size goes past the limit', async () => { const error = await reasonOf(make().download(url('/stream-huge'))); expect(error.reason).toBe('too_large'); await new Promise((resolve) => setTimeout(resolve, 100)); expect(serverClosed).toContain('stream-huge'); }); it('counts the bytes after a compressed answer is unpacked (a compression bomb)', async () => { const error = await reasonOf(make().download(url('/gzip-bomb'))); expect(error.reason).toBe('too_large'); }); }); describe('download: time and the network', () => { it('fails with timeout when the server does not answer', async () => { const error = await reasonOf( make({ timeoutMs: 150 }).download(url('/hang')) ); expect(error.reason).toBe('timeout'); }); it('fails with timeout when the body never ends: the deadline is for the whole image', async () => { const error = await reasonOf( make({ timeoutMs: 300 }).download(url('/drip')) ); expect(error.reason).toBe('timeout'); }); it('fails with network and the code when nothing listens', async () => { const free = http.createServer(); await new Promise((resolve) => free.listen(0, '127.0.0.1', resolve) ); const closedPort = (free.address() as AddressInfo).port; await new Promise((resolve) => free.close(() => resolve())); const error = await reasonOf( make().download(`http://img.test:${closedPort}/a.png`) ); expect([error.reason, error.params]).toEqual([ 'network', { code: 'ECONNREFUSED' } ]); }); it('fails with network when the name does not resolve', async () => { const failing: LookupFunction = (_h, _o, callback) => callback(Object.assign(new Error('nx'), { code: 'ENOTFOUND' }), []); const error = await reasonOf( make({ lookup: failing }).download(url('/ok.png')) ); expect([error.reason, error.params]).toEqual([ 'network', { code: 'ENOTFOUND' } ]); }); }); describe('the deadline timer', () => { it('is cleared when the image is done, whether it worked or not, so that it does not keep the process alive', async () => { const delay = 4321; const started: unknown[] = []; const cleared: unknown[] = []; const realSet = globalThis.setTimeout; const realClear = globalThis.clearTimeout; globalThis.setTimeout = (( fn: () => void, ms?: number, ...args: unknown[] ) => { const handle = realSet(fn, ms, ...args); if (ms === delay) started.push(handle); return handle; }) as typeof setTimeout; globalThis.clearTimeout = (( handle?: Parameters[0] ) => { cleared.push(handle); return realClear(handle); }) as typeof clearTimeout; try { const downloader = make({ timeoutMs: delay }); await downloader.download(url('/ok.png')); await downloader.download(url('/missing')).catch(() => undefined); await downloader.probe(url('/ok.png')); } finally { globalThis.setTimeout = realSet; globalThis.clearTimeout = realClear; } expect(started).toHaveLength(3); for (const handle of started) { expect(cleared).toContain(handle); } }); }); describe('download: private addresses are refused unless the switch is on', () => { beforeEach(() => { privateAllowed = false; }); it('refuses an IP literal', async () => { const error = await reasonOf( make().download(`http://127.0.0.1:${port}/ok.png`) ); expect(error.reason).toBe('blocked'); expect(seen).toHaveLength(0); }); it('refuses the cloud metadata address', async () => { const error = await reasonOf( make().download('http://169.254.169.254/latest/meta-data/') ); expect(error.reason).toBe('blocked'); }); it('refuses a name that resolves to a private address, and makes no request', async () => { const error = await reasonOf(make().download(url('/ok.png'))); expect(error.reason).toBe('blocked'); expect(seen).toHaveLength(0); }); it('refuses localhost', async () => { const error = await reasonOf( make({ lookup: undefined }).download(`http://localhost:${port}/ok.png`) ); expect(error.reason).toBe('blocked'); expect(seen).toHaveLength(0); }); it('refuses NAT64 and 6to4 forms of a private address', async () => { for (const host of ['[64:ff9b::7f00:1]', '[2002:7f00:1::]']) { const error = await reasonOf( make().download(`http://${host}:${port}/ok.png`) ); expect(error.reason).toBe('blocked'); } }); it('lets an IP literal through when the switch is on (the end-to-end suite serves from localhost)', async () => { privateAllowed = true; const image = await make().download(`http://127.0.0.1:${port}/ok.png`); expect(image.contentType).toBe('image/png'); }); it('reads the switch for every request', async () => { const downloader = make(); expect((await reasonOf(downloader.download(url('/ok.png')))).reason).toBe( 'blocked' ); privateAllowed = true; expect((await downloader.download(url('/ok.png'))).contentType).toBe( 'image/png' ); }); }); describe('download: addresses that cannot be read', () => { it.each([ ['ftp://example.com/a.png'], ['file:///etc/passwd'], ['https://user:pw@example.com/a.png'], ['//example.com/a.png'], ['not a url'], [''] ])('fails with invalid_url for %j', async (text) => { expect((await reasonOf(make().download(text))).reason).toBe( 'invalid_url' ); expect(seen).toHaveLength(0); }); }); describe('probe (IMG-05)', () => { it('asks with HEAD and does not read the body', async () => { await make().probe(url('/ok.png')); expect(seen.map((s) => s.method)).toEqual(['HEAD']); }); it.each([['/head-405'], ['/head-501']])( 'asks for the first byte with GET when the server refuses HEAD (%s)', async (path) => { await make().probe(url(path)); expect(seen.map((s) => s.method)).toEqual(['HEAD', 'GET']); expect(seen[1].headers.range).toBe('bytes=0-0'); } ); it('does not read the body of the answer to the GET, however long it is', async () => { const started = Date.now(); await make().probe(url('/head-405-endless')); expect(Date.now() - started).toBeLessThan(2000); }); it('does not ask again when HEAD is refused for another reason', async () => { const error = await reasonOf(make().probe(url('/head-403'))); expect([error.reason, error.params]).toEqual([ 'http_status', { status: 403 } ]); expect(seen.map((s) => s.method)).toEqual(['HEAD']); }); it('fails like a download: status, type, private address, redirects, time', async () => { expect((await reasonOf(make().probe(url('/head-404')))).reason).toBe( 'http_status' ); expect((await reasonOf(make().probe(url('/page')))).reason).toBe( 'not_an_image' ); expect((await reasonOf(make().probe(url('/r4')))).reason).toBe( 'too_many_redirects' ); expect( (await reasonOf(make({ timeoutMs: 150 }).probe(url('/hang')))).reason ).toBe('timeout'); privateAllowed = false; expect((await reasonOf(make().probe(url('/ok.png')))).reason).toBe( 'blocked' ); }); it('follows redirects', async () => { await make().probe(url('/r3')); expect(seen).toHaveLength(4); }); }); });