import * as _ from 'lodash'; import * as Request from 'requestretry'; import * as UserAgent from 'user-agents'; const timeout = 10 * 1000; const retryDelay = 3000; const maxAttempts = 5; export function request2({ url = '', device = 'mobile', headers = {}, ...otherOptions } = {}) { const userAgent = new UserAgent({ deviceCategory: device }); return new Promise(function (resolve, reject) { Request({ url: url, timeout, headers: { 'User-Agent': userAgent.toString(), ...headers, }, maxAttempts, // (default) try 5 times retryDelay, // (default) wait for 5s before trying again // retryStrategy: Request.RetryStrategies.HTTPOrNetworkError // (default) retry on 5xx or network errors retryStrategy: myRetryStrategy,// (default) retry on 5xx or network errors ...otherOptions, }, function (error, resp, body) { if (error) { } handleResp(resolve, reject, error, resp, body); }); }); } export function doRequest(url, { device = 'mobile', headers = {}, ...otherOptions } = {}) { return request2({ url, device, headers, ...otherOptions, }); } export function handleResp(resolve, reject, error, resp, body) { const statusCode = _.get(resp, 'statusCode') || ''; const statusMessage = _.get(resp, 'statusMessage') || ''; if (!error && statusCode == 200) { resolve(body); } else { reject({ statusCode, statusMessage, error }); } } function myRetryStrategy(err, response) { const statusCode = _.get(response, 'statusCode') || ''; return err || statusCode === 502; } export function normalRequest({ url, device = 'mobile', headers = {} }) { const userAgent = new UserAgent({ deviceCategory: device }); return new Promise(function (resolve, reject) { Request({ url: url, timeout, headers: { 'User-Agent': userAgent.toString(), ...headers, }, maxAttempts, // (default) try 5 times retryDelay, // (default) wait for 5s before trying again // retryStrategy: Request.RetryStrategies.HTTPOrNetworkError // (default) retry on 5xx or network errors retryStrategy: myRetryStrategy,// (default) retry on 5xx or network errors }, function (error, resp, body) { if (error) { // console.error('error', error); // console.error('error.stack', error.stack?.toString().split('\n').slice(0, 2).join('\n')); } handleResp(resolve, reject, error, resp, body); }); }); } export function proxyRequest({ url, device = 'mobile' }) { const userAgent = new UserAgent({ deviceCategory: device }); console.log('process.env.LUMINATY_PROXY', !!process.env.LUMINATY_PROXY); return new Promise(function (resolve, reject) { Request({ url: url, timeout, rejectUnauthorized: false, headers: { 'User-Agent': userAgent.toString(), }, maxAttempts, retryDelay, retryStrategy: myRetryStrategy, }, function (error, resp, body) { if (error) { // console.error('error', error); // console.error('error.stack', error.stack?.toString().split('\n').slice(0, 2).join('\n')); const attempts = _.get(resp, 'attempts') || ''; } handleResp(resolve, reject, error, resp, body || ''); }); }); } export function proxyCrawlRequest({ url, device = 'mobile' }) { const userAgent = new UserAgent({ deviceCategory: device }); const newUrl = ``; return new Promise(function (resolve, reject) { Request({ url: newUrl, timeout, rejectUnauthorized: false, headers: { 'User-Agent': userAgent.toString(), }, maxAttempts, retryDelay, retryStrategy: myRetryStrategy, }, function (error, resp, body) { handleResp(resolve, reject, error, resp, body); }); }); }