import { Browser, Page, Target, Viewport } from "puppeteer"; import * as cheerio from 'cheerio'; import fetch from 'node-fetch'; export async function getSkus(detailsPages, browser: Browser, newPage: Target, viewport: Viewport) { const page = await newPage.page(); const skus = [] as any[] for (const detailPage of detailsPages) { const [newDetailPage] = await Promise.all([ browser.waitForTarget(target => target.opener() === newPage), page!.evaluate(detailPage => window.open(detailPage, '_blank'), detailPage) ]); const detailPageInstance = (await newDetailPage.page()) as Page; await detailPageInstance.setViewport(viewport); await detailPageInstance.bringToFront(); // @ts-expect-error const initData = await detailPageInstance!.evaluate(() => window.__INIT_DATA) await detailPageInstance.close() skus.push(initData.globalData.skuModel) } return skus } export async function scrapePage(browser, page, pageNum, keyword, viewport) { console.log(`Scraping page ${pageNum} for keyword "${keyword}"`); const url = encodeURI(`https://s.1688.com/selloffer/offer_search.htm?keywords=${encodeURIComponent(keyword)}&n=y&netType=16&beginPage=${pageNum}#sm-filtbar`); const [newPage] = await Promise.all([ browser.waitForTarget(target => target.opener() === page.target()), page.evaluate(url => window.open(url, '_blank'), url) ]); const pageInstance = await newPage.page(); await pageInstance.setViewport(viewport); await pageInstance.bringToFront(); await pageInstance.waitForSelector('.page-offerlist'); for (let i = 0; i < 5; i++) { await pageInstance.evaluate(() => window.scrollBy(0, 380)); await pageInstance.waitForTimeout(1000); } const content = await pageInstance.content(); const $ = cheerio.load(content); const detailsPages: string[] = [] $('.page-offerlist .feeds-wrapper > a').each((i, elem) => { detailsPages.push(elem.attribs.href) const title = $(elem).find('.title-text').text().trim(); const img_url = $(elem).find('.img').attr('style'); const desc_href = $(elem).find('.img-container a').attr('href'); const price = $(elem).find('.desc-container .price-container .price').text().trim(); const sales_num = $(elem).find('.desc-container .price-container .sale').text().trim(); const company_name = $(elem).find('.company-name a').text().trim(); const company_href = $(elem).find('.company-name a').attr('href'); const company_tag = $(elem).find('.common-company-tag').text().trim(); const shop_repurchase_rate = $(elem).find('.shop-repurchase-rate').text().trim(); const row = [keyword, title, img_url, desc_href, price, sales_num, company_name, company_href, company_tag, shop_repurchase_rate]; console.log(row); }); await getSkus(detailsPages, browser, newPage, viewport) await pageInstance.close(); } export async function addToCart(cart_items, browser: Browser, newPage: Target, viewport: Viewport) { const pageInstance = await newPage.page() const csrfToken = await pageInstance!.evaluate(() => { // @ts-expect-error return window.feloader.getModule('@ali/tdmod-od-gyp-pc-layout-two-columns/index-pc')._require('@ali/rox-csrf-token/index').default.legacyToken() }); // 在 evaluate 之外获取 Cookies 然后传递给 evaluate 内部 const cookies = await pageInstance!.cookies(); // 序列化 Cookies 为请求头的值 const cookieString = cookies.map(cookie => `${cookie.name}=${cookie.value}`).join('; '); try { const result = await Promise.all(cart_items.map(async (item) => { // 设定我们将要使用的 AJAX 请求的参数 const body = `type=offer&cargoIdentity=${item.product.cargoId}&returnType=url&needTotalPrice=false&promotionSwitch=false&_csrf_token=${csrfToken}&t=${Date.now()}purchaseType=&specData=${encodeURIComponent(JSON.stringify(item.skus.map(item => ({ amount: item.amount, specId: item.specId, selectedTradeServices: [] }))))}` const options = { method: 'POST', headers: { 'accept': 'text/plain, */*; q=0.01', 'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', 'bx-v': '2.5.14', 'cache-control': 'no-cache', 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', 'cookie': cookieString, 'origin': 'https://cart.1688.com', 'pragma': 'no-cache', 'priority': 'u=1, i', 'referer': 'https://cart.1688.com/message_proxy_server', 'sec-ch-ua': '"Chromium";v="128", "Not;A=Brand";v="24", "Google Chrome";v="128"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', 'x-requested-with': 'XMLHttpRequest' }, body }; const response = await fetch('https://cart.1688.com/ajax/safe/add_to_cart_list_new.jsx', options); return JSON.parse(await response.text()) })) return result } catch (e) { console.error(e) } }