// 导入必要的库 import puppeteer, { Page } from 'puppeteer'; import { askQuestion, loginTaobao, USER_DATA_DIR, windowDimension } from './common'; // 创建Excel表格 const ws = [ [ 'Page', 'Num', 'title', 'Price', 'Deal', 'Location', 'Shop', 'IsPostFree', 'Title_URL', 'Shop_URL', 'Img_URL' ] ]; // 搜索商品关键词并进行首次爬取 const searchGoods = async (page: Page, keyword: string, startPage: number, endPage: number) => { console.log('正在搜索:', keyword); await page.type('#q', keyword); await page.click('#J_TSearchForm > div.search-button > button'); await new Promise(resolve => setTimeout(resolve,5000)); // 等待可能的滑块验证 if (startPage !== 1) { await page.evaluate(() => { window.scrollTo(0, document.body.scrollHeight); }); await new Promise(resolve => setTimeout(resolve,3000)); await page.type( '#pageContent > div:nth-child(1) > div:nth-child(3) > div:nth-child(4) > div > div > span:nth-child(3) > input', startPage.toString() ); await page.click( '#pageContent > div:nth-child(1) > div:nth-child(3) > div:nth-child(4) > div > div > button:nth-child(3)' ); } await getGoods(page, startPage); for (let i = startPage + 1; i <= endPage; i++) { await pageTurning(page, i); } }; // 翻页函数 const pageTurning = async (page, pageNumber) => { console.log('正在翻页:', pageNumber); const pageNavButton = `#search-content-leftWrap > div.LeftLay--leftContent--AMmPNfB > div.Pagination--pgWrap--kfPsaVv > div > div > div > button:nth-child(${pageNumber})` await page.waitForSelector(pageNavButton); await page.click(pageNavButton); await page.waitForFunction( pageNumber => document.querySelector( '#sortBarWrap > div:nth-child(1) > div:nth-child(2) > div:nth-child(2) > div:nth-child(8) > div > span > em' )?.textContent?.trim() === pageNumber.toString(), {}, pageNumber ); await getGoods(page, pageNumber); }; declare global { interface Element { innerText: string; } } // 获取商品信息 const getGoods = async (page, currentPage) => { await page.waitForTimeout(10000); const goodsList = await page.evaluate(() => { const items = document.querySelectorAll( 'div.PageContent--contentWrap--mep7AEm > div.LeftLay--leftWrap--xBQipVc > div.LeftLay--leftContent--AMmPNfB > div.Content--content--sgSCZ12 > div > a' ); const goodsArray: any[] = []; items.forEach((item, index) => { const title = item.querySelector('.Title--title--jCOPvpf span') ? item.querySelector('.Title--title--jCOPvpf span')?.innerText : ''; const priceInt = item.querySelector('.Price--priceInt--ZlsSi_M') ? item.querySelector('.Price--priceInt--ZlsSi_M')?.innerText : '0'; const priceFloat = item.querySelector('.Price--priceFloat--h2RR0RK') ? item.querySelector('.Price--priceFloat--h2RR0RK')?.innerText : '0.0'; const price = parseFloat(`${priceInt}${priceFloat}`); const deal = item.querySelector('.Price--realSales--FhTZc7U') ? item.querySelector('.Price--realSales--FhTZc7U')?.innerText : ''; const location = item.querySelector('.Price--procity--_7Vt3mX') ? item.querySelector('.Price--procity--_7Vt3mX')?.innerText : ''; const shop = item.querySelector('.ShopInfo--TextAndPic--yH0AZfx a') ? item.querySelector('.ShopInfo--TextAndPic--yH0AZfx a')?.innerText : ''; const postText = item.querySelector('.SalesPoint--subIconWrapper--s6vanNY span') && item .querySelector('.SalesPoint--subIconWrapper--s6vanNY span')?.innerText.includes('包邮') ? '包邮' : '/'; const t_url = item.getAttribute('href'); const shop_url = item.querySelector('.ShopInfo--TextAndPic--yH0AZfx a') ? item.querySelector('.ShopInfo--TextAndPic--yH0AZfx a')?.getAttribute( 'href' ) : ''; const img_url = item.querySelector('.MainPic--mainPicWrapper--iv9Yv90 img') ? item .querySelector('.MainPic--mainPicWrapper--iv9Yv90 img')?.getAttribute('src') : ''; goodsArray.push({ num: index, title: title, price: price, deal: deal, location: location, shop: shop, isPostFree: postText, url: t_url, shop_url: shop_url, img_url: img_url }); }); return goodsArray; }); goodsList.forEach((product) => { console.log(product); const row = [ currentPage, product.num, product.title, product.price, product.deal, product.location, product.shop, product.isPostFree, product.url, product.shop_url, product.img_url ]; ws.push(row); }); }; // 开始爬取 export const goodsList = async () => { const browser = await puppeteer.launch({ headless: false, userDataDir: USER_DATA_DIR, defaultViewport: { width: windowDimension[0], height: windowDimension[1] }, args: [ "--no-sandbox", "--proxy-server=https=192.168.31.121:1080", "--ignore-certificate-errors", "--ignore-certificate-errors-spki-list", `--window-size=${windowDimension.join(',')}`, ] }); try { const keyword = await askQuestion('输入搜索的商品关键词Keyword:',' 头盔'); const startPage = parseInt(await askQuestion('输入爬取的起始页PageStart:','1')); const endPage = parseInt(await askQuestion('输入爬取的终止页PageEnd:','2')); // rl.close(); const page = await loginTaobao(); // 开始爬取数据 await searchGoods(page, keyword, startPage, endPage); } catch (error) { console.error('Crawer_main函数报错:', error); } finally { if (browser) { await browser.close(); } } };