import axios from "axios"; import { WbPlatform } from "../entities/platform"; import { Spilder, SpilderError } from "./spilder"; export class WeiboSpilder extends Spilder { createTopicArticleLongTextDetail(id: string): Promise { const url = `https://weibo.com/ajax/statuses/longtext?id=${id}`; return axios.get(url, { headers: { cookie: this.cookies } }).then(res => { return res.data; }) } async createAccount(uid: number): Promise { return await Promise.all([ this.getUserInfo(uid), this.getUserDetail(uid) ]).then(([info, detail]) => { return { ...info.data, ...detail.data } }) } private getUserInfo(uid: number) { const url = `https://weibo.com/ajax/profile/info?uid=${uid}`; console.log(url) return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data); } private getUserDetail(uid: number) { const url = `https://weibo.com/ajax/profile/detail?uid=${uid}`; return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data); } async setCookies(cookies: any[]): Promise { if (this.page) await this.page.setCookie(...cookies); } async createLoginQrCode(): Promise { const url = `https://weibo.com`; const ctx = this.browser.defaultBrowserContext(); await ctx.overridePermissions(url, ['geolocation']) const isLogin = await this.isLogin(); if (isLogin) { return; } if (!this.page) { return; } this.page.setJavaScriptEnabled(true); this.page.setViewport({ width: 1920, height: 768 }); try { await this.page.goto(url, { waitUntil: ['networkidle2'] }); await this.delay(20); const href = this.page.url(); if (href === 'https://weibo.com/') { await this.updateLoginInfo(); throw new SpilderError(200, '账号已登录') } if (href.startsWith('https://weibo.com/newlogin')) { } else { /** * 先跳转拿到cookie */ await this.page.waitForNavigation(); throw new Error(`获取登录二维码失败`) } // https://passport.weibo.com/visitor/visitor?entry=miniblog&a=enter&url=https%3A%2F%2Fweibo.com%2F&domain=.weibo.com&ua=php-sso_sdk_client-0.6.36&_rand=1658565674.705 // #app > div:nth-child(4) > div.woo-modal-main > div > div.woo-box-flex.woo-box-alignCenter.woo-box-justifyCenter.LoginPop_main_SAOC0 > div > div > div.woo-box-flex.woo-box-alignCenter.woo-box-justifyBetween.LoginPop_crbox_237wO > div.LoginPop_mabox_3Lyr6 > img const loginCardBtnJpU1 = await this.page.waitForSelector('.LoginCard_btn_Jp_u1'); if (loginCardBtnJpU1) { await loginCardBtnJpU1.click(); await this.delay(200); const img = await this.page.waitForSelector('div.LoginPop_mabox_3Lyr6 > img').catch(e => { return null; }); if (!img) { throw new Error(`获取登录二维码失败`) } let loginImage = await this.page.$eval('div.LoginPop_mabox_3Lyr6 > img', (img) => (img as HTMLImageElement).src); if (loginImage.startsWith(`https://weibo.com/newlogin`)) { throw new Error(`获取登录二维码失败`) } return loginImage; } } finally { // await p.close(); } } async checkScanStatus(): Promise { if (this.page) { try { const tip = await this.page.waitForSelector('span.woo-tip-text > div.LoginPop_t1_2fuX8', { timeout: 1000 }) if (tip) { const txt = await this.page.$eval('span.woo-tip-text > div.LoginPop_t1_2fuX8', (div) => { return (div as HTMLDivElement).innerText; }); return txt; } else { throw new Error(`获取扫码信息失败`) } } catch (e: any) { console.log(e.message) throw e; } } } async updateLoginInfo() { if (this.page) { const data = await this.page.$eval('.woo-tab-nav a:nth-child(5)', (nav) => { const linkElement = nav as HTMLAnchorElement; const href = linkElement ? linkElement.href : ""; const reg = /\/u\/(.*)/ const result = href.match(reg); const uid = result ? result[1] : ''; const nameElement = nav.querySelector('.woo-tab-item-main') as HTMLDivElement; const name = nameElement.ariaLabel as string; const imageElement = nav.querySelector('img') as HTMLImageElement; // div.woo-box-flex.woo-box-alignCenter.woo-box-justifyCenter.Nav_mid_5pisP > div > div.woo-box-flex.woo-tab-nav > a:nth-child(5) > div > div > div > div > img return { uid, name: name ? name : '', avatar: imageElement ? imageElement.src : '' } }); if (data.uid) { this.platform.uid = data.uid; this.platform.nickname = data.name; this.platform.status = 1; const cookies = await this.page.cookies('https://weibo.com/') this.platform.cookies = cookies; await this.db.manager.save(WbPlatform, this.platform); console.info(`登录状态更新成功`) return this.platform; } } } async loginSuccess(): Promise { if (this.page) { try { await this.page.waitForNavigation(); } catch (e) { } return this.updateLoginInfo(); } } /** * 检查登录状态 * @param uid 用户id * @returns */ async isLogin(): Promise { try { const res = await axios.get(`https://weibo.com/ajax/profile/info?uid=${this.platform.uid}`, { headers: { cookie: this.cookies } }).then(r => r.data).then(data => { return data.ok === 1; }).catch(e => { return false; }); return res; } catch (e) { return false; } } /** * 搜索符合条件的主题分页连接 * @param keyword * @returns */ async createTopicPageLinks(keyword: string) { const url = `https://s.weibo.com/topic?q=${decodeURIComponent(keyword)}&pagetype=topic&topic=1&Refer=weibo_topic`; const page = await this.browser.newPage(); try { await page.goto(url, { waitUntil: ['networkidle2'] }); await page.waitForSelector('#pl_feedlist_index'); const count = await page.$eval('#pl_feedlist_index', (feedlist) => { const pageElement = feedlist.querySelector('div.m-page > div > span > ul > li:last-child > a') as HTMLAnchorElement; const pageText = pageElement ? pageElement.innerText : ''; const re = /第(.*?)页/ const matchRes = pageText.match(re); let count: number = 1; if (matchRes) { count = Number(matchRes[1]) } return count; }); const links: string[] = []; for (let i = 1; i <= count; i++) { links.push(`${url}&page=${i}`) } return { links, keyword, count }; } catch (e) { return { links: [`${url}`], keyword, count: 1 } } finally { await page.close(); } } /** * 创建主题连接 * @param link 分页连接 * @returns { list: 评论列表连接, detail: 详情连接 } */ async createTopicLinks(link: string) { const page = await this.browser.newPage(); try { await page.goto(link, { waitUntil: ['networkidle2'] }); await page.waitForSelector('#pl_feedlist_index'); const links = await page.$eval('#pl_feedlist_index', (feedlist) => { const cards = feedlist.querySelectorAll('.card'); const links: { list: string, detail: string }[] = []; cards.forEach(card => { const titleElement = card.querySelector('div.info > div > a') as HTMLAnchorElement; if (titleElement) { const title = titleElement ? titleElement.innerText : ''; links.push({ list: `https://s.weibo.com/weibo?q=${encodeURIComponent(title)}`, detail: `https://m.s.weibo.com/vtopic/detail?q=${encodeURIComponent(title)}` }); } }); return links; }) return links; } finally { await page.close(); } } /** * 获取主题详情 * @param link 主题详情连接 */ async createTopicDetail(link: string) { const href = decodeURIComponent(link) const res = href.match('q=(.*)') let q = ``; if (res) { q = res[1]; } const url = `https://m.s.weibo.com/ajax_topic/detail?q=${encodeURIComponent(q)}`; return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).then(data => { return data.data; }) } /** * 获取主题帖子所有分页连接 * @param url * @returns */ async createTopicArticlePageLinks(url: string) { const p = await this.browser.newPage(); try { await p.goto(url, { waitUntil: ['networkidle2'] }); await p.waitForSelector('#pl_feedlist_index'); const count = await p.$eval('#pl_feedlist_index', feedlist => { const pageElement = feedlist.querySelector('div.m-page > div > span > ul > li:last-child > a') as HTMLAnchorElement; const pageText = pageElement ? pageElement.innerText : ''; const re = /第(.*?)页/ const matchRes = pageText.match(re); let count: number = 1; if (matchRes) { count = Number(matchRes[1]) } return count; }); const links: string[] = []; for (let i = 1; i <= count; i++) { links.push(`${url}&page=${i}`) } return links; } catch (e) { return [ `${url}` ] } finally { await p.close(); } } /** * 获取主题某页文章连接 */ async createTopicArticleLinks(page: string) { const p = await this.browser.newPage(); try { await p.goto(page, { waitUntil: ['networkidle2'] }); await p.waitForSelector('#pl_feedlist_index'); const links = await p.$eval('#pl_feedlist_index', feedlist => { const items = feedlist.querySelectorAll('.card'); const links: string[] = []; items.forEach(item => { const aidLink = item.querySelector('div.card-feed > div.content > p.from > a:nth-child(1)') as HTMLAnchorElement; if (aidLink) { const link = aidLink.href; links.push(link); } }) return links; }); return links; } finally { await p.close(); } } /** * 获取文章详情 * @param link * @returns */ async createTopicArticleDetail(link: string) { //https://weibo.com/7467277921/LDTaIgeYS?refer_flag=1001030103_ const res = link.match(/weibo\.com\/(.*?)\/(.*?)\?/) let id = ``; if (res) { id = res[2]; } if (id.length > 0) { const url = `https://weibo.com/ajax/statuses/show?id=${id}` return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).catch(async e => { const { response } = e; console.error(`createTopicArticleDetail-${url}-${e.message}-${link}`, response.data); if (response && response.status === 400) { const { data } = response; if (data) { const message: string = data.message; if (message.includes('系统繁忙')) { throw e; } if (message.includes('无法访问')) { return; } } } await this.updatePlatform(); throw e; }) } } async updatePlatform() { const platform = await this.db.manager.findOne(WbPlatform, { where: { id: this.platform.id } }) if (platform) { this.platform = platform; this.cookies = this.platform.cookies.map(c => `${c.name}=${c.value}`).join(';') } } async loadComments(id: number, uid: number): Promise { const params: any = { is_reload: 1, id: Number(id), is_show_bulletin: 2, is_mix: 0, count: 100, uid: Number(uid) } const p = Object.keys(params).map(k => { return `${k}=${params[k]}` }).join('&'); const url = `https://weibo.com/ajax/statuses/buildComments?${p}` return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).catch(async e => { const { response } = e; console.error(`loadComments-${url}-${e.message}`, response.data); if (response && response.status === 400) { const { data } = response; if (data) { const message: string = data.message; if (message.includes('系统繁忙')) { throw e; } if (message.includes('无法访问')) { return; } } } await this.updatePlatform(); throw e; }) } async loadMoreComments(id: number, uid: number, max_id: number): Promise { const params: any = { flow: 0, is_reload: 1, id, is_show_bulletin: 2, is_mix: 0, max_id: max_id, count: 200, uid: uid }; const p = Object.keys(params).map(k => { return `${k}=${params[k]}` }).join('&'); const url = `https://weibo.com/ajax/statuses/buildComments?${p}` return await axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).catch(async e => { const { response } = e; console.error(`loadMoreComments-${url}-${e.message}`, response.data); if (response && response.status === 400) { const { data } = response; if (data) { const message: string = data.message; if (message.includes('系统繁忙')) { throw e; } if (message.includes('无法访问')) { return; } } } await this.updatePlatform(); throw e; }) } async loadChildComments(id: number, uid: number) { const params: any = { is_reload: 1, id, is_show_bulletin: 2, is_mix: 1, fetch_level: 1, max_id: 0, count: 200, uid } const p = Object.keys(params).map(k => { return `${k}=${params[k]}` }).join('&'); const url = `https://weibo.com/ajax/statuses/buildComments?${p}` return axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).catch(async e => { const { response } = e; console.error(`loadChildComments-${url}-${e.message}`, response.data); if (response && response.status === 400) { const { data } = response; if (data) { const message: string = data.message; if (message.includes('系统繁忙')) { throw e; } if (message.includes('无法访问')) { return; } } } await this.updatePlatform(); throw e; }); } async loadMoreChildComments(id: number, uid: number, max_id: number) { const params: any = { flow: 0, is_reload: 1, id, is_show_bulletin: 2, is_mix: 1, fetch_level: 1, max_id, count: 20, uid } const p = Object.keys(params).map(k => { return `${k}=${params[k]}` }).join('&'); const url = `https://weibo.com/ajax/statuses/buildComments?${p}` return await axios.get(url, { headers: { cookie: this.cookies } }).then(res => res.data).catch(async e => { const { response } = e; await this.updatePlatform(); console.error(`loadMoreChildComments-${url}-${e.message}`, response.data); if (response && response.status === 400) { const { data } = response; if (typeof data === 'string') { if (data.includes('请稍后尝试刷新')) { throw e; } } if (data) { const message: string = data.message; if (message.includes('系统繁忙')) { throw e; } if (message.includes('无法访问')) { return; } if (message.includes('获取数据失败(ApiError)')) { return; } if (message.includes('invalid weibo user!')) { return; } } } throw e; }) } }