extractPageContent.mjs - Documentation

import estimateVisibleText from './estimateVisibleText.mjs'


//可見文字低於此字數才啟動Shadow DOM穿透
//
//本值與inspectHtml.mjs之EMPTY_VISIBLE_MAX**必須相同**: 後者決定「低於多少判為空內容」。
//兩者一致時, 會被判空的頁面正好就是會啟動穿透的頁面; 不一致即開縫,
//出現「判空但不穿透」(升級後仍空, 白費一階)或「穿透了卻仍被判空」之頁面。
//此約束由 unit-inspectHtml 之「兩處門檻須一致」鎖住
let SHADOW_VISIBLE_THRESHOLD = 200

//Shadow DOM取得之文字低於此字數視為無效, 仍回傳原HTML
let SHADOW_MIN_CHARS = 50


//轉義HTML特殊字元
function _esc(s) {
    return String(s).replace(/[<>&]/g, (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;' }[c]))
}


/**
 * 由Playwright之page提取網頁HTML，可見文字過少時穿透Shadow DOM
 *
 * 先取page.content()，若估算可見文字已達門檻則直接回傳；否則遞迴穿透Shadow DOM取得
 * 各層innerText，再依換行切段重組為含article之簡易HTML，供Readability解析。
 *
 * 回傳一併標明內容形態：合成內容之標籤結構已被剝除，其可信的判識證據與原始文件不同，
 * 故由本函數（唯一知道走了哪個出口者）標明，而非由下游猜測
 *
 * @param {Object} page 輸入Playwright之page物件
 * @returns {Promise} 回傳Promise，resolve回傳{html,contentKind}物件，contentKind為'raw'或'synthesized'
 * @example
 *
 * import { chromium } from 'playwright'
 * import extractPageContent from './src/extractPageContent.mjs'
 *
 * let test = async () => {
 *
 *     let browser = await chromium.launch({ headless: true, channel: 'chrome' })
 *     let page = await browser.newPage()
 *     await page.goto('https://example.com/')
 *     let { html, contentKind } = await extractPageContent(page)
 *     console.log(html.length, contentKind)
 *     // => 234 'raw'
 *     await browser.close()
 *
 * }
 * await test()
 *     .catch((err) => {
 *         console.log(err)
 *     })
 *
 */
async function extractPageContent(page) {

    let html = await page.content()
    let visible = estimateVisibleText(html)
    if (visible.length >= SHADOW_VISIBLE_THRESHOLD) {
        return { html, contentKind: 'raw' }
    }

    //穿透Shadow DOM
    let shadow = await page.evaluate(() => {
        function getDeepInnerText(el) {
            if (!el) {
                return ''
            }
            let text = ''
            if (el.shadowRoot) {
                for (let child of el.shadowRoot.children) {
                    text += getDeepInnerText(child)
                }
                return text
            }
            if (el.children && el.children.length > 0) {
                for (let child of el.children) {
                    text += getDeepInnerText(child)
                }
                return text
            }
            return el.innerText || ''
        }
        return getDeepInnerText(document.body || document.documentElement)
    })

    if (!shadow || shadow.length < SHADOW_MIN_CHARS) {
        return { html, contentKind: 'raw' }
    }

    //由Shadow DOM內文重組為簡易HTML
    let title = await page.title().catch(() => '')
    let titleEsc = _esc(title || '')
    let bodyEsc = _esc(shadow)
    let synth = `<!DOCTYPE html><html><head><title>${titleEsc}</title></head><body><article>${bodyEsc.split(/\n+/).map((p) => `<p>${p}</p>`).join('\n')}</article></body></html>`
    return { html: synth, contentKind: 'synthesized' }
}


export { SHADOW_VISIBLE_THRESHOLD }
export default extractPageContent