{"version":3,"sources":["../src/modules/lastChapterInfo.ts","../src/utils/lastChapterInfoScraper.ts","../src/utils/titleCleaner.ts","../src/modules/ptwxz/chapterScraper.ts","../src/utils/scraper.ts","../src/modules/ptwxz/tocScraper.ts","../src/modules/snShu/chapterScraper.ts","../src/modules/snShu/tocScraper.ts"],"sourcesContent":["import { isUrlString } from \"is-url-online\";\nimport type { Page } from \"playwright-chromium\";\nimport { lastChapterInfoScraper } from \"../utils/lastChapterInfoScraper\";\nimport { titleCleaner } from \"../utils/titleCleaner\";\n\ntype Selector = {\n  sourceUrl: string;\n  numberSelector: string;\n  linkSelector: string;\n  titleSelector: string;\n  page: Page;\n};\n\nexport const lastChapterInfo = async ({\n  linkSelector,\n  numberSelector,\n  sourceUrl,\n  titleSelector,\n  page,\n}: Selector): Promise<{\n  link: string;\n  number: number;\n  title: string;\n}> => {\n  const content = await lastChapterInfoScraper(\n    sourceUrl,\n    linkSelector,\n    numberSelector,\n    titleSelector,\n    page\n  );\n\n  if (content.url === \"\") {\n    throw new Error(\"No link found\");\n  }\n\n  if (!content.number || !content.title) {\n    throw new Error(\"No number or title found\");\n  }\n  const urlString = isUrlString(content.url);\n\n  let { url } = content;\n  if (!urlString) {\n    const { href: absoluteUrl } = new URL(url, sourceUrl);\n\n    url = absoluteUrl;\n  }\n\n  const { number, title } = content;\n\n  return {\n    link: url,\n    number,\n    title: titleCleaner(title.trim()),\n  };\n};\n","import { toArabicString } from \"chinese-numbers-to-arabic\";\nimport type { Page } from \"playwright-chromium\";\nimport type { TocItem } from \"./types\";\n\nexport const lastChapterInfoScraper = async (\n  link: string,\n  linkSelector: string,\n  numberSelector: string,\n  titleSelector: string,\n  page: Page\n): Promise<TocItem> => {\n  await page.goto(link);\n\n  const url = await page.$eval(linkSelector, (elm) => elm.getAttribute(\"href\"));\n\n  if (!url) {\n    throw new Error(\"No link found\");\n  }\n\n  const numberString = await page.$eval(\n    numberSelector,\n    (elm) => elm.textContent\n  );\n\n  if (!numberString) {\n    throw new Error(\"No number found\");\n  }\n\n  const title = await page.$eval(titleSelector, (elm) => elm.textContent);\n\n  if (!title) {\n    throw new Error(\"No title found\");\n  }\n\n  const num = toArabicString(numberString);\n\n  const number = Number.parseInt(num.replace(/^\\D+/gu, \"\"));\n\n  return {\n    url,\n    number,\n    title: title.trim(),\n  };\n};\n","const wordsTobeRemoved = new Set([\"chapter\", \"-\", \":\"]);\n\nconst firstLettersToBeRemoved = new Set([\"第\", \"#\", \"the\"]);\n\nexport const titleCleaner = (title: string): string => {\n  const words = title.split(\" \").filter((word) => word.length > 0);\n\n  // This will handle mainly the english translations, like in wuxiaworld.\n  if (words[0].toLowerCase() === \"chapter\") {\n    words.shift();\n    words.shift();\n  }\n\n  // this will handle lnmtl, and both type of chinese titles\n  if (firstLettersToBeRemoved.has(words[0][0].toLowerCase())) {\n    words.shift();\n  }\n\n  const cleanedWords = words.filter(\n    (word) => !wordsTobeRemoved.has(word.toLowerCase())\n  );\n\n  return cleanedWords.join(\" \");\n};\n","import { load } from \"cheerio\";\nimport type { Page } from \"playwright-chromium\";\nimport { scraper } from \"../../utils/scraper\";\n\nexport const ptwxzChapterScraper = async (\n  url: string,\n  page: Page\n): Promise<string[]> => {\n  const content = await scraper(url, page);\n\n  if (!content) {\n    throw new Error(\"No content found for the page\");\n  }\n\n  const $ = load(content);\n\n  const chapterText = $(\"#content\")\n    .contents()\n    .filter((_: number, element: { type: string }) => element.type === \"text\")\n    .map((index: number, element) => $(element).text())\n    .get();\n\n  const chapterOutput: string[] = [];\n\n  chapterText.forEach((str) => {\n    const trimmedStr = str.trim();\n    if (trimmedStr.length > 0) {\n      chapterOutput.push(trimmedStr);\n    }\n  });\n\n  return chapterOutput;\n};\n","import type { Page } from \"playwright-chromium\";\n\nexport const scraper = async (url: string, page: Page): Promise<string> => {\n  await page.goto(url);\n  return page.content();\n};\n","import { toArabicString } from \"chinese-numbers-to-arabic\";\nimport type { Page } from \"playwright-chromium\";\nimport type { QuickScraperOutput } from \"quick-scraper\";\nimport { quickScraperHeadless } from \"quick-scraper\";\nimport { titleCleaner } from \"../../utils/titleCleaner\";\nimport type { TocItem } from \"../../utils/types\";\n\nexport const ptwxzTocScraper = async (\n  url: string,\n  page: Page\n): Promise<TocItem[]> => {\n  let data: QuickScraperOutput;\n\n  try {\n    data = await quickScraperHeadless({\n      url,\n      options: {\n        chapters: {\n          selector: \".centent > ul> li > a\",\n          listItem: true,\n          href: true,\n        },\n      },\n      page,\n    });\n  } catch (error) {\n    throw new Error(`Error while scraping - ${error as string}`);\n  }\n\n  const chaptersArray = data.data.chapters.lists;\n\n  const chaptersData = [];\n\n  if (chaptersArray?.length === 0 || !chaptersArray) {\n    throw new Error(\"Some error happened while scraping\");\n  }\n\n  for (const chapter of chaptersArray) {\n    const chapterNumberString = toArabicString(\n      chapter.text?.split(\" \")[0] as string\n    );\n\n    if (!chapterNumberString.startsWith(\"第\")) {\n      continue;\n    }\n\n    const chapterNumber = chapterNumberString.replace(/\\D/gu, \"\");\n    const chapterTitle = titleCleaner(chapter.text as string);\n    const chapterUrl = chapter.href;\n\n    if (chapterNumber && chapterTitle && chapterUrl) {\n      chaptersData.push({\n        number: Number.parseInt(chapterNumber),\n        title: titleCleaner(chapterTitle),\n        url: chapterUrl,\n      });\n    }\n  }\n  return chaptersData;\n};\n","import { load } from \"cheerio\";\nimport type { Page } from \"playwright-chromium\";\nimport { scraper } from \"../../utils/scraper\";\n\nexport const snChapterScraper = async (\n  url: string,\n  page: Page\n): Promise<string[]> => {\n  const content = await scraper(url, page);\n\n  if (!content) {\n    throw new Error(\"No content found for the page\");\n  }\n\n  const $ = load(content);\n\n  const chapterText = $(\".txtnav\")\n    .contents()\n    .filter((_: number, element: { type: string }) => element.type === \"text\")\n    .map((_index: number, element) => $(element).text())\n    .get();\n\n  const chapterOutput: string[] = [];\n\n  chapterText.forEach((str) => {\n    const trimmedStr = str.trim();\n    if (trimmedStr.length > 0) {\n      chapterOutput.push(trimmedStr);\n    }\n  });\n\n  return chapterOutput;\n};\n","import { toArabicString } from \"chinese-numbers-to-arabic\";\nimport type { Page } from \"playwright-chromium\";\nimport type { QuickScraperOutput } from \"quick-scraper\";\nimport { quickScraperHeadless } from \"quick-scraper\";\nimport { scraper } from \"../../utils/scraper\";\nimport { titleCleaner } from \"../../utils/titleCleaner\";\nimport type { TocItem } from \"../../utils/types\";\n\nexport const snTocScraper = async (\n  url: string,\n  page: Page\n): Promise<TocItem[]> => {\n  const content = await scraper(url, page);\n\n  if (!content) {\n    throw new Error(\"No content found for the page\");\n  }\n\n  let data: QuickScraperOutput;\n  try {\n    data = await quickScraperHeadless({\n      url,\n      options: {\n        chapters: {\n          selector: \"#catalog > ul> li > a\",\n          listItem: true,\n          href: true,\n        },\n      },\n      page,\n    });\n  } catch (error) {\n    console.log(error);\n    throw new Error(\"Error while scraping\");\n  }\n\n  const chaptersArray = data.data.chapters.lists;\n\n  const chaptersData = [];\n\n  if (chaptersArray?.length === 0 || !chaptersArray) {\n    throw new Error(\"Some error happened while scraping\");\n  }\n\n  for (const chapter of chaptersArray) {\n    const chapterNumberString = toArabicString(\n      chapter.text?.split(\" \")[0] as string\n    );\n\n    if (!chapterNumberString.startsWith(\"第\")) {\n      continue;\n    }\n\n    const chapterNumber = chapterNumberString.replace(/\\D/gu, \"\");\n    const chapterTitle = titleCleaner(chapter.text as string);\n    const chapterUrl = chapter.href;\n\n    if (chapterNumber && chapterTitle && chapterUrl) {\n      chaptersData.push({\n        number: Number.parseInt(chapterNumber),\n        title: chapterTitle,\n        url: chapterUrl,\n      });\n    }\n  }\n  return chaptersData;\n};\n"],"mappings":";AAAA;;;ACAA;AAIO,IAAM,yBAAyB,OACpC,MACA,cACA,gBACA,eACA,SACqB;AACrB,QAAM,KAAK,KAAK;AAEhB,QAAM,MAAM,MAAM,KAAK,MAAM,cAAc,CAAC,QAAQ,IAAI,aAAa;AAErE,MAAI,CAAC,KAAK;AACR,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,eAAe,MAAM,KAAK,MAC9B,gBACA,CAAC,QAAQ,IAAI;AAGf,MAAI,CAAC,cAAc;AACjB,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,QAAQ,MAAM,KAAK,MAAM,eAAe,CAAC,QAAQ,IAAI;AAE3D,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,MAAM,eAAe;AAE3B,QAAM,SAAS,OAAO,SAAS,IAAI,QAAQ,UAAU;AAErD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,MAAM;AAAA;AAAA;;;ACzCjB,IAAM,mBAAmB,oBAAI,IAAI,CAAC,WAAW,KAAK;AAElD,IAAM,0BAA0B,oBAAI,IAAI,CAAC,UAAK,KAAK;AAE5C,IAAM,eAAe,CAAC,UAA0B;AACrD,QAAM,QAAQ,MAAM,MAAM,KAAK,OAAO,CAAC,SAAS,KAAK,SAAS;AAG9D,MAAI,MAAM,GAAG,kBAAkB,WAAW;AACxC,UAAM;AACN,UAAM;AAAA;AAIR,MAAI,wBAAwB,IAAI,MAAM,GAAG,GAAG,gBAAgB;AAC1D,UAAM;AAAA;AAGR,QAAM,eAAe,MAAM,OACzB,CAAC,SAAS,CAAC,iBAAiB,IAAI,KAAK;AAGvC,SAAO,aAAa,KAAK;AAAA;;;AFTpB,IAAM,kBAAkB,OAAO;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,MAKI;AACJ,QAAM,UAAU,MAAM,uBACpB,WACA,cACA,gBACA,eACA;AAGF,MAAI,QAAQ,QAAQ,IAAI;AACtB,UAAM,IAAI,MAAM;AAAA;AAGlB,MAAI,CAAC,QAAQ,UAAU,CAAC,QAAQ,OAAO;AACrC,UAAM,IAAI,MAAM;AAAA;AAElB,QAAM,YAAY,YAAY,QAAQ;AAEtC,MAAI,EAAE,QAAQ;AACd,MAAI,CAAC,WAAW;AACd,UAAM,EAAE,MAAM,gBAAgB,IAAI,IAAI,KAAK;AAE3C,UAAM;AAAA;AAGR,QAAM,EAAE,QAAQ,UAAU;AAE1B,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,OAAO,aAAa,MAAM;AAAA;AAAA;;;AGrD9B;;;ACEO,IAAM,UAAU,OAAO,KAAa,SAAgC;AACzE,QAAM,KAAK,KAAK;AAChB,SAAO,KAAK;AAAA;;;ADAP,IAAM,sBAAsB,OACjC,KACA,SACsB;AACtB,QAAM,UAAU,MAAM,QAAQ,KAAK;AAEnC,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,IAAI,KAAK;AAEf,QAAM,cAAc,EAAE,YACnB,WACA,OAAO,CAAC,GAAW,YAA8B,QAAQ,SAAS,QAClE,IAAI,CAAC,OAAe,YAAY,EAAE,SAAS,QAC3C;AAEH,QAAM,gBAA0B;AAEhC,cAAY,QAAQ,CAAC,QAAQ;AAC3B,UAAM,aAAa,IAAI;AACvB,QAAI,WAAW,SAAS,GAAG;AACzB,oBAAc,KAAK;AAAA;AAAA;AAIvB,SAAO;AAAA;;;AE/BT;AAGA;AAIO,IAAM,kBAAkB,OAC7B,KACA,SACuB;AAVzB;AAWE,MAAI;AAEJ,MAAI;AACF,WAAO,MAAM,qBAAqB;AAAA,MAChC;AAAA,MACA,SAAS;AAAA,QACP,UAAU;AAAA,UACR,UAAU;AAAA,UACV,UAAU;AAAA,UACV,MAAM;AAAA;AAAA;AAAA,MAGV;AAAA;AAAA,WAEK,OAAP;AACA,UAAM,IAAI,MAAM,0BAA0B;AAAA;AAG5C,QAAM,gBAAgB,KAAK,KAAK,SAAS;AAEzC,QAAM,eAAe;AAErB,MAAI,gDAAe,YAAW,KAAK,CAAC,eAAe;AACjD,UAAM,IAAI,MAAM;AAAA;AAGlB,aAAW,WAAW,eAAe;AACnC,UAAM,sBAAsB,gBAC1B,cAAQ,SAAR,mBAAc,MAAM,KAAK;AAG3B,QAAI,CAAC,oBAAoB,WAAW,WAAM;AACxC;AAAA;AAGF,UAAM,gBAAgB,oBAAoB,QAAQ,QAAQ;AAC1D,UAAM,eAAe,aAAa,QAAQ;AAC1C,UAAM,aAAa,QAAQ;AAE3B,QAAI,iBAAiB,gBAAgB,YAAY;AAC/C,mBAAa,KAAK;AAAA,QAChB,QAAQ,OAAO,SAAS;AAAA,QACxB,OAAO,aAAa;AAAA,QACpB,KAAK;AAAA;AAAA;AAAA;AAIX,SAAO;AAAA;;;AC1DT;AAIO,IAAM,mBAAmB,OAC9B,KACA,SACsB;AACtB,QAAM,UAAU,MAAM,QAAQ,KAAK;AAEnC,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,IAAI,MAAK;AAEf,QAAM,cAAc,EAAE,WACnB,WACA,OAAO,CAAC,GAAW,YAA8B,QAAQ,SAAS,QAClE,IAAI,CAAC,QAAgB,YAAY,EAAE,SAAS,QAC5C;AAEH,QAAM,gBAA0B;AAEhC,cAAY,QAAQ,CAAC,QAAQ;AAC3B,UAAM,aAAa,IAAI;AACvB,QAAI,WAAW,SAAS,GAAG;AACzB,oBAAc,KAAK;AAAA;AAAA;AAIvB,SAAO;AAAA;;;AC/BT;AAGA;AAKO,IAAM,eAAe,OAC1B,KACA,SACuB;AAXzB;AAYE,QAAM,UAAU,MAAM,QAAQ,KAAK;AAEnC,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM;AAAA;AAGlB,MAAI;AACJ,MAAI;AACF,WAAO,MAAM,sBAAqB;AAAA,MAChC;AAAA,MACA,SAAS;AAAA,QACP,UAAU;AAAA,UACR,UAAU;AAAA,UACV,UAAU;AAAA,UACV,MAAM;AAAA;AAAA;AAAA,MAGV;AAAA;AAAA,WAEK,OAAP;AACA,YAAQ,IAAI;AACZ,UAAM,IAAI,MAAM;AAAA;AAGlB,QAAM,gBAAgB,KAAK,KAAK,SAAS;AAEzC,QAAM,eAAe;AAErB,MAAI,gDAAe,YAAW,KAAK,CAAC,eAAe;AACjD,UAAM,IAAI,MAAM;AAAA;AAGlB,aAAW,WAAW,eAAe;AACnC,UAAM,sBAAsB,gBAC1B,cAAQ,SAAR,mBAAc,MAAM,KAAK;AAG3B,QAAI,CAAC,oBAAoB,WAAW,WAAM;AACxC;AAAA;AAGF,UAAM,gBAAgB,oBAAoB,QAAQ,QAAQ;AAC1D,UAAM,eAAe,aAAa,QAAQ;AAC1C,UAAM,aAAa,QAAQ;AAE3B,QAAI,iBAAiB,gBAAgB,YAAY;AAC/C,mBAAa,KAAK;AAAA,QAChB,QAAQ,OAAO,SAAS;AAAA,QACxB,OAAO;AAAA,QACP,KAAK;AAAA;AAAA;AAAA;AAIX,SAAO;AAAA;","names":[]}