##\
# This file is a part of the CIDRAM package.
# Homepage: https://cidram.github.io/
#
# CIDRAM COPYRIGHT 2016 and beyond by Caleb Mazalevskis (Maikuolan).
#
# License: GNU/GPLv2
# @see LICENSE.txt
#
# This file: Verification data (last modified: 2023.11.17).
##/

# Additional references: https://udger.com/resources/ua-list/crawlers
Search Engine Verification:
 # https://developer.amazon.com/support/amazonbot
 # https://github.com/CIDRAM/CIDRAM/issues/317
 # Updated: 2022.06.20
 Amazonbot:
  User Agent: "amazonbot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".crawl.amazonbot.amazon"

 # https://discussions.apple.com/thread/7090135
 # https://support.apple.com/en-gb/HT204683
 # Updated: 2020.02.07 / Checked: 2022.02.18
 Applebot:
  User Agent: "applebot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".applebot.apple.com"

 # https://help.baidu.com/question?prod_en=master&class=Baiduspider
 # https://help.baidu.com/question?prod_id=99&class=476&id=3001
 # Updated: 2020.02.07 / Checked: 2023.04.06
 Baidu:
  Bypass flag: "Flag-Bypass-Baidu-Check"
  User Agent: "baiduspider"
  Method: "dnsReverseForward"
  Valid domains:
   - ".baidu.com"
   - ".baidu.jp"

 # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot
 # https://github.com/CIDRAM/CIDRAM/issues/242
 # Updated: 2020.02.07 / Checked: 2022.02.18
 Bingbot:
  Bypass flag: "Flag-Bypass-Bingbot-Check"
  User Agent Pattern: "~(?:msn|bing)bot|bingpreview~"
  Method: "dnsReverseForward"
  Valid domains:
   - ".search.msn.com"

 # https://duckduckgo.com/duckduckbot
 # Updated: 2023.11.17
 DuckDuckBot:
  User Agent Pattern: "~duckduck(?:go-favicons-)?bot~"
  Method: "uaIpMatch"
  Valid domains:
   - "20.12.141.99"
   - "20.40.133.240"
   - "20.49.136.28"
   - "20.50.168.91"
   - "20.185.79.15"
   - "20.185.79.47"
   - "20.191.45.212"
   - "20.193.45.113"
   - "40.76.162.191"
   - "40.76.162.208"
   - "40.76.162.247"
   - "40.76.163.7"
   - "40.76.163.23"
   - "40.76.173.151"
   - "40.80.242.63"
   - "40.88.21.235"
   - "51.107.40.209"
   - "51.116.131.221"
   - "51.120.48.122"
   - "52.142.24.149"
   - "52.142.26.175"

 # https://support.google.com/webmasters/answer/80553?hl=en
 # Updated: 2018.07.14 / Checked: 2022.02.18
 Googlebot:
  Bypass flag: "Flag-Bypass-Googlebot-Check"
  User Agent: "googlebot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".google.com"
   - ".googlebot.com"

 # https://www.mojeek.com/bot.html
 # Updated: 2020.02.07 / Checked: 2022.02.18
 MojeekBot:
  User Agent: "mojeekbot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".mojeek.com"

 # https://neeva.com/neevabot
 # Updated: 2023.02.28
 Neevabot:
  User Agent: "neevabot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".amazonaws.com"
   - ".neevabot.com"

 # https://aspiegel.com/petalbot
 # https://github.com/CIDRAM/CIDRAM/issues/254
 # https://webmaster.petalsearch.com/site/petalbot
 # Updated: 2021.10.01 / Checked: 2022.02.18
 PetalBot:
  Bypass flag: "Flag-Bypass-PetalBot-Check"
  User Agent: "petalbot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".aspiegel.com"
   - ".petalsearch.com"

 # https://help.qwant.com/bot/
 # Updated: 2020.02.07 / Checked: 2022.02.18
 Qwantify:
  User Agent: "qwantify/bleriot"
  Method: "dnsReverseForward"
  Valid domains:
   - "search.qwant.com"

 # https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler/
 # Updated: 2020.02.07 / Checked: 2022.02.18
 SeznamBot:
  User Agent: "seznambot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".seznam.cz"

 # https://www.sogou.com/docs/help/webmasters.htm#07
 # Updated: 2018.07.16 / Checked: 2023.04.06
 Sogou:
  Bypass flag: "Flag-Bypass-Sogou-Check"
  User Agent: "sogou"
  Method: "dnsReverseForward"
  Valid domains:
   - ".sogou.com"

 # https://help.yahoo.com/help/us/ysearch/slurp
 # Updated: 2020.02.07 / Checked: 2022.02.18
 Yahoo:
  User Agent: "slurp"
  Method: "dnsReverseForward"
  Valid domains:
   - ".crawl.yahoo.net"
   - ".yse.yahoo.net"

 # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml
 # Updated: 2018.07.14 / Checked: 2023.04.06
 Yandex:
  Bypass flag: "Flag-Bypass-Yandex-Check"
  User Agent: "yandex"
  Method: "dnsReverseForward"
  Valid domains:
   - ".yandex.com"
   - ".yandex.net"
   - ".yandex.ru"

 # https://udger.com/resources/ua-list/bot-detail?bot=YoudaoBot#id1507
 # Updated: 2020.02.07 / Checked: 2022.02.18
 YoudaoBot:
  User Agent: "youdaobot"
  Method: "dnsReverseForward"
  Valid domains:
   - ".crawler.youdao.com"

Social Media Verification:
 # https://udger.com/resources/ua-list/bot-detail?bot=Embedly#id22674
 # Updated: 2018.07.14 / Checked: 2022.02.18
 Embedly:
  User Agent: "embedly"
  Method: "dnsReverseForward"
  Reverse only: true
  Valid domains:
   - "embed.ly"

 # https://developers.facebook.com/docs/sharing/webmasters/crawler/
 # https://github.com/CIDRAM/CIDRAM/issues/170
 # https://github.com/CIDRAM/CIDRAM/issues/263
 # Updated: 2022.02.18 / Checked: 2023.11.17
 Facebook external hit:
  User Agent Pattern: "~facebook(?:catalog|externalhit)/(?!1\.1 facebot twitterbot/1\.0)~"
  Method: "uaAsnMatch"
  Valid domains:
   - "32934"

 # https://github.com/CIDRAM/CIDRAM/issues/253
 # https://github.com/CIDRAM/CIDRAM/issues/322
 # https://help.pinterest.com/en/business/article/pinterest-crawler
 # Updated: 2020.02.07 / Checked: 2023.11.17
 Pinterest:
  User Agent: "pinterest"
  Method: "dnsReverseForward"
  Valid domains:
   - ".pinterest.com"
   - ".pinterestcrawler.com"

 # https://udger.com/resources/ua-list/bot-detail?bot=Twitterbot#id6168
 # https://github.com/CIDRAM/CIDRAM/issues/263
 # https://github.com/CIDRAM/CIDRAM/issues/341
 # Updated: 2022.10.01 / Checked: 2023.07.28
 Twitterbot:
  User Agent Pattern: "~(?<!^telegrambot \(like )(?<!^facebookexternalhit/1\.1 facebot )twitterbot~"
  Method: "dnsReverseForward"
  Valid domains:
   - ".twttr.com"

 # https://github.com/CIDRAM/CIDRAM/issues/422
 # https://robotsdb.de/snap-url-preview-service-bot-snapchat/
 # https://udger.com/resources/ua-list/bot-detail?bot=Snap+URL+Preview+Service
 # Updated: 2023.02.28
 Snapchat:
  User Agent Pattern: "~snapchat|developers\.snap\.com/robots$~"
  Method: "dnsReverseForward"
  Valid domains:
   - ".amazonaws.com"

Other Verification:
 # https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers
 # https://github.com/CIDRAM/CIDRAM/issues/233
 # Updated: 2021.10.01 / Checked: 2022.02.18
 AdSense:
  User Agent: "mediapartners-google"
  Method: "dnsReverseForward"
  Valid domains:
   - ".google.com"
   - ".googlebot.com"

 # https://adbot.amazon.com/index.html
 # https://github.com/CIDRAM/CIDRAM/issues/260
 # Updated: 2021.10.01 / Checked: 2022.02.18
 AmazonAdBot:
  User Agent Pattern: "~amazonadbot|adbot\.amazon\.com~"
  Method: "dnsReverseForward"
  Valid domains:
   - ".amazonadbot.com"

 # https://platform.openai.com/docs/plugins/bot
 # Updated: 2023.08.10
 ChatGPT-User:
  User Agent: "chatgpt-user"
  Method: "uaCidrMatch"
  Valid domains:
   - "23.98.142.176/28"

 # https://platform.openai.com/docs/gptbot
 # https://openai.com/gptbot-ranges.txt
 # Updated: 2023.08.10
 GPTBot:
  User Agent: "gptbot"
  Method: "uaCidrMatch"
  Valid domains:
   - "20.15.240.64/27"
   - "20.15.240.96/28"
   - "20.15.240.176/28"
   - "20.15.241.0/28"
   - "20.15.242.128/27"
   - "20.15.242.192/28"
   - "40.83.2.64/28"

 # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
 # Updated: 2021.03.13 / Checked: 2023.11.17
 Grapeshot:
  User Agent: "grapeshot"
  Method: "uaCidrMatch"
  Valid domains:
   - "132.145.9.5/32"
   - "132.145.11.125/32"
   - "132.145.14.70/32"
   - "132.145.15.209/32"
   - "132.145.64.33/32"
   - "132.145.66.116/32"
   - "132.145.66.156/32"
   - "132.145.67.248/32"
   - "140.238.81.78/32"
   - "140.238.83.181/32"
   - "140.238.94.137/32"
   - "140.238.95.47/32"
   - "140.238.95.199/32"
   - "148.64.56.64/28"
   - "148.64.56.112/28"
   - "152.67.128.219/32"
   - "152.67.137.35/32"
   - "152.67.138.180/32"