##\
# This file is a part of the CIDRAM package.
# Homepage: https://cidram.github.io/
#
# CIDRAM COPYRIGHT 2016 and beyond by Caleb Mazalevskis (Maikuolan).
#
# License: GNU/GPLv2
# @see LICENSE.txt
#
# This file: Verification data for search engines and social media (last modified: 2020.02.07).
##/

# Additional references: https://udger.com/resources/ua-list/crawlers
Search Engine Verification:

 # https://discussions.apple.com/thread/7090135
 Applebot:
  User Agent: "applebot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".applebot.apple.com"

 # https://help.baidu.com/question?prod_en=master&class=Baiduspider
 Baiduspider/百度:
  Bypass Flag: "Flag-Bypass-Baidu-Check"
  User Agent: "baidu"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".baidu.com"
   - ".baidu.jp"

 # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot
 Bingbot:
  Bypass Flag: "Flag-Bypass-Bingbot-Check"
  User Agent Pattern: "~(?:msn|bing)bot|bingpreview~"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".search.msn.com"

 # https://duckduckgo.com/duckduckbot
 DuckDuckBot:
  User Agent Pattern: "~duckduck(?:go-favicons-)?bot~"
  Closure: "UA-IP-Match"
  Can Modify Trackable: false
  Valid Domains:
   - "23.21.227.69"
   - "50.16.241.113"
   - "50.16.241.114"
   - "50.16.241.117"
   - "50.16.247.234"
   - "52.5.190.19"
   - "52.204.97.54"
   - "54.197.234.188"
   - "54.208.100.253"
   - "54.208.102.37"
   - "107.21.1.8"

 # https://support.google.com/webmasters/answer/80553?hl=en
 Googlebot:
  Bypass Flag: "Flag-Bypass-Googlebot-Check"
  User Agent: "googlebot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".googlebot.com"
   - ".google.com"

 # https://www.mojeek.com/bot.html
 MojeekBot:
  User Agent: "mojeekbot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".mojeek.com"

 # https://www.mojeek.com/bot.html
 Qwantify/Bleriot:
  User Agent: "qwantify/bleriot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - "search.qwant.com"

 # https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler/
 SeznamBot:
  User Agent: "seznambot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".seznam.cz"

 # https://www.sogou.com/docs/help/webmasters.htm#07
 Sogou/搜狗:
  User Agent: "sogou"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".sogou.com"

 # https://help.yahoo.com/help/us/ysearch/slurp
 Yahoo/Slurp:
  User Agent: "slurp"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".crawl.yahoo.net"
   - ".yse.yahoo.net"

 # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml
 Yandex/Яндекс:
  Bypass Flag: "Flag-Bypass-Yandex-Check"
  User Agent: "yandex"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".yandex.com"
   - ".yandex.net"
   - ".yandex.ru"

 # https://udger.com/resources/ua-list/bot-detail?bot=YoudaoBot#id1507
 YoudaoBot:
  User Agent: "youdaobot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".crawler.youdao.com"

Social Media Verification:

 # https://udger.com/resources/ua-list/bot-detail?bot=Embedly#id22674
 Embedly:
  User Agent: "embedly"
  Closure: "DNS-Reverse-Forward"
  Reverse Only: true
  Can Modify Trackable: false
  Valid Domains:
   - "embed.ly"

 # https://developers.facebook.com/docs/sharing/webmasters/crawler/
 Facebook external hit:
  User Agent: "facebookexternalhit/"
  Closure: "UA-ASN-Match"
  Valid Domains:
   - "32934"

 # https://www.grapeshot.com/crawler/
 GrapeshotCrawler:
  User Agent: "grapeshot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".grapeshot.co.uk"

 # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
 Pinterest:
  User Agent: "pinterest"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".pinterest.com"

 # https://udger.com/resources/ua-list/bot-detail?bot=Twitterbot#id6168
 Twitterbot:
  User Agent: "twitterbot"
  Closure: "DNS-Reverse-Forward"
  Valid Domains:
   - ".twttr.com"