build(update): crawlers

plantaclaus · Oct 23, 2023 · f3aa5ea · f3aa5ea
1 parent 590442a
commit f3aa5ea
Showing 1 changed file with 1 addition and 53 deletions.
diff --git a/index.json b/index.json
@@ -1,18 +1,5 @@
 [
  "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)",
- "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (compatible; adidxbot/2.0; http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (compatible; adidxbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (compatible; bingbot/2.0; http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm) SitemapProbe",
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm)",
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm) Safari/537.36",
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http:https://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36",
  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http:https://www.linkedin.com)",
  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http:https://www.linkedin.com)",
  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http:https://www.linkedin.com)",
@@ -28,44 +15,5 @@
  "msnbot/1.0 (+http:https://search.msn.com/msnbot.htm)",
  "msnbot/1.1 (+http:https://search.msn.com/msnbot.htm)",
  "msnbot/2.0b (+http:https://search.msn.com/msnbot.htm)",
- "msnbot/2.0b (+http:https://search.msn.com/msnbot.htm).",
- "msnbot/2.0b (+http:https://search.msn.com/msnbot.htm)._",
- "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http:https://yacy.net/bot.html",
- "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
- "Mozilla/5.0 (compatible; bnf.fr_bot; +http:https://bibnum.bnf.fr/robot/bnf.html)",
- "Mozilla/5.0 (compatible; bnf.fr_bot; +http:https://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)",
- "Mozilla/5.0 (compatible; memorybot/1.21.14 +http:https://mignify.com/bot.html)",
- "Mozilla/5.0 (compatible; XoviBot/2.0; +http:https://www.xovibot.net/)",
- "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)",
- "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)"
+ "msnbot/2.0b (+http:https://search.msn.com/msnbot.htm)."
 ]