build(update): crawlers

plantaclaus · Oct 16, 2023 · b55b43c · b55b43c
1 parent e4be3b2
commit b55b43c
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/index.json b/index.json
@@ -53,12 +53,19 @@
  "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http:https://yacy.net/bot.html",
+ "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http:https://yacy.net/bot.html",
+ "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
  "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http:https://yacy.net/bot.html",
  "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http:https://yacy.net/bot.html",
- "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http:https://yacy.net/bot.html"
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http:https://bibnum.bnf.fr/robot/bnf.html)",
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http:https://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)",
+ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http:https://mignify.com/bot.html)",
+ "Mozilla/5.0 (compatible; XoviBot/2.0; +http:https://www.xovibot.net/)",
+ "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)",
+ "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)"
 ]