From 180cddec97c3f71608169097db24a34b598711c7 Mon Sep 17 00:00:00 2001 From: CRGMB Date: Wed, 10 Jan 2024 15:39:28 +0000 Subject: [PATCH 1/3] Updated some entries in the file COUNTER_Robots_list and added 1636 new entries, updated the CHANGES.md with the new bots and changes and corrections to the convert_to_txt file. --- CHANGES.md | 1641 +++++ COUNTER_Robots_list.json | 10031 +++++++++++++++++++++++++++- convert_to_txt | 2 +- generated/COUNTER_Robots_list.txt | 1641 +++++ 4 files changed, 13273 insertions(+), 42 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d959cee..05240b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -110,3 +110,1644 @@ Processed 3 following merge requests : ## July 27th, 2017 - First release, this is the latest list provided by COUNTER + +## January 10th, 2024 +- Added user-agents: + - &as_qdr=all + - */Nutch-0.9-dev + - 192.comAgent + - 222.122.15.190 + - 2Bone_LinkChkr/1.0 l + - 7331230CB97FADECF052 + - 8484 Boston Project + - 8LEGS + - A-Online Search + - ADmantX Platform Sem + - AHC/2.0 + - ASPSeek/1.2.5 + - ASPSeek/1.2.x + - ASPSeek/1.2.xa + - ASPSeek/1.2.xxpre + - ASPseek/1.2.9d + - ASPseek/1.2.xx + - ASSORT/0.10 + - AV Fetch 1.0 + - AVSearch-1.0(peter.t + - AVSearch-2.0-fusionI + - AVSearch-3.0(AltaVis + - About/0.1libwww-perl + - Accoona-AI-Agent/1.1 + - Acorn/Nutch-0.9 (Non + - ActiveBookmark 1.x + - AdobeUxTechC4-Async/ + - AgentName/0.1 libwww + - AideRSS/1.0 (aiderss + - Aladin/3.324 + - Allesklar/0.1 libwww + - Allrati/1.1 (+) + - AltaVista Intranet V + - Amazon CloudFront + - AmiTCP Miami (AmigaO + - Anonymizer/1.1 + - AnswerChase PROve x. + - AnswerChase x.0 + - AnzwersCrawl/2.0 (an + - Apache (internal dum + - Apache-HttpAsyncClie + - Apache-HttpClient/4. + - Apache-HttpClient/UN + - Aport + - Arachnoidea (arachno + - Asahina-Antenna/1.x + - Asahina-Antenna/1.x + - Ask.24x.Info + - Atomic_Email_Hunter/ + - Atomz/1.0 + - BIGLOTRON (Beta 2;GN + - BMCLIENT + - BMLAUNCHER + - BPImageWalker/2.0 (w + - BSDSeek/1.0 + - BStop.BravoBrian.it + - BTWebClient/180B(970 + - BeamMachine/0.5 (dea + - BeebwareDirectory/v0 + - Big Fish v1.0 + - BigBrother/1.6e + - Bilbo/2.3b-UNIX + - BlackBerry9000/4.6.0 + - BlackWidow + - Blackboard Safeassig + - BlogPulseLive (suppo + - Blogpulse (info@blog + - Bobby/4.0.x RPT-HTTP + - Bookdog/x.x + - BravoBrian bstop.bra + - BullsEye + - BunnySlippers + - CB/Nutch-1.7 + - CFNetwork/x.x + - CHttpClient by Open + - CLIPish Pro2 10.21 ( + - COAST WebMaster Pro/ + - CamelHttpStream/1.0 + - Cancer Information a + - Chat Catcher/1.0 + - China Local Browse 2 + - Chitika ContentHit 1 + - Claymont.com + - ColdFusion (Bookmark + - Combine/x.0 + - Contact + - Cookie%20Stumbler/22 + - CrawlConvera0.1 (Cra + - Custo x.x (www.netwu + - CyberSpyder Link Tes + - DBrowse 1.4b + - DBrowse 1.4d + - DTAAgent + - Dart Communications + - DataCha0s/2.0 + - DataFountains/DMOZ D + - Datafeedwatch/2.1.x + - Deep Link Calculator + - DeepIndex + - DeepIndex (www.en.de + - DeepIndexer.ca + - DevComponents.com Ht + - Diamond/x.0 + - DigOut4U + - Digger/1.0 JDK/1.3.0 + - Disqus/1.0 + - DittoSpyder + - DoCoMo/1.0/Nxxxi/c10 + - DoCoMo/2.0 P900iV(c1 + - EARTHCOM.info/1.x [w + - EARTHCOM.info/1.xbet + - EBrowse 1.4b + - EasyDL/3.xx + - EchO!/2.0 + - Educate Search VxB + - EldoS TimelyWeb/3.x + - EmailWolf 1.00 + - Embedly +support@emb + - Enfish Tracker + - Enterprise_Search/1. + - Evernote Clip Resolv + - Exalead NG/MimeLive + - Expired Domain Sleut + - ExtractorPro + - EyeCatcher (Download + - FARK.com link verifi + - FLATARTS_FAVICO + - FaEdit/2.0.x + - FavOrg + - Favorites Sweeper v. + - Feed24.com + - FeedForAll rss2html. + - FeedValidator/1.3 + - Filtrbox/1.0 + - FindAnISP.com_ISP_Fi + - Firefly/1.0 + - Firefox (kastaneta03 + - Firefox_1.0.6 (kaspa + - FirstGov.gov Search + - Flexum/2.0 + - FocusedSampler/1.0 + - Franklin Locator 1.8 + - FuseBulb.Com + - GigablastOpenSource/ + - Go 1.1 package http + - Go-http-client/1.1 + - Go-http-client/2.0 + - GoForIt.com + - Goofer/0.2 + - GrapeFX/0.3 libwww/5 + - Gromit/1.0 + - Guestbook Auto Submi + - Gulliver/1.3 + - Guzzle/4.2.3 curl/7. + - HLoader + - HPL/Nutch-0.9 - + - Haansoft + - Harvest-NG/1.0.2 + - Hatena Antenna/0.3 + - Hatena Pagetitle Age + - Hatena::Bookmark/2.0 + - Hippias/0.9 Beta + - HitList + - Hivemind + - HomePageSearch(hpsea + - Html Link Validator + - Hybrid/1.2 [en] (OS + - IAArchiver-1.0 + - IBM WebExplorer /v0. + - IEFav172Free + - INFOMINE/8.0 Adders + - INFOMINE/8.0 RemoteS + - IP*Works! V5 HTTP/S + - ISC Systems iRc Sear + - IWAgent/ 1.0 - www.b + - Icarus6 - (contact: + - ImageWalker/2.0 (www + - Incutio HttpClient v + - IncyWincy data gathe + - Industry Program 1.0 + - Inet library + - InetURL/1.0 + - InfoLink/1.x + - InfoSeek Sidewinder/ + - Infoaxe./Nutch-0.9 + - Infoseek SideWinder/ + - Inktomi Search + - Insitor.com search a + - Insitornaut + - Internet Ninja x.0 + - InternetLinkAgent/3. + - InternetSeer.com + - IrssiUrlLog/0.2 + - JDK/1.1 + - Jack + - Jakarta Commons-Http + - Java 1.1 + - Java Browser + - Java/1.4.1_01 + - Java/1.4.1_04 + - Java/1.5.0_16 + - Java/1.6.0_04 + - Java/1.6.0_07 + - Java/1.6.0_18 + - Java/1.6.0_22 + - Java/1.6.0_24 + - Java/1.6.0_25 + - Java/1.6.0_26 + - Java/1.6.0_27 + - Java/1.6.0_29 + - Java/1.6.0_30 + - Java/1.6.0_31 + - Java/1.6.0_32 + - Java/1.6.0_33 + - Java/1.6.0_34 + - Java/1.6.0_35 + - Java/1.6.0_39 + - Java/1.6.0_45 + - Java/1.6.0_51 + - Java/1.7.0_03 + - Java/1.7.0_09 + - Java/1.7.0_15 + - Java/1.7.0_17 + - Java/1.7.0_21 + - Java/1.7.0_25 + - Java/1.7.0_40 + - Java/1.7.0_45 + - Java/1.7.0_51 + - Java/1.7.0_55 + - Java/1.7.0_60 + - Java/1.7.0_65 + - Java/1.7.0_71 + - Java/1.7.0_75 + - Java/1.7.0_80 + - Java/1.8.0_20 + - Java/1.8.0_25 + - Java/1.8.0_31 + - Java/1.8.0_40 + - Java/1.8.0_45 + - Java/1.8.0_51 + - Java1.0.21.0 + - Java1.1.xx.x + - Java1.3.0rc1 + - Java1.3.x + - Java1.4.0 + - Jersey/2.25.1 (HttpU + - Jetty/9.3.z-SNAPSHOT + - Jigsaw/2.2.x W3C_CSS + - Jigsaw/2.3.0 W3C_CSS + - Junkie/1.0 + - KE_1.0/2.0 libwww/5. + - KIT-Fireball/2.0 + - KnowItAll(knowitall@ + - Knowledge.com/0.x + - KummHttp/1.1 (compat + - LARBIN-EXPERIMENTAL + - LEIA/2.90 + - LWP::Simple/5.22 + - LWP::Simple/5.36 + - LWP::Simple/5.48 + - LWP::Simple/5.50 + - LWP::Simple/5.51 + - LWP::Simple/5.53 + - LWP::Simple/5.63 + - LWP::Simple/5.803 + - Lachesis + - Libby_1.1/libwww-per + - Lincoln State Web Br + - Link Valet Online 1. + - LinkAlarm/2.x + - LinkPimpin v1.0 + - LinkProver 2.1 + - LinkSonar/1.35 + - LinkSweeper/1.x + - LinkWalker + - LinkWalker/2.0 + - Look.com + - Lynx/2.8.4rel.1 libw + - MARTINI + - MFC Foundation Class + - MFC_Tear_Sample + - MSIE-5.13 (larbin@un + - MSNPTC/1.0 + - MVAClient + - Mac Finder 1.0.xx + - Mag-Net + - Mail.Ru/1.0 + - MantraAgent + - MapoftheInternet.com + - Marketwave Hit List + - Martini + - Marvin v0.3 + - MasterSeek + - Mata Hari/2.00 + - MediaSearch/0.1 + - Mediapartners-Google + - MegaSheep v1.0 (www. + - MeltwaterNews www.me + - Mercator-1.x + - Mercator-2.0 + - Mercator-Scrub-1.1 + - Mercury/894 CFNetwor + - MetaURI API/2.0 +met + - MicroBaz + - Microsoft Data Acces + - Microsoft Small Busi + - Microsoft URL Contro + - Microsoft Windows Ne + - MiracleAlphaTest + - Missauga Locate 1.0. + - Missigua Locator 1.9 + - Missouri College Bro + - Mister PiX version.d + - Mister Pix II 2.02a + - Miva (AlgoFeedback@m + - Mizzu Labs 2.2 + - MnogoSearch/3.2.xx + - Mnogosearch-3.1.21 + - Mo College 1.9 + - More yacy user agent + - Mozi! + - MultiText/0.1 + - My WinHTTP Connectio + - MyNutchTest/Nutch-1. + - NASA Search 1.0 + - NEC Research Agent - + - NEC-Hayek/1.0 + - NG/4.0.1229 + - NICO/1.0 + - NING/1.0 + - NationalDirectoryAdd + - NativeHost + - NetLookout/2.24 + - NetMechanic Vx.0 + - NetSprint -- 2.0 + - NetZippy + - Nocilla/1.0 + - Nsauditor/1.x + - Nuzzel + - Nuzzel/876 CFNetwork + - Octora Beta - www.oc + - Oculus360/Nutch-1.7 + - OliverPerry + - OpenAcoon v4.0.x (ww + - OpenISearch/1.x (www + - OpenOffice/4.1.1 + - OpenWebIndex/Nutch-1 + - Openstat/0.1 + - Opera/6.01 (larbin@u + - Oracle Ultra Search + - OzySoftware.com + - PBrowse 1.4b + - PEval 1.4b + - PHP/3.x.xx + - PHP/4.0.4pl1 + - PHP/4.0.6 + - PHP/4.1.1 + - PHP/4.1.2 + - POE-Component-Client + - Pcore-HTTP/v0.40.3 + - PeopleChat/Search_En + - PicoSearch/1.0 + - Piffany_Web_Scraper_ + - PingALink Monitoring + - Pita + - Poirot + - Pompos/1.x pompos@il + - Popdexter/1.0 + - Port Huron Labs + - PostFavorites + - PostRank/2.0 (postra + - PrivacyFinder/1.1 + - Program Shareware 1. + - Progressive Download + - Project XP5 [2.03.07 + - PubCrawl (pubcrawl.s + - Pulsepoint XT3 web s + - PycURL/7.13.2 + - PycURL/7.15.5 + - PycURL/7.16.4 + - PycURL/7.18.0 + - PycURL/7.18.2 + - PycURL/7.19.0 + - PycURL/7.19.3 + - PycURL/7.19.5 + - PycURL/7.19.7 + - PycURL/7.23.1 + - Python-urllib/1.17 + - Python-urllib/1.1x + - Python-urllib/2.0a1 + - Python-urllib/2.5 + - Python-urllib/2.6 + - Python-urllib/2.7 + - Python-urllib/3.1 + - Python-urllib/3.2 + - Python-urllib/3.3 + - Python-urllib/3.4 + - Python-urllib/3.5 + - Python-urllib/3.6 + - QPCreep Test Rig ( W + - Qseero v1.0.0 + - QuepasaCreep v0.9.1x + - QueryN Metasearch + - Qwantify/1.0 + - R6_VoteReader(www.ra + - REBOL Core 2.x.x.x.x + - RPT-HTTPClient/0.3-x + - RepoMonkey Bait & Ta + - RoboCrawl (www.canad + - Robozilla/1.0 + - Rotondo/3.1 libwww/5 + - Ruby + - Rumours-Agent + - SMRF URL Expander + - Scooter-3.0.EU + - Scooter-3.0.FS + - Scooter-3.0.HD + - Scooter-3.0.VNS + - Scooter-3.0QI + - Scooter-3.2 + - Scooter-3.2.BT + - Scooter-3.2.DIL + - Scooter-3.2.EX + - Scooter-3.2.JT + - Scooter-3.2.NIV + - Scooter-3.2.SF0 + - Scooter-3.2.snippet + - Scooter-3.3dev + - Scooter-ARS-1.1 + - Scooter-ARS-1.1-ih + - Scooter-W3-1.0 + - Scooter-W3.1.2 + - Scooter/1.0 + - Scooter/1.0 scooter@ + - Scooter/1.1 (custom) + - Scooter/2.0 G.R.A.B. + - Scooter/3.3 + - Scooter/3.3.QA.pczuk + - Scooter/3.3.vscooter + - Scooter/3.3_SF + - Scooter2_Mercator_x- + - Scooter_bh0-3.0.3 + - Scooter_trk3-3.0.3 + - ScoutAbout + - Search+ + - Search-Engine-Studio + - SearchGuild/DMOZ/Exp + - SearchGuild_DMOZ_Exp + - SearchSight.com + - SearchTone2.0 - IDEA + - Seeker.lookseek.com + - Silk/1.0 + - SiteBar/x.x + - SiteRecon+(xx) + - SiteTruth.com site r + - SiteXpert + - Skywalker/0.1 (Skywa + - Slarp/0.1 + - Slider_Search_v1-de + - Slurpy Verifier/1.0 + - SlySearch (slysearch + - SnoopRob/x.x + - Snoopy v1.xx + - Snoopy_v0.xx + - Sphere Scout&v4.0 - + - Sphider + - Spida/0.1 + - Spinne/2.0 + - Spinne/2.0 med + - Spinne/2.0 med_AH + - Sqworm/2.9.81-BETA ( + - Sqworm/2.9.85-BETA ( + - Sqworm/2.9.89-BETA ( + - StackRambler/2.0 + - StackRambler/2.0 (MS + - StackRambler/x.x + - SyncIT/x.x + - Szukacz/1.x + - TE + - TJvMultiHttpGrabber + - Tagyu Agent/1.0 + - TeamSoft WinInet Com + - Teoma MP + - The European Library + - Thinklab (thinklab.c + - TinEye + - Toutatis x-xx.x (hop + - Toutatis x.x (hoppa. + - Toutatis x.x-x + - Traackr.com + - Twisted PageGetter + - Twitter/6.18.3 CFNet + - Twitturly / v0.x + - Tycoon Agent/Nutch-1 + - TygoProwler + - UCWEB/2.0(Linux; U; + - UDM + - URLBase/6.x + - URLBlaze + - UdmSearch/3.1.x + - Ultraseek + - Under the Rainbow 2. + - UofTDB_experiment (l + - Upflow/1.0 + - VYU2 (GNU; OpenRISC) + - Vagabondo/1.x MT (we + - Vagabondo/2.0 MT + - Vagabondo/2.0 MT (we + - Vagabondo/3.0 (webag + - Validator.nu/LV + - VayalaCreep-v0.0.1 ( + - Vayala|Creep-v0.0.1 + - VeryGoodSearch.com.D + - W3C-mobileOK/DDC-1.0 + - W3C_Unicorn/1.0 + - W3C_Validator/1.3 + - W3C_Validator/1.xxx + - WDG_Validator/1.1 + - WEP Search 00 + - WFARC + - WIRE/0.22 (Linux; x8 + - WST-Plagiarism + - WWWC/1.0x + - Wada.vn Vietnamese S + - Waypath Scout v2.x - + - WeSEE + - WeSEE:Search + - Web Link Validator 1 + - Web Snooper + - WebBug/5.x + - WebCompass 2.0 + - WebCorp/1.0 + - WebSearch.COM.AU; + - WebStat/1.0 (Unix; b + - WebTrends/3.0 (WinNT + - WebVac (webmaster@pi + - WebVal/1.0 + - WebVulnCrawl.unknown + - WebWatcherMonitor/2. + - Webclipping.com + - Webdup/0.9 + - Weblog Attitude Diff + - WebsiteWorth v1.0 + - Webspinne/1.0 webmas + - Webverzeichnis.de - + - Wells Search II + - Wget/1.14 (linux-gnu + - WhatsApp/2.12.15/i + - WhatsApp/2.12.16/i + - WhatsApp/2.12.17/i + - WhatsApp/2.12.449 A + - WhatsApp/2.12.453 A + - WhatsApp/2.12.510 A + - WhatsApp/2.12.540 A + - WhatsApp/2.12.548 A + - WhatsApp/2.12.555 A + - WhatsApp/2.12.556 A + - WhatsApp/2.16.1/i + - WhatsApp/2.16.13 A + - WhatsApp/2.16.2/i + - WhatsApp/2.16.42 A + - WhatsApp/2.16.57 A + - Whatsup/x.x + - WhizBang! Lab + - WinHTTP Example/1.0 + - WinampMPEG/2.00 (lar + - WincerSong Agent v1. + - WorQmada/1.0 + - WordupInfoSearch/1.0 + - WorldLight + - Wusage/x.0@boutell.c + - Wwlib/Linux + - XMLSlurp/0.1 libwww- + - Xenu Link Sleuth 1.x + - Xenu Link Sleuth/1.3 + - Xenu's Link Sleuth 1 + - Xylix + - Y!J-SRD/1.0 + - Yahoo Pipes 1.0 + - Yahoo! Mindset + - Yahoo-Test/4.0 + - YahooCacheSystem + - YahooCacheSystem; Ya + - YahooSeeker/CafeKels + - Yanadoo + - Yandex/1.01.001 (com + - Zearchit + - ZendHttpClient + - Zeus ThemeSite Viewe + - Zeus xxxxx Webster P + - Ziggy -- The Clown F + - Zippy v2.0 - Zippyfi + - a.pr-cy.ru + - about.ask.com + - about.depspid.net + - accoona + - aihit.com + - amzn_assoc + - app.hypefactors.com + - appie 1.1 (www.walhe + - archive.org + - ask.24x.info + - asterias/2.0 + - augurfind + - augurnfind V-1.x + - autohttp + - autowebdir 1.1 (www. + - axios/0.18.0 + - bCentral Billing Pos + - bdcindexer_2.6.2 (re + - binlar_2.6.3 binlar2 + - binlar_2.6.3 binlar_ + - binlar_2.6.3 larbin2 + - binlar_2.6.3 phanend + - binlar_2.6.3 test@mg + - blacklistdeclassified.net + - bookmarkbase.com + - bulkfeeds.net + - bumblebee@relevare.c + - cantor.ee.ucla.edu + - carleson/1.0 + - ccubee/x.x + - cfetch/1.0 + - cg-eye interactive + - cobitsprobe + - coccoc/1.0 () + - code.google.com + - codegator + - cohalinglit.wordpress.com + - collage.cgi/1.xx + - combine.it.lth.se + - combine/0.0 + - computerorgs.com + - content.easybib.com + - coralcdn.org + - cosco.hiit.fi + - crawler + - cs.daum.net + - csci_b659/0.13 + - curl/7.18.2 (x86_64- + - curl/7.19.7 (x86_64- + - curl/7.21.4 (univers + - curl/7.37.1 + - dcrawl/1.0 + - dds explorer v1.0 be + - deepak-USC/ISI + - deusu.org + - developers.google.com + - dhcibr.wordpress.com + - digg.com + - dir.com + - drupal.org + - dznet.com + - e-sense 1.0 ea(www.v + - eStyleSearch 4 (comp + - ec2linkfinder + - ecairn.com + - ejupiter.com + - enter4u.eu + - exactseek.com + - example-masto-instance.org + - exploringdh.wordpress.com + - facebookexternalhit/ + - feed.moo.jp + - feedafever.com; + - flipboard.com + - flunky + - fly/6.01 libwww/4.0D + - free-downloads.net d + - fulltext.seznam.cz + - gazz/x.x (gazz@nttrd + - geourl/2.0b2 + - getpocket.com + - gigabaz.com + - github.com + - google-xrawler + - googlebot + - gossamer-threads.com + - grub-client + - grub.org + - guess.scritch.org + - haste.kytoon.com + - help.coccoc.com + - help.coccoc.vn + - help.goo.ne.jp + - help.hepburnhome.ca + - help.yahoo.co.jp + - help.yahoo.com + - hgrepurl/1.0 + - holmes/3.9 (onet.pl) + - holmes/3.9 (someurl. + - holmes/x.x + - htdig/3.1.5 + - htdig/3.1.5 (infosys + - htdig/3.1.5 (root@lo + - htdig/3.1.5 (webmast + - htdig/3.1.6 (mathieu + - htdig/3.1.6 (unconfi + - htdig/3.1.x (root@lo + - http://www.mabontlan + - http://www.monogol.d + - http://www.timelyweb + - http_get + - httpunit/1.x + - i.stanford.edu + - iFeed.jp/2.0 (www.ps + - ia_archiver-web.arch + - ia_archiver/1.6 + - ia_archiver/8.0 (Win + - ia_archiver/8.1 (Win + - ia_archiver/8.2 (Win + - ia_archiver/8.8 (Win + - ia_archiver/8.9 (Lin + - ia_archiver/8.9 (Win + - icerocket.com; + - ichiro/2.0 (ichiro@n + - ichiro/x.0 (ichiro@n + - ideare - SignSite/1. + - iearthworm/1.0, iear + - igde.ru + - ilial/Nutch-0.9-dev + - innovationblog.com + - integrity/1.6 + - integrity/5 + - ivia.ucr.edu + - jack + - journster.com + - knil + - kulturarw3/0.1 + - larbin (samualt9@big + - larbin_2.1.1 larbin2 + - larbin_2.2.0 (crawl@ + - larbin_2.2.1_de_Vien + - larbin_2.2.2 (sugaya + - larbin_2.2.2_guillau + - larbin_2.5.0 (larbin + - larbin_2.6.0 (larbin + - larbin_2.6.1 (larbin + - larbin_2.6.1 larbin2 + - larbin_2.6.2 (hamasa + - larbin_2.6.2 (larbin + - larbin_2.6.2 (liston + - larbin_2.6.2 (pierre + - larbin_2.6.2 (pimena + - larbin_2.6.2 (tom@le + - larbin_2.6.2 (vitalb + - larbin_2.6.2 dthunen + - larbin_2.6.2 kalou@k + - larbin_2.6.2 larbin2 + - larbin_2.6.2 larbin@ + - larbin_2.6.2 listonA + - larbin_2.6.2 pierre@ + - larbin_2.6.2 vitalbo + - larbin_2.6.3 (larbin + - larbin_2.6.3 (wgao@g + - larbin_2.6.3 capveg@ + - larbin_2.6.3 ghary@s + - larbin_2.6.3 gqnmgsp + - larbin_2.6.3 larbin2 + - larbin_2.6.3 tangyi8 + - larbin_2.6.3 zumesun + - larbin_2.6_basileoca + - larbin_test (nobody@ + - larbin_test nobody@a + - larbin_xy250 larbin2 + - law.di.unimi.it + - lc/$ROADS::Version l + - letscrawl.com + - libWeb/clsHTTP -- hi + - libwww-perl/5.41 + - libwww-perl/5.45 + - libwww-perl/5.48 + - libwww-perl/5.50 + - libwww-perl/5.52 FP/ + - libwww-perl/5.53 + - libwww-perl/5.63 + - libwww-perl/5.64 + - libwww-perl/5.65 + - libwww-perl/5.800 + - libwww/5.3.2 + - linkdex.com/v2.0 + - ltx71.com + - lunefran.wordpress.com + - lwp-trivial/1.32 + - lwp-trivial/1.33 + - lwp-trivial/1.34 + - lwp-trivial/1.35 + - lwp-trivial/1.36 + - lwp-trivial/1.38 + - lwp-trivial/1.41 + - mailto:webcraft@bea. + - mbblog14.wordpress.com + - metatagsdir.com + - miniflux.net + - misc.yahoo.com.cn + - moget/x.x (moget@goo + - mogimogi/1.0 + - moiNAG 0.02 + - muckrack.com + - myDaemon + - net-promoter.com + - netarkivet.dk + - newsearchengine (Thi + - newsme/1.0; feedback + - newspaper/0.1.0.7 + - newspaper/0.2.5 + - newspaper/0.2.6 + - newstin.com + - nlp.uned.es + - noyona_0_1 + - nsyght.com/Nutch-1.0 + - nsyght.com/Nutch-x.x + - nutch-1.4/Nutch-1.4 + - nutchsearch/Nutch-0. + - nymesis.com + - oegp v. 1.3.0 + - okhttp/2.5.0 + - okhttp/2.7.5 + - okhttp/3.2.0 + - okhttp/3.5.0 + - omgili.com + - ostermiller.org + - owler-agent + - ozelot/2.7.3 (Search + - page2rss.com + - pandora.nla.gov.au + - panscient.com + - parsijoo.ir + - pauillac.inria.fr + - pd02_1.0.0 pd02_1.0. + - pear.php.net + - people.eng.unimelb.edu.au + - phpcrawl + - pompos.iliad.fr + - psycheclone + - publisher.yahoo.com + - python-requests/0.13 + - python-requests/1.0. + - python-requests/2.18 + - python-requests/2.3. + - r.hatena.ne.jp + - rabaz (rabaz at giga + - ranksonic.com + - rico/0.1 + - riddler.io + - rjpower.org + - robot + - sagool.jp + - scooter-venus-3.0.vn + - scoutmaster + - scrapy.org + - search.ch V1.4 + - search.denmex.com + - search.goo.ne.jp + - search.google.com + - search.yahoo.com + - seekda.com + - semanticdiscovery/0. + - sentry.io + - silk/1.0 + - sitebar.org + - siteexplorer.info + - smartwit.com + - sna-0.0.1 (mikemuzio + - sna-0.0.1 mikeelliot + - sohu agent + - sohu-search + - sp.ask.com + - suchbaer.de + - summify.com + - support.embed.ly + - surf + - suzuran + - szukaj.onet.pl + - tags2dir.com + - tagword.com + - targetblaster.com/0. + - teoma_agent1 + - therarestwords.com + - toCrawl/UrlDispatche + - tuezilla.de + - twikle.com + - twurly.org + - unknownght.com Web S + - users.cs.cf.ac.uk + - versus.integis.ch + - vias.ncsa.uiuc.edu + - visionutils/0.2 + - vk.com + - voyager-hc/1.0 + - voyager/1.0 + - web-bekannt (Version + - web2express.org + - webGobbler/1.x.x + - webagent.wise-guys.nl + - webalgo.iit.cnr.it + - webbandit/4.xx.0 + - webcollage/1.114 + - webcollage/1.117 + - webcollage/1.125 + - webcollage/1.129 + - webcollage/1.93 + - webcollage/1.xx + - webcrawl.net + - webdatastats.com + - webglimpse.net + - webhack + - websearchbench.cs.uni-dortmund.de + - websitealert.net + - webster-internet.de + - webster.healeys.net + - whatUseek_winona/3.0 + - wiki.creativecommons.org + - wish.slis.tsukuba.ac.jp + - wkd.iis.sinica.edu.tw + - wortschatz.uni-leipzig.de + - ws.daum.net + - www.Abolimba.de + - www.WISEnut.com + - www.abonti.com + - www.ackerm.com + - www.activtourist.com + - www.agada.de + - www.ahmadsoftware.co + - www.answerbus.com + - www.ant.com + - www.aol.com + - www.aramabeta.com; + - www.arianna.it + - www.arocom.de + - www.asaha.com + - www.axxus.de + - www.ay-up.com + - www.ba.be + - www.blaiz.net + - www.bloglines.com + - www.bortzmeyer.org + - www.brandverity.com + - www.buddybuzz.net + - www.buscaplus.com + - www.business-socket. + - www.canadiancontent.net + - www.chepi.net + - www.chilkatsoft.com + - www.cloudflare.com + - www.cnet.com + - www.cougarsearch.com + - www.covario.com + - www.crim.ca + - www.cs.washington.edu + - www.cs.washington.edu; + - www.dataparksearch.org + - www.de.com + - www.dead-links.com + - www.die-kraehe.de + - www.dnsdigger.com + - www.earthcom.info + - www.eurekster.com + - www.europarchive.org + - www.eventax.de + - www.everatom.com + - www.evri.com + - www.exensa.com + - www.exif-search.com + - www.facebook.com + - www.favcollector.com + - www.feeddemon.com + - www.feedhub.com + - www.feedly.com + - www.feedmap.net + - www.fileboost.net + - www.findpal.com + - www.floorstransformed.com + - www.flyindex.net + - www.followsite.com + - www.genieo.com + - www.globalspec.com + - www.goguides.org + - www.google.com + - www.greaterera.com + - www.hermitsearch.com + - www.htmlvalidator.com + - www.hyperix.com + - www.ianett.com + - www.icerocket.com + - www.illumit.com + - www.iltrovatore.it + - www.inktomi.com + - www.intelliseek.com + - www.ip2location.com + - www.iskanie.com + - www.jordomedia.com + - www.kalooga.com; + - www.kototoi.org + - www.l3s.de + - www.lan4lano.net + - www.lipperhey.com + - www.look.com + - www.lw01.com + - www.matuschek.net + - www.megaindex.ru + - www.megite.com + - www.meta-spinner.de + - www.mixdata.com + - www.moreover.com + - www.neomo.de + - www.netluchs.de + - www.newsgator.com; + - www.ng-search.com + - www.page-store.com + - www.pagebull.com + - www.pingalink.com + - www.pingdom.com + - www.plinki.com; + - www.powerset.com + - www.prairielandconsulting.com + - www.proximic.com + - www.pubblisito.com + - www.qango.com + - www.quickiwiki.com; + - www.qwant.com + - www.rawgrunt.com + - www.relsoftware.com + - www.retrevo.com + - www.samsung.com + - www.scifihifi.com + - www.scoutjet.com + - www.scrubtheweb.com + - www.searchme.com + - www.securecomputing.com + - www.semager.de + - www.shopwiki.com + - www.siigle.com + - www.simpy.com + - www.sitesearch.ca + - www.sitesell.com + - www.sixxs.net + - www.skaffe.com + - www.skyrocket.gr.jp + - www.sli-systems.com + - www.slider.com + - www.slysearch.com + - www.somusical.com + - www.suchen.de + - www.sygol.com + - www.sync2it.com + - www.sysomos.com + - www.targetgroups.net + - www.teradex.com + - www.tkensaku.com + - www.truveo.com + - www.tumblr.com + - www.ukwizz.com + - www.unchaos.com + - www.updated.com; + - www.urltrends.com + - www.vakes.com + - www.versions-project.org + - www.verzamelgids.nl + - www.wavefire.com; + - www.webarchiv.cz + - www.worio.com + - www.wowlemmings.com + - www.xirq.com; + - www.xrayz.co.uk + - www.yama.info.waseda.ac.jp + - www.yoono.com + - www.zedzo.com + - www4mail/2.x libwww- + - wwwster/1.x (Beta- m + - xmlset_roodkcableoj2 + - yacy (www.yacy.net; + - yarienavoir.net/0.2 + - yggdrasil/Nutch-0.9 + - yoogliFetchAgent/0.1 + - Googlebot\/ + - Googlebot-Mobile + - Googlebot-Image + - Googlebot-News + - Googlebot-Video + - AdsBot-Google([^-]|$) + - AdsBot-Google-Mobile + - Feedfetcher-Google + - Mediapartners-Google + - Mediapartners \(Googlebot\) + - APIs-Google + - Google-InspectionTool + - Storebot-Google + - GoogleOther + - bingbot + - Slurp + - [wW]get + - LinkedInBot + - Python-urllib + - python-requests + - aiohttp + - httpx + - libwww-perl + - httpunit + - nutch + - Go-http-client + - phpcrawl + - msnbot + - jyxobot + - FAST-WebCrawler + - FAST Enterprise Crawler + - BIGLOTRON + - Teoma + - convera + - seekbot + - Gigabot + - Gigablast + - exabot + - ia_archiver + - GingerCrawler + - webmon + - HTTrack + - grub.org + - UsineNouvelleCrawler + - antibot + - netresearchserver + - speedy + - fluffy + - findlink + - msrbot + - panscient + - yacybot + - AISearchBot + - ips-agent + - tagoobot + - MJ12bot + - woriobot + - yanga + - buzzbot + - mlbot + - yandex\.com\/bots + - purebot + - Linguee Bot + - CyberPatrol + - voilabot + - Baiduspider + - citeseerxbot + - spbot + - twengabot + - postrank + - Turnitin + - scribdbot + - page2rss + - sitebot + - linkdex + - Adidxbot + - ezooms + - dotbot + - Mail.RU_Bot + - discobot + - heritrix + - findthatfile + - europarchive.org + - NerdByNature.Bot + - sistrix crawler + - Ahrefs(Bot|SiteAudit) + - fuelbot + - CrunchBot + - IndeedBot + - mappydata + - woobot + - ZoominfoBot + - PrivacyAwareBot + - Multiviewbot + - SWIMGBot + - Grobbot + - eright + - Apercite + - semanticbot + - Aboundex + - domaincrawler + - wbsearchbot + - summify + - CCBot + - edisterbot + - SeznamBot + - ec2linkfinder + - gslfbot + - aiHitBot + - intelium_bot + - facebookexternalhit + - Yeti + - RetrevoPageAnalyzer + - lb-spider + - Sogou + - lssbot + - careerbot + - wotbox + - wocbot + - ichiro + - DuckDuckBot + - lssrocketcrawler + - drupact + - webcompanycrawler + - acoonbot + - openindexspider + - gnam gnam spider + - web-archive-net.com.bot + - backlinkcrawler + - coccoc + - integromedb + - content crawler spider + - toplistbot + - it2media-domain-crawler + - ip-web-crawler.com + - siteexplorer.info + - elisabot + - proximic + - changedetection + - arabot + - WeSEE:Search + - niki-bot + - CrystalSemanticsBot + - rogerbot + - 360Spider + - psbot + - InterfaxScanBot + - CC Metadata Scaper + - g00g1e.net + - GrapeshotCrawler + - urlappendbot + - brainobot + - fr-crawler + - binlar + - SimpleCrawler + - Twitterbot + - cXensebot + - smtbot + - bnf.fr_bot + - A6-Indexer + - ADmantX + - Facebot + - OrangeBot\/ + - memorybot + - AdvBot + - MegaIndex + - SemanticScholarBot + - ltx71 + - nerdybot + - xovibot + - BUbiNG + - Qwantify + - archive.org_bot + - Applebot + - TweetmemeBot + - crawler4j + - findxbot + - S[eE][mM]rushBot + - yoozBot + - lipperhey + - Y!J + - Domain Re-Animator Bot + - AddThis + - Screaming Frog SEO Spider + - MetaURI + - Scrapy + - Livelap[bB]ot + - OpenHoseBot + - CapsuleChecker + - collection@infegy.com + - IstellaBot + - DeuSu\/ + - betaBot + - Cliqzbot\/ + - MojeekBot\/ + - netEstate NE Crawler + - SafeSearch microdata crawler + - Gluten Free Crawler\/ + - Sonic + - Sysomos + - Trove + - deadlinkchecker + - Slack-ImgProxy + - Embedly + - RankActiveLinkBot + - iskanie + - SafeDNSBot + - SkypeUriPreview + - Veoozbot + - Slackbot + - redditbot + - datagnionbot + - Google-Adwords-Instant + - adbeat_bot + - WhatsApp + - contxbot + - pinterest.com.bot + - electricmonk + - GarlikCrawler + - BingPreview\/ + - vebidoobot + - FemtosearchBot + - Yahoo Link Preview + - MetaJobBot + - DomainStatsBot + - mindUpBot + - Daum\/ + - Jugendschutzprogramm-Crawler + - Xenu Link Sleuth + - Pcore-HTTP + - moatbot + - KosmioBot + - [pP]ingdom + - AppInsights + - PhantomJS + - Gowikibot + - PiplBot + - Discordbot + - TelegramBot + - Jetslide + - newsharecounts + - James BOT + - Bark[rR]owler + - TinEye + - SocialRankIOBot + - trendictionbot + - Ocarinabot + - epicbot + - Primalbot + - DuckDuckGo-Favicons-Bot + - GnowitNewsbot + - Leikibot + - LinkArchiver + - YaK\/ + - PaperLiBot + - Digg Deeper + - dcrawl + - Snacktory + - AndersPinkBot + - Fyrebot + - EveryoneSocialBot + - Mediatoolkitbot + - Luminator-robots + - ExtLinksBot + - SurveyBot + - NING\/ + - okhttp + - Nuzzel + - omgili + - PocketParser + - YisouSpider + - um-LN + - ToutiaoSpider + - MuckRack + - Jamie's Spider + - AHC\/ + - NetcraftSurveyAgent + - Laserlikebot + - ^Apache-HttpClient + - AppEngine-Google + - Jetty + - Upflow + - Thinklab + - Traackr.com + - Twurly + - Mastodon + - http_get + - DnyzBot + - botify + - 007ac9 Crawler + - BehloolBot + - BrandVerity + - check_http + - BDCbot + - ZumBot + - EZID + - ICC-Crawler + - ArchiveBot + - ^LCC + - filterdb.iss.net\/crawler + - BLP_bbot + - BomboraBot + - Buck\/ + - Companybook-Crawler + - Genieo + - magpie-crawler + - MeltwaterNews + - Moreover + - newspaper\/ + - ScoutJet + - (^| )sentry\/ + - StorygizeBot + - UptimeRobot + - OutclicksBot + - seoscanners + - Hatena + - Google Web Preview + - MauiBot + - AlphaBot + - SBL-BOT + - IAS crawler + - adscanner + - Netvibes + - acapbot + - Baidu-YunGuanCe + - bitlybot + - blogmuraBot + - Bot.AraTurka.com + - bot-pge.chlooe.com + - BoxcarBot + - BTWebClient + - ContextAd Bot + - Digincore bot + - Disqus + - Feedly + - Fetch\/ + - Fever + - Flamingo_SearchEngine + - FlipboardProxy + - g2reader-bot + - G2 Web Services + - imrbot + - K7MLWCBot + - Kemvibot + - Landau-Media-Spider + - linkapediabot + - vkShare + - Siteimprove.com + - BLEXBot\/ + - DareBoost + - ZuperlistBot\/ + - Miniflux\/ + - Feedspot + - Diffbot\/ + - SEOkicks + - tracemyfile + - Nimbostratus-Bot + - zgrab + - PR-CY.RU + - AdsTxtCrawler + - Datafeedwatch + - Zabbix + - TangibleeBot + - google-xrawler + - axios + - Amazon CloudFront + - Pulsepoint + - CloudFlare-AlwaysOnline + - Google-Structured-Data-Testing-Tool + - WordupInfoSearch + - WebDataStats + - HttpUrlConnection + - Seekport Crawler + - ZoomBot + - VelenPublicWebCrawler + - MoodleBot + - jpg-newsbot + - outbrain + - W3C_Validator + - Validator\.nu + - W3C-checklink + - W3C-mobileOK + - W3C_I18n-Checker + - FeedValidator + - W3C_CSS_Validator + - W3C_Unicorn + - Google-PhysicalWeb + - Blackboard + - ICBot\/ + - BazQux + - Twingly + - Rivva + - Experibot + - awesomecrawler + - Dataprovider.com + - GroupHigh\/ + - theoldreader.com + - AnyEvent + - Uptimebot\.org + - Nmap Scripting Engine + - 2ip.ru + - Clickagy + - Caliperbot + - MBCrawler + - online-webceo-bot + - B2B Bot + - AddSearchBot + - Google Favicon + - HubSpot + - Chrome-Lighthouse + - HeadlessChrome + - CheckMarkNetwork\/ + - www\.uptime\.com + - Streamline3Bot\/ + - serpstatbot\/ + - MixnodeCache\/ + - ^curl + - SimpleScraper + - RSSingBot + - Jooblebot + - fedoraplanet + - Friendica + - NextCloud + - Tiny Tiny RSS + - RegionStuttgartBot + - Bytespider + - Datanyze + - Google-Site-Verification + - TrendsmapResolver + - tweetedtimes + - NTENTbot + - Gwene + - SimplePie + - SearchAtlas + - Superfeedr + - feedbot + - UT-Dorkbot + - Amazonbot + - SerendeputyBot + - Eyeotabot + - officestorebot + - Neticle Crawler + - SurdotlyBot + - LinkisBot + - AwarioSmartBot + - AwarioRssBot + - RyteBot + - FreeWebMonitoring SiteChecker + - AspiegelBot + - NAVER Blog Rssbot + - zenback bot + - SentiBot + - Domains Project\/ + - Pandalytics + - VKRobot + - bidswitchbot + - tigerbot + - NIXStatsbot + - Atom Feed Robot + - [Cc]urebot + - PagePeeker\/ + - Vigil\/ + - rssbot\/ + - startmebot\/ + - JobboerseBot + - seewithkids + - NINJA bot + - Cutbot + - BublupBot + - BrandONbot + - RidderBot + - Taboolabot + - Dubbotbot + - FindITAnswersbot + - infoobot + - Refindbot + - BlogTraffic\/\d\.\d+ Feed-Fetcher + - SeobilityBot + - Cincraw + - Dragonbot + - VoluumDSP-content-bot + - FreshRSS + - BitBot + - ^PHP-Curl-Class + - Google-Certificates-Bridge + - centurybot + - Viber + - e\.ventures Investment Crawler + - evc-batch + - PetalBot + - virustotal + - (^| )PTST\/ + - minicrawler + - Cookiebot + - trovitBot + - seostar\.co + - IonCrawl + - Uptime-Kuma + - SeekportBot + - FreshpingBot + - Feedbin + - CriteoBot + - Snap URL Preview Service + - Better Uptime Bot + - RuxitSynthetic + - Google-Read-Aloud + - Valve\/Steam + - OdklBot\/ + - GPTBot + - YandexRenderResourcesBot\/ + - LightspeedSystemsCrawler + - ev-crawler\/ + - BitSightBot\/ + - woorankreview\/ + - Google-Safety + - AwarioBot + - DataForSeoBot + - Linespider + - WellKnownBot + - A Patent Crawler + - StractBot + - search\.marginalia\.nu + - YouBot + - Nicecrawler + - Neevabot + - BrightEdge Crawler + - SiteCheckerBotCrawler + - TombaPublicWebCrawler + - CrawlyProjectCrawler + - KomodiaBot + - KStandBot + - CISPA Webcrawler + - MTRobot + - hyscore.io + - AlexandriaOrgBot + - 2ip bot + - Yellowbrandprotectionbot + - SEOlizer + - vuhuvBot + - INETDEX-BOT + - Synapse + - t3versionsBot + - deepnoc + - Cocolyzebot + - hypestat + - ReverseEngineeringBot + - sempi.tech + - Iframely + - MetaInspector + - node-fetch + - lkxscan + - python-opengraph + - OpenGraphCheck + - developers.google.com\/\+\/web\/snippet + - SenutoBot + - MaCoCu + - NewsBlur + - inoreader + - NetSystemsResearch + - PageThing + - WordPress\/ +- Added `addition_date` and `instances` for the new items where possible. +- Executed and updated the convert_to_txt to write the file in the `generated/` directory diff --git a/COUNTER_Robots_list.json b/COUNTER_Robots_list.json index 40df507..df5c89d 100644 --- a/COUNTER_Robots_list.json +++ b/COUNTER_Robots_list.json @@ -45,16 +45,31 @@ }, { "pattern": "AddThis", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "url": "https://www.addthis.com", + "instances": [ + "AddThis.com robot tech.support@clearspring.com" + ], + "addition_date": "2015-06-02" }, { "pattern": "A6-Indexer", "last_changed": "2019-11-19", - "description": "A6 Corp's bot" + "description": "A6 Corp's bot", + "addition_date": "2014-12-05", + "url": "http://www.a6corp.com/a6-web-scraping-policy/", + "instances": [ + "A6-Indexer" + ] }, { "pattern": "ADmantX", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2014-12-05", + "url": "http://www.admantx.com", + "instances": [ + "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" + ] }, { "pattern": "alexa", @@ -148,7 +163,15 @@ }, { "pattern": "binlar", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2014-09-12", + "instances": [ + "binlar_2.6.3 binlar2.6.3@unspecified.mail", + "binlar_2.6.3 binlar_2.6.3@unspecified.mail", + "binlar_2.6.3 larbin2.6.3@unspecified.mail", + "binlar_2.6.3 phanendra_kalapala@McAfee.com", + "binlar_2.6.3 test@mgmt.mic" + ] }, { "pattern": "bjaaland", @@ -186,7 +209,11 @@ "pattern": "BUbiNG", "last_changed": "2018-02-07", "description": "BUbiNG bot scrapes the internet for contacts and news articles that would be of interest to their students and university.", - "url": "http://law.di.unimi.it/BUbiNG.html" + "url": "http://law.di.unimi.it/BUbiNG.html", + "instances": [ + "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" + ], + "addition_date": "2015-04-06" }, { "pattern": "bwh3_user_agent", @@ -243,7 +270,12 @@ }, { "pattern": "collection@infegy.com", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "url": "http://infegy.com/", + "instances": [ + "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" + ], + "addition_date": "2016-01-03" }, { "pattern": "com\\.plumanalytics", @@ -263,7 +295,11 @@ }, { "pattern": "convera", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" + ], + "url": "http://ews.converasearch.com/crawl.htm" }, { "pattern": "core", @@ -312,7 +348,13 @@ }, { "pattern": "DeuSu\\/", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-01-23", + "url": "https://deusu.de/robot.html", + "instances": [ + "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", + "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" + ] }, { "pattern": "Dispatch\\/\\d", @@ -378,7 +420,14 @@ }, { "pattern": "Embedly", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-04-25", + "url": "http://support.embed.ly", + "instances": [ + "Embedly +support@embed.ly", + "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", + "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" + ] }, { "pattern": "EThOS\\+\\(British\\+Library\\)", @@ -403,7 +452,12 @@ }, { "pattern": "Feedbin", - "last_changed": "2019-01-22" + "last_changed": "2019-01-22", + "addition_date": "2022-11-05", + "url": "https://feedbin.com/", + "instances": [ + "Feedbin feed-id:2005098 - 2 subscribers" + ] }, { "pattern": "feedburner", @@ -431,7 +485,10 @@ }, { "pattern": "findthatfile", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2011-06-21", + "url": "http://www.findthatfile.com/", + "instances": [] }, { "pattern": "^FileDown$", @@ -461,7 +518,11 @@ "pattern": "Genieo", "last_changed": "2018-09-05", "Listed as a bad bot by Distil Networks": "2018-09-05", - "url": "https://www.distilnetworks.com/bot-directory/bot/genieo/" + "url": "http://www.genieo.com/webfilter.html", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" + ] }, { "pattern": "GetRight", @@ -474,7 +535,7 @@ { "pattern": "GigablastOpenSource", "last_changed": "2020-07-02" - }, + }, { "pattern": "G-i-g-a-b-o-t", "last_changed": "2018-02-15", @@ -522,7 +583,33 @@ }, { "pattern": "heritrix", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2011-06-21", + "url": "https://github.com/internetarchive/heritrix3/wiki", + "instances": [ + "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", + "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", + "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" + ] }, { "pattern": "holmes", @@ -540,7 +627,13 @@ "pattern": "HeadlessChrome", "last_changed": "2021-12-07", "description": "Headless Chrome is used to automate Chrome requests for testing purposes.", - "url": "https://developers.google.com/web/updates/2017/04/headless-chrome" + "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", + "addition_date": "2019-06-17", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" + ] }, { "pattern": "HttpComponents\\/1.1", @@ -561,7 +654,12 @@ { "pattern": "httpx", "last_changed": "2021-11-23", - "url": "https://github.com/projectdiscovery/httpx" + "url": "https://www.python-httpx.org", + "addition_date": " 2019/12/23", + "instances": [ + "python-httpx/0.16.1", + "python-httpx/0.13.0.dev1" + ] }, { "pattern": "httrack", @@ -569,11 +667,34 @@ }, { "pattern": "ia_archiver", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", + "ia_archiver-web.archive.org" + ] }, { "pattern": "ichiro", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2012-08-28", + "url": "http://help.goo.ne.jp/help/article/1142", + "instances": [ + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", + "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/2.0 (ichiro@nttr.co.jp)", + "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", + "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" + ] }, { "pattern": "iktomi", @@ -611,7 +732,12 @@ }, { "pattern": "iskanie", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-09-02", + "url": "http://www.iskanie.com", + "instances": [ + "iskanie (+http://www.iskanie.com)" + ] }, { "pattern": "^java\\/\\d{1,2}.\\d", @@ -695,7 +821,15 @@ }, { "pattern": "lipperhey", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "url": "http://www.lipperhey.com/", + "instances": [ + "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" + ], + "addition_date": "2015-08-26" }, { "pattern": "livejournal\\.com", @@ -711,7 +845,12 @@ }, { "pattern": "ltx71", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "url": "http://ltx71.com/", + "instances": [ + "ltx71 - (http://ltx71.com/)" + ], + "addition_date": "2015-04-04" }, { "pattern": "lwp", @@ -724,7 +863,11 @@ { "pattern": "MaCoCu", "last_changed": "2021-11-23", - "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/" + "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" + ] }, { "pattern": "mail\\.ru", @@ -747,7 +890,11 @@ "pattern": "MetaInspector", "last_changed": "2022-05-04", "description": "Ruby gem for web scraping purposes.", - "url": "https://github.com/metainspector/metainspector" + "url": "https://github.com/jaimeiniesta/metainspector", + "addition_date": "2023-09-08", + "instances": [ + "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" + ] }, { "pattern": "MetaURI[\\+\\s]API\\/\\d\\.\\d", @@ -879,7 +1026,11 @@ }, { "pattern": "nutch", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", + "istellabot-nutch/Nutch-1.10" + ] }, { "pattern": "^oaDOI$", @@ -898,12 +1049,20 @@ { "pattern": "OgScrper", "last_changed": "2020-07-02" - }, + }, { "pattern": "okhttp", "last_changed": "2020-02-25", "description": "okhttp is a Java HTTP client library.", - "url": "http://square.github.io/okhttp/" + "url": "http://square.github.io/okhttp/", + "addition_date": "2017-11-02", + "instances": [ + "okhttp/2.5.0", + "okhttp/2.7.5", + "okhttp/3.2.0", + "okhttp/3.5.0", + "okhttp/4.1.0" + ] }, { "pattern": "onetszukaj", @@ -919,7 +1078,10 @@ }, { "pattern": "panscient", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "panscient.com" + ] }, { "pattern": "parsijoo", @@ -932,7 +1094,13 @@ }, { "pattern": "Pcore-HTTP", - "last_changed": "2019-11-19" + "last_changed": "2019-11-19", + "addition_date": "2017-08-19", + "instances": [ + "Pcore-HTTP/v0.40.3", + "Pcore-HTTP/v0.44.0" + ], + "url": "https://bitbucket.org/softvisio/pcore/overview" }, { "pattern": "pear\\.php\\.net", @@ -974,7 +1142,13 @@ }, { "pattern": "proximic", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2013-09-12", + "url": "http://www.proximic.com/info/spider.php", + "instances": [ + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" + ] }, { "pattern": "PycURL", @@ -986,7 +1160,15 @@ }, { "pattern": "Qwantify", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "url": "https://www.qwant.com/", + "instances": [ + "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", + "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", + "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", + "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" + ], + "addition_date": "2015-04-06" }, { "pattern": "rambler", @@ -1047,7 +1229,11 @@ { "pattern": "ScoutJet", "last_changed": "2019-02-12", - "url": "http://www.scoutjet.com/" + "url": "http://www.scoutjet.com/", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" + ] }, { "pattern": "^scrutiny\\/\\d", @@ -1067,7 +1253,11 @@ }, { "pattern": "SkypeUriPreview", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-10-10", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" + ] }, { "pattern": "slurp", @@ -1079,7 +1269,14 @@ }, { "pattern": "speedy", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", + "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" + ] }, { "pattern": "sqlmap", @@ -1097,7 +1294,12 @@ }, { "pattern": "summify", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2012-01-04", + "url": "http://summify.com", + "instances": [ + "Summify (Summify/1.0.1; +http://summify.com)" + ] }, { "pattern": "sunrise", @@ -1105,7 +1307,12 @@ }, { "pattern": "Sysomos", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-02-08", + "url": "http://www.sysomos.com", + "instances": [ + "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" + ] }, { "pattern": "T\\-H\\-U\\-N\\-D\\-E\\-R\\-S\\-T\\-O\\-N\\-E", @@ -1121,7 +1328,12 @@ }, { "pattern": "Teoma", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "instances": [ + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" + ], + "url": "http://about.ask.com/en/docs/about/webmasters.shtml" }, { "pattern": "The[\\+\\s]Knowledge[\\+\\s]AI", @@ -1142,13 +1354,21 @@ }, { "pattern": "Trove", - "last_changed": "2017-08-08" + "last_changed": "2017-08-08", + "addition_date": "2016-02-08", + "url": "http://www.trove.com", + "instances": [] }, { "pattern": "Turnitin", "last_changed": "2020-07-01", "description": "This robot collects content from the Internet for the sole purpose of helping educational institutions prevent plagiarism.", - "url": "https://turnitin.com/robot/crawlerinfo.html" + "url": "http://www.turnitin.com", + "addition_date": "2010-09-26", + "instances": [ + "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", + "Turnitin (https://bit.ly/2UvnfoQ)" + ] }, { "pattern": "twiceler", @@ -1287,7 +1507,36 @@ }, { "pattern": "WhatsApp", - "last_changed": "2021-03-30" + "last_changed": "2021-03-30", + "addition_date": "2016-11-15", + "url": "https://www.whatsapp.com/", + "instances": [ + "WhatsApp", + "WhatsApp/0.3.4479 N", + "WhatsApp/0.3.4679 N", + "WhatsApp/0.3.4941 N", + "WhatsApp/2.12.15/i", + "WhatsApp/2.12.16/i", + "WhatsApp/2.12.17/i", + "WhatsApp/2.12.449 A", + "WhatsApp/2.12.453 A", + "WhatsApp/2.12.510 A", + "WhatsApp/2.12.540 A", + "WhatsApp/2.12.548 A", + "WhatsApp/2.12.555 A", + "WhatsApp/2.12.556 A", + "WhatsApp/2.16.1/i", + "WhatsApp/2.16.13 A", + "WhatsApp/2.16.2/i", + "WhatsApp/2.16.42 A", + "WhatsApp/2.16.57 A", + "WhatsApp/2.19.92 i", + "WhatsApp/2.19.175 A", + "WhatsApp/2.19.244 A", + "WhatsApp/2.19.258 A", + "WhatsApp/2.19.308 A", + "WhatsApp/2.19.330 A" + ] }, { "pattern": "wordpress", @@ -1333,7 +1582,11 @@ { "pattern": "Zabbix", "last_changed": "2021-11-23", - "url": "https://github.com/zabbix/zabbix" + "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring", + "addition_date": "2018-09-05", + "instances": [ + "Zabbix" + ] }, { "pattern": "ZoteroTranslationServer", @@ -1353,5 +1606,9701 @@ "pattern": "7siters", "last_changed": "2020-02-25", "url": "https://7ooo.ru/siters" + }, + { + "pattern": "&as_qdr=all", + "last_changed": "2024-01-02" + }, + { + "pattern": "*/Nutch-0.9-dev", + "last_changed": "2024-01-02" + }, + { + "pattern": "192.comAgent", + "last_changed": "2024-01-02" + }, + { + "pattern": "222.122.15.190", + "last_changed": "2024-01-02" + }, + { + "pattern": "2Bone_LinkChkr/1.0 l", + "last_changed": "2024-01-02" + }, + { + "pattern": "7331230CB97FADECF052", + "last_changed": "2024-01-02" + }, + { + "pattern": "8484 Boston Project ", + "last_changed": "2024-01-02" + }, + { + "pattern": "8LEGS", + "last_changed": "2024-01-02" + }, + { + "pattern": "A-Online Search", + "last_changed": "2024-01-02" + }, + { + "pattern": "ADmantX Platform Sem", + "last_changed": "2024-01-02" + }, + { + "pattern": "AHC/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPSeek/1.2.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPSeek/1.2.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPSeek/1.2.xa", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPSeek/1.2.xxpre", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPseek/1.2.9d", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASPseek/1.2.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "ASSORT/0.10", + "last_changed": "2024-01-02" + }, + { + "pattern": "AV Fetch 1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "AVSearch-1.0(peter.t", + "last_changed": "2024-01-02" + }, + { + "pattern": "AVSearch-2.0-fusionI", + "last_changed": "2024-01-02" + }, + { + "pattern": "AVSearch-3.0(AltaVis", + "last_changed": "2024-01-02" + }, + { + "pattern": "About/0.1libwww-perl", + "last_changed": "2024-01-02" + }, + { + "pattern": "Accoona-AI-Agent/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Acorn/Nutch-0.9 (Non", + "last_changed": "2024-01-02" + }, + { + "pattern": "ActiveBookmark 1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "AdobeUxTechC4-Async/", + "last_changed": "2024-01-02" + }, + { + "pattern": "AgentName/0.1 libwww", + "last_changed": "2024-01-02" + }, + { + "pattern": "AideRSS/1.0 (aiderss", + "last_changed": "2024-01-02" + }, + { + "pattern": "Aladin/3.324", + "last_changed": "2024-01-02" + }, + { + "pattern": "Allesklar/0.1 libwww", + "last_changed": "2024-01-02" + }, + { + "pattern": "Allrati/1.1 (+)", + "last_changed": "2024-01-02" + }, + { + "pattern": "AltaVista Intranet V", + "last_changed": "2024-01-02" + }, + { + "pattern": "Amazon CloudFront", + "last_changed": "2024-01-02", + "addition_date": "2018-09-07", + "instances": [ + "Amazon CloudFront" + ], + "url": "https://aws.amazon.com/cloudfront/" + }, + { + "pattern": "AmiTCP Miami (AmigaO", + "last_changed": "2024-01-02" + }, + { + "pattern": "Anonymizer/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "AnswerChase PROve x.", + "last_changed": "2024-01-02" + }, + { + "pattern": "AnswerChase x.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "AnzwersCrawl/2.0 (an", + "last_changed": "2024-01-02" + }, + { + "pattern": "Apache (internal dum", + "last_changed": "2024-01-02" + }, + { + "pattern": "Apache-HttpAsyncClie", + "last_changed": "2024-01-02" + }, + { + "pattern": "Apache-HttpClient/4.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Apache-HttpClient/UN", + "last_changed": "2024-01-02" + }, + { + "pattern": "Aport", + "last_changed": "2024-01-02" + }, + { + "pattern": "Arachnoidea (arachno", + "last_changed": "2024-01-02" + }, + { + "pattern": "Asahina-Antenna/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Asahina-Antenna/1.x ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Ask.24x.Info", + "last_changed": "2024-01-02" + }, + { + "pattern": "Atomic_Email_Hunter/", + "last_changed": "2024-01-02" + }, + { + "pattern": "Atomz/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "BIGLOTRON (Beta 2;GN", + "last_changed": "2024-01-02" + }, + { + "pattern": "BMCLIENT", + "last_changed": "2024-01-02" + }, + { + "pattern": "BMLAUNCHER", + "last_changed": "2024-01-02" + }, + { + "pattern": "BPImageWalker/2.0 (w", + "last_changed": "2024-01-02" + }, + { + "pattern": "BSDSeek/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "BStop.BravoBrian.it ", + "last_changed": "2024-01-02" + }, + { + "pattern": "BTWebClient/180B(970", + "last_changed": "2024-01-02" + }, + { + "pattern": "BeamMachine/0.5 (dea", + "last_changed": "2024-01-02" + }, + { + "pattern": "BeebwareDirectory/v0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Big Fish v1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "BigBrother/1.6e", + "last_changed": "2024-01-02" + }, + { + "pattern": "Bilbo/2.3b-UNIX", + "last_changed": "2024-01-02" + }, + { + "pattern": "BlackBerry9000/4.6.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "BlackWidow", + "last_changed": "2024-01-02" + }, + { + "pattern": "Blackboard Safeassig", + "last_changed": "2024-01-02" + }, + { + "pattern": "BlogPulseLive (suppo", + "last_changed": "2024-01-02" + }, + { + "pattern": "Blogpulse (info@blog", + "last_changed": "2024-01-02" + }, + { + "pattern": "Bobby/4.0.x RPT-HTTP", + "last_changed": "2024-01-02" + }, + { + "pattern": "Bookdog/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "BravoBrian bstop.bra", + "last_changed": "2024-01-02" + }, + { + "pattern": "BullsEye", + "last_changed": "2024-01-02" + }, + { + "pattern": "BunnySlippers", + "last_changed": "2024-01-02" + }, + { + "pattern": "CB/Nutch-1.7", + "last_changed": "2024-01-02" + }, + { + "pattern": "CFNetwork/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "CHttpClient by Open ", + "last_changed": "2024-01-02" + }, + { + "pattern": "CLIPish Pro2 10.21 (", + "last_changed": "2024-01-02" + }, + { + "pattern": "COAST WebMaster Pro/", + "last_changed": "2024-01-02" + }, + { + "pattern": "CamelHttpStream/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Cancer Information a", + "last_changed": "2024-01-02" + }, + { + "pattern": "Chat Catcher/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "China Local Browse 2", + "last_changed": "2024-01-02" + }, + { + "pattern": "Chitika ContentHit 1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Claymont.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "ColdFusion (Bookmark", + "last_changed": "2024-01-02" + }, + { + "pattern": "Combine/x.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Contact", + "last_changed": "2024-01-02" + }, + { + "pattern": "Cookie%20Stumbler/22", + "last_changed": "2024-01-02" + }, + { + "pattern": "CrawlConvera0.1 (Cra", + "last_changed": "2024-01-02" + }, + { + "pattern": "Custo x.x (www.netwu", + "last_changed": "2024-01-02" + }, + { + "pattern": "CyberSpyder Link Tes", + "last_changed": "2024-01-02" + }, + { + "pattern": "DBrowse 1.4b", + "last_changed": "2024-01-02" + }, + { + "pattern": "DBrowse 1.4d", + "last_changed": "2024-01-02" + }, + { + "pattern": "DTAAgent", + "last_changed": "2024-01-02" + }, + { + "pattern": "Dart Communications ", + "last_changed": "2024-01-02" + }, + { + "pattern": "DataCha0s/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "DataFountains/DMOZ D", + "last_changed": "2024-01-02" + }, + { + "pattern": "Datafeedwatch/2.1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Deep Link Calculator", + "last_changed": "2024-01-02" + }, + { + "pattern": "DeepIndex", + "last_changed": "2024-01-02" + }, + { + "pattern": "DeepIndex (www.en.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "DeepIndexer.ca", + "last_changed": "2024-01-02" + }, + { + "pattern": "DevComponents.com Ht", + "last_changed": "2024-01-02" + }, + { + "pattern": "Diamond/x.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "DigOut4U", + "last_changed": "2024-01-02" + }, + { + "pattern": "Digger/1.0 JDK/1.3.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Disqus/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "DittoSpyder", + "last_changed": "2024-01-02" + }, + { + "pattern": "DoCoMo/1.0/Nxxxi/c10", + "last_changed": "2024-01-02" + }, + { + "pattern": "DoCoMo/2.0 P900iV(c1", + "last_changed": "2024-01-02" + }, + { + "pattern": "EARTHCOM.info/1.x [w", + "last_changed": "2024-01-02" + }, + { + "pattern": "EARTHCOM.info/1.xbet", + "last_changed": "2024-01-02" + }, + { + "pattern": "EBrowse 1.4b", + "last_changed": "2024-01-02" + }, + { + "pattern": "EasyDL/3.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "EchO!/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Educate Search VxB", + "last_changed": "2024-01-02" + }, + { + "pattern": "EldoS TimelyWeb/3.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "EmailWolf 1.00", + "last_changed": "2024-01-02" + }, + { + "pattern": "Embedly +support@emb", + "last_changed": "2024-01-02" + }, + { + "pattern": "Enfish Tracker", + "last_changed": "2024-01-02" + }, + { + "pattern": "Enterprise_Search/1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Evernote Clip Resolv", + "last_changed": "2024-01-02" + }, + { + "pattern": "Exalead NG/MimeLive ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Expired Domain Sleut", + "last_changed": "2024-01-02" + }, + { + "pattern": "ExtractorPro", + "last_changed": "2024-01-02" + }, + { + "pattern": "EyeCatcher (Download", + "last_changed": "2024-01-02" + }, + { + "pattern": "FARK.com link verifi", + "last_changed": "2024-01-02" + }, + { + "pattern": "FLATARTS_FAVICO", + "last_changed": "2024-01-02" + }, + { + "pattern": "FaEdit/2.0.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "FavOrg", + "last_changed": "2024-01-02" + }, + { + "pattern": "Favorites Sweeper v.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Feed24.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "FeedForAll rss2html.", + "last_changed": "2024-01-02" + }, + { + "pattern": "FeedValidator/1.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Filtrbox/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "FindAnISP.com_ISP_Fi", + "last_changed": "2024-01-02" + }, + { + "pattern": "Firefly/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Firefox (kastaneta03", + "last_changed": "2024-01-02" + }, + { + "pattern": "Firefox_1.0.6 (kaspa", + "last_changed": "2024-01-02" + }, + { + "pattern": "FirstGov.gov Search ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Flexum/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "FocusedSampler/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Franklin Locator 1.8", + "last_changed": "2024-01-02" + }, + { + "pattern": "FuseBulb.Com", + "last_changed": "2024-01-02" + }, + { + "pattern": "GigablastOpenSource/", + "last_changed": "2024-01-02" + }, + { + "pattern": "Go 1.1 package http", + "last_changed": "2024-01-02" + }, + { + "pattern": "Go-http-client/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Go-http-client/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "GoForIt.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Goofer/0.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "GrapeFX/0.3 libwww/5", + "last_changed": "2024-01-02" + }, + { + "pattern": "Gromit/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Guestbook Auto Submi", + "last_changed": "2024-01-02" + }, + { + "pattern": "Gulliver/1.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Guzzle/4.2.3 curl/7.", + "last_changed": "2024-01-02" + }, + { + "pattern": "HLoader", + "last_changed": "2024-01-02" + }, + { + "pattern": "HPL/Nutch-0.9 -", + "last_changed": "2024-01-02" + }, + { + "pattern": "Haansoft", + "last_changed": "2024-01-02" + }, + { + "pattern": "Harvest-NG/1.0.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hatena Antenna/0.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hatena Pagetitle Age", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hatena::Bookmark/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hippias/0.9 Beta", + "last_changed": "2024-01-02" + }, + { + "pattern": "HitList", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hivemind", + "last_changed": "2024-01-02" + }, + { + "pattern": "HomePageSearch(hpsea", + "last_changed": "2024-01-02" + }, + { + "pattern": "Html Link Validator ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Hybrid/1.2 [en] (OS ", + "last_changed": "2024-01-02" + }, + { + "pattern": "IAArchiver-1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "IBM WebExplorer /v0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "IEFav172Free", + "last_changed": "2024-01-02" + }, + { + "pattern": "INFOMINE/8.0 Adders", + "last_changed": "2024-01-02" + }, + { + "pattern": "INFOMINE/8.0 RemoteS", + "last_changed": "2024-01-02" + }, + { + "pattern": "IP*Works! V5 HTTP/S ", + "last_changed": "2024-01-02" + }, + { + "pattern": "ISC Systems iRc Sear", + "last_changed": "2024-01-02" + }, + { + "pattern": "IWAgent/ 1.0 - www.b", + "last_changed": "2024-01-02" + }, + { + "pattern": "Icarus6 - (contact: ", + "last_changed": "2024-01-02" + }, + { + "pattern": "ImageWalker/2.0 (www", + "last_changed": "2024-01-02" + }, + { + "pattern": "Incutio HttpClient v", + "last_changed": "2024-01-02" + }, + { + "pattern": "IncyWincy data gathe", + "last_changed": "2024-01-02" + }, + { + "pattern": "Industry Program 1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Inet library", + "last_changed": "2024-01-02" + }, + { + "pattern": "InetURL/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "InfoLink/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "InfoSeek Sidewinder/", + "last_changed": "2024-01-02" + }, + { + "pattern": "Infoaxe./Nutch-0.9", + "last_changed": "2024-01-02" + }, + { + "pattern": "Infoseek SideWinder/", + "last_changed": "2024-01-02" + }, + { + "pattern": "Inktomi Search", + "last_changed": "2024-01-02" + }, + { + "pattern": "Insitor.com search a", + "last_changed": "2024-01-02" + }, + { + "pattern": "Insitornaut", + "last_changed": "2024-01-02" + }, + { + "pattern": "Internet Ninja x.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "InternetLinkAgent/3.", + "last_changed": "2024-01-02" + }, + { + "pattern": "InternetSeer.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "IrssiUrlLog/0.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "JDK/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jack", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jakarta Commons-Http", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java 1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java Browser", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.4.1_01", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.4.1_04", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.5.0_16", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_04", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_07", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_18", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_22", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_24", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_25", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_26", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_27", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_29", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_30", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_31", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_32", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_33", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_34", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_35", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_39", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_45", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.6.0_51", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_03", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_09", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_15", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_17", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_21", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_25", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_40", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_45", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_51", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_55", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_60", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_65", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_71", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_75", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.7.0_80", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_20", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_25", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_31", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_40", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_45", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java/1.8.0_51", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java1.0.21.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java1.1.xx.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java1.3.0rc1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java1.3.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Java1.4.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jersey/2.25.1 (HttpU", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jetty/9.3.z-SNAPSHOT", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jigsaw/2.2.x W3C_CSS", + "last_changed": "2024-01-02" + }, + { + "pattern": "Jigsaw/2.3.0 W3C_CSS", + "last_changed": "2024-01-02" + }, + { + "pattern": "Junkie/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "KE_1.0/2.0 libwww/5.", + "last_changed": "2024-01-02" + }, + { + "pattern": "KIT-Fireball/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "KnowItAll(knowitall@", + "last_changed": "2024-01-02" + }, + { + "pattern": "Knowledge.com/0.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "KummHttp/1.1 (compat", + "last_changed": "2024-01-02" + }, + { + "pattern": "LARBIN-EXPERIMENTAL ", + "last_changed": "2024-01-02" + }, + { + "pattern": "LEIA/2.90", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.22", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.36", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.48", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.50", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.51", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.53", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.63", + "last_changed": "2024-01-02" + }, + { + "pattern": "LWP::Simple/5.803", + "last_changed": "2024-01-02" + }, + { + "pattern": "Lachesis", + "last_changed": "2024-01-02" + }, + { + "pattern": "Libby_1.1/libwww-per", + "last_changed": "2024-01-02" + }, + { + "pattern": "Lincoln State Web Br", + "last_changed": "2024-01-02" + }, + { + "pattern": "Link Valet Online 1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkAlarm/2.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkPimpin v1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkProver 2.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkSonar/1.35", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkSweeper/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkWalker", + "last_changed": "2024-01-02" + }, + { + "pattern": "LinkWalker/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Look.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Lynx/2.8.4rel.1 libw", + "last_changed": "2024-01-02" + }, + { + "pattern": "MARTINI", + "last_changed": "2024-01-02" + }, + { + "pattern": "MFC Foundation Class", + "last_changed": "2024-01-02" + }, + { + "pattern": "MFC_Tear_Sample", + "last_changed": "2024-01-02" + }, + { + "pattern": "MSIE-5.13 (larbin@un", + "last_changed": "2024-01-02" + }, + { + "pattern": "MSNPTC/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "MVAClient", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mac Finder 1.0.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mag-Net", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mail.Ru/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "MantraAgent", + "last_changed": "2024-01-02" + }, + { + "pattern": "MapoftheInternet.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Marketwave Hit List", + "last_changed": "2024-01-02" + }, + { + "pattern": "Martini", + "last_changed": "2024-01-02" + }, + { + "pattern": "Marvin v0.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "MasterSeek", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mata Hari/2.00", + "last_changed": "2024-01-02" + }, + { + "pattern": "MediaSearch/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mediapartners-Google", + "last_changed": "2024-01-02", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "Mediapartners-Google", + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" + ] + }, + { + "pattern": "MegaSheep v1.0 (www.", + "last_changed": "2024-01-02" + }, + { + "pattern": "MeltwaterNews www.me", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mercator-1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mercator-2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mercator-Scrub-1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mercury/894 CFNetwor", + "last_changed": "2024-01-02" + }, + { + "pattern": "MetaURI API/2.0 +met", + "last_changed": "2024-01-02" + }, + { + "pattern": "MicroBaz", + "last_changed": "2024-01-02" + }, + { + "pattern": "Microsoft Data Acces", + "last_changed": "2024-01-02" + }, + { + "pattern": "Microsoft Small Busi", + "last_changed": "2024-01-02" + }, + { + "pattern": "Microsoft URL Contro", + "last_changed": "2024-01-02" + }, + { + "pattern": "Microsoft Windows Ne", + "last_changed": "2024-01-02" + }, + { + "pattern": "MiracleAlphaTest", + "last_changed": "2024-01-02" + }, + { + "pattern": "Missauga Locate 1.0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Missigua Locator 1.9", + "last_changed": "2024-01-02" + }, + { + "pattern": "Missouri College Bro", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mister PiX version.d", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mister Pix II 2.02a", + "last_changed": "2024-01-02" + }, + { + "pattern": "Miva (AlgoFeedback@m", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mizzu Labs 2.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "MnogoSearch/3.2.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mnogosearch-3.1.21", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mo College 1.9", + "last_changed": "2024-01-02" + }, + { + "pattern": "More yacy user agent", + "last_changed": "2024-01-02" + }, + { + "pattern": "Mozi!", + "last_changed": "2024-01-02" + }, + { + "pattern": "MultiText/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "My WinHTTP Connectio", + "last_changed": "2024-01-02" + }, + { + "pattern": "MyNutchTest/Nutch-1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "NASA Search 1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NEC Research Agent -", + "last_changed": "2024-01-02" + }, + { + "pattern": "NEC-Hayek/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NG/4.0.1229", + "last_changed": "2024-01-02" + }, + { + "pattern": "NICO/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NING/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NationalDirectoryAdd", + "last_changed": "2024-01-02" + }, + { + "pattern": "NativeHost", + "last_changed": "2024-01-02" + }, + { + "pattern": "NetLookout/2.24", + "last_changed": "2024-01-02" + }, + { + "pattern": "NetMechanic Vx.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NetSprint -- 2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "NetZippy", + "last_changed": "2024-01-02" + }, + { + "pattern": "Nocilla/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Nsauditor/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Nuzzel", + "last_changed": "2024-01-02", + "addition_date": "2017-11-02", + "instances": [ + "Nuzzel" + ] + }, + { + "pattern": "Nuzzel/876 CFNetwork", + "last_changed": "2024-01-02" + }, + { + "pattern": "Octora Beta - www.oc", + "last_changed": "2024-01-02" + }, + { + "pattern": "Oculus360/Nutch-1.7", + "last_changed": "2024-01-02" + }, + { + "pattern": "OliverPerry", + "last_changed": "2024-01-02" + }, + { + "pattern": "OpenAcoon v4.0.x (ww", + "last_changed": "2024-01-02" + }, + { + "pattern": "OpenISearch/1.x (www", + "last_changed": "2024-01-02" + }, + { + "pattern": "OpenOffice/4.1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "OpenWebIndex/Nutch-1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Openstat/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Opera/6.01 (larbin@u", + "last_changed": "2024-01-02" + }, + { + "pattern": "Oracle Ultra Search", + "last_changed": "2024-01-02" + }, + { + "pattern": "OzySoftware.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "PBrowse 1.4b", + "last_changed": "2024-01-02" + }, + { + "pattern": "PEval 1.4b", + "last_changed": "2024-01-02" + }, + { + "pattern": "PHP/3.x.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "PHP/4.0.4pl1", + "last_changed": "2024-01-02" + }, + { + "pattern": "PHP/4.0.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "PHP/4.1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "PHP/4.1.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "POE-Component-Client", + "last_changed": "2024-01-02" + }, + { + "pattern": "Pcore-HTTP/v0.40.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "PeopleChat/Search_En", + "last_changed": "2024-01-02" + }, + { + "pattern": "PicoSearch/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Piffany_Web_Scraper_", + "last_changed": "2024-01-02" + }, + { + "pattern": "PingALink Monitoring", + "last_changed": "2024-01-02" + }, + { + "pattern": "Pita", + "last_changed": "2024-01-02" + }, + { + "pattern": "Poirot", + "last_changed": "2024-01-02" + }, + { + "pattern": "Pompos/1.x pompos@il", + "last_changed": "2024-01-02" + }, + { + "pattern": "Popdexter/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Port Huron Labs", + "last_changed": "2024-01-02" + }, + { + "pattern": "PostFavorites", + "last_changed": "2024-01-02" + }, + { + "pattern": "PostRank/2.0 (postra", + "last_changed": "2024-01-02" + }, + { + "pattern": "PrivacyFinder/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Program Shareware 1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Progressive Download", + "last_changed": "2024-01-02" + }, + { + "pattern": "Project XP5 [2.03.07", + "last_changed": "2024-01-02" + }, + { + "pattern": "PubCrawl (pubcrawl.s", + "last_changed": "2024-01-02" + }, + { + "pattern": "Pulsepoint XT3 web s", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.13.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.15.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.16.4", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.18.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.18.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.19.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.19.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.19.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.19.7", + "last_changed": "2024-01-02" + }, + { + "pattern": "PycURL/7.23.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/1.17", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/1.1x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/2.0a1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/2.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/2.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/2.7", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.4", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "Python-urllib/3.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "QPCreep Test Rig ( W", + "last_changed": "2024-01-02" + }, + { + "pattern": "Qseero v1.0.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "QuepasaCreep v0.9.1x", + "last_changed": "2024-01-02" + }, + { + "pattern": "QueryN Metasearch", + "last_changed": "2024-01-02" + }, + { + "pattern": "Qwantify/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "R6_VoteReader(www.ra", + "last_changed": "2024-01-02" + }, + { + "pattern": "REBOL Core 2.x.x.x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "RPT-HTTPClient/0.3-x", + "last_changed": "2024-01-02" + }, + { + "pattern": "RepoMonkey Bait & Ta", + "last_changed": "2024-01-02" + }, + { + "pattern": "RoboCrawl (www.canad", + "last_changed": "2024-01-02" + }, + { + "pattern": "Robozilla/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Rotondo/3.1 libwww/5", + "last_changed": "2024-01-02" + }, + { + "pattern": "Ruby", + "last_changed": "2024-01-02" + }, + { + "pattern": "Rumours-Agent", + "last_changed": "2024-01-02" + }, + { + "pattern": "SMRF URL Expander", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.0.EU", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.0.FS", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.0.HD", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.0.VNS", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.0QI", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.BT", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.DIL", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.EX", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.JT", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.NIV", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.SF0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.2.snippet", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-3.3dev", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-ARS-1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-ARS-1.1-ih", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-W3-1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter-W3.1.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/1.0 scooter@", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/1.1 (custom)", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/2.0 G.R.A.B.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/3.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/3.3.QA.pczuk", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/3.3.vscooter", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter/3.3_SF", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter2_Mercator_x-", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter_bh0-3.0.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Scooter_trk3-3.0.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "ScoutAbout", + "last_changed": "2024-01-02" + }, + { + "pattern": "Search+", + "last_changed": "2024-01-02" + }, + { + "pattern": "Search-Engine-Studio", + "last_changed": "2024-01-02" + }, + { + "pattern": "SearchGuild/DMOZ/Exp", + "last_changed": "2024-01-02" + }, + { + "pattern": "SearchGuild_DMOZ_Exp", + "last_changed": "2024-01-02" + }, + { + "pattern": "SearchSight.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "SearchTone2.0 - IDEA", + "last_changed": "2024-01-02" + }, + { + "pattern": "Seeker.lookseek.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Silk/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "SiteBar/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "SiteRecon+(xx)", + "last_changed": "2024-01-02" + }, + { + "pattern": "SiteTruth.com site r", + "last_changed": "2024-01-02" + }, + { + "pattern": "SiteXpert", + "last_changed": "2024-01-02" + }, + { + "pattern": "Skywalker/0.1 (Skywa", + "last_changed": "2024-01-02" + }, + { + "pattern": "Slarp/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Slider_Search_v1-de", + "last_changed": "2024-01-02" + }, + { + "pattern": "Slurpy Verifier/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "SlySearch (slysearch", + "last_changed": "2024-01-02" + }, + { + "pattern": "SnoopRob/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Snoopy v1.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "Snoopy_v0.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "Sphere Scout&v4.0 - ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Sphider", + "last_changed": "2024-01-02" + }, + { + "pattern": "Spida/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Spinne/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Spinne/2.0 med", + "last_changed": "2024-01-02" + }, + { + "pattern": "Spinne/2.0 med_AH", + "last_changed": "2024-01-02" + }, + { + "pattern": "Sqworm/2.9.81-BETA (", + "last_changed": "2024-01-02" + }, + { + "pattern": "Sqworm/2.9.85-BETA (", + "last_changed": "2024-01-02" + }, + { + "pattern": "Sqworm/2.9.89-BETA (", + "last_changed": "2024-01-02" + }, + { + "pattern": "StackRambler/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "StackRambler/2.0 (MS", + "last_changed": "2024-01-02" + }, + { + "pattern": "StackRambler/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "SyncIT/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Szukacz/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "TE", + "last_changed": "2024-01-02" + }, + { + "pattern": "TJvMultiHttpGrabber ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Tagyu Agent/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "TeamSoft WinInet Com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Teoma MP", + "last_changed": "2024-01-02" + }, + { + "pattern": "The European Library", + "last_changed": "2024-01-02" + }, + { + "pattern": "Thinklab (thinklab.c", + "last_changed": "2024-01-02" + }, + { + "pattern": "TinEye", + "last_changed": "2024-01-02", + "addition_date": "2017-10-14", + "url": "http://www.tineye.com/crawler.html", + "instances": [ + "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", + "TinEye/1.1 (http://tineye.com/crawler.html)" + ] + }, + { + "pattern": "Toutatis x-xx.x (hop", + "last_changed": "2024-01-02" + }, + { + "pattern": "Toutatis x.x (hoppa.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Toutatis x.x-x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Traackr.com", + "last_changed": "2024-01-02", + "addition_date": "2017-11-02", + "url": "Traackr.com", + "instances": [ + "Traackr.com" + ] + }, + { + "pattern": "Twisted PageGetter", + "last_changed": "2024-01-02" + }, + { + "pattern": "Twitter/6.18.3 CFNet", + "last_changed": "2024-01-02" + }, + { + "pattern": "Twitturly / v0.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Tycoon Agent/Nutch-1", + "last_changed": "2024-01-02" + }, + { + "pattern": "TygoProwler", + "last_changed": "2024-01-02" + }, + { + "pattern": "UCWEB/2.0(Linux; U; ", + "last_changed": "2024-01-02" + }, + { + "pattern": "UDM", + "last_changed": "2024-01-02" + }, + { + "pattern": "URLBase/6.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "URLBlaze", + "last_changed": "2024-01-02" + }, + { + "pattern": "UdmSearch/3.1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Ultraseek", + "last_changed": "2024-01-02" + }, + { + "pattern": "Under the Rainbow 2.", + "last_changed": "2024-01-02" + }, + { + "pattern": "UofTDB_experiment (l", + "last_changed": "2024-01-02" + }, + { + "pattern": "Upflow/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "VYU2 (GNU; OpenRISC)", + "last_changed": "2024-01-02" + }, + { + "pattern": "Vagabondo/1.x MT (we", + "last_changed": "2024-01-02" + }, + { + "pattern": "Vagabondo/2.0 MT", + "last_changed": "2024-01-02" + }, + { + "pattern": "Vagabondo/2.0 MT (we", + "last_changed": "2024-01-02" + }, + { + "pattern": "Vagabondo/3.0 (webag", + "last_changed": "2024-01-02" + }, + { + "pattern": "Validator.nu/LV", + "last_changed": "2024-01-02" + }, + { + "pattern": "VayalaCreep-v0.0.1 (", + "last_changed": "2024-01-02" + }, + { + "pattern": "Vayala|Creep-v0.0.1 ", + "last_changed": "2024-01-02" + }, + { + "pattern": "VeryGoodSearch.com.D", + "last_changed": "2024-01-02" + }, + { + "pattern": "W3C-mobileOK/DDC-1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "W3C_Unicorn/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "W3C_Validator/1.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "W3C_Validator/1.xxx ", + "last_changed": "2024-01-02" + }, + { + "pattern": "WDG_Validator/1.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "WEP Search 00", + "last_changed": "2024-01-02" + }, + { + "pattern": "WFARC", + "last_changed": "2024-01-02" + }, + { + "pattern": "WIRE/0.22 (Linux; x8", + "last_changed": "2024-01-02" + }, + { + "pattern": "WST-Plagiarism", + "last_changed": "2024-01-02" + }, + { + "pattern": "WWWC/1.0x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Wada.vn Vietnamese S", + "last_changed": "2024-01-02" + }, + { + "pattern": "Waypath Scout v2.x -", + "last_changed": "2024-01-02" + }, + { + "pattern": "WeSEE", + "last_changed": "2024-01-02" + }, + { + "pattern": "WeSEE:Search", + "last_changed": "2024-01-02", + "addition_date": "2013-11-18", + "instances": [ + "WeSEE:Search", + "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" + ] + }, + { + "pattern": "Web Link Validator 1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Web Snooper", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebBug/5.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebCompass 2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebCorp/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebSearch.COM.AU;", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebStat/1.0 (Unix; b", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebTrends/3.0 (WinNT", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebVac (webmaster@pi", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebVal/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebVulnCrawl.unknown", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebWatcherMonitor/2.", + "last_changed": "2024-01-02" + }, + { + "pattern": "Webclipping.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Webdup/0.9", + "last_changed": "2024-01-02" + }, + { + "pattern": "Weblog Attitude Diff", + "last_changed": "2024-01-02" + }, + { + "pattern": "WebsiteWorth v1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Webspinne/1.0 webmas", + "last_changed": "2024-01-02" + }, + { + "pattern": "Webverzeichnis.de - ", + "last_changed": "2024-01-02" + }, + { + "pattern": "Wells Search II", + "last_changed": "2024-01-02" + }, + { + "pattern": "Wget/1.14 (linux-gnu", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.15/i", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.16/i", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.17/i", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.449 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.453 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.510 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.540 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.548 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.555 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.12.556 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.16.1/i", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.16.13 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.16.2/i", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.16.42 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhatsApp/2.16.57 A", + "last_changed": "2024-01-02" + }, + { + "pattern": "Whatsup/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "WhizBang! Lab", + "last_changed": "2024-01-02" + }, + { + "pattern": "WinHTTP Example/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WinampMPEG/2.00 (lar", + "last_changed": "2024-01-02" + }, + { + "pattern": "WincerSong Agent v1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "WorQmada/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WordupInfoSearch/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "WorldLight", + "last_changed": "2024-01-02" + }, + { + "pattern": "Wusage/x.0@boutell.c", + "last_changed": "2024-01-02" + }, + { + "pattern": "Wwlib/Linux", + "last_changed": "2024-01-02" + }, + { + "pattern": "XMLSlurp/0.1 libwww-", + "last_changed": "2024-01-02" + }, + { + "pattern": "Xenu Link Sleuth 1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "Xenu Link Sleuth/1.3", + "last_changed": "2024-01-02" + }, + { + "pattern": "Xenu's Link Sleuth 1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Xylix", + "last_changed": "2024-01-02" + }, + { + "pattern": "Y!J-SRD/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Yahoo Pipes 1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "Yahoo! Mindset", + "last_changed": "2024-01-02" + }, + { + "pattern": "Yahoo-Test/4.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "YahooCacheSystem", + "last_changed": "2024-01-02" + }, + { + "pattern": "YahooCacheSystem; Ya", + "last_changed": "2024-01-02" + }, + { + "pattern": "YahooSeeker/CafeKels", + "last_changed": "2024-01-02" + }, + { + "pattern": "Yanadoo", + "last_changed": "2024-01-02" + }, + { + "pattern": "Yandex/1.01.001 (com", + "last_changed": "2024-01-02" + }, + { + "pattern": "Zearchit", + "last_changed": "2024-01-02" + }, + { + "pattern": "ZendHttpClient", + "last_changed": "2024-01-02" + }, + { + "pattern": "Zeus ThemeSite Viewe", + "last_changed": "2024-01-02" + }, + { + "pattern": "Zeus xxxxx Webster P", + "last_changed": "2024-01-02" + }, + { + "pattern": "Ziggy -- The Clown F", + "last_changed": "2024-01-02" + }, + { + "pattern": "Zippy v2.0 - Zippyfi", + "last_changed": "2024-01-02" + }, + { + "pattern": "a.pr-cy.ru", + "last_changed": "2024-01-02" + }, + { + "pattern": "about.ask.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "about.depspid.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "accoona", + "last_changed": "2024-01-02" + }, + { + "pattern": "aihit.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "amzn_assoc", + "last_changed": "2024-01-02" + }, + { + "pattern": "app.hypefactors.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "appie 1.1 (www.walhe", + "last_changed": "2024-01-02" + }, + { + "pattern": "archive.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "ask.24x.info", + "last_changed": "2024-01-02" + }, + { + "pattern": "asterias/2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "augurfind", + "last_changed": "2024-01-02" + }, + { + "pattern": "augurnfind V-1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "autohttp", + "last_changed": "2024-01-02" + }, + { + "pattern": "autowebdir 1.1 (www.", + "last_changed": "2024-01-02" + }, + { + "pattern": "axios/0.18.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "bCentral Billing Pos", + "last_changed": "2024-01-02" + }, + { + "pattern": "bdcindexer_2.6.2 (re", + "last_changed": "2024-01-02" + }, + { + "pattern": "binlar_2.6.3 binlar2", + "last_changed": "2024-01-02" + }, + { + "pattern": "binlar_2.6.3 binlar_", + "last_changed": "2024-01-02" + }, + { + "pattern": "binlar_2.6.3 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "binlar_2.6.3 phanend", + "last_changed": "2024-01-02" + }, + { + "pattern": "binlar_2.6.3 test@mg", + "last_changed": "2024-01-02" + }, + { + "pattern": "blacklistdeclassified.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "bookmarkbase.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "bulkfeeds.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "bumblebee@relevare.c", + "last_changed": "2024-01-02" + }, + { + "pattern": "cantor.ee.ucla.edu", + "last_changed": "2024-01-02" + }, + { + "pattern": "carleson/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "ccubee/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "cfetch/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "cg-eye interactive", + "last_changed": "2024-01-02" + }, + { + "pattern": "cobitsprobe", + "last_changed": "2024-01-02" + }, + { + "pattern": "coccoc/1.0 ()", + "last_changed": "2024-01-02" + }, + { + "pattern": "code.google.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "codegator", + "last_changed": "2024-01-02" + }, + { + "pattern": "cohalinglit.wordpress.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "collage.cgi/1.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "combine.it.lth.se", + "last_changed": "2024-01-02" + }, + { + "pattern": "combine/0.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "computerorgs.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "content.easybib.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "coralcdn.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "cosco.hiit.fi", + "last_changed": "2024-01-02" + }, + { + "pattern": "crawler", + "last_changed": "2024-01-02" + }, + { + "pattern": "cs.daum.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "csci_b659/0.13", + "last_changed": "2024-01-02" + }, + { + "pattern": "curl/7.18.2 (x86_64-", + "last_changed": "2024-01-02" + }, + { + "pattern": "curl/7.19.7 (x86_64-", + "last_changed": "2024-01-02" + }, + { + "pattern": "curl/7.21.4 (univers", + "last_changed": "2024-01-02" + }, + { + "pattern": "curl/7.37.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "dcrawl/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "dds explorer v1.0 be", + "last_changed": "2024-01-02" + }, + { + "pattern": "deepak-USC/ISI", + "last_changed": "2024-01-02" + }, + { + "pattern": "deusu.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "developers.google.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "dhcibr.wordpress.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "digg.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "dir.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "drupal.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "dznet.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "e-sense 1.0 ea(www.v", + "last_changed": "2024-01-02" + }, + { + "pattern": "eStyleSearch 4 (comp", + "last_changed": "2024-01-02" + }, + { + "pattern": "ec2linkfinder", + "last_changed": "2024-01-02", + "addition_date": "2012-03-22", + "instances": [ + "ec2linkfinder" + ] + }, + { + "pattern": "ecairn.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "ejupiter.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "enter4u.eu", + "last_changed": "2024-01-02" + }, + { + "pattern": "exactseek.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "example-masto-instance.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "exploringdh.wordpress.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "facebookexternalhit/", + "last_changed": "2024-01-02" + }, + { + "pattern": "feed.moo.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "feedafever.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "flipboard.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "flunky", + "last_changed": "2024-01-02" + }, + { + "pattern": "fly/6.01 libwww/4.0D", + "last_changed": "2024-01-02" + }, + { + "pattern": "free-downloads.net d", + "last_changed": "2024-01-02" + }, + { + "pattern": "fulltext.seznam.cz", + "last_changed": "2024-01-02" + }, + { + "pattern": "gazz/x.x (gazz@nttrd", + "last_changed": "2024-01-02" + }, + { + "pattern": "geourl/2.0b2", + "last_changed": "2024-01-02" + }, + { + "pattern": "getpocket.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "gigabaz.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "github.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "google-xrawler", + "last_changed": "2024-01-02", + "addition_date": "2018-09-05", + "instances": [ + "google-xrawler" + ], + "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for" + }, + { + "pattern": "googlebot", + "last_changed": "2024-01-02" + }, + { + "pattern": "gossamer-threads.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "grub-client", + "last_changed": "2024-01-02" + }, + { + "pattern": "grub.org", + "last_changed": "2024-01-02", + "instances": [ + "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" + ] + }, + { + "pattern": "guess.scritch.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "haste.kytoon.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.coccoc.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.coccoc.vn", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.goo.ne.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.hepburnhome.ca", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.yahoo.co.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "help.yahoo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "hgrepurl/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "holmes/3.9 (onet.pl)", + "last_changed": "2024-01-02" + }, + { + "pattern": "holmes/3.9 (someurl.", + "last_changed": "2024-01-02" + }, + { + "pattern": "holmes/x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.5 (infosys", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.5 (root@lo", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.5 (webmast", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.6 (mathieu", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.6 (unconfi", + "last_changed": "2024-01-02" + }, + { + "pattern": "htdig/3.1.x (root@lo", + "last_changed": "2024-01-02" + }, + { + "pattern": "http://www.mabontlan", + "last_changed": "2024-01-02" + }, + { + "pattern": "http://www.monogol.d", + "last_changed": "2024-01-02" + }, + { + "pattern": "http://www.timelyweb", + "last_changed": "2024-01-02" + }, + { + "pattern": "http_get", + "last_changed": "2024-01-02", + "addition_date": "2017-11-02", + "instances": [ + "http_get" + ] + }, + { + "pattern": "httpunit/1.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "i.stanford.edu", + "last_changed": "2024-01-02" + }, + { + "pattern": "iFeed.jp/2.0 (www.ps", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver-web.arch", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/1.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.0 (Win", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.1 (Win", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.2 (Win", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.8 (Win", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.9 (Lin", + "last_changed": "2024-01-02" + }, + { + "pattern": "ia_archiver/8.9 (Win", + "last_changed": "2024-01-02" + }, + { + "pattern": "icerocket.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "ichiro/2.0 (ichiro@n", + "last_changed": "2024-01-02" + }, + { + "pattern": "ichiro/x.0 (ichiro@n", + "last_changed": "2024-01-02" + }, + { + "pattern": "ideare - SignSite/1.", + "last_changed": "2024-01-02" + }, + { + "pattern": "iearthworm/1.0, iear", + "last_changed": "2024-01-02" + }, + { + "pattern": "igde.ru", + "last_changed": "2024-01-02" + }, + { + "pattern": "ilial/Nutch-0.9-dev", + "last_changed": "2024-01-02" + }, + { + "pattern": "innovationblog.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "integrity/1.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "integrity/5", + "last_changed": "2024-01-02" + }, + { + "pattern": "ivia.ucr.edu", + "last_changed": "2024-01-02" + }, + { + "pattern": "jack", + "last_changed": "2024-01-02" + }, + { + "pattern": "journster.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "knil", + "last_changed": "2024-01-02" + }, + { + "pattern": "kulturarw3/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin (samualt9@big", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.1.1 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.2.0 (crawl@", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.2.1_de_Vien", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.2.2 (sugaya", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.2.2_guillau", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.5.0 (larbin", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.0 (larbin", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.1 (larbin", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.1 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (hamasa", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (larbin", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (liston", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (pierre", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (pimena", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (tom@le", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 (vitalb", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 dthunen", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 kalou@k", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 larbin@", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 listonA", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 pierre@", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.2 vitalbo", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 (larbin", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 (wgao@g", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 capveg@", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 ghary@s", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 gqnmgsp", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 tangyi8", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6.3 zumesun", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_2.6_basileoca", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_test (nobody@", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_test nobody@a", + "last_changed": "2024-01-02" + }, + { + "pattern": "larbin_xy250 larbin2", + "last_changed": "2024-01-02" + }, + { + "pattern": "law.di.unimi.it", + "last_changed": "2024-01-02" + }, + { + "pattern": "lc/$ROADS::Version l", + "last_changed": "2024-01-02" + }, + { + "pattern": "letscrawl.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "libWeb/clsHTTP -- hi", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.41", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.45", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.48", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.50", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.52 FP/", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.53", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.63", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.64", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.65", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww-perl/5.800", + "last_changed": "2024-01-02" + }, + { + "pattern": "libwww/5.3.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "linkdex.com/v2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "ltx71.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "lunefran.wordpress.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.32", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.33", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.34", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.35", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.36", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.38", + "last_changed": "2024-01-02" + }, + { + "pattern": "lwp-trivial/1.41", + "last_changed": "2024-01-02" + }, + { + "pattern": "mailto:webcraft@bea.", + "last_changed": "2024-01-02" + }, + { + "pattern": "mbblog14.wordpress.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "metatagsdir.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "miniflux.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "misc.yahoo.com.cn", + "last_changed": "2024-01-02" + }, + { + "pattern": "moget/x.x (moget@goo", + "last_changed": "2024-01-02" + }, + { + "pattern": "mogimogi/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "moiNAG 0.02", + "last_changed": "2024-01-02" + }, + { + "pattern": "muckrack.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "myDaemon", + "last_changed": "2024-01-02" + }, + { + "pattern": "net-promoter.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "netarkivet.dk", + "last_changed": "2024-01-02" + }, + { + "pattern": "newsearchengine (Thi", + "last_changed": "2024-01-02" + }, + { + "pattern": "newsme/1.0; feedback", + "last_changed": "2024-01-02" + }, + { + "pattern": "newspaper/0.1.0.7", + "last_changed": "2024-01-02" + }, + { + "pattern": "newspaper/0.2.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "newspaper/0.2.6", + "last_changed": "2024-01-02" + }, + { + "pattern": "newstin.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "nlp.uned.es", + "last_changed": "2024-01-02" + }, + { + "pattern": "noyona_0_1", + "last_changed": "2024-01-02" + }, + { + "pattern": "nsyght.com/Nutch-1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "nsyght.com/Nutch-x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "nutch-1.4/Nutch-1.4", + "last_changed": "2024-01-02" + }, + { + "pattern": "nutchsearch/Nutch-0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "nymesis.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "oegp v. 1.3.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "okhttp/2.5.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "okhttp/2.7.5", + "last_changed": "2024-01-02" + }, + { + "pattern": "okhttp/3.2.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "okhttp/3.5.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "omgili.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "ostermiller.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "owler-agent", + "last_changed": "2024-01-02" + }, + { + "pattern": "ozelot/2.7.3 (Search", + "last_changed": "2024-01-02" + }, + { + "pattern": "page2rss.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "pandora.nla.gov.au", + "last_changed": "2024-01-02" + }, + { + "pattern": "panscient.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "parsijoo.ir", + "last_changed": "2024-01-02" + }, + { + "pattern": "pauillac.inria.fr", + "last_changed": "2024-01-02" + }, + { + "pattern": "pd02_1.0.0 pd02_1.0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "pear.php.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "people.eng.unimelb.edu.au", + "last_changed": "2024-01-02" + }, + { + "pattern": "phpcrawl", + "last_changed": "2024-01-02", + "addition_date": "2012-09/17", + "url": "http://phpcrawl.cuab.de/", + "instances": [ + "phpcrawl" + ] + }, + { + "pattern": "pompos.iliad.fr", + "last_changed": "2024-01-02" + }, + { + "pattern": "psycheclone", + "last_changed": "2024-01-02" + }, + { + "pattern": "publisher.yahoo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "python-requests/0.13", + "last_changed": "2024-01-02" + }, + { + "pattern": "python-requests/1.0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "python-requests/2.18", + "last_changed": "2024-01-02" + }, + { + "pattern": "python-requests/2.3.", + "last_changed": "2024-01-02" + }, + { + "pattern": "r.hatena.ne.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "rabaz (rabaz at giga", + "last_changed": "2024-01-02" + }, + { + "pattern": "ranksonic.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "rico/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "riddler.io", + "last_changed": "2024-01-02" + }, + { + "pattern": "rjpower.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "robot", + "last_changed": "2024-01-02" + }, + { + "pattern": "sagool.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "scooter-venus-3.0.vn", + "last_changed": "2024-01-02" + }, + { + "pattern": "scoutmaster", + "last_changed": "2024-01-02" + }, + { + "pattern": "scrapy.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "search.ch V1.4", + "last_changed": "2024-01-02" + }, + { + "pattern": "search.denmex.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "search.goo.ne.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "search.google.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "search.yahoo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "seekda.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "semanticdiscovery/0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "sentry.io", + "last_changed": "2024-01-02" + }, + { + "pattern": "silk/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "sitebar.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "siteexplorer.info", + "last_changed": "2024-01-02", + "addition_date": "2013-05-01", + "instances": [ + "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", + "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" + ] + }, + { + "pattern": "smartwit.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "sna-0.0.1 (mikemuzio", + "last_changed": "2024-01-02" + }, + { + "pattern": "sna-0.0.1 mikeelliot", + "last_changed": "2024-01-02" + }, + { + "pattern": "sohu agent", + "last_changed": "2024-01-02" + }, + { + "pattern": "sohu-search", + "last_changed": "2024-01-02" + }, + { + "pattern": "sp.ask.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "suchbaer.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "summify.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "support.embed.ly", + "last_changed": "2024-01-02" + }, + { + "pattern": "surf", + "last_changed": "2024-01-02" + }, + { + "pattern": "suzuran", + "last_changed": "2024-01-02" + }, + { + "pattern": "szukaj.onet.pl", + "last_changed": "2024-01-02" + }, + { + "pattern": "tags2dir.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "tagword.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "targetblaster.com/0.", + "last_changed": "2024-01-02" + }, + { + "pattern": "teoma_agent1", + "last_changed": "2024-01-02" + }, + { + "pattern": "therarestwords.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "toCrawl/UrlDispatche", + "last_changed": "2024-01-02" + }, + { + "pattern": "tuezilla.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "twikle.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "twurly.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "unknownght.com Web S", + "last_changed": "2024-01-02" + }, + { + "pattern": "users.cs.cf.ac.uk", + "last_changed": "2024-01-02" + }, + { + "pattern": "versus.integis.ch", + "last_changed": "2024-01-02" + }, + { + "pattern": "vias.ncsa.uiuc.edu", + "last_changed": "2024-01-02" + }, + { + "pattern": "visionutils/0.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "vk.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "voyager-hc/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "voyager/1.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "web-bekannt (Version", + "last_changed": "2024-01-02" + }, + { + "pattern": "web2express.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "webGobbler/1.x.x", + "last_changed": "2024-01-02" + }, + { + "pattern": "webagent.wise-guys.nl", + "last_changed": "2024-01-02" + }, + { + "pattern": "webalgo.iit.cnr.it", + "last_changed": "2024-01-02" + }, + { + "pattern": "webbandit/4.xx.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.114", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.117", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.125", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.129", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.93", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcollage/1.xx", + "last_changed": "2024-01-02" + }, + { + "pattern": "webcrawl.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "webdatastats.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "webglimpse.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "webhack", + "last_changed": "2024-01-02" + }, + { + "pattern": "websearchbench.cs.uni-dortmund.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "websitealert.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "webster-internet.de ", + "last_changed": "2024-01-02" + }, + { + "pattern": "webster.healeys.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "whatUseek_winona/3.0", + "last_changed": "2024-01-02" + }, + { + "pattern": "wiki.creativecommons.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "wish.slis.tsukuba.ac.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "wkd.iis.sinica.edu.tw", + "last_changed": "2024-01-02" + }, + { + "pattern": "wortschatz.uni-leipzig.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "ws.daum.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.Abolimba.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.WISEnut.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.abonti.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ackerm.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.activtourist.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.agada.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ahmadsoftware.co", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.answerbus.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ant.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.aol.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.aramabeta.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.arianna.it", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.arocom.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.asaha.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.axxus.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ay-up.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ba.be", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.blaiz.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.bloglines.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.bortzmeyer.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.brandverity.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.buddybuzz.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.buscaplus.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.business-socket.", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.canadiancontent.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.chepi.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.chilkatsoft.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.cloudflare.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.cnet.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.cougarsearch.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.covario.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.crim.ca", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.cs.washington.edu", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.cs.washington.edu;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.dataparksearch.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.de.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.dead-links.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.die-kraehe.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.dnsdigger.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.earthcom.info", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.eurekster.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.europarchive.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.eventax.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.everatom.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.evri.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.exensa.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.exif-search.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.facebook.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.favcollector.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.feeddemon.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.feedhub.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.feedly.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.feedmap.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.fileboost.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.findpal.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.floorstransformed.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.flyindex.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.followsite.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.genieo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.globalspec.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.goguides.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.google.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.greaterera.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.hermitsearch.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.htmlvalidator.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.hyperix.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ianett.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.icerocket.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.illumit.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.iltrovatore.it", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.inktomi.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.intelliseek.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ip2location.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.iskanie.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.jordomedia.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.kalooga.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.kototoi.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.l3s.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.lan4lano.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.lipperhey.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.look.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.lw01.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.matuschek.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.megaindex.ru", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.megite.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.meta-spinner.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.mixdata.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.moreover.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.neomo.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.netluchs.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.newsgator.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ng-search.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.page-store.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.pagebull.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.pingalink.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.pingdom.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.plinki.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.powerset.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.prairielandconsulting.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.proximic.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.pubblisito.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.qango.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.quickiwiki.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.qwant.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.rawgrunt.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.relsoftware.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.retrevo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.samsung.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.scifihifi.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.scoutjet.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.scrubtheweb.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.searchme.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.securecomputing.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.semager.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.shopwiki.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.siigle.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.simpy.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sitesearch.ca", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sitesell.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sixxs.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.skaffe.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.skyrocket.gr.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sli-systems.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.slider.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.slysearch.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.somusical.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.suchen.de", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sygol.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sync2it.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.sysomos.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.targetgroups.net", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.teradex.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.tkensaku.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.truveo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.tumblr.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.ukwizz.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.unchaos.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.updated.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.urltrends.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.vakes.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.versions-project.org", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.verzamelgids.nl", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.wavefire.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.webarchiv.cz", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.worio.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.wowlemmings.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.xirq.com;", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.xrayz.co.uk", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.yama.info.waseda.ac.jp", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.yoono.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www.zedzo.com", + "last_changed": "2024-01-02" + }, + { + "pattern": "www4mail/2.x libwww-", + "last_changed": "2024-01-02" + }, + { + "pattern": "wwwster/1.x (Beta- m", + "last_changed": "2024-01-02" + }, + { + "pattern": "xmlset_roodkcableoj2", + "last_changed": "2024-01-02" + }, + { + "pattern": "yacy (www.yacy.net; ", + "last_changed": "2024-01-02" + }, + { + "pattern": "yarienavoir.net/0.2", + "last_changed": "2024-01-02" + }, + { + "pattern": "yggdrasil/Nutch-0.9 ", + "last_changed": "2024-01-02" + }, + { + "pattern": "yoogliFetchAgent/0.1", + "last_changed": "2024-01-02" + }, + { + "pattern": "Googlebot\\/", + "url": "http://www.google.com/bot.html", + "instances": [ + "Googlebot/2.1 (+http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36" + ] + }, + { + "pattern": "Googlebot-Mobile", + "instances": [ + "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" + ] + }, + { + "pattern": "Googlebot-Image", + "instances": [ + "Googlebot-Image/1.0" + ] + }, + { + "pattern": "Googlebot-News", + "instances": [ + "Googlebot-News" + ] + }, + { + "pattern": "Googlebot-Video", + "instances": [ + "Googlebot-Video/1.0" + ] + }, + { + "pattern": "AdsBot-Google([^-]|$)", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "AdsBot-Google (+http://www.google.com/adsbot.html)" + ] + }, + { + "pattern": "AdsBot-Google-Mobile", + "addition_date": "2017-08-21", + "url": "https://support.google.com/adwords/answer/2404197", + "instances": [ + "AdsBot-Google-Mobile-Apps", + "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Feedfetcher-Google", + "addition_date": "2018-06-27", + "url": "https://support.google.com/webmasters/answer/178852", + "instances": [ + "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Mediapartners-Google", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "Mediapartners-Google", + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" + ] + }, + { + "pattern": "Mediapartners \\(Googlebot\\)", + "addition_date": "2017-08-08", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "APIs-Google", + "addition_date": "2017-08-08", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-InspectionTool", + "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "instances": [ + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", + "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" + ] + }, + { + "pattern": "Storebot-Google", + "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" + ] + }, + { + "pattern": "GoogleOther", + "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "instances": [ + "GoogleOther" + ] + }, + { + "pattern": "bingbot", + "url": "http://www.bing.com/bingbot.htm", + "instances": [ + "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36" + ] + }, + { + "pattern": "Slurp", + "url": "http://help.yahoo.com/help/us/ysearch/slurp", + "instances": [ + "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", + "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" + ] + }, + { + "pattern": "[wW]get", + "instances": [ + "WGETbot/1.0 (+http://wget.alanreed.org)", + "Wget/1.14 (linux-gnu)", + "Wget/1.20.3 (linux-gnu)" + ] + }, + { + "pattern": "LinkedInBot", + "instances": [ + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)" + ] + }, + { + "pattern": "Python-urllib", + "instances": [ + "Python-urllib/1.17", + "Python-urllib/2.5", + "Python-urllib/2.6", + "Python-urllib/2.7", + "Python-urllib/3.1", + "Python-urllib/3.2", + "Python-urllib/3.3", + "Python-urllib/3.4", + "Python-urllib/3.5", + "Python-urllib/3.6", + "Python-urllib/3.7" + ] + }, + { + "pattern": "python-requests", + "addition_date": "2018-05-27", + "instances": [ + "python-requests/2.9.2", + "python-requests/2.11.1", + "python-requests/2.18.4", + "python-requests/2.19.1", + "python-requests/2.20.0", + "python-requests/2.21.0", + "python-requests/2.22.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "aiohttp", + "addition_date": "2019-12-23", + "instances": [ + "Python/3.9 aiohttp/3.7.3", + "Python/3.8 aiohttp/3.7.2", + "Python/3.7 aiohttp/3.6.2a2" + ], + "url": "https://docs.aiohttp.org/en/stable/", + "last_changed": "2024-01-10" + }, + { + "pattern": "httpx", + "addition_date": " 2019/12/23", + "instances": [ + "python-httpx/0.16.1", + "python-httpx/0.13.0.dev1" + ], + "url": "https://www.python-httpx.org" + }, + { + "pattern": "libwww-perl", + "instances": [ + "2Bone_LinkChecker/1.0 libwww-perl/6.03", + "2Bone_LinkChkr/1.0 libwww-perl/6.03", + "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" + ] + }, + { + "pattern": "httpunit", + "instances": [ + "httpunit/1.x" + ] + }, + { + "pattern": "nutch", + "instances": [ + "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", + "istellabot-nutch/Nutch-1.10" + ] + }, + { + "pattern": "Go-http-client", + "addition_date": "2016-03-26", + "url": "https://golang.org/pkg/net/http/", + "instances": [ + "Go-http-client/1.1", + "Go-http-client/2.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "phpcrawl", + "addition_date": "2012-09/17", + "url": "http://phpcrawl.cuab.de/", + "instances": [ + "phpcrawl" + ] + }, + { + "pattern": "msnbot", + "url": "http://search.msn.com/msnbot.htm", + "instances": [ + "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", + "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", + "librabot/1.0 (+http://search.msn.com/msnbot.htm)", + "librabot/2.0 (+http://search.msn.com/msnbot.htm)", + "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", + "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", + "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", + "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" + ] + }, + { + "pattern": "jyxobot", + "instances": [] + }, + { + "pattern": "FAST-WebCrawler", + "instances": [ + "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.8" + ] + }, + { + "pattern": "FAST Enterprise Crawler", + "instances": [ + "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", + "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" + ] + }, + { + "pattern": "BIGLOTRON", + "instances": [ + "BIGLOTRON (Beta 2;GNU/Linux)" + ] + }, + { + "pattern": "Teoma", + "instances": [ + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" + ], + "url": "http://about.ask.com/en/docs/about/webmasters.shtml" + }, + { + "pattern": "convera", + "instances": [ + "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" + ], + "url": "http://ews.converasearch.com/crawl.htm" + }, + { + "pattern": "seekbot", + "instances": [ + "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" + ], + "url": "http://www.seekbot.net/bot.html" + }, + { + "pattern": "Gigabot", + "instances": [ + "Gigabot/1.0", + "Gigabot/2.0 (http://www.gigablast.com/spider.html)" + ], + "url": "http://www.gigablast.com/spider.html" + }, + { + "pattern": "Gigablast", + "instances": [ + "GigablastOpenSource/1.0" + ], + "url": "https://github.com/gigablast/open-source-search-engine" + }, + { + "pattern": "exabot", + "instances": [ + "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", + "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" + ] + }, + { + "pattern": "ia_archiver", + "instances": [ + "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", + "ia_archiver-web.archive.org" + ] + }, + { + "pattern": "GingerCrawler", + "instances": [ + "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" + ] + }, + { + "pattern": "webmon ", + "instances": [] + }, + { + "pattern": "HTTrack", + "instances": [ + "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" + ] + }, + { + "pattern": "grub.org", + "instances": [ + "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" + ] + }, + { + "pattern": "UsineNouvelleCrawler", + "instances": [] + }, + { + "pattern": "antibot", + "instances": [] + }, + { + "pattern": "netresearchserver", + "instances": [] + }, + { + "pattern": "speedy", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", + "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" + ] + }, + { + "pattern": "fluffy", + "instances": [] + }, + { + "pattern": "findlink", + "instances": [ + "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", + "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" + ] + }, + { + "pattern": "msrbot", + "instances": [] + }, + { + "pattern": "panscient", + "instances": [ + "panscient.com" + ] + }, + { + "pattern": "yacybot", + "instances": [ + "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", + "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html" + ] + }, + { + "pattern": "AISearchBot", + "instances": [] + }, + { + "pattern": "ips-agent", + "instances": [ + "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", + "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" + ] + }, + { + "pattern": "tagoobot", + "instances": [] + }, + { + "pattern": "MJ12bot", + "instances": [ + "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" + ] + }, + { + "pattern": "woriobot", + "instances": [ + "Mozilla/5.0 (compatible; woriobot +http://worio.com)", + "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" + ] + }, + { + "pattern": "yanga", + "instances": [ + "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" + ] + }, + { + "pattern": "buzzbot", + "instances": [ + "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" + ] + }, + { + "pattern": "mlbot", + "instances": [ + "MLBot (www.metadatalabs.com/mlbot)" + ] + }, + { + "pattern": "yandex\\.com\\/bots", + "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", + "instances": [ + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots", + "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots yabs01)", + "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMetrika/4.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots", + "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMarket/2.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)", + "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)" + ], + "addition_date": "2015-04-14", + "last_changed": "2024-01-10" + }, + { + "pattern": "purebot", + "addition_date": "2010-01-19", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "Linguee Bot", + "addition_date": "2010-01-26", + "url": "http://www.linguee.com/bot", + "instances": [ + "Linguee Bot (http://www.linguee.com/bot)", + "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CyberPatrol", + "addition_date": "2010-02-11", + "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", + "instances": [ + "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "voilabot", + "addition_date": "2010-05-18", + "instances": [ + "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Baiduspider", + "addition_date": "2010-07-15", + "url": "http://www.baidu.jp/spider/", + "instances": [ + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", + "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "citeseerxbot", + "addition_date": "2010-07-17", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "spbot", + "addition_date": "2010-07-31", + "url": "http://www.seoprofiler.com/bot", + "instances": [ + "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "twengabot", + "addition_date": "2010-08-03", + "url": "http://www.twenga.com/bot.html", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "postrank", + "addition_date": "2010-08-03", + "url": "http://www.postrank.com", + "instances": [ + "PostRank/2.0 (postrank.com)", + "PostRank/2.0 (postrank.com; 1 subscribers)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Turnitin", + "addition_date": "2010-09-26", + "url": "http://www.turnitin.com", + "instances": [ + "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", + "Turnitin (https://bit.ly/2UvnfoQ)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "scribdbot", + "addition_date": "2010-09-28", + "url": "http://www.scribd.com", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "page2rss", + "addition_date": "2010-10-07", + "url": "http://www.page2rss.com", + "instances": [ + "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "sitebot", + "addition_date": "2010-12-15", + "url": "http://www.sitebot.org", + "instances": [ + "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "linkdex", + "addition_date": "2011-01-06", + "url": "http://www.linkdex.com", + "instances": [ + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", + "linkdex.com/v2.0", + "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Adidxbot", + "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx", + "instances": [] + }, + { + "pattern": "ezooms", + "addition_date": "2011-04-27", + "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", + "instances": [ + "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "dotbot", + "addition_date": "2011-04-27", + "instances": [ + "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", + "dotbot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Mail.RU_Bot", + "addition_date": "2011-04-27", + "instances": [ + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", + "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/", + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "discobot", + "addition_date": "2011-05-03", + "url": "http://discoveryengine.com/discobot.html", + "instances": [ + "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", + "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", + "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "heritrix", + "addition_date": "2011-06-21", + "url": "https://github.com/internetarchive/heritrix3/wiki", + "instances": [ + "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", + "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", + "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "findthatfile", + "addition_date": "2011-06-21", + "url": "http://www.findthatfile.com/", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "europarchive.org", + "addition_date": "2011-06-21", + "url": "", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "NerdByNature.Bot", + "addition_date": "2011-07-12", + "url": "http://www.nerdbynature.net/bot", + "instances": [ + "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "sistrix crawler", + "addition_date": "2011-08-02", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "Ahrefs(Bot|SiteAudit)", + "addition_date": "2011-08-28", + "instances": [ + "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "fuelbot", + "addition_date": "2018-06-28", + "instances": [ + "fuelbot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CrunchBot", + "addition_date": "2018-06-28", + "instances": [ + "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "IndeedBot", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "mappydata", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "woobot", + "addition_date": "2018-06-28", + "instances": [ + "woobot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ZoominfoBot", + "addition_date": "2018-06-28", + "instances": [ + "ZoominfoBot (zoominfobot at zoominfo dot com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "PrivacyAwareBot", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Multiviewbot", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SWIMGBot", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Grobbot", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "eright", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Apercite", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "semanticbot", + "addition_date": "2018-06-28", + "instances": [ + "semanticbot", + "semanticbot (info@semanticaudience.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Aboundex", + "addition_date": "2011-09-28", + "url": "http://www.aboundex.com/crawler/", + "instances": [ + "Aboundex/0.2 (http://www.aboundex.com/crawler/)", + "Aboundex/0.3 (http://www.aboundex.com/crawler/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "domaincrawler", + "addition_date": "2011-10-21", + "instances": [ + "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "wbsearchbot", + "addition_date": "2011-12-21", + "url": "http://www.warebay.com/bot.html", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "summify", + "addition_date": "2012-01-04", + "url": "http://summify.com", + "instances": [ + "Summify (Summify/1.0.1; +http://summify.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CCBot", + "addition_date": "2012-02-05", + "url": "http://www.commoncrawl.org/bot.html", + "instances": [ + "CCBot/2.0 (http://commoncrawl.org/faq/)", + "CCBot/2.0 (https://commoncrawl.org/faq/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "edisterbot", + "addition_date": "2012-02-25", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "SeznamBot", + "addition_date": "2012-03-14", + "instances": [ + "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ec2linkfinder", + "addition_date": "2012-03-22", + "instances": [ + "ec2linkfinder" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "gslfbot", + "addition_date": "2012-04-03", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "aiHitBot", + "addition_date": "2012-04-16", + "instances": [ + "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "intelium_bot", + "addition_date": "2012-05-07", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "facebookexternalhit", + "addition_date": "2012-05-07", + "instances": [ + "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", + "facebookexternalhit/1.1", + "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" + ], + "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Yeti", + "addition_date": "2012-05-07", + "url": "http://naver.me/bot", + "instances": [ + "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "RetrevoPageAnalyzer", + "addition_date": "2012-05-07", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "lb-spider", + "addition_date": "2012-05-07", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "Sogou", + "addition_date": "2012-05-13", + "url": "http://www.sogou.com/docs/help/webmasters.htm#07", + "instances": [ + "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", + "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", + "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "lssbot", + "addition_date": "2012-05-15", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "careerbot", + "addition_date": "2012-05-23", + "url": "http://www.career-x.de/bot.html", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "wotbox", + "addition_date": "2012-06-12", + "url": "http://www.wotbox.com", + "instances": [ + "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", + "Wotbox/2.01 (+http://www.wotbox.com/bot/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "wocbot", + "addition_date": "2012-07-25", + "url": "http://www.wocodi.com/crawler", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "ichiro", + "addition_date": "2012-08-28", + "url": "http://help.goo.ne.jp/help/article/1142", + "instances": [ + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", + "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/2.0 (ichiro@nttr.co.jp)", + "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", + "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "DuckDuckBot", + "addition_date": "2012-09-19", + "url": "http://duckduckgo.com/duckduckbot.html", + "instances": [ + "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", + "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", + "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)", + "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "lssrocketcrawler", + "addition_date": "2012-09-24", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "drupact", + "addition_date": "2012-09-27", + "url": "http://www.arocom.de/drupact", + "instances": [ + "drupact/0.7; http://www.arocom.de/drupact" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "webcompanycrawler", + "addition_date": "2012-10-03", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "acoonbot", + "addition_date": "2012-10-07", + "url": "http://www.acoon.de/robot.asp", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "openindexspider", + "addition_date": "2012-10-26", + "url": "http://www.openindex.io/en/webmasters/spider.html", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "gnam gnam spider", + "addition_date": "2012-10-31", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "web-archive-net.com.bot", + "instances": [] + }, + { + "pattern": "backlinkcrawler", + "addition_date": "2013-01-04", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "coccoc", + "addition_date": "2013-01-04", + "url": "http://help.coccoc.vn/", + "instances": [ + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", + "coccoc", + "coccoc/1.0 ()", + "coccoc/1.0 (http://help.coccoc.com/)", + "coccoc/1.0 (http://help.coccoc.vn/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "integromedb", + "addition_date": "2013-01-10", + "url": "http://www.integromedb.org/Crawler", + "instances": [ + "www.integromedb.org/Crawler" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "content crawler spider", + "addition_date": "2013-01-11", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "toplistbot", + "addition_date": "2013-02-05", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "it2media-domain-crawler", + "addition_date": "2013-03-12", + "instances": [ + "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", + "it2media-domain-crawler/2.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ip-web-crawler.com", + "addition_date": "2013-03-22", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "siteexplorer.info", + "addition_date": "2013-05-01", + "instances": [ + "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", + "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "elisabot", + "addition_date": "2013-06-27", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "proximic", + "addition_date": "2013-09-12", + "url": "http://www.proximic.com/info/spider.php", + "instances": [ + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "changedetection", + "addition_date": "2013-09-13", + "url": "http://www.changedetection.com/bot.html", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "arabot", + "addition_date": "2013-10-09", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "WeSEE:Search", + "addition_date": "2013-11-18", + "instances": [ + "WeSEE:Search", + "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "niki-bot", + "addition_date": "2014-01-01", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "CrystalSemanticsBot", + "addition_date": "2014-02-17", + "url": "http://www.crystalsemantics.com/user-agent/", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "rogerbot", + "addition_date": "2014-02-28", + "url": "http://moz.com/help/pro/what-is-rogerbot-", + "instances": [ + "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", + "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", + "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "360Spider", + "addition_date": "2014-03-14", + "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "psbot", + "addition_date": "2014-03-31", + "url": "http://www.picsearch.com/bot.html", + "instances": [ + "psbot-image (+http://www.picsearch.com/bot.html)", + "psbot-page (+http://www.picsearch.com/bot.html)", + "psbot/0.1 (+http://www.picsearch.com/bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "InterfaxScanBot", + "addition_date": "2014-03-31", + "url": "http://scan-interfax.ru", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "CC Metadata Scaper", + "addition_date": "2014-04-01", + "url": "http://wiki.creativecommons.org/Metadata_Scraper", + "instances": [ + "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "g00g1e.net", + "addition_date": "2014-04-01", + "url": "http://www.g00g1e.net/", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "GrapeshotCrawler", + "addition_date": "2014-04-01", + "url": "http://www.grapeshot.co.uk/crawler.php", + "instances": [ + "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "urlappendbot", + "addition_date": "2014-05-10", + "url": "http://www.profound.net/urlappendbot.html", + "instances": [ + "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "brainobot", + "addition_date": "2014-06-24", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "fr-crawler", + "addition_date": "2014-07-31", + "instances": [ + "Mozilla/5.0 (compatible; fr-crawler/1.1)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "binlar", + "addition_date": "2014-09-12", + "instances": [ + "binlar_2.6.3 binlar2.6.3@unspecified.mail", + "binlar_2.6.3 binlar_2.6.3@unspecified.mail", + "binlar_2.6.3 larbin2.6.3@unspecified.mail", + "binlar_2.6.3 phanendra_kalapala@McAfee.com", + "binlar_2.6.3 test@mgmt.mic" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SimpleCrawler", + "addition_date": "2014-09-12", + "instances": [ + "SimpleCrawler/0.1" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Twitterbot", + "addition_date": "2014-09-12", + "url": "https://dev.twitter.com/cards/getting-started", + "instances": [ + "Twitterbot/0.1", + "Twitterbot/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "cXensebot", + "addition_date": "2014-10-05", + "instances": [ + "cXensebot/1.1a" + ], + "url": "http://www.cxense.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "smtbot", + "addition_date": "2014-10-04", + "instances": [ + "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", + "SMTBot (similartech.com/smtbot)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)" + ], + "url": "http://www.similartech.com/smtbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "bnf.fr_bot", + "addition_date": "2014-11-18", + "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", + "instances": [ + "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", + "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "A6-Indexer", + "addition_date": "2014-12-05", + "url": "http://www.a6corp.com/a6-web-scraping-policy/", + "instances": [ + "A6-Indexer" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ADmantX", + "addition_date": "2014-12-05", + "url": "http://www.admantx.com", + "instances": [ + "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Facebot", + "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl", + "addition_date": "2014-12-30", + "instances": [ + "Facebot/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "OrangeBot\\/", + "instances": [ + "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" + ], + "addition_date": "2015-01-12", + "last_changed": "2024-01-10" + }, + { + "pattern": "memorybot", + "url": "http://mignify.com/bot.htm", + "instances": [ + "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" + ], + "addition_date": "2015-02-01", + "last_changed": "2024-01-10" + }, + { + "pattern": "AdvBot", + "url": "http://advbot.net/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" + ], + "addition_date": "2015-02-01", + "last_changed": "2024-01-10" + }, + { + "pattern": "MegaIndex", + "url": "https://www.megaindex.ru/?tab=linkAnalyze", + "instances": [ + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" + ], + "addition_date": "2015-03-28", + "last_changed": "2024-01-10" + }, + { + "pattern": "SemanticScholarBot", + "url": "https://www.semanticscholar.org/crawler", + "instances": [ + "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", + "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" + ], + "addition_date": "2015-03-28", + "last_changed": "2024-01-10" + }, + { + "pattern": "ltx71", + "url": "http://ltx71.com/", + "instances": [ + "ltx71 - (http://ltx71.com/)" + ], + "addition_date": "2015-04-04", + "last_changed": "2024-01-10" + }, + { + "pattern": "nerdybot", + "url": "http://nerdybot.com/", + "instances": [ + "nerdybot" + ], + "addition_date": "2015-04-05", + "last_changed": "2024-01-10" + }, + { + "pattern": "xovibot", + "url": "http://www.xovibot.net/", + "instances": [ + "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" + ], + "addition_date": "2015-04-05", + "last_changed": "2024-01-10" + }, + { + "pattern": "BUbiNG", + "url": "http://law.di.unimi.it/BUbiNG.html", + "instances": [ + "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" + ], + "addition_date": "2015-04-06", + "last_changed": "2024-01-10" + }, + { + "pattern": "Qwantify", + "url": "https://www.qwant.com/", + "instances": [ + "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", + "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", + "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", + "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" + ], + "addition_date": "2015-04-06", + "last_changed": "2024-01-10" + }, + { + "pattern": "archive.org_bot", + "url": "http://www.archive.org/details/archive.org_bot", + "depends_on": [ + "heritrix" + ], + "instances": [ + "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" + ], + "addition_date": "2015-04-14", + "last_changed": "2024-01-10" + }, + { + "pattern": "Applebot", + "url": "http://www.apple.com/go/applebot", + "addition_date": "2015-04-15", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "TweetmemeBot", + "url": "http://datasift.com/bot.html", + "instances": [ + "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" + ], + "addition_date": "2015-04-15", + "last_changed": "2024-01-10" + }, + { + "pattern": "crawler4j", + "url": "https://github.com/yasserg/crawler4j", + "instances": [ + "crawler4j (http://code.google.com/p/crawler4j/)", + "crawler4j (https://github.com/yasserg/crawler4j/)" + ], + "addition_date": "2015-05-07", + "last_changed": "2024-01-10" + }, + { + "pattern": "findxbot", + "url": "http://www.findxbot.com", + "instances": [ + "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" + ], + "addition_date": "2015-05-07", + "last_changed": "2024-01-10" + }, + { + "pattern": "S[eE][mM]rushBot", + "url": "http://www.semrush.com/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", + "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", + "SEMrushBot" + ], + "addition_date": "2015-05-26", + "last_changed": "2024-01-10" + }, + { + "pattern": "yoozBot", + "url": "http://yooz.ir", + "instances": [ + "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" + ], + "addition_date": "2015-05-26", + "last_changed": "2024-01-10" + }, + { + "pattern": "lipperhey", + "url": "http://www.lipperhey.com/", + "instances": [ + "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" + ], + "addition_date": "2015-08-26", + "last_changed": "2024-01-10" + }, + { + "pattern": "Y!J", + "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", + "instances": [ + "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", + "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" + ], + "addition_date": "2015-05-26", + "last_changed": "2024-01-10" + }, + { + "pattern": "Domain Re-Animator Bot", + "url": "http://domainreanimator.com", + "instances": [ + "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" + ], + "addition_date": "2015-04-14", + "last_changed": "2024-01-10" + }, + { + "pattern": "AddThis", + "url": "https://www.addthis.com", + "instances": [ + "AddThis.com robot tech.support@clearspring.com" + ], + "addition_date": "2015-06-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "Screaming Frog SEO Spider", + "url": "http://www.screamingfrog.co.uk/seo-spider", + "instances": [ + "Screaming Frog SEO Spider/5.1" + ], + "addition_date": "2016-01-08", + "last_changed": "2024-01-10" + }, + { + "pattern": "MetaURI", + "url": "http://www.useragentstring.com/MetaURI_id_17683.php", + "instances": [ + "MetaURI API/2.0 +metauri.com" + ], + "addition_date": "2016-01-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "Scrapy", + "url": "http://scrapy.org/", + "instances": [ + "Scrapy/1.0.3 (+http://scrapy.org)" + ], + "addition_date": "2016-01-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "Livelap[bB]ot", + "url": "http://site.livelap.com/crawler", + "instances": [ + "LivelapBot/0.2 (http://site.livelap.com/crawler)", + "Livelapbot/0.1" + ], + "addition_date": "2016-01-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "OpenHoseBot", + "url": "http://www.openhose.org/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" + ], + "addition_date": "2016-01-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "CapsuleChecker", + "url": "http://www.capsulink.com/about", + "instances": [ + "CapsuleChecker (http://www.capsulink.com/)" + ], + "addition_date": "2016-01-02", + "last_changed": "2024-01-10" + }, + { + "pattern": "collection@infegy.com", + "url": "http://infegy.com/", + "instances": [ + "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" + ], + "addition_date": "2016-01-03", + "last_changed": "2024-01-10" + }, + { + "pattern": "IstellaBot", + "url": "http://www.tiscali.it/", + "instances": [ + "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" + ], + "addition_date": "2016-01-09", + "last_changed": "2024-01-10" + }, + { + "pattern": "DeuSu\\/", + "addition_date": "2016-01-23", + "url": "https://deusu.de/robot.html", + "instances": [ + "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", + "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "betaBot", + "addition_date": "2016-01-23", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "Cliqzbot\\/", + "addition_date": "2016-01-23", + "url": "http://cliqz.com/company/cliqzbot", + "instances": [ + "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", + "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", + "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", + "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", + "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "MojeekBot\\/", + "addition_date": "2016-01-23", + "url": "https://www.mojeek.com/bot.html", + "instances": [ + "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", + "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "netEstate NE Crawler", + "addition_date": "2016-01-23", + "url": "http://www.website-datenbank.de/", + "instances": [ + "netEstate NE Crawler (+http://www.sengine.info/)", + "netEstate NE Crawler (+http://www.website-datenbank.de/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SafeSearch microdata crawler", + "addition_date": "2016-01-23", + "url": "https://safesearch.avira.com", + "instances": [ + "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Gluten Free Crawler\\/", + "addition_date": "2016-01-23", + "url": "http://glutenfreepleasure.com/", + "instances": [ + "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Sonic", + "addition_date": "2016-02-08", + "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", + "instances": [ + "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", + "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", + "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Sysomos", + "addition_date": "2016-02-08", + "url": "http://www.sysomos.com", + "instances": [ + "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Trove", + "addition_date": "2016-02-08", + "url": "http://www.trove.com", + "instances": [], + "last_changed": "2024-01-10" + }, + { + "pattern": "deadlinkchecker", + "addition_date": "2016-02-08", + "url": "http://www.deadlinkchecker.com", + "instances": [ + "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", + "www.deadlinkchecker.com XMLHTTP/1.0", + "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Slack-ImgProxy", + "addition_date": "2016-04-25", + "url": "https://api.slack.com/robots", + "instances": [ + "Slack-ImgProxy (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", + "Slack-ImgProxy 149 (+https://api.slack.com/robots)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Embedly", + "addition_date": "2016-04-25", + "url": "http://support.embed.ly", + "instances": [ + "Embedly +support@embed.ly", + "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", + "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "RankActiveLinkBot", + "addition_date": "2016-06-20", + "url": "https://rankactive.com/resources/rankactive-linkbot", + "instances": [ + "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "iskanie", + "addition_date": "2016-09-02", + "url": "http://www.iskanie.com", + "instances": [ + "iskanie (+http://www.iskanie.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SafeDNSBot", + "addition_date": "2016-09-10", + "url": "https://www.safedns.com/searchbot", + "instances": [ + "SafeDNSBot (https://www.safedns.com/searchbot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SkypeUriPreview", + "addition_date": "2016-10-10", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Veoozbot", + "addition_date": "2016-11-03", + "url": "http://www.veooz.com/veoozbot.html", + "instances": [ + "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Slackbot", + "addition_date": "2016-11-03", + "url": "https://api.slack.com/robots", + "instances": [ + "Slackbot-LinkExpanding (+https://api.slack.com/robots)", + "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", + "Slackbot 1.0 (+https://api.slack.com/robots)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "redditbot", + "addition_date": "2016-11-03", + "url": "http://www.reddit.com/feedback", + "instances": [ + "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "datagnionbot", + "addition_date": "2016-11-03", + "url": "http://www.datagnion.com/bot.html", + "instances": [ + "datagnionbot (+http://www.datagnion.com/bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Adwords-Instant", + "addition_date": "2016-11-03", + "url": "http://www.google.com/adsbot.html", + "instances": [ + "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "adbeat_bot", + "addition_date": "2016-11-04", + "instances": [ + "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", + "adbeat_bot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "WhatsApp", + "addition_date": "2016-11-15", + "url": "https://www.whatsapp.com/", + "instances": [ + "WhatsApp", + "WhatsApp/0.3.4479 N", + "WhatsApp/0.3.4679 N", + "WhatsApp/0.3.4941 N", + "WhatsApp/2.12.15/i", + "WhatsApp/2.12.16/i", + "WhatsApp/2.12.17/i", + "WhatsApp/2.12.449 A", + "WhatsApp/2.12.453 A", + "WhatsApp/2.12.510 A", + "WhatsApp/2.12.540 A", + "WhatsApp/2.12.548 A", + "WhatsApp/2.12.555 A", + "WhatsApp/2.12.556 A", + "WhatsApp/2.16.1/i", + "WhatsApp/2.16.13 A", + "WhatsApp/2.16.2/i", + "WhatsApp/2.16.42 A", + "WhatsApp/2.16.57 A", + "WhatsApp/2.19.92 i", + "WhatsApp/2.19.175 A", + "WhatsApp/2.19.244 A", + "WhatsApp/2.19.258 A", + "WhatsApp/2.19.308 A", + "WhatsApp/2.19.330 A" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "contxbot", + "addition_date": "2017-02-25", + "instances": [ + "Mozilla/5.0 (compatible;contxbot/1.0)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "pinterest.com.bot", + "addition_date": "2017-03-03", + "instances": [ + "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", + "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" + ], + "url": "http://www.pinterest.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "electricmonk", + "addition_date": "2017-03-04", + "instances": [ + "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" + ], + "url": "https://www.duedil.com/our-crawler/", + "last_changed": "2024-01-10" + }, + { + "pattern": "GarlikCrawler", + "addition_date": "2017-03-18", + "instances": [ + "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" + ], + "url": "http://garlik.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "BingPreview\\/", + "addition_date": "2017-04-23", + "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", + "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "vebidoobot", + "addition_date": "2017-05-08", + "instances": [ + "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" + ], + "url": "https://blog.vebidoo.de/vebidoobot/", + "last_changed": "2024-01-10" + }, + { + "pattern": "FemtosearchBot", + "addition_date": "2017-05-16", + "instances": [ + "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" + ], + "url": "http://femtosearch.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Yahoo Link Preview", + "addition_date": "2017-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" + ], + "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "MetaJobBot", + "addition_date": "2017-08-16", + "instances": [ + "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" + ], + "url": "http://www.metajob.de/the/crawler", + "last_changed": "2024-01-10" + }, + { + "pattern": "DomainStatsBot", + "addition_date": "2017-08-16", + "instances": [ + "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" + ], + "url": "http://domainstats.io/our-bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "mindUpBot", + "addition_date": "2017-08-16", + "instances": [ + "mindUpBot (datenbutler.de)" + ], + "url": "http://www.datenbutler.de/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Daum\\/", + "addition_date": "2017-08-16", + "instances": [ + "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" + ], + "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966", + "last_changed": "2024-01-10" + }, + { + "pattern": "Jugendschutzprogramm-Crawler", + "addition_date": "2017-08-16", + "instances": [ + "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" + ], + "url": "http://www.jugendschutzprogramm.de", + "last_changed": "2024-01-10" + }, + { + "pattern": "Xenu Link Sleuth", + "addition_date": "2017-08-19", + "instances": [ + "Xenu Link Sleuth/1.3.8" + ], + "url": "http://home.snafu.de/tilman/xenulink.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Pcore-HTTP", + "addition_date": "2017-08-19", + "instances": [ + "Pcore-HTTP/v0.40.3", + "Pcore-HTTP/v0.44.0" + ], + "url": "https://bitbucket.org/softvisio/pcore/overview", + "last_changed": "2024-01-10" + }, + { + "pattern": "moatbot", + "addition_date": "2017-09-16", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" + ], + "url": "https://moat.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "KosmioBot", + "addition_date": "2017-09-16", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" + ], + "url": "http://kosm.io/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "[pP]ingdom", + "addition_date": "2017-09-16", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", + "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", + "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" + ], + "url": "http://www.pingdom.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "AppInsights", + "addition_date": "2019-03-09", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" + ], + "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview", + "last_changed": "2024-01-10" + }, + { + "pattern": "PhantomJS", + "addition_date": "2017-09-18", + "instances": [ + "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" + ], + "url": "http://phantomjs.org/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Gowikibot", + "addition_date": "2017-10-26", + "instances": [ + "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" + ], + "url": "http://www.gowikibot.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "PiplBot", + "addition_date": "2017-10-30", + "instances": [ + "PiplBot (+http://www.pipl.com/bot/)", + "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" + ], + "url": "http://www.pipl.com/bot/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Discordbot", + "addition_date": "2017-09-22", + "url": "https://discordapp.com", + "instances": [ + "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "TelegramBot", + "addition_date": "2017-10-01", + "instances": [ + "TelegramBot (like TwitterBot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Jetslide", + "addition_date": "2017-09-27", + "url": "http://jetsli.de/crawler", + "instances": [ + "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "newsharecounts", + "addition_date": "2017-09-30", + "url": "http://newsharecounts.com/crawler", + "instances": [ + "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "James BOT", + "addition_date": "2017-10-12", + "url": "http://cognitiveseo.com/bot.html", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Bark[rR]owler", + "addition_date": "2017-10-09", + "url": "http://www.exensa.com/crawl", + "instances": [ + "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", + "Barkrowler/0.7 (+http://www.exensa.com/crawl)", + "BarkRowler/0.7 (+http://www.exensa.com/crawling)", + "Barkrowler/0.9 (+http://www.exensa.com/crawl)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "TinEye", + "addition_date": "2017-10-14", + "url": "http://www.tineye.com/crawler.html", + "instances": [ + "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", + "TinEye/1.1 (http://tineye.com/crawler.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SocialRankIOBot", + "addition_date": "2017-10-19", + "url": "http://socialrank.io/about", + "instances": [ + "SocialRankIOBot; http://socialrank.io/about" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "trendictionbot", + "addition_date": "2017-10-30", + "url": "http://www.trendiction.de/bot", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Ocarinabot", + "addition_date": "2017-09-27", + "instances": [ + "Ocarinabot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "epicbot", + "addition_date": "2017-10-31", + "url": "http://www.epictions.com/epicbot", + "instances": [ + "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Primalbot", + "addition_date": "2017-09-27", + "url": "https://www.primal.com", + "instances": [ + "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "DuckDuckGo-Favicons-Bot", + "addition_date": "2017-10-06", + "url": "http://duckduckgo.com", + "instances": [ + "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "GnowitNewsbot", + "addition_date": "2017-10-30", + "url": "http://www.gnowit.com", + "instances": [ + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Leikibot", + "addition_date": "2017-09-24", + "url": "http://www.leiki.com", + "instances": [ + "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "LinkArchiver", + "addition_date": "2017-09-24", + "instances": [ + "@LinkArchiver twitter bot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "YaK\\/", + "addition_date": "2017-09-25", + "url": "http://linkfluence.com", + "instances": [ + "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "PaperLiBot", + "addition_date": "2017-09-25", + "url": "http://support.paper.li/entries/20023257-what-is-paper-li", + "instances": [ + "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", + "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Digg Deeper", + "addition_date": "2017-09-26", + "url": "http://digg.com/about", + "instances": [ + "Digg Deeper/v1 (http://digg.com/about)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "dcrawl", + "addition_date": "2017-09-22", + "instances": [ + "dcrawl/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Snacktory", + "addition_date": "2017-09-23", + "url": "https://github.com/karussell/snacktory", + "instances": [ + "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AndersPinkBot", + "addition_date": "2017-09-24", + "url": "http://anderspink.com/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Fyrebot", + "addition_date": "2017-09-22", + "instances": [ + "Fyrebot/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "EveryoneSocialBot", + "addition_date": "2017-09-22", + "url": "http://everyonesocial.com", + "instances": [ + "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Mediatoolkitbot", + "addition_date": "2017-10-06", + "url": "http://mediatoolkit.com", + "instances": [ + "Mediatoolkitbot (complaints@mediatoolkit.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Luminator-robots", + "addition_date": "2017-09-22", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ExtLinksBot", + "addition_date": "2017-11-02", + "url": "https://extlinks.com/Bot.html", + "instances": [ + "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SurveyBot", + "addition_date": "2017-11-02", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "NING\\/", + "addition_date": "2017-11-02", + "instances": [ + "NING/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "okhttp", + "addition_date": "2017-11-02", + "instances": [ + "okhttp/2.5.0", + "okhttp/2.7.5", + "okhttp/3.2.0", + "okhttp/3.5.0", + "okhttp/4.1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Nuzzel", + "addition_date": "2017-11-02", + "instances": [ + "Nuzzel" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "omgili", + "addition_date": "2017-11-02", + "url": "http://omgili.com", + "instances": [ + "omgili/0.5 +http://omgili.com" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "PocketParser", + "addition_date": "2017-11-02", + "url": "https://getpocket.com/pocketparser_ua", + "instances": [ + "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "YisouSpider", + "addition_date": "2017-11-02", + "instances": [ + "YisouSpider", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "um-LN", + "addition_date": "2017-11-02", + "instances": [ + "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ToutiaoSpider", + "addition_date": "2017-11-02", + "url": "http://web.toutiao.com/media_cooperation/", + "instances": [ + "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "MuckRack", + "addition_date": "2017-11-02", + "url": "http://muckrack.com", + "instances": [ + "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Jamie's Spider", + "addition_date": "2017-11-02", + "url": "http://jamiembrown.com/", + "instances": [ + "Jamie's Spider (http://jamiembrown.com/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AHC\\/", + "addition_date": "2017-11-02", + "instances": [ + "AHC/2.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "NetcraftSurveyAgent", + "addition_date": "2017-11-02", + "instances": [ + "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Laserlikebot", + "addition_date": "2017-11-02", + "instances": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "^Apache-HttpClient", + "addition_date": "2017-11-02", + "instances": [ + "Apache-HttpClient/4.2.3 (java 1.5)", + "Apache-HttpClient/4.2.5 (java 1.5)", + "Apache-HttpClient/4.3.1 (java 1.5)", + "Apache-HttpClient/4.3.3 (java 1.5)", + "Apache-HttpClient/4.3.5 (java 1.5)", + "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", + "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", + "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", + "Apache-HttpClient/4.5.7 (Java/11.0.3)", + "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AppEngine-Google", + "addition_date": "2017-11-02", + "instances": [ + "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Jetty", + "addition_date": "2017-11-02", + "instances": [ + "Jetty/9.3.z-SNAPSHOT" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Upflow", + "addition_date": "2017-11-02", + "instances": [ + "Upflow/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Thinklab", + "addition_date": "2017-11-02", + "url": "thinklab.com", + "instances": [ + "Thinklab (thinklab.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Traackr.com", + "addition_date": "2017-11-02", + "url": "Traackr.com", + "instances": [ + "Traackr.com" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Twurly", + "addition_date": "2017-11-02", + "url": "http://twurly.org", + "instances": [ + "Ruby, Twurly v1.1 (http://twurly.org)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Mastodon", + "addition_date": "2017-11-02", + "instances": [ + "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "http_get", + "addition_date": "2017-11-02", + "instances": [ + "http_get" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "DnyzBot", + "addition_date": "2017-11-20", + "instances": [ + "Mozilla/5.0 (compatible; DnyzBot/1.0)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "botify", + "addition_date": "2018-02-01", + "instances": [ + "Mozilla/5.0 (compatible; botify; http://botify.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "007ac9 Crawler", + "addition_date": "2018-02-09", + "instances": [ + "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BehloolBot", + "addition_date": "2018-02-09", + "instances": [ + "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BrandVerity", + "addition_date": "2018-02-27", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" + ], + "url": "http://www.brandverity.com/why-is-brandverity-visiting-me", + "last_changed": "2024-01-10" + }, + { + "pattern": "check_http", + "addition_date": "2018-02-09", + "instances": [ + "check_http/v2.2.1 (nagios-plugins 2.2.1)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BDCbot", + "addition_date": "2018-02-09", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ZumBot", + "addition_date": "2018-02-09", + "instances": [ + "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "EZID", + "addition_date": "2018-02-09", + "instances": [ + "EZID (EZID link checker; https://ezid.cdlib.org/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ICC-Crawler", + "addition_date": "2018-02-28", + "instances": [ + "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" + ], + "url": "http://ucri.nict.go.jp/en/icccrawler.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "ArchiveBot", + "addition_date": "2018-02-28", + "instances": [ + "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" + ], + "url": "https://github.com/ArchiveTeam/ArchiveBot", + "last_changed": "2024-01-10" + }, + { + "pattern": "^LCC ", + "addition_date": "2018-02-28", + "instances": [ + "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" + ], + "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "filterdb.iss.net\\/crawler", + "addition_date": "2018-03-16", + "instances": [ + "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" + ], + "url": "http://filterdb.iss.net/crawler/", + "last_changed": "2024-01-10" + }, + { + "pattern": "BLP_bbot", + "addition_date": "2018-03-27", + "instances": [ + "BLP_bbot/0.1" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BomboraBot", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" + ], + "url": "http://www.bombora.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Buck\\/", + "addition_date": "2018-03-27", + "instances": [ + "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" + ], + "url": "https://app.hypefactors.com/media-monitoring/about.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Companybook-Crawler", + "addition_date": "2018-03-27", + "instances": [ + "Companybook-Crawler (+https://www.companybooknetworking.com/)" + ], + "url": "https://www.companybooknetworking.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Genieo", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" + ], + "url": "http://www.genieo.com/webfilter.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "magpie-crawler", + "addition_date": "2018-03-27", + "instances": [ + "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" + ], + "url": "http://www.brandwatch.net", + "last_changed": "2024-01-10" + }, + { + "pattern": "MeltwaterNews", + "addition_date": "2018-03-27", + "instances": [ + "MeltwaterNews www.meltwater.com" + ], + "url": "http://www.meltwater.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Moreover", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" + ], + "url": "http://www.moreover.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "newspaper\\/", + "addition_date": "2018-03-27", + "instances": [ + "newspaper/0.1.0.7", + "newspaper/0.2.5", + "newspaper/0.2.6", + "newspaper/0.2.8" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ScoutJet", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" + ], + "url": "http://www.scoutjet.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "(^| )sentry\\/", + "addition_date": "2018-03-27", + "instances": [ + "sentry/8.22.0 (https://sentry.io)" + ], + "url": "https://sentry.io", + "last_changed": "2024-01-10" + }, + { + "pattern": "StorygizeBot", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" + ], + "url": "http://www.storygize.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "UptimeRobot", + "addition_date": "2018-03-27", + "instances": [ + "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" + ], + "url": "http://www.uptimerobot.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "OutclicksBot", + "addition_date": "2018-04-21", + "instances": [ + "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", + "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", + "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", + "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" + ], + "url": "https://www.outclicks.net", + "last_changed": "2024-01-10" + }, + { + "pattern": "seoscanners", + "addition_date": "2018-05-27", + "instances": [ + "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" + ], + "url": "http://www.seoscanners.net/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Hatena", + "addition_date": "2018-05-29", + "instances": [ + "Hatena Antenna/0.3", + "Hatena::Russia::Crawler/0.01", + "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", + "Hatena::Scissors/0.01", + "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", + "Hatena::Fetcher/0.01 (master) Furl/3.13" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Google Web Preview", + "addition_date": "2018-05-31", + "instances": [ + "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "MauiBot", + "addition_date": "2018-06-06", + "instances": [ + "MauiBot (crawler.feedback+wc@gmail.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AlphaBot", + "addition_date": "2018-05-27", + "instances": [ + "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" + ], + "url": "http://alphaseobot.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "SBL-BOT", + "addition_date": "2018-06-06", + "instances": [ + "SBL-BOT (http://sbl.net)" + ], + "url": "http://sbl.net", + "description": "Bot of SoftByte BlackWidow", + "last_changed": "2024-01-10" + }, + { + "pattern": "IAS crawler", + "addition_date": "2018-06-06", + "instances": [ + "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" + ], + "url": "http://integralads.com/site-indexing-policy/", + "description": "Bot of Integral Ad Science, Inc.", + "last_changed": "2024-01-10" + }, + { + "pattern": "adscanner", + "addition_date": "2018-06-24", + "instances": [ + "Mozilla/5.0 (compatible; adscanner/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Netvibes", + "addition_date": "2018-06-24", + "instances": [ + "Netvibes (crawler/bot; http://www.netvibes.com", + "Netvibes (crawler; http://www.netvibes.com)" + ], + "url": "http://www.netvibes.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "acapbot", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", + "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Baidu-YunGuanCe", + "addition_date": "2018-06-27", + "instances": [ + "Baidu-YunGuanCe-Bot(ce.baidu.com)", + "Baidu-YunGuanCe-SLABot(ce.baidu.com)", + "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", + "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", + "Baidu-YunGuanCe-VSBot(ce.baidu.com)" + ], + "url": "https://ce.baidu.com/topic/topic20150908", + "description": "Baidu Cloud Watch", + "last_changed": "2024-01-10" + }, + { + "pattern": "bitlybot", + "addition_date": "2018-06-27", + "instances": [ + "bitlybot/3.0 (+http://bit.ly/)", + "bitlybot/2.0", + "bitlybot" + ], + "url": "http://bit.ly/", + "last_changed": "2024-01-10" + }, + { + "pattern": "blogmuraBot", + "addition_date": "2018-06-27", + "instances": [ + "blogmuraBot (+http://www.blogmura.com)" + ], + "url": "http://www.blogmura.com", + "description": "A blog ranking site which links to blogs on just about every theme possible.", + "last_changed": "2024-01-10" + }, + { + "pattern": "Bot.AraTurka.com", + "addition_date": "2018-06-27", + "instances": [ + "Bot.AraTurka.com/0.0.1" + ], + "url": "http://www.araturka.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "bot-pge.chlooe.com", + "addition_date": "2018-06-27", + "instances": [ + "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BoxcarBot", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" + ], + "url": "https://boxcar.io/", + "last_changed": "2024-01-10" + }, + { + "pattern": "BTWebClient", + "addition_date": "2018-06-27", + "instances": [ + "BTWebClient/180B(9704)" + ], + "url": "http://www.utorrent.com/", + "description": "\u00b5Torrent BitTorrent Client", + "last_changed": "2024-01-10" + }, + { + "pattern": "ContextAd Bot", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", + "ContextAd Bot 1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Digincore bot", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" + ], + "url": "http://www.digincore.com/crawler.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Disqus", + "addition_date": "2018-06-27", + "instances": [ + "Disqus/1.0" + ], + "url": "https://disqus.com/", + "description": "validate and quality check pages.", + "last_changed": "2024-01-10" + }, + { + "pattern": "Feedly", + "addition_date": "2018-06-27", + "instances": [ + "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", + "FeedlyBot/1.0 (http://feedly.com)" + ], + "url": "https://www.feedly.com/fetcher.html", + "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud.", + "last_changed": "2024-01-10" + }, + { + "pattern": "Fetch\\/", + "addition_date": "2018-06-27", + "instances": [ + "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Fever", + "addition_date": "2018-06-27", + "instances": [ + "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" + ], + "url": "http://feedafever.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Flamingo_SearchEngine", + "addition_date": "2018-06-27", + "instances": [ + "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "FlipboardProxy", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" + ], + "url": "https://about.flipboard.com/browserproxy/", + "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application", + "last_changed": "2024-01-10" + }, + { + "pattern": "g2reader-bot", + "addition_date": "2018-06-27", + "instances": [ + "g2reader-bot/1.0 (+http://www.g2reader.com/)" + ], + "url": "http://www.g2reader.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "G2 Web Services", + "addition_date": "2019-03-01", + "instances": [ + "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" + ], + "url": "https://www.g2webservices.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "imrbot", + "addition_date": "2018-06-27", + "instances": [ + "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" + ], + "url": "http://www.mignify.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "K7MLWCBot", + "addition_date": "2018-06-27", + "instances": [ + "K7MLWCBot/1.0 (+http://www.k7computing.com)" + ], + "url": "http://www.k7computing.com", + "description": "Virus scanner", + "last_changed": "2024-01-10" + }, + { + "pattern": "Kemvibot", + "addition_date": "2018-06-27", + "instances": [ + "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" + ], + "url": "http://kemvi.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Landau-Media-Spider", + "addition_date": "2018-06-27", + "instances": [ + "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" + ], + "url": "http://bots.landaumedia.de/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "linkapediabot", + "addition_date": "2018-06-27", + "instances": [ + "linkapediabot (+http://www.linkapedia.com)" + ], + "url": "http://www.linkapedia.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "vkShare", + "addition_date": "2018-07-02", + "instances": [ + "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" + ], + "url": "http://vk.com/dev/Share", + "last_changed": "2024-01-10" + }, + { + "pattern": "Siteimprove.com", + "addition_date": "2018-06-22", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "BLEXBot\\/", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" + ], + "url": "http://webmeup-crawler.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "DareBoost", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" + ], + "url": "https://www.dareboost.com/", + "description": "Bot to test, Analyze and Optimize website", + "last_changed": "2024-01-10" + }, + { + "pattern": "ZuperlistBot\\/", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Miniflux\\/", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", + "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" + ], + "url": "https://miniflux.net", + "description": "Miniflux is a minimalist and opinionated feed reader.", + "last_changed": "2024-01-10" + }, + { + "pattern": "Feedspot", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", + "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" + ], + "url": "http://www.feedspot.com/fs/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Diffbot\\/", + "addition_date": "2018-07-07", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" + ], + "url": "http://www.diffbot.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "SEOkicks", + "addition_date": "2018-08-22", + "instances": [ + "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" + ], + "url": "https://www.seokicks.de/robot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "tracemyfile", + "addition_date": "2018-08-23", + "instances": [ + "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Nimbostratus-Bot", + "addition_date": "2018-08-29", + "instances": [ + "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "zgrab", + "addition_date": "2018-08-30", + "instances": [ + "Mozilla/5.0 zgrab/0.x" + ], + "url": "https://zmap.io/", + "last_changed": "2024-01-10" + }, + { + "pattern": "PR-CY.RU", + "addition_date": "2018-08-30", + "instances": [ + "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" + ], + "url": "https://a.pr-cy.ru/", + "last_changed": "2024-01-10" + }, + { + "pattern": "AdsTxtCrawler", + "addition_date": "2018-08-30", + "instances": [ + "AdsTxtCrawler/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Datafeedwatch", + "addition_date": "2018-09-05", + "instances": [ + "Datafeedwatch/2.1.x" + ], + "url": "https://www.datafeedwatch.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Zabbix", + "addition_date": "2018-09-05", + "instances": [ + "Zabbix" + ], + "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring", + "last_changed": "2024-01-10" + }, + { + "pattern": "TangibleeBot", + "addition_date": "2018-09-05", + "instances": [ + "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" + ], + "url": "http://tangiblee.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "google-xrawler", + "addition_date": "2018-09-05", + "instances": [ + "google-xrawler" + ], + "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for", + "last_changed": "2024-01-10" + }, + { + "pattern": "axios", + "addition_date": "2018-09-06", + "instances": [ + "axios/0.18.0", + "axios/0.19.0" + ], + "url": "https://github.com/axios/axios", + "last_changed": "2024-01-10" + }, + { + "pattern": "Amazon CloudFront", + "addition_date": "2018-09-07", + "instances": [ + "Amazon CloudFront" + ], + "url": "https://aws.amazon.com/cloudfront/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Pulsepoint", + "addition_date": "2018-09-24", + "instances": [ + "Pulsepoint XT3 web scraper" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CloudFlare-AlwaysOnline", + "addition_date": "2018-09-27", + "instances": [ + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" + ], + "url": "https://www.cloudflare.com/always-online/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Structured-Data-Testing-Tool", + "addition_date": "2018-10-02", + "instances": [ + "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", + "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" + ], + "url": "https://search.google.com/structured-data/testing-tool", + "last_changed": "2024-01-10" + }, + { + "pattern": "WordupInfoSearch", + "addition_date": "2018-10-07", + "instances": [ + "WordupInfoSearch/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "WebDataStats", + "addition_date": "2018-10-08", + "instances": [ + "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" + ], + "url": "https://webdatastats.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "HttpUrlConnection", + "addition_date": "2018-10-08", + "instances": [ + "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Seekport Crawler", + "addition_date": "2018-10-08", + "instances": [ + "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" + ], + "url": "http://seekport.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "ZoomBot", + "addition_date": "2018-10-10", + "instances": [ + "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" + ], + "url": "http://suite.seozoom.it/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "VelenPublicWebCrawler", + "addition_date": "2018-10-09", + "instances": [ + "VelenPublicWebCrawler (velen.io)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "MoodleBot", + "addition_date": "2018-10-10", + "instances": [ + "MoodleBot/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "jpg-newsbot", + "addition_date": "2018-10-10", + "instances": [ + "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" + ], + "url": "https://vipnytt.no/bots/", + "last_changed": "2024-01-10" + }, + { + "pattern": "outbrain", + "addition_date": "2018-10-14", + "instances": [ + "Mozilla/5.0 (Java) outbrain" + ], + "url": "https://www.outbrain.com/help/advertisers/invalid-url/", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C_Validator", + "addition_date": "2018-10-14", + "instances": [ + "W3C_Validator/1.3" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "Validator\\.nu", + "addition_date": "2018-10-14", + "instances": [ + "Validator.nu/LV" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C-checklink", + "addition_date": "2018-10-14", + "depends_on": [ + "libwww-perl" + ], + "instances": [ + "W3C-checklink/2.90 libwww-perl/5.64", + "W3C-checklink/3.6.2.3 libwww-perl/5.64", + "W3C-checklink/4.2 [4.20] libwww-perl/5.803", + "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", + "W3C-checklink/4.3 [4.42] libwww-perl/5.805", + "W3C-checklink/4.3 [4.42] libwww-perl/5.808", + "W3C-checklink/4.3 [4.42] libwww-perl/5.820", + "W3C-checklink/4.5 [4.154] libwww-perl/5.823", + "W3C-checklink/4.5 [4.160] libwww-perl/5.823" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C-mobileOK", + "addition_date": "2018-10-14", + "instances": [ + "W3C-mobileOK/DDC-1.0" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C_I18n-Checker", + "addition_date": "2018-10-14", + "instances": [ + "W3C_I18n-Checker/1.0" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "FeedValidator", + "addition_date": "2018-10-14", + "instances": [ + "FeedValidator/1.3" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C_CSS_Validator", + "addition_date": "2018-10-14", + "instances": [ + "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "W3C_Unicorn", + "addition_date": "2018-10-14", + "instances": [ + "W3C_Unicorn/1.0" + ], + "url": "https://validator.w3.org/services", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-PhysicalWeb", + "addition_date": "2018-10-21", + "instances": [ + "Mozilla/5.0 (Google-PhysicalWeb)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Blackboard", + "addition_date": "2018-10-28", + "instances": [ + "Blackboard Safeassign" + ], + "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign", + "last_changed": "2024-01-10" + }, + { + "pattern": "ICBot\\/", + "addition_date": "2018-10-23", + "instances": [ + "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" + ], + "url": "https://ideasandcode.xyz", + "last_changed": "2024-01-10" + }, + { + "pattern": "BazQux", + "addition_date": "2018-10-23", + "instances": [ + "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" + ], + "url": "https://bazqux.com/fetcher", + "last_changed": "2024-01-10" + }, + { + "pattern": "Twingly", + "addition_date": "2018-10-23", + "instances": [ + "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" + ], + "url": "https://twingly.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Rivva", + "addition_date": "2018-10-23", + "instances": [ + "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" + ], + "url": "http://rivva.de", + "last_changed": "2024-01-10" + }, + { + "pattern": "Experibot", + "addition_date": "2018-11-03", + "instances": [ + "Experibot-v2 http://goo.gl/ZAr8wX", + "Experibot-v3 http://goo.gl/ZAr8wX" + ], + "url": "https://amirkr.wixsite.com/experibot", + "last_changed": "2024-01-10" + }, + { + "pattern": "awesomecrawler", + "addition_date": "2018-11-24", + "instances": [ + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Dataprovider.com", + "addition_date": "2018-11-24", + "instances": [ + "Mozilla/5.0 (compatible; Dataprovider.com)" + ], + "url": "https://www.dataprovider.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "GroupHigh\\/", + "addition_date": "2018-11-24", + "instances": [ + "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" + ], + "url": "http://www.grouphigh.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "theoldreader.com", + "addition_date": "2018-12-02", + "instances": [ + "Mozilla/5.0 (compatible; theoldreader.com)" + ], + "url": "https://www.theoldreader.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "AnyEvent", + "addition_date": "2018-12-07", + "instances": [ + "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" + ], + "url": "http://software.schmorp.de/pkg/AnyEvent.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Uptimebot\\.org", + "addition_date": "2019-01-17", + "instances": [ + "Uptimebot.org - Free website monitoring" + ], + "url": "http://uptimebot.org/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Nmap Scripting Engine", + "addition_date": "2019-02-04", + "instances": [ + "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" + ], + "url": "https://nmap.org/book/nse.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "2ip.ru", + "addition_date": "2019-02-12", + "instances": [ + "2ip.ru CMS Detector (https://2ip.ru/cms/)" + ], + "url": "https://2ip.ru/cms/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Clickagy", + "addition_date": "2019-02-19", + "instances": [ + "Clickagy Intelligence Bot v2" + ], + "url": "https://www.clickagy.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Caliperbot", + "addition_date": "2019-03-02", + "instances": [ + "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" + ], + "url": "http://www.conductor.com/caliperbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "MBCrawler", + "addition_date": "2019-03-02", + "instances": [ + "MBCrawler/1.0 (https://monitorbacklinks.com)" + ], + "url": "https://monitorbacklinks.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "online-webceo-bot", + "addition_date": "2019-03-02", + "instances": [ + "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" + ], + "url": "http://online.webceo.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "B2B Bot", + "addition_date": "2019-03-02", + "instances": [ + "B2B Bot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AddSearchBot", + "addition_date": "2019-03-02", + "instances": [ + "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" + ], + "url": "http://www.addsearch.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google Favicon", + "addition_date": "2019-03-14", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "HubSpot", + "addition_date": "2019-04-15", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", + "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", + "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Chrome-Lighthouse", + "addition_date": "2019-03-15", + "instances": [ + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" + ], + "url": "https://developers.google.com/speed/pagespeed/insights", + "last_changed": "2024-01-10" + }, + { + "pattern": "HeadlessChrome", + "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", + "addition_date": "2019-06-17", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CheckMarkNetwork\\/", + "addition_date": "2019-06-30", + "instances": [ + "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" + ], + "url": "https://www.checkmarknetwork.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "www\\.uptime\\.com", + "addition_date": "2019-07-21", + "instances": [ + "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" + ], + "url": "http://www.uptime.com/uptimebot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Streamline3Bot\\/", + "addition_date": "2019-07-21", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" + ], + "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php", + "last_changed": "2024-01-10" + }, + { + "pattern": "serpstatbot\\/", + "addition_date": "2019-07-25", + "instances": [ + "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", + "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" + ], + "url": "http://serpstatbot.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "MixnodeCache\\/", + "addition_date": "2019-08-04", + "instances": [ + "MixnodeCache/1.8(+https://cache.mixnode.com/)" + ], + "url": "https://cache.mixnode.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "^curl", + "addition_date": "2019-08-15", + "instances": [ + "curl", + "curl/7.29.0", + "curl/7.47.0", + "curl/7.54.0", + "curl/7.55.1", + "curl/7.64.0", + "curl/7.64.1", + "curl/7.65.3" + ], + "url": "https://curl.haxx.se/", + "last_changed": "2024-01-10" + }, + { + "pattern": "SimpleScraper", + "addition_date": "2019-08-16", + "instances": [ + "Mozilla/5.0 (compatible; SimpleScraper)" + ], + "url": "https://github.com/ramonkcom/simple-scraper/", + "last_changed": "2024-01-10" + }, + { + "pattern": "RSSingBot", + "addition_date": "2019-09-15", + "instances": [ + "RSSingBot (http://www.rssing.com)" + ], + "url": "http://www.rssing.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Jooblebot", + "addition_date": "2019-09-25", + "instances": [ + "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" + ], + "url": "http://jooble.org/jooble-bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "fedoraplanet", + "addition_date": "2019-09-28", + "instances": [ + "venus/fedoraplanet" + ], + "url": "http://fedoraplanet.org/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Friendica", + "addition_date": "2019-09-28", + "instances": [ + "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" + ], + "url": "https://hoyer.xyz", + "last_changed": "2024-01-10" + }, + { + "pattern": "NextCloud", + "addition_date": "2019-09-30", + "instances": [ + "NextCloud-News/1.0" + ], + "url": "https://nextcloud.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Tiny Tiny RSS", + "addition_date": "2019-10-04", + "instances": [ + "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", + "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", + "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)", + "Tiny Tiny RSS/19.8 (http://tt-rss.org/)" + ], + "url": "http://tt-rss.org/", + "last_changed": "2024-01-10" + }, + { + "pattern": "RegionStuttgartBot", + "addition_date": "2019-10-17", + "instances": [ + "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" + ], + "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Bytespider", + "addition_date": "2019-11-11", + "instances": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider" + ], + "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent", + "last_changed": "2024-01-10" + }, + { + "pattern": "Datanyze", + "addition_date": "2019-11-17", + "instances": [ + "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" + ], + "url": "https://www.datanyze.com/dnyzbot/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Site-Verification", + "addition_date": "2019-12-11", + "instances": [ + "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" + ], + "url": "https://support.google.com/webmasters/answer/9008080", + "last_changed": "2024-01-10" + }, + { + "pattern": "TrendsmapResolver", + "addition_date": "2020-02-24", + "instances": [ + "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" + ], + "url": "https://www.trendsmap.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "tweetedtimes", + "addition_date": "2020-02-24", + "instances": [ + "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" + ], + "url": "https://tweetedtimes.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "NTENTbot", + "addition_date": "2020-02-24", + "instances": [ + "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" + ], + "url": "https://ntent.com/ntentbot/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Gwene", + "addition_date": "2020-02-24", + "instances": [ + "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" + ], + "url": "https://gwene.org", + "last_changed": "2024-01-10" + }, + { + "pattern": "SimplePie", + "addition_date": "2020-02-24", + "instances": [ + "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" + ], + "url": "http://simplepie.org", + "last_changed": "2024-01-10" + }, + { + "pattern": "SearchAtlas", + "addition_date": "2020-03-02", + "instances": [ + "SearchAtlas.com SEO Crawler" + ], + "url": "http://SearchAtlas.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Superfeedr", + "addition_date": "2020-03-02", + "instances": [ + "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" + ], + "url": "http://superfeedr.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "feedbot", + "addition_date": "2020-03-02", + "instances": [ + "wp.com feedbot/1.0 (+https://wp.com)" + ], + "url": "http://wp.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "UT-Dorkbot", + "addition_date": "2020-03-02", + "instances": [ + "UT-Dorkbot/1.0" + ], + "url": "https://security.utexas.edu/dorkbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Amazonbot", + "addition_date": "2020-03-02", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" + ], + "url": "https://developer.amazon.com/support/amazonbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "SerendeputyBot", + "addition_date": "2020-03-02", + "instances": [ + "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" + ], + "url": "http://serendeputy.com/about/serendeputy-bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Eyeotabot", + "addition_date": "2020-03-02", + "instances": [ + "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" + ], + "url": "http://www.eyeota.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "officestorebot", + "addition_date": "2020-03-02", + "instances": [ + "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" + ], + "url": "https://aka.ms/officestorebot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Neticle Crawler", + "addition_date": "2020-03-02", + "instances": [ + "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" + ], + "url": "https://neticle.com/bot/en/", + "last_changed": "2024-01-10" + }, + { + "pattern": "SurdotlyBot", + "addition_date": "2020-03-02", + "instances": [ + "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" + ], + "url": "http://sur.ly/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "LinkisBot", + "addition_date": "2020-03-02", + "instances": [ + "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "AwarioSmartBot", + "addition_date": "2020-03-02", + "instances": [ + "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" + ], + "url": "https://awario.com/bots.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "AwarioRssBot", + "addition_date": "2020-03-02", + "instances": [ + "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" + ], + "url": "https://awario.com/bots.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "RyteBot", + "addition_date": "2020-03-02", + "instances": [ + "RyteBot/1.0.0 (+https://bot.ryte.com/)" + ], + "url": "https://bot.ryte.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "FreeWebMonitoring SiteChecker", + "addition_date": "2020-03-02", + "instances": [ + "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" + ], + "url": "https://www.freewebmonitoring.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "AspiegelBot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" + ], + "url": "https://aspiegel.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "NAVER Blog Rssbot", + "addition_date": "2020-03-16", + "instances": [ + "NAVER Blog Rssbot" + ], + "url": "http://www.naver.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "zenback bot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" + ], + "url": "http://corp.logly.co.jp/", + "last_changed": "2024-01-10" + }, + { + "pattern": "SentiBot", + "addition_date": "2020-03-16", + "instances": [ + "SentiBot www.sentibot.eu (compatible with Googlebot)" + ], + "url": "https://www.sentibot.eu", + "last_changed": "2024-01-10" + }, + { + "pattern": "Domains Project\\/", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" + ], + "url": "https://github.com/tb0hdan/domains", + "last_changed": "2024-01-10" + }, + { + "pattern": "Pandalytics", + "addition_date": "2020-03-16", + "instances": [ + "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" + ], + "url": "https://domainsbot.com/pandalytics/", + "last_changed": "2024-01-10" + }, + { + "pattern": "VKRobot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; VKRobot/1.0)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "bidswitchbot", + "addition_date": "2020-03-16", + "instances": [ + "bidswitchbot/1.0" + ], + "url": "https://www.bidswitch.com/about-us/", + "last_changed": "2024-01-10" + }, + { + "pattern": "tigerbot", + "addition_date": "2020-03-16", + "instances": [ + "tigerbot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "NIXStatsbot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" + ], + "url": "http://www.nixstats.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Atom Feed Robot", + "addition_date": "2020-03-16", + "instances": [ + "RSSMicro.com RSS/Atom Feed Robot" + ], + "url": "https://rssmicro.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "[Cc]urebot", + "addition_date": "2020-03-16", + "instances": [ + "Curebot/1.0", + "curebot-feed-fetcher" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "PagePeeker\\/", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" + ], + "url": "https://pagepeeker.com/robots/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Vigil\\/", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" + ], + "url": "http://vigil-app.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "rssbot\\/", + "addition_date": "2020-03-16", + "instances": [ + "rssbot/1.4.3 (+https://t.me/RustRssBot)" + ], + "url": "https://t.me/RustRssBot", + "last_changed": "2024-01-10" + }, + { + "pattern": "startmebot\\/", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" + ], + "url": "https://start.me/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "JobboerseBot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" + ], + "url": "http://www.jobboerse.com/bot.htm", + "last_changed": "2024-01-10" + }, + { + "pattern": "seewithkids", + "addition_date": "2020-03-16", + "instances": [ + "http://seewithkids.com/bot" + ], + "url": "http://seewithkids.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "NINJA bot", + "addition_date": "2020-03-16", + "instances": [ + "NINJA bot" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Cutbot", + "addition_date": "2020-03-16", + "instances": [ + "Cutbot; 1.5; http://cutbot.net/" + ], + "url": "http://cutbot.net/", + "last_changed": "2024-01-10" + }, + { + "pattern": "BublupBot", + "addition_date": "2020-03-16", + "instances": [ + "BublupBot (+https://www.bublup.com/bublup-bot.html)" + ], + "url": "https://www.bublup.com/bublup-bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "BrandONbot", + "addition_date": "2020-03-16", + "instances": [ + "BrandONbot (http://brandonmedia.net)" + ], + "url": "http://brandonmedia.net", + "last_changed": "2024-01-10" + }, + { + "pattern": "RidderBot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", + "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" + ], + "url": "http://brandonmedia.net", + "last_changed": "2024-01-10" + }, + { + "pattern": "Taboolabot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" + ], + "url": "http://www.taboola.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "Dubbotbot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" + ], + "url": "http://dubbot.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "FindITAnswersbot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" + ], + "url": "http://search.it-influentials.com/bot.htm", + "last_changed": "2024-01-10" + }, + { + "pattern": "infoobot", + "addition_date": "2020-03-16", + "instances": [ + "infoobot/0.1 (https://www.infoo.nl/bot.html)" + ], + "url": "https://www.infoo.nl/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Refindbot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" + ], + "url": "https://refind.com/about", + "last_changed": "2024-01-10" + }, + { + "pattern": "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" + ], + "url": "http://www.blogtraffic.de/rss-bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "SeobilityBot", + "addition_date": "2020-03-16", + "instances": [ + "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" + ], + "url": "https://www.seobility.net/sites/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Cincraw", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" + ], + "url": "http://cincrawdata.net/bot/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Dragonbot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" + ], + "url": "http://www.dragonmetrics.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "VoluumDSP-content-bot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" + ], + "url": "https://codewise.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "FreshRSS", + "addition_date": "2020-03-16", + "instances": [ + "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" + ], + "url": "https://freshrss.org", + "last_changed": "2024-01-10" + }, + { + "pattern": "BitBot", + "addition_date": "2020-03-16", + "instances": [ + "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" + ], + "url": "https://bitbot.dev", + "last_changed": "2024-01-10" + }, + { + "pattern": "^PHP-Curl-Class", + "addition_date": "2020-12-10", + "instances": [ + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" + ], + "url": "https://github.com/php-curl-class/php-curl-class", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Certificates-Bridge", + "addition_date": "2020-12-23", + "instances": [ + "Google-Certificates-Bridge" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "centurybot", + "addition_date": "2022-04-26", + "instances": [ + "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" + ], + "url": "unknown", + "last_changed": "2024-01-10" + }, + { + "pattern": "Viber", + "addition_date": "2021-04-27", + "instances": [ + "Viber" + ], + "url": "https://www.viber.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "e\\.ventures Investment Crawler", + "addition_date": "2021-06-05", + "instances": [ + "e.ventures Investment Crawler (eventures.vc)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "evc-batch", + "addition_date": "2021-06-07", + "instances": [ + "Mozilla/5.0 (compatible; evc-batch/2.0)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "PetalBot", + "addition_date": "2021-06-07", + "instances": [ + "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", + "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" + ], + "url": "https://webmaster.petalsearch.com/site/petalbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "virustotal", + "addition_date": "2021-09-22", + "instances": [ + "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", + "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" + ], + "url": "https://www.virustotal.com/gui/home/url", + "last_changed": "2024-01-10" + }, + { + "pattern": "(^| )PTST\\/", + "addition_date": "2021-12-05", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" + ], + "url": "https://www.webpagetest.org", + "last_changed": "2024-01-10" + }, + { + "pattern": "minicrawler", + "addition_date": "2022-01-12", + "instances": [ + "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" + ], + "url": "https://www.testomato.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Cookiebot", + "addition_date": "2022-01-23", + "url": "https://www.cookiebot.com/", + "instances": [ + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "trovitBot", + "addition_date": "2022-06-08", + "url": "http://www.trovit.com/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "seostar\\.co", + "addition_date": "2022-08-04", + "url": "https://seostar.co/robot/", + "instances": [ + "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "IonCrawl", + "addition_date": "2022-08-04", + "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", + "instances": [ + "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Uptime-Kuma", + "addition_date": "2022-10-17", + "url": "https://uptime.kuma.pet/", + "instances": [ + "Uptime-Kuma/1.18.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "SeekportBot", + "addition_date": "2022-10-17", + "url": "https://bot.seekport.com", + "instances": [ + "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "FreshpingBot", + "addition_date": "2022-10-17", + "url": "https://www.freshworks.com/website-monitoring/", + "instances": [ + "FreshpingBot/1.0 (+https://freshping.io/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Feedbin", + "addition_date": "2022-11-05", + "url": "https://feedbin.com/", + "instances": [ + "Feedbin feed-id:2005098 - 2 subscribers" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "CriteoBot", + "addition_date": "2022-11-13", + "url": "https://www.criteo.com/", + "instances": [ + "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Snap URL Preview Service", + "addition_date": "2022-11-13", + "url": "https://snap.com/", + "instances": [ + "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Better Uptime Bot", + "addition_date": "2022-11-13", + "url": "https://betteruptime.com/", + "instances": [ + "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "RuxitSynthetic", + "addition_date": "2023-02-16", + "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", + "instances": [ + "RuxitSynthetic/1.0" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Read-Aloud", + "addition_date": "2023-02-16", + "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", + "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "Valve\\/Steam", + "addition_date": "2023-05-24", + "instances": [ + "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "OdklBot\\/", + "addition_date": "2023-05-24", + "instances": [ + "OdklBot/1.0 (share@odnoklassniki.ru)", + "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" + ], + "url": "https://odnoklassniki.ru/", + "last_changed": "2024-01-10" + }, + { + "pattern": "GPTBot", + "addition_date": "2023-08-09", + "instances": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" + ], + "url": "https://platform.openai.com/docs/gptbot", + "last_changed": "2024-01-10" + }, + { + "pattern": "YandexRenderResourcesBot\\/", + "addition_date": "2023-08-16", + "instances": [ + "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" + ], + "url": "http://yandex.com/bots", + "last_changed": "2024-01-10" + }, + { + "pattern": "LightspeedSystemsCrawler", + "addition_date": "2023-08-16", + "instances": [ + "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" + ], + "last_changed": "2024-01-10" + }, + { + "pattern": "ev-crawler\\/", + "addition_date": "2023-08-16", + "instances": [ + "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" + ], + "url": "https://headline.com/legal/crawler", + "last_changed": "2024-01-10" + }, + { + "pattern": "BitSightBot\\/", + "addition_date": "2023-08-16", + "instances": [ + "Mozilla/5.0 (compatible; BitSightBot/1.0)" + ], + "url": "https://www.bitsight.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "woorankreview\\/", + "addition_date": "2023-08-16", + "instances": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", + "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" + ], + "url": "https://www.woorank.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Google-Safety", + "addition_date": "2023-08-17", + "instances": [ + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", + "Google-Safety" + ], + "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "last_changed": "2024-01-10" + }, + { + "pattern": "AwarioBot", + "addition_date": "2023-08-23", + "instances": [ + "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" + ], + "url": "https://awario.com/bots.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "DataForSeoBot", + "addition_date": "2023-08-23", + "instances": [ + "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" + ], + "url": "https://dataforseo.com/dataforseo-bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Linespider", + "addition_date": "2023-08-24", + "instances": [ + "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" + ], + "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en", + "last_changed": "2024-01-10" + }, + { + "pattern": "WellKnownBot", + "addition_date": "2023-08-29", + "instances": [ + "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" + ], + "url": "https://well-known.dev/about/#bot)", + "last_changed": "2024-01-10" + }, + { + "pattern": "A Patent Crawler", + "addition_date": "2023-08-29", + "instances": [ + "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" + ], + "url": "http://scitas.epfl.ch/", + "last_changed": "2024-01-10" + }, + { + "pattern": "StractBot", + "addition_date": "2023-09-06", + "instances": [ + "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" + ], + "url": "https://trystract.com/webmasters", + "last_changed": "2024-01-10" + }, + { + "pattern": "search\\.marginalia\\.nu", + "addition_date": "2023-09-08", + "instances": [ + "search.marginalia.nu" + ], + "url": "https://search.marginalia.nu", + "last_changed": "2024-01-10" + }, + { + "pattern": "YouBot", + "addition_date": "2023-09-08", + "instances": [ + "YouBot (+http://www.you.com)" + ], + "url": "https://you.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Nicecrawler", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" + ], + "url": "http://www.nicecrawler.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "Neevabot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" + ], + "url": "https://neeva.com/neevabot", + "last_changed": "2024-01-10" + }, + { + "pattern": "BrightEdge Crawler", + "addition_date": "2023-09-08", + "instances": [ + "BrightEdge Crawler/1.0 (crawler@brightedge.com)" + ], + "url": "https://www.brightedge.com/", + "last_changed": "2024-01-10" + }, + { + "pattern": "SiteCheckerBotCrawler", + "addition_date": "2023-09-08", + "instances": [ + "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" + ], + "url": "http://sitechecker.pro", + "last_changed": "2024-01-10" + }, + { + "pattern": "TombaPublicWebCrawler", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" + ], + "url": "https://tombascraper.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "CrawlyProjectCrawler", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" + ], + "url": "https://crawlyproject.digitaldragon.dev/", + "last_changed": "2024-01-10" + }, + { + "pattern": "KomodiaBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" + ], + "url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler", + "last_changed": "2024-01-10" + }, + { + "pattern": "KStandBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" + ], + "url": "http://url-classification.io", + "last_changed": "2024-01-10" + }, + { + "pattern": "CISPA Webcrawler", + "addition_date": "2023-09-08", + "instances": [ + "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" + ], + "url": "https://vuln-notify-checker.cispa.saarland", + "last_changed": "2024-01-10" + }, + { + "pattern": "MTRobot", + "addition_date": "2023-09-08", + "instances": [ + "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" + ], + "url": "https://metrics-tools.de/robot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "hyscore.io", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" + ], + "url": "https://hyscore.io/crawler/", + "last_changed": "2024-01-10" + }, + { + "pattern": "AlexandriaOrgBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" + ], + "url": "https://www.alexandria.org/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "2ip bot", + "addition_date": "2023-09-08", + "instances": [ + "2ip bot/1.1 (+http://2ip.io)" + ], + "url": "http://2ip.io", + "last_changed": "2024-01-10" + }, + { + "pattern": "Yellowbrandprotectionbot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" + ], + "url": "https://www.yellowbp.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "SEOlizer", + "addition_date": "2023-09-08", + "instances": [ + "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" + ], + "url": "https://www.seolizer.de/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "vuhuvBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" + ], + "url": "http://vuhuv.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "INETDEX-BOT", + "addition_date": "2023-09-08", + "instances": [ + "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" + ], + "url": "https://inetdex.com/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Synapse", + "addition_date": "2023-09-08", + "instances": [ + "Synapse (bot; +https://github.com/matrix-org/synapse)" + ], + "url": "https://github.com/matrix-org/synapse", + "last_changed": "2024-01-10" + }, + { + "pattern": "t3versionsBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" + ], + "url": "https://www.t3versions.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "deepnoc", + "addition_date": "2023-09-08", + "instances": [ + "deepnoc - https://deepnoc.com/bot" + ], + "url": "https://deepnoc.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "Cocolyzebot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" + ], + "url": "https://cocolyze.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "hypestat", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)" + ], + "url": "https://hypestat.com/bot", + "last_changed": "2024-01-10" + }, + { + "pattern": "ReverseEngineeringBot", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" + ], + "url": "https://torus.company/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "sempi.tech", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" + ], + "url": "http://sempi.tech/bot.html", + "last_changed": "2024-01-10" + }, + { + "pattern": "Iframely", + "addition_date": "2023-09-08", + "instances": [ + "Iframely/1.3.1 (+https://iframely.com/docs/about) Atlassian" + ], + "url": "https://iframely.com/docs/about", + "last_changed": "2024-01-10" + }, + { + "pattern": "MetaInspector", + "addition_date": "2023-09-08", + "instances": [ + "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" + ], + "url": "https://github.com/jaimeiniesta/metainspector", + "last_changed": "2024-01-10" + }, + { + "pattern": "node-fetch", + "addition_date": "2023-09-08", + "instances": [ + "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + ], + "url": "https://github.com/bitinn/node-fetch", + "last_changed": "2024-01-10" + }, + { + "pattern": "lkxscan", + "addition_date": "2023-09-08", + "instances": [ + "lkxscan/v0.1.0 (+https://leakix.net) l9explore/v1.0.0 (+https://github.com/LeakIX/l9explore)" + ], + "url": "https://github.com/LeakIX/l9explore", + "last_changed": "2024-01-10" + }, + { + "pattern": "python-opengraph", + "addition_date": "2023-09-08", + "instances": [ + "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" + ], + "url": "https://github.com/jaywink/python-opengraph", + "last_changed": "2024-01-10" + }, + { + "pattern": "OpenGraphCheck", + "addition_date": "2023-09-08", + "instances": [ + "OpenGraphCheck/2.1 (+https://opengraphcheck.com)" + ], + "url": "https://opengraphcheck.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "developers.google.com\\/\\+\\/web\\/snippet", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" + ], + "url": "https://developers.google.com/+/web/snippet", + "last_changed": "2024-01-10" + }, + { + "pattern": "SenutoBot", + "addition_date": "2023-09-08", + "instances": [ + "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" + ], + "url": "https://www.senuto.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "MaCoCu", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" + ], + "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data", + "last_changed": "2024-01-10" + }, + { + "pattern": "NewsBlur", + "addition_date": "2023-09-08", + "instances": [ + "NewsBlur Feed Fetcher - 1 subscriber - http://www.newsblur.com/site/0000000/webpage (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15)" + ], + "url": "http://www.newsblur.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "inoreader", + "addition_date": "2023-09-08", + "instances": [ + "Mozilla/5.0 (compatible; inoreader.com; 1 subscribers)" + ], + "url": "http://inoreader.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "NetSystemsResearch", + "addition_date": "2023-09-08", + "instances": [ + "NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com" + ], + "url": "http://netsystemsresearch.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "PageThing", + "addition_date": "2023-09-08", + "instances": [ + "PageThing http://pagething.com curl www" + ], + "url": "http://pagething.com", + "last_changed": "2024-01-10" + }, + { + "pattern": "WordPress\\/", + "addition_date": "2023-10-24", + "instances": [ + "WordPress/X.X.X; https://example.com" + ], + "url": "https://wordpress.org", + "last_changed": "2024-01-10" } -] +] \ No newline at end of file diff --git a/convert_to_txt b/convert_to_txt index 00fdbf3..bf3799d 100755 --- a/convert_to_txt +++ b/convert_to_txt @@ -1,4 +1,4 @@ #!/bin/sh -cat COUNTER_Robots_list.json | jq -r .[].pattern > COUNTER_Robots_list.txt +cat COUNTER_Robots_list.json | jq -r .[].pattern > generated/COUNTER_Robots_list.txt echo "List generated in COUNTER_Robots_list.txt" diff --git a/generated/COUNTER_Robots_list.txt b/generated/COUNTER_Robots_list.txt index cec34f7..b1c7c86 100644 --- a/generated/COUNTER_Robots_list.txt +++ b/generated/COUNTER_Robots_list.txt @@ -46,6 +46,7 @@ BUbiNG bwh3_user_agent CakePHP celestial +centuryb cfnetwork checklink checkprivacy @@ -136,6 +137,7 @@ ichiro iktomi ilse Indy Library +insomnia ^integrity\/\d internetseer intute @@ -172,6 +174,7 @@ mail\.ru MarcEdit mediapartners-google megite +MetaInspector MetaURI[\+\s]API\/\d\.\d Microsoft(\s|\+)URL(\s|\+)Control Microsoft Office Existence Discovery @@ -197,6 +200,7 @@ nagios ^NetAnts\/\d netcraft netluchs +nettle newspaper\/\d ng\/2\. ^Ning\/\d @@ -232,6 +236,7 @@ rambler ReactorNetty\/\d Readpaper redalert +RestSharp Riddler robozilla rss @@ -314,3 +319,1639 @@ ZoteroTranslationServer zeus zyborg 7siters +&as_qdr=all +*/Nutch-0.9-dev +192.comAgent +222.122.15.190 +2Bone_LinkChkr/1.0 l +7331230CB97FADECF052 +8484 Boston Project +8LEGS +A-Online Search +ADmantX Platform Sem +AHC/2.0 +ASPSeek/1.2.5 +ASPSeek/1.2.x +ASPSeek/1.2.xa +ASPSeek/1.2.xxpre +ASPseek/1.2.9d +ASPseek/1.2.xx +ASSORT/0.10 +AV Fetch 1.0 +AVSearch-1.0(peter.t +AVSearch-2.0-fusionI +AVSearch-3.0(AltaVis +About/0.1libwww-perl +Accoona-AI-Agent/1.1 +Acorn/Nutch-0.9 (Non +ActiveBookmark 1.x +AdobeUxTechC4-Async/ +AgentName/0.1 libwww +AideRSS/1.0 (aiderss +Aladin/3.324 +Allesklar/0.1 libwww +Allrati/1.1 (+) +AltaVista Intranet V +Amazon CloudFront +AmiTCP Miami (AmigaO +Anonymizer/1.1 +AnswerChase PROve x. +AnswerChase x.0 +AnzwersCrawl/2.0 (an +Apache (internal dum +Apache-HttpAsyncClie +Apache-HttpClient/4. +Apache-HttpClient/UN +Aport +Arachnoidea (arachno +Asahina-Antenna/1.x +Asahina-Antenna/1.x +Ask.24x.Info +Atomic_Email_Hunter/ +Atomz/1.0 +BIGLOTRON (Beta 2;GN +BMCLIENT +BMLAUNCHER +BPImageWalker/2.0 (w +BSDSeek/1.0 +BStop.BravoBrian.it +BTWebClient/180B(970 +BeamMachine/0.5 (dea +BeebwareDirectory/v0 +Big Fish v1.0 +BigBrother/1.6e +Bilbo/2.3b-UNIX +BlackBerry9000/4.6.0 +BlackWidow +Blackboard Safeassig +BlogPulseLive (suppo +Blogpulse (info@blog +Bobby/4.0.x RPT-HTTP +Bookdog/x.x +BravoBrian bstop.bra +BullsEye +BunnySlippers +CB/Nutch-1.7 +CFNetwork/x.x +CHttpClient by Open +CLIPish Pro2 10.21 ( +COAST WebMaster Pro/ +CamelHttpStream/1.0 +Cancer Information a +Chat Catcher/1.0 +China Local Browse 2 +Chitika ContentHit 1 +Claymont.com +ColdFusion (Bookmark +Combine/x.0 +Contact +Cookie%20Stumbler/22 +CrawlConvera0.1 (Cra +Custo x.x (www.netwu +CyberSpyder Link Tes +DBrowse 1.4b +DBrowse 1.4d +DTAAgent +Dart Communications +DataCha0s/2.0 +DataFountains/DMOZ D +Datafeedwatch/2.1.x +Deep Link Calculator +DeepIndex +DeepIndex (www.en.de +DeepIndexer.ca +DevComponents.com Ht +Diamond/x.0 +DigOut4U +Digger/1.0 JDK/1.3.0 +Disqus/1.0 +DittoSpyder +DoCoMo/1.0/Nxxxi/c10 +DoCoMo/2.0 P900iV(c1 +EARTHCOM.info/1.x [w +EARTHCOM.info/1.xbet +EBrowse 1.4b +EasyDL/3.xx +EchO!/2.0 +Educate Search VxB +EldoS TimelyWeb/3.x +EmailWolf 1.00 +Embedly +support@emb +Enfish Tracker +Enterprise_Search/1. +Evernote Clip Resolv +Exalead NG/MimeLive +Expired Domain Sleut +ExtractorPro +EyeCatcher (Download +FARK.com link verifi +FLATARTS_FAVICO +FaEdit/2.0.x +FavOrg +Favorites Sweeper v. +Feed24.com +FeedForAll rss2html. +FeedValidator/1.3 +Filtrbox/1.0 +FindAnISP.com_ISP_Fi +Firefly/1.0 +Firefox (kastaneta03 +Firefox_1.0.6 (kaspa +FirstGov.gov Search +Flexum/2.0 +FocusedSampler/1.0 +Franklin Locator 1.8 +FuseBulb.Com +GigablastOpenSource/ +Go 1.1 package http +Go-http-client/1.1 +Go-http-client/2.0 +GoForIt.com +Goofer/0.2 +GrapeFX/0.3 libwww/5 +Gromit/1.0 +Guestbook Auto Submi +Gulliver/1.3 +Guzzle/4.2.3 curl/7. +HLoader +HPL/Nutch-0.9 - +Haansoft +Harvest-NG/1.0.2 +Hatena Antenna/0.3 +Hatena Pagetitle Age +Hatena::Bookmark/2.0 +Hippias/0.9 Beta +HitList +Hivemind +HomePageSearch(hpsea +Html Link Validator +Hybrid/1.2 [en] (OS +IAArchiver-1.0 +IBM WebExplorer /v0. +IEFav172Free +INFOMINE/8.0 Adders +INFOMINE/8.0 RemoteS +IP*Works! V5 HTTP/S +ISC Systems iRc Sear +IWAgent/ 1.0 - www.b +Icarus6 - (contact: +ImageWalker/2.0 (www +Incutio HttpClient v +IncyWincy data gathe +Industry Program 1.0 +Inet library +InetURL/1.0 +InfoLink/1.x +InfoSeek Sidewinder/ +Infoaxe./Nutch-0.9 +Infoseek SideWinder/ +Inktomi Search +Insitor.com search a +Insitornaut +Internet Ninja x.0 +InternetLinkAgent/3. +InternetSeer.com +IrssiUrlLog/0.2 +JDK/1.1 +Jack +Jakarta Commons-Http +Java 1.1 +Java Browser +Java/1.4.1_01 +Java/1.4.1_04 +Java/1.5.0_16 +Java/1.6.0_04 +Java/1.6.0_07 +Java/1.6.0_18 +Java/1.6.0_22 +Java/1.6.0_24 +Java/1.6.0_25 +Java/1.6.0_26 +Java/1.6.0_27 +Java/1.6.0_29 +Java/1.6.0_30 +Java/1.6.0_31 +Java/1.6.0_32 +Java/1.6.0_33 +Java/1.6.0_34 +Java/1.6.0_35 +Java/1.6.0_39 +Java/1.6.0_45 +Java/1.6.0_51 +Java/1.7.0_03 +Java/1.7.0_09 +Java/1.7.0_15 +Java/1.7.0_17 +Java/1.7.0_21 +Java/1.7.0_25 +Java/1.7.0_40 +Java/1.7.0_45 +Java/1.7.0_51 +Java/1.7.0_55 +Java/1.7.0_60 +Java/1.7.0_65 +Java/1.7.0_71 +Java/1.7.0_75 +Java/1.7.0_80 +Java/1.8.0_20 +Java/1.8.0_25 +Java/1.8.0_31 +Java/1.8.0_40 +Java/1.8.0_45 +Java/1.8.0_51 +Java1.0.21.0 +Java1.1.xx.x +Java1.3.0rc1 +Java1.3.x +Java1.4.0 +Jersey/2.25.1 (HttpU +Jetty/9.3.z-SNAPSHOT +Jigsaw/2.2.x W3C_CSS +Jigsaw/2.3.0 W3C_CSS +Junkie/1.0 +KE_1.0/2.0 libwww/5. +KIT-Fireball/2.0 +KnowItAll(knowitall@ +Knowledge.com/0.x +KummHttp/1.1 (compat +LARBIN-EXPERIMENTAL +LEIA/2.90 +LWP::Simple/5.22 +LWP::Simple/5.36 +LWP::Simple/5.48 +LWP::Simple/5.50 +LWP::Simple/5.51 +LWP::Simple/5.53 +LWP::Simple/5.63 +LWP::Simple/5.803 +Lachesis +Libby_1.1/libwww-per +Lincoln State Web Br +Link Valet Online 1. +LinkAlarm/2.x +LinkPimpin v1.0 +LinkProver 2.1 +LinkSonar/1.35 +LinkSweeper/1.x +LinkWalker +LinkWalker/2.0 +Look.com +Lynx/2.8.4rel.1 libw +MARTINI +MFC Foundation Class +MFC_Tear_Sample +MSIE-5.13 (larbin@un +MSNPTC/1.0 +MVAClient +Mac Finder 1.0.xx +Mag-Net +Mail.Ru/1.0 +MantraAgent +MapoftheInternet.com +Marketwave Hit List +Martini +Marvin v0.3 +MasterSeek +Mata Hari/2.00 +MediaSearch/0.1 +Mediapartners-Google +MegaSheep v1.0 (www. +MeltwaterNews www.me +Mercator-1.x +Mercator-2.0 +Mercator-Scrub-1.1 +Mercury/894 CFNetwor +MetaURI API/2.0 +met +MicroBaz +Microsoft Data Acces +Microsoft Small Busi +Microsoft URL Contro +Microsoft Windows Ne +MiracleAlphaTest +Missauga Locate 1.0. +Missigua Locator 1.9 +Missouri College Bro +Mister PiX version.d +Mister Pix II 2.02a +Miva (AlgoFeedback@m +Mizzu Labs 2.2 +MnogoSearch/3.2.xx +Mnogosearch-3.1.21 +Mo College 1.9 +More yacy user agent +Mozi! +MultiText/0.1 +My WinHTTP Connectio +MyNutchTest/Nutch-1. +NASA Search 1.0 +NEC Research Agent - +NEC-Hayek/1.0 +NG/4.0.1229 +NICO/1.0 +NING/1.0 +NationalDirectoryAdd +NativeHost +NetLookout/2.24 +NetMechanic Vx.0 +NetSprint -- 2.0 +NetZippy +Nocilla/1.0 +Nsauditor/1.x +Nuzzel +Nuzzel/876 CFNetwork +Octora Beta - www.oc +Oculus360/Nutch-1.7 +OliverPerry +OpenAcoon v4.0.x (ww +OpenISearch/1.x (www +OpenOffice/4.1.1 +OpenWebIndex/Nutch-1 +Openstat/0.1 +Opera/6.01 (larbin@u +Oracle Ultra Search +OzySoftware.com +PBrowse 1.4b +PEval 1.4b +PHP/3.x.xx +PHP/4.0.4pl1 +PHP/4.0.6 +PHP/4.1.1 +PHP/4.1.2 +POE-Component-Client +Pcore-HTTP/v0.40.3 +PeopleChat/Search_En +PicoSearch/1.0 +Piffany_Web_Scraper_ +PingALink Monitoring +Pita +Poirot +Pompos/1.x pompos@il +Popdexter/1.0 +Port Huron Labs +PostFavorites +PostRank/2.0 (postra +PrivacyFinder/1.1 +Program Shareware 1. +Progressive Download +Project XP5 [2.03.07 +PubCrawl (pubcrawl.s +Pulsepoint XT3 web s +PycURL/7.13.2 +PycURL/7.15.5 +PycURL/7.16.4 +PycURL/7.18.0 +PycURL/7.18.2 +PycURL/7.19.0 +PycURL/7.19.3 +PycURL/7.19.5 +PycURL/7.19.7 +PycURL/7.23.1 +Python-urllib/1.17 +Python-urllib/1.1x +Python-urllib/2.0a1 +Python-urllib/2.5 +Python-urllib/2.6 +Python-urllib/2.7 +Python-urllib/3.1 +Python-urllib/3.2 +Python-urllib/3.3 +Python-urllib/3.4 +Python-urllib/3.5 +Python-urllib/3.6 +QPCreep Test Rig ( W +Qseero v1.0.0 +QuepasaCreep v0.9.1x +QueryN Metasearch +Qwantify/1.0 +R6_VoteReader(www.ra +REBOL Core 2.x.x.x.x +RPT-HTTPClient/0.3-x +RepoMonkey Bait & Ta +RoboCrawl (www.canad +Robozilla/1.0 +Rotondo/3.1 libwww/5 +Ruby +Rumours-Agent +SMRF URL Expander +Scooter-3.0.EU +Scooter-3.0.FS +Scooter-3.0.HD +Scooter-3.0.VNS +Scooter-3.0QI +Scooter-3.2 +Scooter-3.2.BT +Scooter-3.2.DIL +Scooter-3.2.EX +Scooter-3.2.JT +Scooter-3.2.NIV +Scooter-3.2.SF0 +Scooter-3.2.snippet +Scooter-3.3dev +Scooter-ARS-1.1 +Scooter-ARS-1.1-ih +Scooter-W3-1.0 +Scooter-W3.1.2 +Scooter/1.0 +Scooter/1.0 scooter@ +Scooter/1.1 (custom) +Scooter/2.0 G.R.A.B. +Scooter/3.3 +Scooter/3.3.QA.pczuk +Scooter/3.3.vscooter +Scooter/3.3_SF +Scooter2_Mercator_x- +Scooter_bh0-3.0.3 +Scooter_trk3-3.0.3 +ScoutAbout +Search+ +Search-Engine-Studio +SearchGuild/DMOZ/Exp +SearchGuild_DMOZ_Exp +SearchSight.com +SearchTone2.0 - IDEA +Seeker.lookseek.com +Silk/1.0 +SiteBar/x.x +SiteRecon+(xx) +SiteTruth.com site r +SiteXpert +Skywalker/0.1 (Skywa +Slarp/0.1 +Slider_Search_v1-de +Slurpy Verifier/1.0 +SlySearch (slysearch +SnoopRob/x.x +Snoopy v1.xx +Snoopy_v0.xx +Sphere Scout&v4.0 - +Sphider +Spida/0.1 +Spinne/2.0 +Spinne/2.0 med +Spinne/2.0 med_AH +Sqworm/2.9.81-BETA ( +Sqworm/2.9.85-BETA ( +Sqworm/2.9.89-BETA ( +StackRambler/2.0 +StackRambler/2.0 (MS +StackRambler/x.x +SyncIT/x.x +Szukacz/1.x +TE +TJvMultiHttpGrabber +Tagyu Agent/1.0 +TeamSoft WinInet Com +Teoma MP +The European Library +Thinklab (thinklab.c +TinEye +Toutatis x-xx.x (hop +Toutatis x.x (hoppa. +Toutatis x.x-x +Traackr.com +Twisted PageGetter +Twitter/6.18.3 CFNet +Twitturly / v0.x +Tycoon Agent/Nutch-1 +TygoProwler +UCWEB/2.0(Linux; U; +UDM +URLBase/6.x +URLBlaze +UdmSearch/3.1.x +Ultraseek +Under the Rainbow 2. +UofTDB_experiment (l +Upflow/1.0 +VYU2 (GNU; OpenRISC) +Vagabondo/1.x MT (we +Vagabondo/2.0 MT +Vagabondo/2.0 MT (we +Vagabondo/3.0 (webag +Validator.nu/LV +VayalaCreep-v0.0.1 ( +Vayala|Creep-v0.0.1 +VeryGoodSearch.com.D +W3C-mobileOK/DDC-1.0 +W3C_Unicorn/1.0 +W3C_Validator/1.3 +W3C_Validator/1.xxx +WDG_Validator/1.1 +WEP Search 00 +WFARC +WIRE/0.22 (Linux; x8 +WST-Plagiarism +WWWC/1.0x +Wada.vn Vietnamese S +Waypath Scout v2.x - +WeSEE +WeSEE:Search +Web Link Validator 1 +Web Snooper +WebBug/5.x +WebCompass 2.0 +WebCorp/1.0 +WebSearch.COM.AU; +WebStat/1.0 (Unix; b +WebTrends/3.0 (WinNT +WebVac (webmaster@pi +WebVal/1.0 +WebVulnCrawl.unknown +WebWatcherMonitor/2. +Webclipping.com +Webdup/0.9 +Weblog Attitude Diff +WebsiteWorth v1.0 +Webspinne/1.0 webmas +Webverzeichnis.de - +Wells Search II +Wget/1.14 (linux-gnu +WhatsApp/2.12.15/i +WhatsApp/2.12.16/i +WhatsApp/2.12.17/i +WhatsApp/2.12.449 A +WhatsApp/2.12.453 A +WhatsApp/2.12.510 A +WhatsApp/2.12.540 A +WhatsApp/2.12.548 A +WhatsApp/2.12.555 A +WhatsApp/2.12.556 A +WhatsApp/2.16.1/i +WhatsApp/2.16.13 A +WhatsApp/2.16.2/i +WhatsApp/2.16.42 A +WhatsApp/2.16.57 A +Whatsup/x.x +WhizBang! Lab +WinHTTP Example/1.0 +WinampMPEG/2.00 (lar +WincerSong Agent v1. +WorQmada/1.0 +WordupInfoSearch/1.0 +WorldLight +Wusage/x.0@boutell.c +Wwlib/Linux +XMLSlurp/0.1 libwww- +Xenu Link Sleuth 1.x +Xenu Link Sleuth/1.3 +Xenu's Link Sleuth 1 +Xylix +Y!J-SRD/1.0 +Yahoo Pipes 1.0 +Yahoo! Mindset +Yahoo-Test/4.0 +YahooCacheSystem +YahooCacheSystem; Ya +YahooSeeker/CafeKels +Yanadoo +Yandex/1.01.001 (com +Zearchit +ZendHttpClient +Zeus ThemeSite Viewe +Zeus xxxxx Webster P +Ziggy -- The Clown F +Zippy v2.0 - Zippyfi +a.pr-cy.ru +about.ask.com +about.depspid.net +accoona +aihit.com +amzn_assoc +app.hypefactors.com +appie 1.1 (www.walhe +archive.org +ask.24x.info +asterias/2.0 +augurfind +augurnfind V-1.x +autohttp +autowebdir 1.1 (www. +axios/0.18.0 +bCentral Billing Pos +bdcindexer_2.6.2 (re +binlar_2.6.3 binlar2 +binlar_2.6.3 binlar_ +binlar_2.6.3 larbin2 +binlar_2.6.3 phanend +binlar_2.6.3 test@mg +blacklistdeclassified.net +bookmarkbase.com +bulkfeeds.net +bumblebee@relevare.c +cantor.ee.ucla.edu +carleson/1.0 +ccubee/x.x +cfetch/1.0 +cg-eye interactive +cobitsprobe +coccoc/1.0 () +code.google.com +codegator +cohalinglit.wordpress.com +collage.cgi/1.xx +combine.it.lth.se +combine/0.0 +computerorgs.com +content.easybib.com +coralcdn.org +cosco.hiit.fi +crawler +cs.daum.net +csci_b659/0.13 +curl/7.18.2 (x86_64- +curl/7.19.7 (x86_64- +curl/7.21.4 (univers +curl/7.37.1 +dcrawl/1.0 +dds explorer v1.0 be +deepak-USC/ISI +deusu.org +developers.google.com +dhcibr.wordpress.com +digg.com +dir.com +drupal.org +dznet.com +e-sense 1.0 ea(www.v +eStyleSearch 4 (comp +ec2linkfinder +ecairn.com +ejupiter.com +enter4u.eu +exactseek.com +example-masto-instance.org +exploringdh.wordpress.com +facebookexternalhit/ +feed.moo.jp +feedafever.com; +flipboard.com +flunky +fly/6.01 libwww/4.0D +free-downloads.net d +fulltext.seznam.cz +gazz/x.x (gazz@nttrd +geourl/2.0b2 +getpocket.com +gigabaz.com +github.com +google-xrawler +googlebot +gossamer-threads.com +grub-client +grub.org +guess.scritch.org +haste.kytoon.com +help.coccoc.com +help.coccoc.vn +help.goo.ne.jp +help.hepburnhome.ca +help.yahoo.co.jp +help.yahoo.com +hgrepurl/1.0 +holmes/3.9 (onet.pl) +holmes/3.9 (someurl. +holmes/x.x +htdig/3.1.5 +htdig/3.1.5 (infosys +htdig/3.1.5 (root@lo +htdig/3.1.5 (webmast +htdig/3.1.6 (mathieu +htdig/3.1.6 (unconfi +htdig/3.1.x (root@lo +http://www.mabontlan +http://www.monogol.d +http://www.timelyweb +http_get +httpunit/1.x +i.stanford.edu +iFeed.jp/2.0 (www.ps +ia_archiver-web.arch +ia_archiver/1.6 +ia_archiver/8.0 (Win +ia_archiver/8.1 (Win +ia_archiver/8.2 (Win +ia_archiver/8.8 (Win +ia_archiver/8.9 (Lin +ia_archiver/8.9 (Win +icerocket.com; +ichiro/2.0 (ichiro@n +ichiro/x.0 (ichiro@n +ideare - SignSite/1. +iearthworm/1.0, iear +igde.ru +ilial/Nutch-0.9-dev +innovationblog.com +integrity/1.6 +integrity/5 +ivia.ucr.edu +jack +journster.com +knil +kulturarw3/0.1 +larbin (samualt9@big +larbin_2.1.1 larbin2 +larbin_2.2.0 (crawl@ +larbin_2.2.1_de_Vien +larbin_2.2.2 (sugaya +larbin_2.2.2_guillau +larbin_2.5.0 (larbin +larbin_2.6.0 (larbin +larbin_2.6.1 (larbin +larbin_2.6.1 larbin2 +larbin_2.6.2 (hamasa +larbin_2.6.2 (larbin +larbin_2.6.2 (liston +larbin_2.6.2 (pierre +larbin_2.6.2 (pimena +larbin_2.6.2 (tom@le +larbin_2.6.2 (vitalb +larbin_2.6.2 dthunen +larbin_2.6.2 kalou@k +larbin_2.6.2 larbin2 +larbin_2.6.2 larbin@ +larbin_2.6.2 listonA +larbin_2.6.2 pierre@ +larbin_2.6.2 vitalbo +larbin_2.6.3 (larbin +larbin_2.6.3 (wgao@g +larbin_2.6.3 capveg@ +larbin_2.6.3 ghary@s +larbin_2.6.3 gqnmgsp +larbin_2.6.3 larbin2 +larbin_2.6.3 tangyi8 +larbin_2.6.3 zumesun +larbin_2.6_basileoca +larbin_test (nobody@ +larbin_test nobody@a +larbin_xy250 larbin2 +law.di.unimi.it +lc/$ROADS::Version l +letscrawl.com +libWeb/clsHTTP -- hi +libwww-perl/5.41 +libwww-perl/5.45 +libwww-perl/5.48 +libwww-perl/5.50 +libwww-perl/5.52 FP/ +libwww-perl/5.53 +libwww-perl/5.63 +libwww-perl/5.64 +libwww-perl/5.65 +libwww-perl/5.800 +libwww/5.3.2 +linkdex.com/v2.0 +ltx71.com +lunefran.wordpress.com +lwp-trivial/1.32 +lwp-trivial/1.33 +lwp-trivial/1.34 +lwp-trivial/1.35 +lwp-trivial/1.36 +lwp-trivial/1.38 +lwp-trivial/1.41 +mailto:webcraft@bea. +mbblog14.wordpress.com +metatagsdir.com +miniflux.net +misc.yahoo.com.cn +moget/x.x (moget@goo +mogimogi/1.0 +moiNAG 0.02 +muckrack.com +myDaemon +net-promoter.com +netarkivet.dk +newsearchengine (Thi +newsme/1.0; feedback +newspaper/0.1.0.7 +newspaper/0.2.5 +newspaper/0.2.6 +newstin.com +nlp.uned.es +noyona_0_1 +nsyght.com/Nutch-1.0 +nsyght.com/Nutch-x.x +nutch-1.4/Nutch-1.4 +nutchsearch/Nutch-0. +nymesis.com +oegp v. 1.3.0 +okhttp/2.5.0 +okhttp/2.7.5 +okhttp/3.2.0 +okhttp/3.5.0 +omgili.com +ostermiller.org +owler-agent +ozelot/2.7.3 (Search +page2rss.com +pandora.nla.gov.au +panscient.com +parsijoo.ir +pauillac.inria.fr +pd02_1.0.0 pd02_1.0. +pear.php.net +people.eng.unimelb.edu.au +phpcrawl +pompos.iliad.fr +psycheclone +publisher.yahoo.com +python-requests/0.13 +python-requests/1.0. +python-requests/2.18 +python-requests/2.3. +r.hatena.ne.jp +rabaz (rabaz at giga +ranksonic.com +rico/0.1 +riddler.io +rjpower.org +robot +sagool.jp +scooter-venus-3.0.vn +scoutmaster +scrapy.org +search.ch V1.4 +search.denmex.com +search.goo.ne.jp +search.google.com +search.yahoo.com +seekda.com +semanticdiscovery/0. +sentry.io +silk/1.0 +sitebar.org +siteexplorer.info +smartwit.com +sna-0.0.1 (mikemuzio +sna-0.0.1 mikeelliot +sohu agent +sohu-search +sp.ask.com +suchbaer.de +summify.com +support.embed.ly +surf +suzuran +szukaj.onet.pl +tags2dir.com +tagword.com +targetblaster.com/0. +teoma_agent1 +therarestwords.com +toCrawl/UrlDispatche +tuezilla.de +twikle.com +twurly.org +unknownght.com Web S +users.cs.cf.ac.uk +versus.integis.ch +vias.ncsa.uiuc.edu +visionutils/0.2 +vk.com +voyager-hc/1.0 +voyager/1.0 +web-bekannt (Version +web2express.org +webGobbler/1.x.x +webagent.wise-guys.nl +webalgo.iit.cnr.it +webbandit/4.xx.0 +webcollage/1.114 +webcollage/1.117 +webcollage/1.125 +webcollage/1.129 +webcollage/1.93 +webcollage/1.xx +webcrawl.net +webdatastats.com +webglimpse.net +webhack +websearchbench.cs.uni-dortmund.de +websitealert.net +webster-internet.de +webster.healeys.net +whatUseek_winona/3.0 +wiki.creativecommons.org +wish.slis.tsukuba.ac.jp +wkd.iis.sinica.edu.tw +wortschatz.uni-leipzig.de +ws.daum.net +www.Abolimba.de +www.WISEnut.com +www.abonti.com +www.ackerm.com +www.activtourist.com +www.agada.de +www.ahmadsoftware.co +www.answerbus.com +www.ant.com +www.aol.com +www.aramabeta.com; +www.arianna.it +www.arocom.de +www.asaha.com +www.axxus.de +www.ay-up.com +www.ba.be +www.blaiz.net +www.bloglines.com +www.bortzmeyer.org +www.brandverity.com +www.buddybuzz.net +www.buscaplus.com +www.business-socket. +www.canadiancontent.net +www.chepi.net +www.chilkatsoft.com +www.cloudflare.com +www.cnet.com +www.cougarsearch.com +www.covario.com +www.crim.ca +www.cs.washington.edu +www.cs.washington.edu; +www.dataparksearch.org +www.de.com +www.dead-links.com +www.die-kraehe.de +www.dnsdigger.com +www.earthcom.info +www.eurekster.com +www.europarchive.org +www.eventax.de +www.everatom.com +www.evri.com +www.exensa.com +www.exif-search.com +www.facebook.com +www.favcollector.com +www.feeddemon.com +www.feedhub.com +www.feedly.com +www.feedmap.net +www.fileboost.net +www.findpal.com +www.floorstransformed.com +www.flyindex.net +www.followsite.com +www.genieo.com +www.globalspec.com +www.goguides.org +www.google.com +www.greaterera.com +www.hermitsearch.com +www.htmlvalidator.com +www.hyperix.com +www.ianett.com +www.icerocket.com +www.illumit.com +www.iltrovatore.it +www.inktomi.com +www.intelliseek.com +www.ip2location.com +www.iskanie.com +www.jordomedia.com +www.kalooga.com; +www.kototoi.org +www.l3s.de +www.lan4lano.net +www.lipperhey.com +www.look.com +www.lw01.com +www.matuschek.net +www.megaindex.ru +www.megite.com +www.meta-spinner.de +www.mixdata.com +www.moreover.com +www.neomo.de +www.netluchs.de +www.newsgator.com; +www.ng-search.com +www.page-store.com +www.pagebull.com +www.pingalink.com +www.pingdom.com +www.plinki.com; +www.powerset.com +www.prairielandconsulting.com +www.proximic.com +www.pubblisito.com +www.qango.com +www.quickiwiki.com; +www.qwant.com +www.rawgrunt.com +www.relsoftware.com +www.retrevo.com +www.samsung.com +www.scifihifi.com +www.scoutjet.com +www.scrubtheweb.com +www.searchme.com +www.securecomputing.com +www.semager.de +www.shopwiki.com +www.siigle.com +www.simpy.com +www.sitesearch.ca +www.sitesell.com +www.sixxs.net +www.skaffe.com +www.skyrocket.gr.jp +www.sli-systems.com +www.slider.com +www.slysearch.com +www.somusical.com +www.suchen.de +www.sygol.com +www.sync2it.com +www.sysomos.com +www.targetgroups.net +www.teradex.com +www.tkensaku.com +www.truveo.com +www.tumblr.com +www.ukwizz.com +www.unchaos.com +www.updated.com; +www.urltrends.com +www.vakes.com +www.versions-project.org +www.verzamelgids.nl +www.wavefire.com; +www.webarchiv.cz +www.worio.com +www.wowlemmings.com +www.xirq.com; +www.xrayz.co.uk +www.yama.info.waseda.ac.jp +www.yoono.com +www.zedzo.com +www4mail/2.x libwww- +wwwster/1.x (Beta- m +xmlset_roodkcableoj2 +yacy (www.yacy.net; +yarienavoir.net/0.2 +yggdrasil/Nutch-0.9 +yoogliFetchAgent/0.1 +Googlebot\/ +Googlebot-Mobile +Googlebot-Image +Googlebot-News +Googlebot-Video +AdsBot-Google([^-]|$) +AdsBot-Google-Mobile +Feedfetcher-Google +Mediapartners-Google +Mediapartners \(Googlebot\) +APIs-Google +Google-InspectionTool +Storebot-Google +GoogleOther +bingbot +Slurp +[wW]get +LinkedInBot +Python-urllib +python-requests +aiohttp +httpx +libwww-perl +httpunit +nutch +Go-http-client +phpcrawl +msnbot +jyxobot +FAST-WebCrawler +FAST Enterprise Crawler +BIGLOTRON +Teoma +convera +seekbot +Gigabot +Gigablast +exabot +ia_archiver +GingerCrawler +webmon +HTTrack +grub.org +UsineNouvelleCrawler +antibot +netresearchserver +speedy +fluffy +findlink +msrbot +panscient +yacybot +AISearchBot +ips-agent +tagoobot +MJ12bot +woriobot +yanga +buzzbot +mlbot +yandex\.com\/bots +purebot +Linguee Bot +CyberPatrol +voilabot +Baiduspider +citeseerxbot +spbot +twengabot +postrank +Turnitin +scribdbot +page2rss +sitebot +linkdex +Adidxbot +ezooms +dotbot +Mail.RU_Bot +discobot +heritrix +findthatfile +europarchive.org +NerdByNature.Bot +sistrix crawler +Ahrefs(Bot|SiteAudit) +fuelbot +CrunchBot +IndeedBot +mappydata +woobot +ZoominfoBot +PrivacyAwareBot +Multiviewbot +SWIMGBot +Grobbot +eright +Apercite +semanticbot +Aboundex +domaincrawler +wbsearchbot +summify +CCBot +edisterbot +SeznamBot +ec2linkfinder +gslfbot +aiHitBot +intelium_bot +facebookexternalhit +Yeti +RetrevoPageAnalyzer +lb-spider +Sogou +lssbot +careerbot +wotbox +wocbot +ichiro +DuckDuckBot +lssrocketcrawler +drupact +webcompanycrawler +acoonbot +openindexspider +gnam gnam spider +web-archive-net.com.bot +backlinkcrawler +coccoc +integromedb +content crawler spider +toplistbot +it2media-domain-crawler +ip-web-crawler.com +siteexplorer.info +elisabot +proximic +changedetection +arabot +WeSEE:Search +niki-bot +CrystalSemanticsBot +rogerbot +360Spider +psbot +InterfaxScanBot +CC Metadata Scaper +g00g1e.net +GrapeshotCrawler +urlappendbot +brainobot +fr-crawler +binlar +SimpleCrawler +Twitterbot +cXensebot +smtbot +bnf.fr_bot +A6-Indexer +ADmantX +Facebot +OrangeBot\/ +memorybot +AdvBot +MegaIndex +SemanticScholarBot +ltx71 +nerdybot +xovibot +BUbiNG +Qwantify +archive.org_bot +Applebot +TweetmemeBot +crawler4j +findxbot +S[eE][mM]rushBot +yoozBot +lipperhey +Y!J +Domain Re-Animator Bot +AddThis +Screaming Frog SEO Spider +MetaURI +Scrapy +Livelap[bB]ot +OpenHoseBot +CapsuleChecker +collection@infegy.com +IstellaBot +DeuSu\/ +betaBot +Cliqzbot\/ +MojeekBot\/ +netEstate NE Crawler +SafeSearch microdata crawler +Gluten Free Crawler\/ +Sonic +Sysomos +Trove +deadlinkchecker +Slack-ImgProxy +Embedly +RankActiveLinkBot +iskanie +SafeDNSBot +SkypeUriPreview +Veoozbot +Slackbot +redditbot +datagnionbot +Google-Adwords-Instant +adbeat_bot +WhatsApp +contxbot +pinterest.com.bot +electricmonk +GarlikCrawler +BingPreview\/ +vebidoobot +FemtosearchBot +Yahoo Link Preview +MetaJobBot +DomainStatsBot +mindUpBot +Daum\/ +Jugendschutzprogramm-Crawler +Xenu Link Sleuth +Pcore-HTTP +moatbot +KosmioBot +[pP]ingdom +AppInsights +PhantomJS +Gowikibot +PiplBot +Discordbot +TelegramBot +Jetslide +newsharecounts +James BOT +Bark[rR]owler +TinEye +SocialRankIOBot +trendictionbot +Ocarinabot +epicbot +Primalbot +DuckDuckGo-Favicons-Bot +GnowitNewsbot +Leikibot +LinkArchiver +YaK\/ +PaperLiBot +Digg Deeper +dcrawl +Snacktory +AndersPinkBot +Fyrebot +EveryoneSocialBot +Mediatoolkitbot +Luminator-robots +ExtLinksBot +SurveyBot +NING\/ +okhttp +Nuzzel +omgili +PocketParser +YisouSpider +um-LN +ToutiaoSpider +MuckRack +Jamie's Spider +AHC\/ +NetcraftSurveyAgent +Laserlikebot +^Apache-HttpClient +AppEngine-Google +Jetty +Upflow +Thinklab +Traackr.com +Twurly +Mastodon +http_get +DnyzBot +botify +007ac9 Crawler +BehloolBot +BrandVerity +check_http +BDCbot +ZumBot +EZID +ICC-Crawler +ArchiveBot +^LCC +filterdb.iss.net\/crawler +BLP_bbot +BomboraBot +Buck\/ +Companybook-Crawler +Genieo +magpie-crawler +MeltwaterNews +Moreover +newspaper\/ +ScoutJet +(^| )sentry\/ +StorygizeBot +UptimeRobot +OutclicksBot +seoscanners +Hatena +Google Web Preview +MauiBot +AlphaBot +SBL-BOT +IAS crawler +adscanner +Netvibes +acapbot +Baidu-YunGuanCe +bitlybot +blogmuraBot +Bot.AraTurka.com +bot-pge.chlooe.com +BoxcarBot +BTWebClient +ContextAd Bot +Digincore bot +Disqus +Feedly +Fetch\/ +Fever +Flamingo_SearchEngine +FlipboardProxy +g2reader-bot +G2 Web Services +imrbot +K7MLWCBot +Kemvibot +Landau-Media-Spider +linkapediabot +vkShare +Siteimprove.com +BLEXBot\/ +DareBoost +ZuperlistBot\/ +Miniflux\/ +Feedspot +Diffbot\/ +SEOkicks +tracemyfile +Nimbostratus-Bot +zgrab +PR-CY.RU +AdsTxtCrawler +Datafeedwatch +Zabbix +TangibleeBot +google-xrawler +axios +Amazon CloudFront +Pulsepoint +CloudFlare-AlwaysOnline +Google-Structured-Data-Testing-Tool +WordupInfoSearch +WebDataStats +HttpUrlConnection +Seekport Crawler +ZoomBot +VelenPublicWebCrawler +MoodleBot +jpg-newsbot +outbrain +W3C_Validator +Validator\.nu +W3C-checklink +W3C-mobileOK +W3C_I18n-Checker +FeedValidator +W3C_CSS_Validator +W3C_Unicorn +Google-PhysicalWeb +Blackboard +ICBot\/ +BazQux +Twingly +Rivva +Experibot +awesomecrawler +Dataprovider.com +GroupHigh\/ +theoldreader.com +AnyEvent +Uptimebot\.org +Nmap Scripting Engine +2ip.ru +Clickagy +Caliperbot +MBCrawler +online-webceo-bot +B2B Bot +AddSearchBot +Google Favicon +HubSpot +Chrome-Lighthouse +HeadlessChrome +CheckMarkNetwork\/ +www\.uptime\.com +Streamline3Bot\/ +serpstatbot\/ +MixnodeCache\/ +^curl +SimpleScraper +RSSingBot +Jooblebot +fedoraplanet +Friendica +NextCloud +Tiny Tiny RSS +RegionStuttgartBot +Bytespider +Datanyze +Google-Site-Verification +TrendsmapResolver +tweetedtimes +NTENTbot +Gwene +SimplePie +SearchAtlas +Superfeedr +feedbot +UT-Dorkbot +Amazonbot +SerendeputyBot +Eyeotabot +officestorebot +Neticle Crawler +SurdotlyBot +LinkisBot +AwarioSmartBot +AwarioRssBot +RyteBot +FreeWebMonitoring SiteChecker +AspiegelBot +NAVER Blog Rssbot +zenback bot +SentiBot +Domains Project\/ +Pandalytics +VKRobot +bidswitchbot +tigerbot +NIXStatsbot +Atom Feed Robot +[Cc]urebot +PagePeeker\/ +Vigil\/ +rssbot\/ +startmebot\/ +JobboerseBot +seewithkids +NINJA bot +Cutbot +BublupBot +BrandONbot +RidderBot +Taboolabot +Dubbotbot +FindITAnswersbot +infoobot +Refindbot +BlogTraffic\/\d\.\d+ Feed-Fetcher +SeobilityBot +Cincraw +Dragonbot +VoluumDSP-content-bot +FreshRSS +BitBot +^PHP-Curl-Class +Google-Certificates-Bridge +centurybot +Viber +e\.ventures Investment Crawler +evc-batch +PetalBot +virustotal +(^| )PTST\/ +minicrawler +Cookiebot +trovitBot +seostar\.co +IonCrawl +Uptime-Kuma +SeekportBot +FreshpingBot +Feedbin +CriteoBot +Snap URL Preview Service +Better Uptime Bot +RuxitSynthetic +Google-Read-Aloud +Valve\/Steam +OdklBot\/ +GPTBot +YandexRenderResourcesBot\/ +LightspeedSystemsCrawler +ev-crawler\/ +BitSightBot\/ +woorankreview\/ +Google-Safety +AwarioBot +DataForSeoBot +Linespider +WellKnownBot +A Patent Crawler +StractBot +search\.marginalia\.nu +YouBot +Nicecrawler +Neevabot +BrightEdge Crawler +SiteCheckerBotCrawler +TombaPublicWebCrawler +CrawlyProjectCrawler +KomodiaBot +KStandBot +CISPA Webcrawler +MTRobot +hyscore.io +AlexandriaOrgBot +2ip bot +Yellowbrandprotectionbot +SEOlizer +vuhuvBot +INETDEX-BOT +Synapse +t3versionsBot +deepnoc +Cocolyzebot +hypestat +ReverseEngineeringBot +sempi.tech +Iframely +MetaInspector +node-fetch +lkxscan +python-opengraph +OpenGraphCheck +developers.google.com\/\+\/web\/snippet +SenutoBot +MaCoCu +NewsBlur +inoreader +NetSystemsResearch +PageThing +WordPress\/ From ace3e57e7e0303b3a59c5dbfe7c3616180d5c223 Mon Sep 17 00:00:00 2001 From: CRMGB Date: Fri, 26 Jan 2024 09:45:04 +0000 Subject: [PATCH 2/3] Deleted all the java extra patterns which are not needed if you've got this regular expression: '^java\/\d{1,2}.\d' --- COUNTER_Robots_list.json | 196 --------------------------------------- 1 file changed, 196 deletions(-) diff --git a/COUNTER_Robots_list.json b/COUNTER_Robots_list.json index df5c89d..03c01c8 100644 --- a/COUNTER_Robots_list.json +++ b/COUNTER_Robots_list.json @@ -2396,202 +2396,6 @@ "pattern": "Jakarta Commons-Http", "last_changed": "2024-01-02" }, - { - "pattern": "Java 1.1", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java Browser", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.4.1_01", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.4.1_04", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.5.0_16", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_04", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_07", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_18", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_22", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_24", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_25", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_26", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_27", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_29", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_30", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_31", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_32", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_33", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_34", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_35", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_39", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_45", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.6.0_51", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_03", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_09", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_15", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_17", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_21", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_25", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_40", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_45", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_51", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_55", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_60", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_65", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_71", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_75", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.7.0_80", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_20", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_25", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_31", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_40", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_45", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java/1.8.0_51", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java1.0.21.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java1.1.xx.x", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java1.3.0rc1", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java1.3.x", - "last_changed": "2024-01-02" - }, - { - "pattern": "Java1.4.0", - "last_changed": "2024-01-02" - }, { "pattern": "Jersey/2.25.1 (HttpU", "last_changed": "2024-01-02" From b22e6c67ec61056cf3f2addde02d53f27a35dc2b Mon Sep 17 00:00:00 2001 From: CRMGB Date: Mon, 29 Jan 2024 17:59:02 +0000 Subject: [PATCH 3/3] Resolved the issues with COUNTER_Robots_list.json: - Deletion of patterns duplications. - Added the escape for special characters. - Added the right lenght for some patterns. - Deleted patters with a previous match. Updated CHANGES.md and the txt version respectively. --- CHANGES.md | 2085 +++----- COUNTER_Robots_list.json | 8179 ++++++----------------------- generated/COUNTER_Robots_list.txt | 2083 +++----- 3 files changed, 2837 insertions(+), 9510 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 05240b7..633fe17 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -111,1643 +111,866 @@ Processed 3 following merge requests : ## July 27th, 2017 - First release, this is the latest list provided by COUNTER -## January 10th, 2024 +## January 29th, 2024 - Added user-agents: - &as_qdr=all - - */Nutch-0.9-dev - - 192.comAgent - - 222.122.15.190 - - 2Bone_LinkChkr/1.0 l + - 192\.comAgent + - 222\.122\.15\.190 + - 2Bone_LinkChkr\/1\.0\sl - 7331230CB97FADECF052 - - 8484 Boston Project + - 8484\sBoston\sProject - 8LEGS - - A-Online Search - - ADmantX Platform Sem - - AHC/2.0 - - ASPSeek/1.2.5 - - ASPSeek/1.2.x - - ASPSeek/1.2.xa - - ASPSeek/1.2.xxpre - - ASPseek/1.2.9d - - ASPseek/1.2.xx - - ASSORT/0.10 - - AV Fetch 1.0 - - AVSearch-1.0(peter.t - - AVSearch-2.0-fusionI - - AVSearch-3.0(AltaVis - - About/0.1libwww-perl - - Accoona-AI-Agent/1.1 - - Acorn/Nutch-0.9 (Non - - ActiveBookmark 1.x - - AdobeUxTechC4-Async/ - - AgentName/0.1 libwww - - AideRSS/1.0 (aiderss - - Aladin/3.324 - - Allesklar/0.1 libwww - - Allrati/1.1 (+) - - AltaVista Intranet V - - Amazon CloudFront - - AmiTCP Miami (AmigaO - - Anonymizer/1.1 - - AnswerChase PROve x. - - AnswerChase x.0 - - AnzwersCrawl/2.0 (an - - Apache (internal dum + - A-Online\sSearch + - AHC\/2\.0 + - ASPSeek\/1\.2\.5 + - ASPSeek\/1\.2\.x + - ASPseek\/1\.2\.9d + - ASSORT\/0\.10 + - AV\sFetch\s1\.0 + - AVSearch-1\.0 + - AVSearch-2\.0-fusionI + - AVSearch-3\.0 + - Accoona-AI-Agent\/1\.1 + - ActiveBookmark\s1\.x + - AdobeUxTechC4-Async\/ + - Aladin\/3\.324 + - Allrati\/1\.1 + - AltaVista\sIntranet\sV + - Amazon\sCloudFront + - AmiTCP\sMiami + - Anonymizer\/1\.1 + - AnswerChase\sPROve\sx\. + - AnswerChase\sx\.0 + - Apache - Apache-HttpAsyncClie - - Apache-HttpClient/4. - - Apache-HttpClient/UN + - Apache-HttpClient\/4\. + - Apache-HttpClient\/UN - Aport - - Arachnoidea (arachno - - Asahina-Antenna/1.x - - Asahina-Antenna/1.x - - Ask.24x.Info - - Atomic_Email_Hunter/ - - Atomz/1.0 - - BIGLOTRON (Beta 2;GN + - Arachnoidea + - Asahina-Antenna\/1\.x + - Ask\.24x\.Info + - Atomic_Email_Hunter\/ - BMCLIENT - BMLAUNCHER - - BPImageWalker/2.0 (w - - BSDSeek/1.0 - - BStop.BravoBrian.it - - BTWebClient/180B(970 - - BeamMachine/0.5 (dea - - BeebwareDirectory/v0 - - Big Fish v1.0 - - BigBrother/1.6e - - Bilbo/2.3b-UNIX - - BlackBerry9000/4.6.0 + - BPImageWalker\/2\.0 + - BSDSeek\/1\.0 + - BStop\.BravoBrian\.it + - BTWebClient\/180B + - BeamMachine\/0\.5 + - BeebwareDirectory\/v0 + - Big\sFish\sv1\.0 + - BigBrother\/1\.6e + - Bilbo\/2\.3b-UNIX + - BlackBerry9000\/4\.6\.0 - BlackWidow - - Blackboard Safeassig - - BlogPulseLive (suppo - - Blogpulse (info@blog - - Bobby/4.0.x RPT-HTTP - - Bookdog/x.x - - BravoBrian bstop.bra + - Blackboard\sSafeassig + - Bobby\/4\.0\.x\sRPT-HTTP + - Bookdog\/x\.x + - BravoBrian\sbstop\.bra - BullsEye - BunnySlippers - - CB/Nutch-1.7 - - CFNetwork/x.x - - CHttpClient by Open - - CLIPish Pro2 10.21 ( - - COAST WebMaster Pro/ - - CamelHttpStream/1.0 - - Cancer Information a - - Chat Catcher/1.0 - - China Local Browse 2 - - Chitika ContentHit 1 - - Claymont.com - - ColdFusion (Bookmark - - Combine/x.0 + - CHttpClient\sby\sOpen + - CLIPish\sPro2\s10\.21 + - COAST\sWebMaster\sPro\/ + - CamelHttpStream\/1\.0 + - Cancer\sInformation\sa + - Chat\sCatcher\/1\.0 + - China\sLocal\sBrowse\s2 + - Chitika\sContentHit\s1 + - Claymont\.com - Contact - - Cookie%20Stumbler/22 - - CrawlConvera0.1 (Cra - - Custo x.x (www.netwu - - CyberSpyder Link Tes - - DBrowse 1.4b - - DBrowse 1.4d + - Cookie%20Stumbler\/22 + - CyberSpyder\sLink\sTes + - DBrowse\s1\.4b + - DBrowse\s1\.4d - DTAAgent - - Dart Communications - - DataCha0s/2.0 - - DataFountains/DMOZ D - - Datafeedwatch/2.1.x - - Deep Link Calculator + - Dart\sCommunications + - DataFountains\/DMOZ\sD + - Datafeedwatch\/2\.1\.x + - Deep\sLink\sCalculator - DeepIndex - - DeepIndex (www.en.de - - DeepIndexer.ca - - DevComponents.com Ht - - Diamond/x.0 + - DeepIndexer\.ca + - DevComponents\.com\sHt + - Diamond\/x\.0 - DigOut4U - - Digger/1.0 JDK/1.3.0 - - Disqus/1.0 + - Digger\/1\.0\sJDK\/1\.3\.0 + - Disqus\/1\.0 - DittoSpyder - - DoCoMo/1.0/Nxxxi/c10 - - DoCoMo/2.0 P900iV(c1 - - EARTHCOM.info/1.x [w - - EARTHCOM.info/1.xbet - - EBrowse 1.4b - - EasyDL/3.xx - - EchO!/2.0 - - Educate Search VxB - - EldoS TimelyWeb/3.x - - EmailWolf 1.00 - - Embedly +support@emb - - Enfish Tracker - - Enterprise_Search/1. - - Evernote Clip Resolv - - Exalead NG/MimeLive - - Expired Domain Sleut + - EARTHCOM\.info\/1\.x\s\[w + - EARTHCOM\.info\/1\.xbet + - EBrowse\s1\.4b + - EchO!\/2\.0 + - Educate\sSearch\sVxB + - EldoS\sTimelyWeb\/3\.x + - Enfish\sTracker + - Enterprise_Search\/1\. + - Evernote\sClip\sResolv + - Exalead\sNG\/MimeLive + - Expired\sDomain\sSleut - ExtractorPro - - EyeCatcher (Download - - FARK.com link verifi + - EyeCatcher + - FARK\.com\slink\sverifi - FLATARTS_FAVICO - - FaEdit/2.0.x - - FavOrg - - Favorites Sweeper v. - - Feed24.com - - FeedForAll rss2html. - - FeedValidator/1.3 - - Filtrbox/1.0 - - FindAnISP.com_ISP_Fi - - Firefly/1.0 - - Firefox (kastaneta03 - - Firefox_1.0.6 (kaspa - - FirstGov.gov Search - - Flexum/2.0 - - FocusedSampler/1.0 - - Franklin Locator 1.8 - - FuseBulb.Com - - GigablastOpenSource/ - - Go 1.1 package http - - Go-http-client/1.1 - - Go-http-client/2.0 - - GoForIt.com - - Goofer/0.2 - - GrapeFX/0.3 libwww/5 - - Gromit/1.0 - - Guestbook Auto Submi - - Gulliver/1.3 - - Guzzle/4.2.3 curl/7. + - FaEdit\/2\.0\.x + - Favorites\sSweeper\sv\. + - Feed24\.com + - Filtrbox\/1\.0 + - FindAnISP\.com_ISP_Fi + - Firefly\/1\.0 + - Firefox + - Firefox_1\.0\.6 + - FirstGov\.gov\sSearch + - Flexum\/2\.0 + - FocusedSampler\/1\.0 + - Franklin\sLocator\s1\.8 + - FuseBulb\.Com + - Go\s1\.1\spackage\shttp + - Go-http-client\/1\.1 + - Go-http-client\/2\.0 + - GoForIt\.com + - Goofer\/0\.2 + - Gromit\/1\.0 + - Guestbook\sAuto\sSubmi - HLoader - - HPL/Nutch-0.9 - - Haansoft - - Harvest-NG/1.0.2 - - Hatena Antenna/0.3 - - Hatena Pagetitle Age - - Hatena::Bookmark/2.0 - - Hippias/0.9 Beta + - Hatena\sAntenna\/0\.3 + - Hatena\sPagetitle\sAge + - Hatena::Bookmark\/2\.0 + - Hippias\/0\.9\sBeta - HitList - Hivemind - - HomePageSearch(hpsea - - Html Link Validator - - Hybrid/1.2 [en] (OS - - IAArchiver-1.0 - - IBM WebExplorer /v0. + - HomePageSearch + - Hybrid\/1\.2\s\[en] + - IAArchiver-1\.0 + - IBM\sWebExplorer\s\/v0\. - IEFav172Free - - INFOMINE/8.0 Adders - - INFOMINE/8.0 RemoteS - - IP*Works! V5 HTTP/S - - ISC Systems iRc Sear - - IWAgent/ 1.0 - www.b - - Icarus6 - (contact: - - ImageWalker/2.0 (www - - Incutio HttpClient v - - IncyWincy data gathe - - Industry Program 1.0 - - Inet library - - InetURL/1.0 - - InfoLink/1.x - - InfoSeek Sidewinder/ - - Infoaxe./Nutch-0.9 - - Infoseek SideWinder/ - - Inktomi Search - - Insitor.com search a + - INFOMINE\/8\.0\sAdders + - INFOMINE\/8\.0\sRemoteS + - IP\*Works!\sV5\sHTTP\/S + - ISC\sSystems\siRc\sSear + - IWAgent\/\s1\.0\s-\swww\.b + - Icarus6\s- + - ImageWalker\/2\.0 + - Incutio\sHttpClient\sv + - IncyWincy\sdata\sgathe + - Industry\sProgram\s1\.0 + - Inet\slibrary + - InetURL\/1\.0 + - InfoLink\/1\.x + - InfoSeek\sSidewinder\/ + - Infoseek\sSideWinder\/ + - Inktomi\sSearch + - Insitor\.com\ssearch\sa - Insitornaut - - Internet Ninja x.0 - - InternetLinkAgent/3. - - InternetSeer.com - - IrssiUrlLog/0.2 - - JDK/1.1 + - Internet\sNinja\sx\.0 + - InternetLinkAgent\/3\. + - JDK\/1\.1 - Jack - - Jakarta Commons-Http - - Java 1.1 - - Java Browser - - Java/1.4.1_01 - - Java/1.4.1_04 - - Java/1.5.0_16 - - Java/1.6.0_04 - - Java/1.6.0_07 - - Java/1.6.0_18 - - Java/1.6.0_22 - - Java/1.6.0_24 - - Java/1.6.0_25 - - Java/1.6.0_26 - - Java/1.6.0_27 - - Java/1.6.0_29 - - Java/1.6.0_30 - - Java/1.6.0_31 - - Java/1.6.0_32 - - Java/1.6.0_33 - - Java/1.6.0_34 - - Java/1.6.0_35 - - Java/1.6.0_39 - - Java/1.6.0_45 - - Java/1.6.0_51 - - Java/1.7.0_03 - - Java/1.7.0_09 - - Java/1.7.0_15 - - Java/1.7.0_17 - - Java/1.7.0_21 - - Java/1.7.0_25 - - Java/1.7.0_40 - - Java/1.7.0_45 - - Java/1.7.0_51 - - Java/1.7.0_55 - - Java/1.7.0_60 - - Java/1.7.0_65 - - Java/1.7.0_71 - - Java/1.7.0_75 - - Java/1.7.0_80 - - Java/1.8.0_20 - - Java/1.8.0_25 - - Java/1.8.0_31 - - Java/1.8.0_40 - - Java/1.8.0_45 - - Java/1.8.0_51 - - Java1.0.21.0 - - Java1.1.xx.x - - Java1.3.0rc1 - - Java1.3.x - - Java1.4.0 - - Jersey/2.25.1 (HttpU - - Jetty/9.3.z-SNAPSHOT - - Jigsaw/2.2.x W3C_CSS - - Jigsaw/2.3.0 W3C_CSS - - Junkie/1.0 - - KE_1.0/2.0 libwww/5. - - KIT-Fireball/2.0 - - KnowItAll(knowitall@ - - Knowledge.com/0.x - - KummHttp/1.1 (compat - - LARBIN-EXPERIMENTAL - - LEIA/2.90 - - LWP::Simple/5.22 - - LWP::Simple/5.36 - - LWP::Simple/5.48 - - LWP::Simple/5.50 - - LWP::Simple/5.51 - - LWP::Simple/5.53 - - LWP::Simple/5.63 - - LWP::Simple/5.803 + - Jakarta\sCommons-Http + - Java1\.0\.21\.0 + - Java1\.1\.xx\.x + - Java1\.3\.0rc1 + - Java1\.3\.x + - Java1\.4\.0 + - Jersey\/2\.25\.1 + - Jetty\/9\.3\.z-SNAPSHOT + - Jigsaw\/2\.2\.x\sW3C_CSS + - Jigsaw\/2\.3\.0\sW3C_CSS + - Junkie\/1\.0 + - KIT-Fireball\/2\.0 + - KnowItAll + - Knowledge\.com\/0\.x + - KummHttp\/1\.1 + - LEIA\/2\.90 - Lachesis - - Libby_1.1/libwww-per - - Lincoln State Web Br - - Link Valet Online 1. - - LinkAlarm/2.x - - LinkPimpin v1.0 - - LinkProver 2.1 - - LinkSonar/1.35 - - LinkSweeper/1.x - - LinkWalker - - LinkWalker/2.0 - - Look.com - - Lynx/2.8.4rel.1 libw + - Lincoln\sState\sWeb\sBr + - Link\sValet\sOnline\s1\. + - LinkAlarm\/2\.x + - LinkPimpin\sv1\.0 + - LinkProver\s2\.1 + - LinkSonar\/1\.35 + - LinkSweeper\/1\.x + - Look\.com + - Lynx\/2\.8\.4rel\.1\slibw - MARTINI - - MFC Foundation Class + - MFC\sFoundation\sClass - MFC_Tear_Sample - - MSIE-5.13 (larbin@un - - MSNPTC/1.0 + - MSIE-5\.13 + - MSNPTC\/1\.0 - MVAClient - - Mac Finder 1.0.xx + - Mac\sFinder\s1\.0\.xx - Mag-Net - - Mail.Ru/1.0 - MantraAgent - - MapoftheInternet.com - - Marketwave Hit List + - MapoftheInternet\.com + - Marketwave\sHit\sList - Martini - - Marvin v0.3 + - Marvin\sv0\.3 - MasterSeek - - Mata Hari/2.00 - - MediaSearch/0.1 - - Mediapartners-Google - - MegaSheep v1.0 (www. - - MeltwaterNews www.me - - Mercator-1.x - - Mercator-2.0 - - Mercator-Scrub-1.1 - - Mercury/894 CFNetwor - - MetaURI API/2.0 +met + - Mata\sHari\/2\.00 + - MediaSearch\/0\.1 + - MegaSheep\sv1\.0 + - MeltwaterNews\swww\.me + - Mercator-1\.x + - Mercator-2\.0 + - Mercator-Scrub-1\.1 + - Mercury\/894\sCFNetwor + - MetaURI\sAPI\/2\.0\s\+met - MicroBaz - - Microsoft Data Acces - - Microsoft Small Busi - - Microsoft URL Contro - - Microsoft Windows Ne + - Microsoft\sData\sAcces + - Microsoft\sSmall\sBusi + - Microsoft\sURL\sContro + - Microsoft\sWindows\sNe - MiracleAlphaTest - - Missauga Locate 1.0. - - Missigua Locator 1.9 - - Missouri College Bro - - Mister PiX version.d - - Mister Pix II 2.02a - - Miva (AlgoFeedback@m - - Mizzu Labs 2.2 - - MnogoSearch/3.2.xx - - Mnogosearch-3.1.21 - - Mo College 1.9 - - More yacy user agent + - Missauga\sLocate\s1\.0\. + - Missigua\sLocator\s1\.9 + - Missouri\sCollege\sBro + - Mister\sPiX\sversion\.d + - Mister\sPix\sII\s2\.02a + - Miva + - Mizzu\sLabs\s2\.2 + - Mo\sCollege\s1\.9 - Mozi! - - MultiText/0.1 - - My WinHTTP Connectio - - MyNutchTest/Nutch-1. - - NASA Search 1.0 - - NEC Research Agent - - - NEC-Hayek/1.0 - - NG/4.0.1229 - - NICO/1.0 - - NING/1.0 + - MultiText\/0\.1 + - My\sWinHTTP\sConnectio + - NASA\sSearch\s1\.0 + - NEC\sResearch\sAgent\s- + - NEC-Hayek\/1\.0 + - NG\/4\.0\.1229 + - NICO\/1\.0 + - NING\/1\.0 - NationalDirectoryAdd - NativeHost - - NetLookout/2.24 - - NetMechanic Vx.0 - - NetSprint -- 2.0 + - NetLookout\/2\.24 + - NetMechanic\sVx\.0 + - NetSprint\s--\s2\.0 - NetZippy - - Nocilla/1.0 - - Nsauditor/1.x + - Nocilla\/1\.0 + - Nsauditor\/1\.x - Nuzzel - - Nuzzel/876 CFNetwork - - Octora Beta - www.oc - - Oculus360/Nutch-1.7 + - Octora\sBeta\s-\swww\.oc - OliverPerry - - OpenAcoon v4.0.x (ww - - OpenISearch/1.x (www - - OpenOffice/4.1.1 - - OpenWebIndex/Nutch-1 - - Openstat/0.1 - - Opera/6.01 (larbin@u - - Oracle Ultra Search - - OzySoftware.com - - PBrowse 1.4b - - PEval 1.4b - - PHP/3.x.xx - - PHP/4.0.4pl1 - - PHP/4.0.6 - - PHP/4.1.1 - - PHP/4.1.2 + - OpenAcoon\sv4\.0\.x + - OpenISearch\/1\.x + - OpenOffice\/4\.1\.1 + - Openstat\/0\.1 + - Opera\/6\.01 + - Oracle\sUltra\sSearch + - OzySoftware\.com + - PBrowse\s1\.4b + - PEval\s1\.4b - POE-Component-Client - - Pcore-HTTP/v0.40.3 - - PeopleChat/Search_En - - PicoSearch/1.0 + - PeopleChat\/Search_En + - PicoSearch\/1\.0 - Piffany_Web_Scraper_ - - PingALink Monitoring + - PingALink\sMonitoring - Pita - Poirot - - Pompos/1.x pompos@il - - Popdexter/1.0 - - Port Huron Labs + - Pompos\/1\.x\spompos@il + - Popdexter\/1\.0 + - Port\sHuron\sLabs - PostFavorites - - PostRank/2.0 (postra - - PrivacyFinder/1.1 - - Program Shareware 1. - - Progressive Download - - Project XP5 [2.03.07 - - PubCrawl (pubcrawl.s - - Pulsepoint XT3 web s - - PycURL/7.13.2 - - PycURL/7.15.5 - - PycURL/7.16.4 - - PycURL/7.18.0 - - PycURL/7.18.2 - - PycURL/7.19.0 - - PycURL/7.19.3 - - PycURL/7.19.5 - - PycURL/7.19.7 - - PycURL/7.23.1 - - Python-urllib/1.17 - - Python-urllib/1.1x - - Python-urllib/2.0a1 - - Python-urllib/2.5 - - Python-urllib/2.6 - - Python-urllib/2.7 - - Python-urllib/3.1 - - Python-urllib/3.2 - - Python-urllib/3.3 - - Python-urllib/3.4 - - Python-urllib/3.5 - - Python-urllib/3.6 - - QPCreep Test Rig ( W - - Qseero v1.0.0 - - QuepasaCreep v0.9.1x - - QueryN Metasearch - - Qwantify/1.0 - - R6_VoteReader(www.ra - - REBOL Core 2.x.x.x.x - - RPT-HTTPClient/0.3-x - - RepoMonkey Bait & Ta - - RoboCrawl (www.canad - - Robozilla/1.0 - - Rotondo/3.1 libwww/5 + - PostRank\/2\.0 + - PrivacyFinder\/1\.1 + - Program\sShareware\s1\. + - Progressive\sDownload + - Project\sXP5\s\[2\.03\.07 + - Pulsepoint\sXT3\sweb\ss + - QPCreep\sTest\sRig + - Qseero\sv1\.0\.0 + - QuepasaCreep\sv0\.9\.1x + - QueryN\sMetasearch + - R6_VoteReader + - RPT-HTTPClient\/0\.3-x + - RepoMonkey\sBait\s&\sTa - Ruby - Rumours-Agent - - SMRF URL Expander - - Scooter-3.0.EU - - Scooter-3.0.FS - - Scooter-3.0.HD - - Scooter-3.0.VNS - - Scooter-3.0QI - - Scooter-3.2 - - Scooter-3.2.BT - - Scooter-3.2.DIL - - Scooter-3.2.EX - - Scooter-3.2.JT - - Scooter-3.2.NIV - - Scooter-3.2.SF0 - - Scooter-3.2.snippet - - Scooter-3.3dev - - Scooter-ARS-1.1 - - Scooter-ARS-1.1-ih - - Scooter-W3-1.0 - - Scooter-W3.1.2 - - Scooter/1.0 - - Scooter/1.0 scooter@ - - Scooter/1.1 (custom) - - Scooter/2.0 G.R.A.B. - - Scooter/3.3 - - Scooter/3.3.QA.pczuk - - Scooter/3.3.vscooter - - Scooter/3.3_SF - - Scooter2_Mercator_x- - - Scooter_bh0-3.0.3 - - Scooter_trk3-3.0.3 + - SMRF\sURL\sExpander - ScoutAbout - - Search+ + - Search\+ - Search-Engine-Studio - - SearchGuild/DMOZ/Exp + - SearchGuild\/DMOZ\/Exp - SearchGuild_DMOZ_Exp - - SearchSight.com - - SearchTone2.0 - IDEA - - Seeker.lookseek.com - - Silk/1.0 - - SiteBar/x.x - - SiteRecon+(xx) - - SiteTruth.com site r + - SearchSight\.com + - SearchTone2\.0\s-\sIDEA + - Seeker\.lookseek\.com + - Silk\/1\.0 + - SiteBar\/x\.x + - SiteRecon\+ + - SiteTruth\.com\ssite\sr - SiteXpert - - Skywalker/0.1 (Skywa - - Slarp/0.1 + - Skywalker\/0\.1 + - Slarp\/0\.1 - Slider_Search_v1-de - - Slurpy Verifier/1.0 - - SlySearch (slysearch - - SnoopRob/x.x - - Snoopy v1.xx - - Snoopy_v0.xx - - Sphere Scout&v4.0 - + - SlySearch + - SnoopRob\/x\.x + - Snoopy\sv1\.xx + - Snoopy_v0\.xx + - Sphere\sScout&v4\.0\s- - Sphider - - Spida/0.1 - - Spinne/2.0 - - Spinne/2.0 med - - Spinne/2.0 med_AH - - Sqworm/2.9.81-BETA ( - - Sqworm/2.9.85-BETA ( - - Sqworm/2.9.89-BETA ( - - StackRambler/2.0 - - StackRambler/2.0 (MS - - StackRambler/x.x - - SyncIT/x.x - - Szukacz/1.x + - Spida\/0\.1 + - Spinne\/2\.0 + - Spinne\/2\.0\smed + - Spinne\/2\.0\smed_AH + - SyncIT\/x\.x + - Szukacz\/1\.x - TE - - TJvMultiHttpGrabber - - Tagyu Agent/1.0 - - TeamSoft WinInet Com - - Teoma MP - - The European Library - - Thinklab (thinklab.c + - TJvMultiHttpGrabber + - Tagyu\sAgent\/1\.0 + - TeamSoft\sWinInet\sCom + - The\sEuropean\sLibrary + - Thinklab - TinEye - - Toutatis x-xx.x (hop - - Toutatis x.x (hoppa. - - Toutatis x.x-x - - Traackr.com - - Twisted PageGetter - - Twitter/6.18.3 CFNet - - Twitturly / v0.x - - Tycoon Agent/Nutch-1 + - Toutatis\sx-xx\.x + - Toutatis\sx\.x + - Toutatis\sx\.x-x + - Traackr\.com + - Twisted\sPageGetter + - Twitter\/6\.18\.3\sCFNet + - Twitturly\s\/\sv0\.x - TygoProwler - - UCWEB/2.0(Linux; U; + - UCWEB\/2\.0 - UDM - - URLBase/6.x + - URLBase\/6\.x - URLBlaze - - UdmSearch/3.1.x - - Ultraseek - - Under the Rainbow 2. - - UofTDB_experiment (l - - Upflow/1.0 - - VYU2 (GNU; OpenRISC) - - Vagabondo/1.x MT (we - - Vagabondo/2.0 MT - - Vagabondo/2.0 MT (we - - Vagabondo/3.0 (webag - - Validator.nu/LV - - VayalaCreep-v0.0.1 ( - - Vayala|Creep-v0.0.1 - - VeryGoodSearch.com.D - - W3C-mobileOK/DDC-1.0 - - W3C_Unicorn/1.0 - - W3C_Validator/1.3 - - W3C_Validator/1.xxx - - WDG_Validator/1.1 - - WEP Search 00 + - UdmSearch\/3\.1\.x + - Under\sthe\sRainbow\s2\. + - UofTDB_experiment + - Upflow\/1\.0 + - VYU2 + - Vagabondo\/1\.x\sMT + - Vagabondo\/2\.0\sMT + - Vagabondo\/3\.0 + - VayalaCreep-v0\.0\.1 + - Vayala\|Creep-v0\.0\.1 + - VeryGoodSearch\.com\.D + - W3C-mobileOK\/DDC-1\.0 + - W3C_Unicorn\/1\.0 + - WEP\sSearch\s00 - WFARC - - WIRE/0.22 (Linux; x8 + - WIRE\/0\.22 - WST-Plagiarism - - WWWC/1.0x - - Wada.vn Vietnamese S - - Waypath Scout v2.x - + - WWWC\/1\.0x + - Wada\.vn\sVietnamese\sS + - Waypath\sScout\sv2\.x\s- - WeSEE - WeSEE:Search - - Web Link Validator 1 - - Web Snooper - - WebBug/5.x - - WebCompass 2.0 - - WebCorp/1.0 - - WebSearch.COM.AU; - - WebStat/1.0 (Unix; b - - WebTrends/3.0 (WinNT - - WebVac (webmaster@pi - - WebVal/1.0 - - WebVulnCrawl.unknown - - WebWatcherMonitor/2. - - Webclipping.com - - Webdup/0.9 - - Weblog Attitude Diff - - WebsiteWorth v1.0 - - Webspinne/1.0 webmas - - Webverzeichnis.de - - - Wells Search II - - Wget/1.14 (linux-gnu - - WhatsApp/2.12.15/i - - WhatsApp/2.12.16/i - - WhatsApp/2.12.17/i - - WhatsApp/2.12.449 A - - WhatsApp/2.12.453 A - - WhatsApp/2.12.510 A - - WhatsApp/2.12.540 A - - WhatsApp/2.12.548 A - - WhatsApp/2.12.555 A - - WhatsApp/2.12.556 A - - WhatsApp/2.16.1/i - - WhatsApp/2.16.13 A - - WhatsApp/2.16.2/i - - WhatsApp/2.16.42 A - - WhatsApp/2.16.57 A - - Whatsup/x.x - - WhizBang! Lab - - WinHTTP Example/1.0 - - WinampMPEG/2.00 (lar - - WincerSong Agent v1. - - WorQmada/1.0 - - WordupInfoSearch/1.0 + - Web\sSnooper + - WebBug\/5\.x + - WebCompass\s2\.0 + - WebCorp\/1\.0 + - WebSearch\.COM\.AU; + - WebStat\/1\.0 + - WebTrends\/3\.0 + - WebVac + - WebVal\/1\.0 + - WebWatcherMonitor\/2\. + - Webclipping\.com + - Webdup\/0\.9 + - Weblog\sAttitude\sDiff + - WebsiteWorth\sv1\.0 + - Webspinne\/1\.0\swebmas + - Webverzeichnis\.de\s- + - Wells\sSearch\sII + - Whatsup\/x\.x + - WhizBang!\sLab + - WinHTTP\sExample\/1\.0 + - WinampMPEG\/2\.00 + - WincerSong\sAgent\sv1\. + - WorQmada\/1\.0 + - WordupInfoSearch\/1\.0 - WorldLight - - Wusage/x.0@boutell.c - - Wwlib/Linux - - XMLSlurp/0.1 libwww- - - Xenu Link Sleuth 1.x - - Xenu Link Sleuth/1.3 - - Xenu's Link Sleuth 1 + - Wusage\/x\.0@boutell\.c + - Wwlib\/Linux - Xylix - - Y!J-SRD/1.0 - - Yahoo Pipes 1.0 - - Yahoo! Mindset - - Yahoo-Test/4.0 - - YahooCacheSystem - - YahooCacheSystem; Ya - - YahooSeeker/CafeKels - Yanadoo - - Yandex/1.01.001 (com - Zearchit - ZendHttpClient - - Zeus ThemeSite Viewe - - Zeus xxxxx Webster P - - Ziggy -- The Clown F - - Zippy v2.0 - Zippyfi - - a.pr-cy.ru - - about.ask.com - - about.depspid.net + - Ziggy\s--\sThe\sClown\sF + - Zippy\sv2\.0\s-\sZippyfi + - a\.pr-cy\.ru + - about\.ask\.com + - about\.depspid\.net - accoona - - aihit.com + - aihit\.com - amzn_assoc - - app.hypefactors.com - - appie 1.1 (www.walhe - - archive.org - - ask.24x.info - - asterias/2.0 + - app\.hypefactors\.com + - archive\.org + - ask\.24x\.info - augurfind - - augurnfind V-1.x + - augurnfind\sV-1\.x - autohttp - - autowebdir 1.1 (www. - - axios/0.18.0 - - bCentral Billing Pos - - bdcindexer_2.6.2 (re - - binlar_2.6.3 binlar2 - - binlar_2.6.3 binlar_ - - binlar_2.6.3 larbin2 - - binlar_2.6.3 phanend - - binlar_2.6.3 test@mg - - blacklistdeclassified.net - - bookmarkbase.com - - bulkfeeds.net - - bumblebee@relevare.c - - cantor.ee.ucla.edu - - carleson/1.0 - - ccubee/x.x - - cfetch/1.0 - - cg-eye interactive + - autowebdir\s1\.1 + - axios\/0\.18\.0 + - bCentral\sBilling\sPos + - bdcindexer_2\.6\.2 + - blacklistdeclassified\.net + - bookmarkbase\.com + - bulkfeeds\.net + - bumblebee@relevare\.c + - cantor\.ee\.ucla\.edu + - carleson\/1\.0 + - ccubee\/x\.x + - cfetch\/1\.0 + - cg-eye\sinteractive - cobitsprobe - - coccoc/1.0 () - - code.google.com - codegator - - cohalinglit.wordpress.com - - collage.cgi/1.xx - - combine.it.lth.se - - combine/0.0 - - computerorgs.com - - content.easybib.com - - coralcdn.org - - cosco.hiit.fi - - crawler - - cs.daum.net - - csci_b659/0.13 - - curl/7.18.2 (x86_64- - - curl/7.19.7 (x86_64- - - curl/7.21.4 (univers - - curl/7.37.1 - - dcrawl/1.0 - - dds explorer v1.0 be - - deepak-USC/ISI - - deusu.org - - developers.google.com - - dhcibr.wordpress.com - - digg.com - - dir.com - - drupal.org - - dznet.com - - e-sense 1.0 ea(www.v - - eStyleSearch 4 (comp + - collage\.cgi\/1\.xx + - computerorgs\.com + - content\.easybib\.com + - coralcdn\.org + - cosco\.hiit\.fi + - cs\.daum\.net + - csci_b659\/0\.13 + - dds\sexplorer\sv1\.0\sbe + - deepak-USC\/ISI + - deusu\.org + - digg\.com + - dir\.com + - dznet\.com + - e-sense\s1\.0\sea + - eStyleSearch\s4 - ec2linkfinder - - ecairn.com - - ejupiter.com - - enter4u.eu - - exactseek.com - - example-masto-instance.org - - exploringdh.wordpress.com - - facebookexternalhit/ - - feed.moo.jp - - feedafever.com; - - flipboard.com + - ecairn\.com + - ejupiter\.com + - enter4u\.eu + - exactseek\.com + - example-masto-instance\.org + - feed\.moo\.jp + - feedafever\.com; + - flipboard\.com - flunky - - fly/6.01 libwww/4.0D - - free-downloads.net d - - fulltext.seznam.cz - - gazz/x.x (gazz@nttrd - - geourl/2.0b2 - - getpocket.com - - gigabaz.com - - github.com - - google-xrawler - - googlebot - - gossamer-threads.com - - grub-client - - grub.org - - guess.scritch.org - - haste.kytoon.com - - help.coccoc.com - - help.coccoc.vn - - help.goo.ne.jp - - help.hepburnhome.ca - - help.yahoo.co.jp - - help.yahoo.com - - hgrepurl/1.0 - - holmes/3.9 (onet.pl) - - holmes/3.9 (someurl. - - holmes/x.x - - htdig/3.1.5 - - htdig/3.1.5 (infosys - - htdig/3.1.5 (root@lo - - htdig/3.1.5 (webmast - - htdig/3.1.6 (mathieu - - htdig/3.1.6 (unconfi - - htdig/3.1.x (root@lo - - http://www.mabontlan - - http://www.monogol.d - - http://www.timelyweb + - free-downloads\.net\sd + - gazz\/x\.x + - geourl\/2\.0b2 + - getpocket\.com + - gigabaz\.com + - github\.com + - gossamer-threads\.com + - guess\.scritch\.org + - haste\.kytoon\.com + - help\.coccoc\.com + - help\.coccoc\.vn + - help\.goo\.ne\.jp + - help\.hepburnhome\.ca + - hgrepurl\/1\.0 + - http:\/\/www\.mabontlan + - http:\/\/www\.monogol\.d + - http:\/\/www\.timelyweb - http_get - - httpunit/1.x - - i.stanford.edu - - iFeed.jp/2.0 (www.ps - - ia_archiver-web.arch - - ia_archiver/1.6 - - ia_archiver/8.0 (Win - - ia_archiver/8.1 (Win - - ia_archiver/8.2 (Win - - ia_archiver/8.8 (Win - - ia_archiver/8.9 (Lin - - ia_archiver/8.9 (Win - - icerocket.com; - - ichiro/2.0 (ichiro@n - - ichiro/x.0 (ichiro@n - - ideare - SignSite/1. - - iearthworm/1.0, iear - - igde.ru - - ilial/Nutch-0.9-dev - - innovationblog.com - - integrity/1.6 - - integrity/5 - - ivia.ucr.edu + - httpunit\/1\.x + - i\.stanford\.edu + - iFeed\.jp\/2\.0 + - icerocket\.com; + - ideare\s-\sSignSite\/1\. + - igde\.ru + - innovationblog\.com + - integrity\/1\.6 + - integrity\/5 + - ivia\.ucr\.edu - jack - - journster.com + - journster\.com - knil - - kulturarw3/0.1 - - larbin (samualt9@big - - larbin_2.1.1 larbin2 - - larbin_2.2.0 (crawl@ - - larbin_2.2.1_de_Vien - - larbin_2.2.2 (sugaya - - larbin_2.2.2_guillau - - larbin_2.5.0 (larbin - - larbin_2.6.0 (larbin - - larbin_2.6.1 (larbin - - larbin_2.6.1 larbin2 - - larbin_2.6.2 (hamasa - - larbin_2.6.2 (larbin - - larbin_2.6.2 (liston - - larbin_2.6.2 (pierre - - larbin_2.6.2 (pimena - - larbin_2.6.2 (tom@le - - larbin_2.6.2 (vitalb - - larbin_2.6.2 dthunen - - larbin_2.6.2 kalou@k - - larbin_2.6.2 larbin2 - - larbin_2.6.2 larbin@ - - larbin_2.6.2 listonA - - larbin_2.6.2 pierre@ - - larbin_2.6.2 vitalbo - - larbin_2.6.3 (larbin - - larbin_2.6.3 (wgao@g - - larbin_2.6.3 capveg@ - - larbin_2.6.3 ghary@s - - larbin_2.6.3 gqnmgsp - - larbin_2.6.3 larbin2 - - larbin_2.6.3 tangyi8 - - larbin_2.6.3 zumesun - - larbin_2.6_basileoca - - larbin_test (nobody@ - - larbin_test nobody@a - - larbin_xy250 larbin2 - - law.di.unimi.it - - lc/$ROADS::Version l - - letscrawl.com - - libWeb/clsHTTP -- hi - - libwww-perl/5.41 - - libwww-perl/5.45 - - libwww-perl/5.48 - - libwww-perl/5.50 - - libwww-perl/5.52 FP/ - - libwww-perl/5.53 - - libwww-perl/5.63 - - libwww-perl/5.64 - - libwww-perl/5.65 - - libwww-perl/5.800 - - libwww/5.3.2 - - linkdex.com/v2.0 - - ltx71.com - - lunefran.wordpress.com - - lwp-trivial/1.32 - - lwp-trivial/1.33 - - lwp-trivial/1.34 - - lwp-trivial/1.35 - - lwp-trivial/1.36 - - lwp-trivial/1.38 - - lwp-trivial/1.41 - - mailto:webcraft@bea. - - mbblog14.wordpress.com - - metatagsdir.com - - miniflux.net - - misc.yahoo.com.cn - - moget/x.x (moget@goo - - mogimogi/1.0 - - moiNAG 0.02 - - muckrack.com + - kulturarw3\/0\.1 + - law\.di\.unimi\.it + - lc\/\$ROADS::Version\sl + - libWeb\/clsHTTP\s--\shi + - linkdex\.com\/v2\.0 + - mailto:webcraft@bea\. + - metatagsdir\.com + - miniflux\.net + - mogimogi\/1\.0 + - moiNAG\s0\.02 + - muckrack\.com - myDaemon - - net-promoter.com - - netarkivet.dk - - newsearchengine (Thi - - newsme/1.0; feedback - - newspaper/0.1.0.7 - - newspaper/0.2.5 - - newspaper/0.2.6 - - newstin.com - - nlp.uned.es + - net-promoter\.com + - netarkivet\.dk + - newsearchengine + - newsme\/1\.0;\sfeedback + - newspaper\/0\.1\.0\.7 + - newspaper\/0\.2\.5 + - newspaper\/0\.2\.6 + - newstin\.com + - nlp\.uned\.es - noyona_0_1 - - nsyght.com/Nutch-1.0 - - nsyght.com/Nutch-x.x - - nutch-1.4/Nutch-1.4 - - nutchsearch/Nutch-0. - - nymesis.com - - oegp v. 1.3.0 - - okhttp/2.5.0 - - okhttp/2.7.5 - - okhttp/3.2.0 - - okhttp/3.5.0 - - omgili.com - - ostermiller.org + - nymesis\.com + - oegp\sv\.\s1\.3\.0 + - omgili\.com + - ostermiller\.org - owler-agent - - ozelot/2.7.3 (Search - - page2rss.com - - pandora.nla.gov.au - - panscient.com - - parsijoo.ir - - pauillac.inria.fr - - pd02_1.0.0 pd02_1.0. - - pear.php.net - - people.eng.unimelb.edu.au - - phpcrawl - - pompos.iliad.fr + - ozelot\/2\.7\.3 + - pandora\.nla\.gov\.au + - pauillac\.inria\.fr + - pd02_1\.0\.0\spd02_1\.0\. + - people\.eng\.unimelb\.edu\.au + - pompos\.iliad\.fr - psycheclone - - publisher.yahoo.com - - python-requests/0.13 - - python-requests/1.0. - - python-requests/2.18 - - python-requests/2.3. - - r.hatena.ne.jp - - rabaz (rabaz at giga - - ranksonic.com - - rico/0.1 - - riddler.io - - rjpower.org - - robot - - sagool.jp - - scooter-venus-3.0.vn + - r\.hatena\.ne\.jp + - rabaz + - ranksonic\.com + - rico\/0\.1 + - rjpower\.org + - sagool\.jp - scoutmaster - - scrapy.org - - search.ch V1.4 - - search.denmex.com - - search.goo.ne.jp - - search.google.com - - search.yahoo.com - - seekda.com - - semanticdiscovery/0. - - sentry.io - - silk/1.0 - - sitebar.org - - siteexplorer.info - - smartwit.com - - sna-0.0.1 (mikemuzio - - sna-0.0.1 mikeelliot - - sohu agent + - scrapy\.org + - search\.ch\sV1\.4 + - search\.denmex\.com + - search\.goo\.ne\.jp + - seekda\.com + - semanticdiscovery\/0\. + - sentry\.io + - silk\/1\.0 + - sitebar\.org + - siteexplorer\.info + - smartwit\.com + - sna-0\.0\.1 + - sna-0\.0\.1\smikeelliot + - sohu\sagent - sohu-search - - sp.ask.com - - suchbaer.de - - summify.com - - support.embed.ly + - sp\.ask\.com + - suchbaer\.de + - support\.embed\.ly - surf - suzuran - - szukaj.onet.pl - - tags2dir.com - - tagword.com - - targetblaster.com/0. - - teoma_agent1 - - therarestwords.com - - toCrawl/UrlDispatche - - tuezilla.de - - twikle.com - - twurly.org - - unknownght.com Web S - - users.cs.cf.ac.uk - - versus.integis.ch - - vias.ncsa.uiuc.edu - - visionutils/0.2 - - vk.com - - voyager-hc/1.0 - - voyager/1.0 - - web-bekannt (Version - - web2express.org - - webGobbler/1.x.x - - webagent.wise-guys.nl - - webalgo.iit.cnr.it - - webbandit/4.xx.0 - - webcollage/1.114 - - webcollage/1.117 - - webcollage/1.125 - - webcollage/1.129 - - webcollage/1.93 - - webcollage/1.xx - - webcrawl.net - - webdatastats.com - - webglimpse.net + - szukaj\.onet\.pl + - tags2dir\.com + - tagword\.com + - targetblaster\.com\/0\. + - therarestwords\.com + - tuezilla\.de + - twikle\.com + - twurly\.org + - unknownght\.com\sWeb\sS + - users\.cs\.cf\.ac\.uk + - versus\.integis\.ch + - vias\.ncsa\.uiuc\.edu + - visionutils\/0\.2 + - vk\.com + - voyager-hc\/1\.0 + - web-bekannt + - web2express\.org + - webGobbler\/1\.x\.x + - webagent\.wise-guys\.nl + - webalgo\.iit\.cnr\.it + - webbandit\/4\.xx\.0 + - webdatastats\.com + - webglimpse\.net - webhack - - websearchbench.cs.uni-dortmund.de - - websitealert.net - - webster-internet.de - - webster.healeys.net - - whatUseek_winona/3.0 - - wiki.creativecommons.org - - wish.slis.tsukuba.ac.jp - - wkd.iis.sinica.edu.tw - - wortschatz.uni-leipzig.de - - ws.daum.net - - www.Abolimba.de - - www.WISEnut.com - - www.abonti.com - - www.ackerm.com - - www.activtourist.com - - www.agada.de - - www.ahmadsoftware.co - - www.answerbus.com - - www.ant.com - - www.aol.com - - www.aramabeta.com; - - www.arianna.it - - www.arocom.de - - www.asaha.com - - www.axxus.de - - www.ay-up.com - - www.ba.be - - www.blaiz.net - - www.bloglines.com - - www.bortzmeyer.org - - www.brandverity.com - - www.buddybuzz.net - - www.buscaplus.com - - www.business-socket. - - www.canadiancontent.net - - www.chepi.net - - www.chilkatsoft.com - - www.cloudflare.com - - www.cnet.com - - www.cougarsearch.com - - www.covario.com - - www.crim.ca - - www.cs.washington.edu - - www.cs.washington.edu; - - www.dataparksearch.org - - www.de.com - - www.dead-links.com - - www.die-kraehe.de - - www.dnsdigger.com - - www.earthcom.info - - www.eurekster.com - - www.europarchive.org - - www.eventax.de - - www.everatom.com - - www.evri.com - - www.exensa.com - - www.exif-search.com - - www.facebook.com - - www.favcollector.com - - www.feeddemon.com - - www.feedhub.com - - www.feedly.com - - www.feedmap.net - - www.fileboost.net - - www.findpal.com - - www.floorstransformed.com - - www.flyindex.net - - www.followsite.com - - www.genieo.com - - www.globalspec.com - - www.goguides.org - - www.google.com - - www.greaterera.com - - www.hermitsearch.com - - www.htmlvalidator.com - - www.hyperix.com - - www.ianett.com - - www.icerocket.com - - www.illumit.com - - www.iltrovatore.it - - www.inktomi.com - - www.intelliseek.com - - www.ip2location.com - - www.iskanie.com - - www.jordomedia.com - - www.kalooga.com; - - www.kototoi.org - - www.l3s.de - - www.lan4lano.net - - www.lipperhey.com - - www.look.com - - www.lw01.com - - www.matuschek.net - - www.megaindex.ru - - www.megite.com - - www.meta-spinner.de - - www.mixdata.com - - www.moreover.com - - www.neomo.de - - www.netluchs.de - - www.newsgator.com; - - www.ng-search.com - - www.page-store.com - - www.pagebull.com - - www.pingalink.com - - www.pingdom.com - - www.plinki.com; - - www.powerset.com - - www.prairielandconsulting.com - - www.proximic.com - - www.pubblisito.com - - www.qango.com - - www.quickiwiki.com; - - www.qwant.com - - www.rawgrunt.com - - www.relsoftware.com - - www.retrevo.com - - www.samsung.com - - www.scifihifi.com - - www.scoutjet.com - - www.scrubtheweb.com - - www.searchme.com - - www.securecomputing.com - - www.semager.de - - www.shopwiki.com - - www.siigle.com - - www.simpy.com - - www.sitesearch.ca - - www.sitesell.com - - www.sixxs.net - - www.skaffe.com - - www.skyrocket.gr.jp - - www.sli-systems.com - - www.slider.com - - www.slysearch.com - - www.somusical.com - - www.suchen.de - - www.sygol.com - - www.sync2it.com - - www.sysomos.com - - www.targetgroups.net - - www.teradex.com - - www.tkensaku.com - - www.truveo.com - - www.tumblr.com - - www.ukwizz.com - - www.unchaos.com - - www.updated.com; - - www.urltrends.com - - www.vakes.com - - www.versions-project.org - - www.verzamelgids.nl - - www.wavefire.com; - - www.webarchiv.cz - - www.worio.com - - www.wowlemmings.com - - www.xirq.com; - - www.xrayz.co.uk - - www.yama.info.waseda.ac.jp - - www.yoono.com - - www.zedzo.com - - www4mail/2.x libwww- - - wwwster/1.x (Beta- m + - websearchbench\.cs\.uni-dortmund\.de + - websitealert\.net + - webster-internet\.de + - webster\.healeys\.net + - whatUseek_winona\/3\.0 + - wiki\.creativecommons\.org + - wish\.slis\.tsukuba\.ac\.jp + - wkd\.iis\.sinica\.edu\.tw + - wortschatz\.uni-leipzig\.de + - ws\.daum\.net + - www\.Abolimba\.de + - www\.WISEnut\.com + - www\.abonti\.com + - www\.ackerm\.com + - www\.activtourist\.com + - www\.agada\.de + - www\.ahmadsoftware\.co + - www\.answerbus\.com + - www\.ant\.com + - www\.aol\.com + - www\.aramabeta\.com; + - www\.arianna\.it + - www\.arocom\.de + - www\.asaha\.com + - www\.axxus\.de + - www\.ay-up\.com + - www\.ba\.be + - www\.blaiz\.net + - www\.bortzmeyer\.org + - www\.brandverity\.com + - www\.buddybuzz\.net + - www\.buscaplus\.com + - www\.business-socket\. + - www\.canadiancontent\.net + - www\.chepi\.net + - www\.chilkatsoft\.com + - www\.cloudflare\.com + - www\.cnet\.com + - www\.cougarsearch\.com + - www\.covario\.com + - www\.crim\.ca + - www\.cs\.washington\.edu + - www\.cs\.washington\.edu; + - www\.de\.com + - www\.dead-links\.com + - www\.die-kraehe\.de + - www\.dnsdigger\.com + - www\.earthcom\.info + - www\.eurekster\.com + - www\.europarchive\.org + - www\.eventax\.de + - www\.everatom\.com + - www\.evri\.com + - www\.exensa\.com + - www\.exif-search\.com + - www\.facebook\.com + - www\.favcollector\.com + - www\.feeddemon\.com + - www\.feedhub\.com + - www\.feedly\.com + - www\.feedmap\.net + - www\.fileboost\.net + - www\.findpal\.com + - www\.floorstransformed\.com + - www\.flyindex\.net + - www\.followsite\.com + - www\.globalspec\.com + - www\.goguides\.org + - www\.greaterera\.com + - www\.hermitsearch\.com + - www\.hyperix\.com + - www\.ianett\.com + - www\.icerocket\.com + - www\.illumit\.com + - www\.iltrovatore\.it + - www\.inktomi\.com + - www\.intelliseek\.com + - www\.ip2location\.com + - www\.jordomedia\.com + - www\.kalooga\.com; + - www\.kototoi\.org + - www\.l3s\.de + - www\.lan4lano\.net + - www\.look\.com + - www\.lw01\.com + - www\.matuschek\.net + - www\.megaindex\.ru + - www\.meta-spinner\.de + - www\.mixdata\.com + - www\.moreover\.com + - www\.neomo\.de + - www\.newsgator\.com; + - www\.ng-search\.com + - www\.page-store\.com + - www\.pagebull\.com + - www\.pingalink\.com + - www\.pingdom\.com + - www\.plinki\.com; + - www\.powerset\.com + - www\.prairielandconsulting\.com + - www\.pubblisito\.com + - www\.qango\.com + - www\.quickiwiki\.com; + - www\.qwant\.com + - www\.rawgrunt\.com + - www\.relsoftware\.com + - www\.retrevo\.com + - www\.samsung\.com + - www\.scifihifi\.com + - www\.scrubtheweb\.com + - www\.searchme\.com + - www\.securecomputing\.com + - www\.semager\.de + - www\.shopwiki\.com + - www\.siigle\.com + - www\.simpy\.com + - www\.sitesearch\.ca + - www\.sitesell\.com + - www\.sixxs\.net + - www\.skaffe\.com + - www\.skyrocket\.gr\.jp + - www\.sli-systems\.com + - www\.slider\.com + - www\.slysearch\.com + - www\.somusical\.com + - www\.suchen\.de + - www\.sygol\.com + - www\.sync2it\.com + - www\.targetgroups\.net + - www\.teradex\.com + - www\.tkensaku\.com + - www\.truveo\.com + - www\.tumblr\.com + - www\.ukwizz\.com + - www\.unchaos\.com + - www\.updated\.com; + - www\.urltrends\.com + - www\.vakes\.com + - www\.versions-project\.org + - www\.verzamelgids\.nl + - www\.wavefire\.com; + - www\.webarchiv\.cz + - www\.worio\.com + - www\.wowlemmings\.com + - www\.xirq\.com; + - www\.xrayz\.co\.uk + - www\.yama\.info\.waseda\.ac\.jp + - www\.yoono\.com + - www\.zedzo\.com + - wwwster\/1\.x - xmlset_roodkcableoj2 - - yacy (www.yacy.net; - - yarienavoir.net/0.2 - - yggdrasil/Nutch-0.9 - - yoogliFetchAgent/0.1 - - Googlebot\/ - - Googlebot-Mobile - - Googlebot-Image - - Googlebot-News - - Googlebot-Video - - AdsBot-Google([^-]|$) - - AdsBot-Google-Mobile - - Feedfetcher-Google - - Mediapartners-Google - - Mediapartners \(Googlebot\) - - APIs-Google - - Google-InspectionTool - - Storebot-Google - - GoogleOther - - bingbot - - Slurp - - [wW]get - - LinkedInBot - - Python-urllib - - python-requests + - yarienavoir\.net\/0\.2 + - yoogliFetchAgent\/0\.1 + - Mediapartners\s\\ + - \[wW]get - aiohttp - - httpx - - libwww-perl - httpunit - - nutch - Go-http-client - - phpcrawl - - msnbot - - jyxobot - - FAST-WebCrawler - - FAST Enterprise Crawler - - BIGLOTRON - - Teoma - - convera - - seekbot - - Gigabot - Gigablast - - exabot - - ia_archiver - - GingerCrawler - - webmon - - HTTrack - - grub.org - - UsineNouvelleCrawler - - antibot - netresearchserver - - speedy - fluffy - findlink - - msrbot - - panscient - - yacybot - - AISearchBot - ips-agent - - tagoobot - - MJ12bot - - woriobot - yanga - - buzzbot - - mlbot - - yandex\.com\/bots - - purebot - - Linguee Bot - CyberPatrol - - voilabot - - Baiduspider - - citeseerxbot - - spbot - - twengabot - postrank - - Turnitin - - scribdbot - - page2rss - - sitebot - linkdex - - Adidxbot - ezooms - - dotbot - - Mail.RU_Bot - - discobot - - heritrix - - findthatfile - - europarchive.org - - NerdByNature.Bot - - sistrix crawler - - Ahrefs(Bot|SiteAudit) - - fuelbot - - CrunchBot - - IndeedBot + - europarchive\.org + - Ahrefs - mappydata - - woobot - - ZoominfoBot - - PrivacyAwareBot - - Multiviewbot - - SWIMGBot - - Grobbot - eright - Apercite - - semanticbot - Aboundex - - domaincrawler - - wbsearchbot - - summify - - CCBot - - edisterbot - - SeznamBot - - ec2linkfinder - - gslfbot - - aiHitBot - - intelium_bot - facebookexternalhit - Yeti - RetrevoPageAnalyzer - - lb-spider - - Sogou - - lssbot - - careerbot - wotbox - - wocbot - - ichiro - - DuckDuckBot - - lssrocketcrawler - drupact - - webcompanycrawler - - acoonbot - - openindexspider - - gnam gnam spider - - web-archive-net.com.bot - - backlinkcrawler - coccoc - integromedb - - content crawler spider - - toplistbot - - it2media-domain-crawler - - ip-web-crawler.com - - siteexplorer.info - - elisabot - - proximic - changedetection - - arabot - - WeSEE:Search - - niki-bot - - CrystalSemanticsBot - - rogerbot - - 360Spider - - psbot - - InterfaxScanBot - - CC Metadata Scaper - - g00g1e.net - - GrapeshotCrawler - - urlappendbot - - brainobot - - fr-crawler - - binlar - - SimpleCrawler - - Twitterbot - - cXensebot - - smtbot - - bnf.fr_bot - - A6-Indexer - - ADmantX - - Facebot - - OrangeBot\/ - - memorybot - - AdvBot + - CC\sMetadata\sScaper + - g00g1e\.net - MegaIndex - - SemanticScholarBot - - ltx71 - - nerdybot - - xovibot - - BUbiNG - - Qwantify - - archive.org_bot - - Applebot - - TweetmemeBot - - crawler4j - - findxbot - - S[eE][mM]rushBot - - yoozBot - - lipperhey - - Y!J - - Domain Re-Animator Bot - - AddThis - - Screaming Frog SEO Spider - MetaURI - Scrapy - - Livelap[bB]ot - - OpenHoseBot + - Livelap\[bB]ot - CapsuleChecker - - collection@infegy.com - - IstellaBot - - DeuSu\/ - - betaBot - - Cliqzbot\/ - - MojeekBot\/ - - netEstate NE Crawler - - SafeSearch microdata crawler - - Gluten Free Crawler\/ + - collection@infegy\.com + - DeuSu\\/ - Sonic - - Sysomos - - Trove - deadlinkchecker - Slack-ImgProxy - - Embedly - - RankActiveLinkBot - - iskanie - - SafeDNSBot - - SkypeUriPreview - - Veoozbot - - Slackbot - - redditbot - - datagnionbot - - Google-Adwords-Instant - - adbeat_bot - - WhatsApp - - contxbot - - pinterest.com.bot - electricmonk - - GarlikCrawler - - BingPreview\/ - - vebidoobot - - FemtosearchBot - - Yahoo Link Preview - - MetaJobBot - - DomainStatsBot - - mindUpBot - - Daum\/ - - Jugendschutzprogramm-Crawler - - Xenu Link Sleuth - - Pcore-HTTP - - moatbot - - KosmioBot - - [pP]ingdom + - Daum\\/ + - \[pP]ingdom - AppInsights - PhantomJS - - Gowikibot - - PiplBot - - Discordbot - - TelegramBot - Jetslide - newsharecounts - - James BOT - - Bark[rR]owler - - TinEye - - SocialRankIOBot - - trendictionbot - - Ocarinabot - - epicbot - - Primalbot - - DuckDuckGo-Favicons-Bot - - GnowitNewsbot - - Leikibot + - Bark\[rR]owler - LinkArchiver - - YaK\/ - - PaperLiBot - - Digg Deeper - - dcrawl + - YaK\\/ + - Digg\sDeeper - Snacktory - - AndersPinkBot - - Fyrebot - - EveryoneSocialBot - - Mediatoolkitbot - - Luminator-robots - - ExtLinksBot - - SurveyBot - - NING\/ - - okhttp - - Nuzzel + - NING\\/ - omgili - PocketParser - - YisouSpider - um-LN - - ToutiaoSpider - MuckRack - - Jamie's Spider - - AHC\/ - - NetcraftSurveyAgent - - Laserlikebot - - ^Apache-HttpClient - - AppEngine-Google + - AHC\\/ + - \^Apache-HttpClient - Jetty - Upflow - - Thinklab - - Traackr.com - Twurly - Mastodon - - http_get - - DnyzBot - - botify - - 007ac9 Crawler - - BehloolBot - BrandVerity - check_http - - BDCbot - - ZumBot - EZID - - ICC-Crawler - - ArchiveBot - - ^LCC - - filterdb.iss.net\/crawler - - BLP_bbot - - BomboraBot - - Buck\/ - - Companybook-Crawler - - Genieo - - magpie-crawler + - \^LCC + - Buck\\/ - MeltwaterNews - Moreover - - newspaper\/ - - ScoutJet - - (^| )sentry\/ - - StorygizeBot - - UptimeRobot - - OutclicksBot + - newspaper\\/ + - - seoscanners - Hatena - - Google Web Preview - - MauiBot - - AlphaBot - - SBL-BOT - - IAS crawler - adscanner - Netvibes - - acapbot - - Baidu-YunGuanCe - - bitlybot - - blogmuraBot - - Bot.AraTurka.com - - bot-pge.chlooe.com - - BoxcarBot - BTWebClient - - ContextAd Bot - - Digincore bot - Disqus - Feedly - - Fetch\/ + - Fetch\\/ - Fever - Flamingo_SearchEngine - FlipboardProxy - - g2reader-bot - - G2 Web Services - - imrbot - - K7MLWCBot - - Kemvibot - - Landau-Media-Spider - - linkapediabot + - G2\sWeb\sServices - vkShare - - Siteimprove.com - - BLEXBot\/ + - Siteimprove\.com - DareBoost - - ZuperlistBot\/ - - Miniflux\/ + - Miniflux\\/ - Feedspot - - Diffbot\/ - SEOkicks - tracemyfile - - Nimbostratus-Bot - zgrab - - PR-CY.RU - - AdsTxtCrawler + - PR-CY\.RU - Datafeedwatch - - Zabbix - - TangibleeBot - - google-xrawler - axios - - Amazon CloudFront - Pulsepoint - CloudFlare-AlwaysOnline - - Google-Structured-Data-Testing-Tool - WordupInfoSearch - WebDataStats - HttpUrlConnection - - Seekport Crawler - - ZoomBot - - VelenPublicWebCrawler - - MoodleBot - - jpg-newsbot - outbrain - - W3C_Validator - - Validator\.nu - - W3C-checklink - W3C-mobileOK - W3C_I18n-Checker - - FeedValidator - - W3C_CSS_Validator - W3C_Unicorn - - Google-PhysicalWeb - Blackboard - - ICBot\/ - BazQux - Twingly - Rivva - - Experibot - - awesomecrawler - - Dataprovider.com - - GroupHigh\/ - - theoldreader.com + - Dataprovider\.com + - GroupHigh\\/ + - theoldreader\.com - AnyEvent - - Uptimebot\.org - - Nmap Scripting Engine - - 2ip.ru + - Nmap\sScripting\sEngine + - 2ip\.ru - Clickagy - - Caliperbot - - MBCrawler - - online-webceo-bot - - B2B Bot - - AddSearchBot - - Google Favicon - HubSpot - Chrome-Lighthouse - - HeadlessChrome - - CheckMarkNetwork\/ - - www\.uptime\.com - - Streamline3Bot\/ - - serpstatbot\/ - - MixnodeCache\/ - - ^curl + - CheckMarkNetwork\\/ + - www\\\.uptime\\\.com + - MixnodeCache\\/ + - \^curl - SimpleScraper - - RSSingBot - - Jooblebot - fedoraplanet - Friendica - NextCloud - - Tiny Tiny RSS - - RegionStuttgartBot - - Bytespider - Datanyze - - Google-Site-Verification - TrendsmapResolver - tweetedtimes - - NTENTbot - Gwene - SimplePie - SearchAtlas - Superfeedr - - feedbot - - UT-Dorkbot - - Amazonbot - - SerendeputyBot - - Eyeotabot - - officestorebot - - Neticle Crawler - - SurdotlyBot - - LinkisBot - - AwarioSmartBot - - AwarioRssBot - - RyteBot - - FreeWebMonitoring SiteChecker - - AspiegelBot - - NAVER Blog Rssbot - - zenback bot - - SentiBot - - Domains Project\/ + - Domains\sProject\\/ - Pandalytics - - VKRobot - - bidswitchbot - - tigerbot - - NIXStatsbot - - Atom Feed Robot - - [Cc]urebot - - PagePeeker\/ - - Vigil\/ - - rssbot\/ - - startmebot\/ - - JobboerseBot + - PagePeeker\\/ + - Vigil\\/ - seewithkids - - NINJA bot - - Cutbot - - BublupBot - - BrandONbot - - RidderBot - - Taboolabot - - Dubbotbot - - FindITAnswersbot - - infoobot - - Refindbot - - BlogTraffic\/\d\.\d+ Feed-Fetcher - - SeobilityBot + - BlogTraffic\\/\\d\\\.\\d\+\sFeed-Fetcher - Cincraw - - Dragonbot - - VoluumDSP-content-bot - - FreshRSS - - BitBot - - ^PHP-Curl-Class - - Google-Certificates-Bridge - - centurybot + - \^PHP-Curl-Class - Viber - - e\.ventures Investment Crawler - evc-batch - - PetalBot - virustotal - - (^| )PTST\/ - - minicrawler - - Cookiebot - - trovitBot - - seostar\.co - - IonCrawl + - seostar\\\.co - Uptime-Kuma - - SeekportBot - - FreshpingBot - - Feedbin - - CriteoBot - - Snap URL Preview Service - - Better Uptime Bot + - Snap\sURL\sPreview\sService - RuxitSynthetic - - Google-Read-Aloud - - Valve\/Steam - - OdklBot\/ - - GPTBot - - YandexRenderResourcesBot\/ - - LightspeedSystemsCrawler - - ev-crawler\/ - - BitSightBot\/ - - woorankreview\/ - - Google-Safety - - AwarioBot - - DataForSeoBot - - Linespider - - WellKnownBot - - A Patent Crawler - - StractBot - - search\.marginalia\.nu - - YouBot - - Nicecrawler - - Neevabot - - BrightEdge Crawler - - SiteCheckerBotCrawler - - TombaPublicWebCrawler - - CrawlyProjectCrawler - - KomodiaBot - - KStandBot - - CISPA Webcrawler - - MTRobot - - hyscore.io - - AlexandriaOrgBot - - 2ip bot - - Yellowbrandprotectionbot + - Valve\\/Steam + - woorankreview\\/ + - search\\\.marginalia\\\.nu - SEOlizer - - vuhuvBot - - INETDEX-BOT - Synapse - - t3versionsBot - deepnoc - - Cocolyzebot - hypestat - - ReverseEngineeringBot - - sempi.tech + - sempi\.tech - Iframely - - MetaInspector - node-fetch - lkxscan - - python-opengraph - OpenGraphCheck - - developers.google.com\/\+\/web\/snippet - - SenutoBot - - MaCoCu - NewsBlur - inoreader - NetSystemsResearch - PageThing - - WordPress\/ - Added `addition_date` and `instances` for the new items where possible. - Executed and updated the convert_to_txt to write the file in the `generated/` directory diff --git a/COUNTER_Robots_list.json b/COUNTER_Robots_list.json index 03c01c8..e7fb5f1 100644 --- a/COUNTER_Robots_list.json +++ b/COUNTER_Robots_list.json @@ -518,11 +518,7 @@ "pattern": "Genieo", "last_changed": "2018-09-05", "Listed as a bad bot by Distil Networks": "2018-09-05", - "url": "http://www.genieo.com/webfilter.html", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" - ] + "url": "https://www.distilnetworks.com/bot-directory/bot/genieo/" }, { "pattern": "GetRight", @@ -535,7 +531,7 @@ { "pattern": "GigablastOpenSource", "last_changed": "2020-07-02" - }, + }, { "pattern": "G-i-g-a-b-o-t", "last_changed": "2018-02-15", @@ -1049,7 +1045,7 @@ { "pattern": "OgScrper", "last_changed": "2020-07-02" - }, + }, { "pattern": "okhttp", "last_changed": "2020-02-25", @@ -1612,19 +1608,15 @@ "last_changed": "2024-01-02" }, { - "pattern": "*/Nutch-0.9-dev", - "last_changed": "2024-01-02" - }, - { - "pattern": "192.comAgent", + "pattern": "192\\.comAgent", "last_changed": "2024-01-02" }, { - "pattern": "222.122.15.190", + "pattern": "222\\.122\\.15\\.190", "last_changed": "2024-01-02" }, { - "pattern": "2Bone_LinkChkr/1.0 l", + "pattern": "2Bone_LinkChkr\\/1\\.0\\sl", "last_changed": "2024-01-02" }, { @@ -1632,7 +1624,7 @@ "last_changed": "2024-01-02" }, { - "pattern": "8484 Boston Project ", + "pattern": "8484\\sBoston\\sProject", "last_changed": "2024-01-02" }, { @@ -1640,9281 +1632,4252 @@ "last_changed": "2024-01-02" }, { - "pattern": "A-Online Search", + "pattern": "A-Online\\sSearch", "last_changed": "2024-01-02" }, { - "pattern": "ADmantX Platform Sem", + "pattern": "AHC\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "AHC/2.0", + "pattern": "ASPSeek\\/1\\.2\\.5", "last_changed": "2024-01-02" }, { - "pattern": "ASPSeek/1.2.5", + "pattern": "ASPSeek\\/1\\.2\\.x", "last_changed": "2024-01-02" }, { - "pattern": "ASPSeek/1.2.x", + "pattern": "ASPseek\\/1\\.2\\.9d", "last_changed": "2024-01-02" }, { - "pattern": "ASPSeek/1.2.xa", + "pattern": "ASSORT\\/0\\.10", "last_changed": "2024-01-02" }, { - "pattern": "ASPSeek/1.2.xxpre", + "pattern": "AV\\sFetch\\s1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "ASPseek/1.2.9d", + "pattern": "AVSearch-1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "ASPseek/1.2.xx", + "pattern": "AVSearch-2\\.0-fusionI", "last_changed": "2024-01-02" }, { - "pattern": "ASSORT/0.10", + "pattern": "AVSearch-3\\.0", "last_changed": "2024-01-02" }, { - "pattern": "AV Fetch 1.0", + "pattern": "Accoona-AI-Agent\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "AVSearch-1.0(peter.t", + "pattern": "ActiveBookmark\\s1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "AVSearch-2.0-fusionI", + "pattern": "AdobeUxTechC4-Async\\/", "last_changed": "2024-01-02" }, { - "pattern": "AVSearch-3.0(AltaVis", + "pattern": "Aladin\\/3\\.324", "last_changed": "2024-01-02" }, { - "pattern": "About/0.1libwww-perl", + "pattern": "Allrati\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Accoona-AI-Agent/1.1", + "pattern": "AltaVista\\sIntranet\\sV", "last_changed": "2024-01-02" }, { - "pattern": "Acorn/Nutch-0.9 (Non", - "last_changed": "2024-01-02" + "pattern": "Amazon\\sCloudFront", + "last_changed": "2024-01-02", + "addition_date": "2018-09-07", + "instances": [ + "Amazon CloudFront" + ], + "url": "https://aws.amazon.com/cloudfront/" }, { - "pattern": "ActiveBookmark 1.x", + "pattern": "AmiTCP\\sMiami", "last_changed": "2024-01-02" }, { - "pattern": "AdobeUxTechC4-Async/", + "pattern": "Anonymizer\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "AgentName/0.1 libwww", + "pattern": "AnswerChase\\sPROve\\sx\\.", "last_changed": "2024-01-02" }, { - "pattern": "AideRSS/1.0 (aiderss", + "pattern": "AnswerChase\\sx\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Aladin/3.324", + "pattern": "Apache", "last_changed": "2024-01-02" }, { - "pattern": "Allesklar/0.1 libwww", + "pattern": "Apache-HttpAsyncClie", "last_changed": "2024-01-02" }, { - "pattern": "Allrati/1.1 (+)", + "pattern": "Apache-HttpClient\\/4\\.", "last_changed": "2024-01-02" }, { - "pattern": "AltaVista Intranet V", + "pattern": "Apache-HttpClient\\/UN", "last_changed": "2024-01-02" }, { - "pattern": "Amazon CloudFront", - "last_changed": "2024-01-02", - "addition_date": "2018-09-07", - "instances": [ - "Amazon CloudFront" - ], - "url": "https://aws.amazon.com/cloudfront/" + "pattern": "Aport", + "last_changed": "2024-01-02" }, { - "pattern": "AmiTCP Miami (AmigaO", + "pattern": "Arachnoidea", "last_changed": "2024-01-02" }, { - "pattern": "Anonymizer/1.1", + "pattern": "Asahina-Antenna\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "AnswerChase PROve x.", + "pattern": "Ask\\.24x\\.Info", "last_changed": "2024-01-02" }, { - "pattern": "AnswerChase x.0", + "pattern": "Atomic_Email_Hunter\\/", "last_changed": "2024-01-02" }, { - "pattern": "AnzwersCrawl/2.0 (an", + "pattern": "BMCLIENT", "last_changed": "2024-01-02" }, { - "pattern": "Apache (internal dum", + "pattern": "BMLAUNCHER", "last_changed": "2024-01-02" }, { - "pattern": "Apache-HttpAsyncClie", + "pattern": "BPImageWalker\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Apache-HttpClient/4.", + "pattern": "BSDSeek\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Apache-HttpClient/UN", + "pattern": "BStop\\.BravoBrian\\.it", "last_changed": "2024-01-02" }, { - "pattern": "Aport", + "pattern": "BTWebClient\\/180B", "last_changed": "2024-01-02" }, { - "pattern": "Arachnoidea (arachno", + "pattern": "BeamMachine\\/0\\.5", "last_changed": "2024-01-02" }, { - "pattern": "Asahina-Antenna/1.x", + "pattern": "BeebwareDirectory\\/v0", "last_changed": "2024-01-02" }, { - "pattern": "Asahina-Antenna/1.x ", + "pattern": "Big\\sFish\\sv1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Ask.24x.Info", + "pattern": "BigBrother\\/1\\.6e", "last_changed": "2024-01-02" }, { - "pattern": "Atomic_Email_Hunter/", + "pattern": "Bilbo\\/2\\.3b-UNIX", "last_changed": "2024-01-02" }, { - "pattern": "Atomz/1.0", + "pattern": "BlackBerry9000\\/4\\.6\\.0", "last_changed": "2024-01-02" }, { - "pattern": "BIGLOTRON (Beta 2;GN", + "pattern": "BlackWidow", "last_changed": "2024-01-02" }, { - "pattern": "BMCLIENT", + "pattern": "Blackboard\\sSafeassig", "last_changed": "2024-01-02" }, { - "pattern": "BMLAUNCHER", + "pattern": "Bobby\\/4\\.0\\.x\\sRPT-HTTP", "last_changed": "2024-01-02" }, { - "pattern": "BPImageWalker/2.0 (w", + "pattern": "Bookdog\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "BSDSeek/1.0", + "pattern": "BravoBrian\\sbstop\\.bra", "last_changed": "2024-01-02" }, { - "pattern": "BStop.BravoBrian.it ", + "pattern": "BullsEye", "last_changed": "2024-01-02" }, { - "pattern": "BTWebClient/180B(970", + "pattern": "BunnySlippers", "last_changed": "2024-01-02" }, { - "pattern": "BeamMachine/0.5 (dea", + "pattern": "CHttpClient\\sby\\sOpen", "last_changed": "2024-01-02" }, { - "pattern": "BeebwareDirectory/v0", + "pattern": "CLIPish\\sPro2\\s10\\.21", "last_changed": "2024-01-02" }, { - "pattern": "Big Fish v1.0", + "pattern": "COAST\\sWebMaster\\sPro\\/", "last_changed": "2024-01-02" }, { - "pattern": "BigBrother/1.6e", + "pattern": "CamelHttpStream\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Bilbo/2.3b-UNIX", + "pattern": "Cancer\\sInformation\\sa", "last_changed": "2024-01-02" }, { - "pattern": "BlackBerry9000/4.6.0", + "pattern": "Chat\\sCatcher\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "BlackWidow", + "pattern": "China\\sLocal\\sBrowse\\s2", "last_changed": "2024-01-02" }, { - "pattern": "Blackboard Safeassig", + "pattern": "Chitika\\sContentHit\\s1", "last_changed": "2024-01-02" }, { - "pattern": "BlogPulseLive (suppo", + "pattern": "Claymont\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Blogpulse (info@blog", + "pattern": "Contact", "last_changed": "2024-01-02" }, { - "pattern": "Bobby/4.0.x RPT-HTTP", + "pattern": "Cookie%20Stumbler\\/22", "last_changed": "2024-01-02" }, { - "pattern": "Bookdog/x.x", + "pattern": "CyberSpyder\\sLink\\sTes", "last_changed": "2024-01-02" }, { - "pattern": "BravoBrian bstop.bra", + "pattern": "DBrowse\\s1\\.4b", "last_changed": "2024-01-02" }, { - "pattern": "BullsEye", + "pattern": "DBrowse\\s1\\.4d", "last_changed": "2024-01-02" }, { - "pattern": "BunnySlippers", + "pattern": "DTAAgent", "last_changed": "2024-01-02" }, { - "pattern": "CB/Nutch-1.7", + "pattern": "Dart\\sCommunications", "last_changed": "2024-01-02" }, { - "pattern": "CFNetwork/x.x", + "pattern": "DataFountains\\/DMOZ\\sD", "last_changed": "2024-01-02" }, { - "pattern": "CHttpClient by Open ", + "pattern": "Datafeedwatch\\/2\\.1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "CLIPish Pro2 10.21 (", + "pattern": "Deep\\sLink\\sCalculator", "last_changed": "2024-01-02" }, { - "pattern": "COAST WebMaster Pro/", + "pattern": "DeepIndex", "last_changed": "2024-01-02" }, { - "pattern": "CamelHttpStream/1.0", + "pattern": "DeepIndexer\\.ca", "last_changed": "2024-01-02" }, { - "pattern": "Cancer Information a", + "pattern": "DevComponents\\.com\\sHt", "last_changed": "2024-01-02" }, { - "pattern": "Chat Catcher/1.0", + "pattern": "Diamond\\/x\\.0", "last_changed": "2024-01-02" }, { - "pattern": "China Local Browse 2", + "pattern": "DigOut4U", "last_changed": "2024-01-02" }, { - "pattern": "Chitika ContentHit 1", + "pattern": "Digger\\/1\\.0\\sJDK\\/1\\.3\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Claymont.com", + "pattern": "Disqus\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "ColdFusion (Bookmark", + "pattern": "DittoSpyder", "last_changed": "2024-01-02" }, { - "pattern": "Combine/x.0", + "pattern": "EARTHCOM\\.info\\/1\\.x\\s\\[w", "last_changed": "2024-01-02" }, { - "pattern": "Contact", + "pattern": "EARTHCOM\\.info\\/1\\.xbet", "last_changed": "2024-01-02" }, { - "pattern": "Cookie%20Stumbler/22", + "pattern": "EBrowse\\s1\\.4b", "last_changed": "2024-01-02" }, { - "pattern": "CrawlConvera0.1 (Cra", + "pattern": "EchO!\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Custo x.x (www.netwu", + "pattern": "Educate\\sSearch\\sVxB", "last_changed": "2024-01-02" }, { - "pattern": "CyberSpyder Link Tes", + "pattern": "EldoS\\sTimelyWeb\\/3\\.x", "last_changed": "2024-01-02" }, { - "pattern": "DBrowse 1.4b", + "pattern": "Enfish\\sTracker", "last_changed": "2024-01-02" }, { - "pattern": "DBrowse 1.4d", + "pattern": "Enterprise_Search\\/1\\.", "last_changed": "2024-01-02" }, { - "pattern": "DTAAgent", + "pattern": "Evernote\\sClip\\sResolv", "last_changed": "2024-01-02" }, { - "pattern": "Dart Communications ", + "pattern": "Exalead\\sNG\\/MimeLive", "last_changed": "2024-01-02" }, { - "pattern": "DataCha0s/2.0", + "pattern": "Expired\\sDomain\\sSleut", "last_changed": "2024-01-02" }, { - "pattern": "DataFountains/DMOZ D", + "pattern": "ExtractorPro", "last_changed": "2024-01-02" }, { - "pattern": "Datafeedwatch/2.1.x", + "pattern": "EyeCatcher", "last_changed": "2024-01-02" }, { - "pattern": "Deep Link Calculator", + "pattern": "FARK\\.com\\slink\\sverifi", "last_changed": "2024-01-02" }, { - "pattern": "DeepIndex", + "pattern": "FLATARTS_FAVICO", "last_changed": "2024-01-02" }, { - "pattern": "DeepIndex (www.en.de", + "pattern": "FaEdit\\/2\\.0\\.x", "last_changed": "2024-01-02" }, { - "pattern": "DeepIndexer.ca", + "pattern": "Favorites\\sSweeper\\sv\\.", "last_changed": "2024-01-02" }, { - "pattern": "DevComponents.com Ht", + "pattern": "Feed24\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Diamond/x.0", + "pattern": "Filtrbox\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "DigOut4U", + "pattern": "FindAnISP\\.com_ISP_Fi", "last_changed": "2024-01-02" }, { - "pattern": "Digger/1.0 JDK/1.3.0", + "pattern": "Firefly\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Disqus/1.0", + "pattern": "Firefox", "last_changed": "2024-01-02" }, { - "pattern": "DittoSpyder", + "pattern": "Firefox_1\\.0\\.6", "last_changed": "2024-01-02" }, { - "pattern": "DoCoMo/1.0/Nxxxi/c10", + "pattern": "FirstGov\\.gov\\sSearch", "last_changed": "2024-01-02" }, { - "pattern": "DoCoMo/2.0 P900iV(c1", + "pattern": "Flexum\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "EARTHCOM.info/1.x [w", + "pattern": "FocusedSampler\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "EARTHCOM.info/1.xbet", + "pattern": "Franklin\\sLocator\\s1\\.8", "last_changed": "2024-01-02" }, { - "pattern": "EBrowse 1.4b", + "pattern": "FuseBulb\\.Com", "last_changed": "2024-01-02" }, { - "pattern": "EasyDL/3.xx", + "pattern": "Go\\s1\\.1\\spackage\\shttp", "last_changed": "2024-01-02" }, { - "pattern": "EchO!/2.0", + "pattern": "Go-http-client\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Educate Search VxB", + "pattern": "Go-http-client\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "EldoS TimelyWeb/3.x", + "pattern": "GoForIt\\.com", "last_changed": "2024-01-02" }, { - "pattern": "EmailWolf 1.00", + "pattern": "Goofer\\/0\\.2", "last_changed": "2024-01-02" }, { - "pattern": "Embedly +support@emb", + "pattern": "Gromit\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Enfish Tracker", + "pattern": "Guestbook\\sAuto\\sSubmi", "last_changed": "2024-01-02" }, { - "pattern": "Enterprise_Search/1.", + "pattern": "HLoader", "last_changed": "2024-01-02" }, { - "pattern": "Evernote Clip Resolv", + "pattern": "Haansoft", "last_changed": "2024-01-02" }, { - "pattern": "Exalead NG/MimeLive ", + "pattern": "Hatena\\sAntenna\\/0\\.3", "last_changed": "2024-01-02" }, { - "pattern": "Expired Domain Sleut", + "pattern": "Hatena\\sPagetitle\\sAge", "last_changed": "2024-01-02" }, { - "pattern": "ExtractorPro", + "pattern": "Hatena::Bookmark\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "EyeCatcher (Download", + "pattern": "Hippias\\/0\\.9\\sBeta", "last_changed": "2024-01-02" }, { - "pattern": "FARK.com link verifi", + "pattern": "HitList", "last_changed": "2024-01-02" }, { - "pattern": "FLATARTS_FAVICO", + "pattern": "Hivemind", "last_changed": "2024-01-02" }, { - "pattern": "FaEdit/2.0.x", + "pattern": "HomePageSearch", "last_changed": "2024-01-02" }, { - "pattern": "FavOrg", + "pattern": "Hybrid\\/1\\.2\\s\\[en]", "last_changed": "2024-01-02" }, { - "pattern": "Favorites Sweeper v.", + "pattern": "IAArchiver-1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Feed24.com", + "pattern": "IBM\\sWebExplorer\\s\\/v0\\.", "last_changed": "2024-01-02" }, { - "pattern": "FeedForAll rss2html.", + "pattern": "IEFav172Free", "last_changed": "2024-01-02" }, { - "pattern": "FeedValidator/1.3", + "pattern": "INFOMINE\\/8\\.0\\sAdders", "last_changed": "2024-01-02" }, { - "pattern": "Filtrbox/1.0", + "pattern": "INFOMINE\\/8\\.0\\sRemoteS", "last_changed": "2024-01-02" }, { - "pattern": "FindAnISP.com_ISP_Fi", + "pattern": "IP\\*Works!\\sV5\\sHTTP\\/S", "last_changed": "2024-01-02" }, { - "pattern": "Firefly/1.0", + "pattern": "ISC\\sSystems\\siRc\\sSear", "last_changed": "2024-01-02" }, { - "pattern": "Firefox (kastaneta03", + "pattern": "IWAgent\\/\\s1\\.0\\s-\\swww\\.b", "last_changed": "2024-01-02" }, { - "pattern": "Firefox_1.0.6 (kaspa", + "pattern": "Icarus6\\s-", "last_changed": "2024-01-02" }, { - "pattern": "FirstGov.gov Search ", + "pattern": "ImageWalker\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Flexum/2.0", + "pattern": "Incutio\\sHttpClient\\sv", "last_changed": "2024-01-02" }, { - "pattern": "FocusedSampler/1.0", + "pattern": "IncyWincy\\sdata\\sgathe", "last_changed": "2024-01-02" }, { - "pattern": "Franklin Locator 1.8", + "pattern": "Industry\\sProgram\\s1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "FuseBulb.Com", + "pattern": "Inet\\slibrary", "last_changed": "2024-01-02" }, { - "pattern": "GigablastOpenSource/", + "pattern": "InetURL\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Go 1.1 package http", + "pattern": "InfoLink\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Go-http-client/1.1", + "pattern": "InfoSeek\\sSidewinder\\/", "last_changed": "2024-01-02" }, { - "pattern": "Go-http-client/2.0", + "pattern": "Infoseek\\sSideWinder\\/", "last_changed": "2024-01-02" }, { - "pattern": "GoForIt.com", + "pattern": "Inktomi\\sSearch", "last_changed": "2024-01-02" }, { - "pattern": "Goofer/0.2", + "pattern": "Insitor\\.com\\ssearch\\sa", "last_changed": "2024-01-02" }, { - "pattern": "GrapeFX/0.3 libwww/5", + "pattern": "Insitornaut", "last_changed": "2024-01-02" }, { - "pattern": "Gromit/1.0", + "pattern": "Internet\\sNinja\\sx\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Guestbook Auto Submi", + "pattern": "InternetLinkAgent\\/3\\.", "last_changed": "2024-01-02" }, { - "pattern": "Gulliver/1.3", + "pattern": "JDK\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Guzzle/4.2.3 curl/7.", + "pattern": "Jack", "last_changed": "2024-01-02" }, { - "pattern": "HLoader", + "pattern": "Jakarta\\sCommons-Http", "last_changed": "2024-01-02" }, { - "pattern": "HPL/Nutch-0.9 -", + "pattern": "Java1\\.0\\.21\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Haansoft", + "pattern": "Java1\\.1\\.xx\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Harvest-NG/1.0.2", + "pattern": "Java1\\.3\\.0rc1", "last_changed": "2024-01-02" }, { - "pattern": "Hatena Antenna/0.3", + "pattern": "Java1\\.3\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Hatena Pagetitle Age", + "pattern": "Java1\\.4\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Hatena::Bookmark/2.0", + "pattern": "Jersey\\/2\\.25\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Hippias/0.9 Beta", + "pattern": "Jetty\\/9\\.3\\.z-SNAPSHOT", "last_changed": "2024-01-02" }, { - "pattern": "HitList", + "pattern": "Jigsaw\\/2\\.2\\.x\\sW3C_CSS", "last_changed": "2024-01-02" }, { - "pattern": "Hivemind", + "pattern": "Jigsaw\\/2\\.3\\.0\\sW3C_CSS", "last_changed": "2024-01-02" }, { - "pattern": "HomePageSearch(hpsea", + "pattern": "Junkie\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Html Link Validator ", + "pattern": "KIT-Fireball\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Hybrid/1.2 [en] (OS ", + "pattern": "KnowItAll", "last_changed": "2024-01-02" }, { - "pattern": "IAArchiver-1.0", + "pattern": "Knowledge\\.com\\/0\\.x", "last_changed": "2024-01-02" }, { - "pattern": "IBM WebExplorer /v0.", + "pattern": "KummHttp\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "IEFav172Free", + "pattern": "LEIA\\/2\\.90", "last_changed": "2024-01-02" }, { - "pattern": "INFOMINE/8.0 Adders", + "pattern": "Lachesis", "last_changed": "2024-01-02" }, { - "pattern": "INFOMINE/8.0 RemoteS", + "pattern": "Lincoln\\sState\\sWeb\\sBr", "last_changed": "2024-01-02" }, { - "pattern": "IP*Works! V5 HTTP/S ", + "pattern": "Link\\sValet\\sOnline\\s1\\.", "last_changed": "2024-01-02" }, { - "pattern": "ISC Systems iRc Sear", + "pattern": "LinkAlarm\\/2\\.x", "last_changed": "2024-01-02" }, { - "pattern": "IWAgent/ 1.0 - www.b", + "pattern": "LinkPimpin\\sv1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Icarus6 - (contact: ", + "pattern": "LinkProver\\s2\\.1", "last_changed": "2024-01-02" }, { - "pattern": "ImageWalker/2.0 (www", + "pattern": "LinkSonar\\/1\\.35", "last_changed": "2024-01-02" }, { - "pattern": "Incutio HttpClient v", + "pattern": "LinkSweeper\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "IncyWincy data gathe", + "pattern": "Look\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Industry Program 1.0", + "pattern": "Lynx\\/2\\.8\\.4rel\\.1\\slibw", "last_changed": "2024-01-02" }, { - "pattern": "Inet library", + "pattern": "MARTINI", "last_changed": "2024-01-02" }, { - "pattern": "InetURL/1.0", + "pattern": "MFC\\sFoundation\\sClass", "last_changed": "2024-01-02" }, { - "pattern": "InfoLink/1.x", + "pattern": "MFC_Tear_Sample", "last_changed": "2024-01-02" }, { - "pattern": "InfoSeek Sidewinder/", + "pattern": "MSIE-5\\.13", "last_changed": "2024-01-02" }, { - "pattern": "Infoaxe./Nutch-0.9", + "pattern": "MSNPTC\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Infoseek SideWinder/", + "pattern": "MVAClient", "last_changed": "2024-01-02" }, { - "pattern": "Inktomi Search", + "pattern": "Mac\\sFinder\\s1\\.0\\.xx", "last_changed": "2024-01-02" }, { - "pattern": "Insitor.com search a", + "pattern": "Mag-Net", "last_changed": "2024-01-02" }, { - "pattern": "Insitornaut", + "pattern": "MantraAgent", "last_changed": "2024-01-02" }, { - "pattern": "Internet Ninja x.0", + "pattern": "MapoftheInternet\\.com", "last_changed": "2024-01-02" }, { - "pattern": "InternetLinkAgent/3.", + "pattern": "Marketwave\\sHit\\sList", "last_changed": "2024-01-02" }, { - "pattern": "InternetSeer.com", + "pattern": "Martini", "last_changed": "2024-01-02" }, { - "pattern": "IrssiUrlLog/0.2", + "pattern": "Marvin\\sv0\\.3", "last_changed": "2024-01-02" }, { - "pattern": "JDK/1.1", + "pattern": "MasterSeek", "last_changed": "2024-01-02" }, { - "pattern": "Jack", + "pattern": "Mata\\sHari\\/2\\.00", "last_changed": "2024-01-02" }, { - "pattern": "Jakarta Commons-Http", + "pattern": "MediaSearch\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Jersey/2.25.1 (HttpU", + "pattern": "MegaSheep\\sv1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Jetty/9.3.z-SNAPSHOT", + "pattern": "MeltwaterNews\\swww\\.me", "last_changed": "2024-01-02" }, { - "pattern": "Jigsaw/2.2.x W3C_CSS", + "pattern": "Mercator-1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Jigsaw/2.3.0 W3C_CSS", + "pattern": "Mercator-2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Junkie/1.0", + "pattern": "Mercator-Scrub-1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "KE_1.0/2.0 libwww/5.", + "pattern": "Mercury\\/894\\sCFNetwor", "last_changed": "2024-01-02" }, { - "pattern": "KIT-Fireball/2.0", + "pattern": "MetaURI\\sAPI\\/2\\.0\\s\\+met", "last_changed": "2024-01-02" }, { - "pattern": "KnowItAll(knowitall@", + "pattern": "MicroBaz", "last_changed": "2024-01-02" }, { - "pattern": "Knowledge.com/0.x", + "pattern": "Microsoft\\sData\\sAcces", "last_changed": "2024-01-02" }, { - "pattern": "KummHttp/1.1 (compat", + "pattern": "Microsoft\\sSmall\\sBusi", "last_changed": "2024-01-02" }, { - "pattern": "LARBIN-EXPERIMENTAL ", + "pattern": "Microsoft\\sURL\\sContro", "last_changed": "2024-01-02" }, { - "pattern": "LEIA/2.90", + "pattern": "Microsoft\\sWindows\\sNe", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.22", + "pattern": "MiracleAlphaTest", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.36", + "pattern": "Missauga\\sLocate\\s1\\.0\\.", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.48", + "pattern": "Missigua\\sLocator\\s1\\.9", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.50", + "pattern": "Missouri\\sCollege\\sBro", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.51", + "pattern": "Mister\\sPiX\\sversion\\.d", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.53", + "pattern": "Mister\\sPix\\sII\\s2\\.02a", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.63", + "pattern": "Miva", "last_changed": "2024-01-02" }, { - "pattern": "LWP::Simple/5.803", + "pattern": "Mizzu\\sLabs\\s2\\.2", "last_changed": "2024-01-02" }, { - "pattern": "Lachesis", + "pattern": "Mo\\sCollege\\s1\\.9", "last_changed": "2024-01-02" }, { - "pattern": "Libby_1.1/libwww-per", + "pattern": "Mozi!", "last_changed": "2024-01-02" }, { - "pattern": "Lincoln State Web Br", + "pattern": "MultiText\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Link Valet Online 1.", + "pattern": "My\\sWinHTTP\\sConnectio", "last_changed": "2024-01-02" }, { - "pattern": "LinkAlarm/2.x", + "pattern": "NASA\\sSearch\\s1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "LinkPimpin v1.0", + "pattern": "NEC\\sResearch\\sAgent\\s-", "last_changed": "2024-01-02" }, { - "pattern": "LinkProver 2.1", + "pattern": "NEC-Hayek\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "LinkSonar/1.35", + "pattern": "NG\\/4\\.0\\.1229", "last_changed": "2024-01-02" }, { - "pattern": "LinkSweeper/1.x", + "pattern": "NICO\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "LinkWalker", + "pattern": "NING\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "LinkWalker/2.0", + "pattern": "NationalDirectoryAdd", "last_changed": "2024-01-02" }, { - "pattern": "Look.com", + "pattern": "NativeHost", "last_changed": "2024-01-02" }, { - "pattern": "Lynx/2.8.4rel.1 libw", + "pattern": "NetLookout\\/2\\.24", "last_changed": "2024-01-02" }, { - "pattern": "MARTINI", + "pattern": "NetMechanic\\sVx\\.0", "last_changed": "2024-01-02" }, { - "pattern": "MFC Foundation Class", + "pattern": "NetSprint\\s--\\s2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "MFC_Tear_Sample", + "pattern": "NetZippy", "last_changed": "2024-01-02" }, { - "pattern": "MSIE-5.13 (larbin@un", + "pattern": "Nocilla\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "MSNPTC/1.0", + "pattern": "Nsauditor\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "MVAClient", - "last_changed": "2024-01-02" + "pattern": "Nuzzel", + "last_changed": "2024-01-02", + "addition_date": "2017-11-02", + "instances": [ + "Nuzzel" + ] }, { - "pattern": "Mac Finder 1.0.xx", + "pattern": "Octora\\sBeta\\s-\\swww\\.oc", "last_changed": "2024-01-02" }, { - "pattern": "Mag-Net", + "pattern": "OliverPerry", "last_changed": "2024-01-02" }, { - "pattern": "Mail.Ru/1.0", + "pattern": "OpenAcoon\\sv4\\.0\\.x", "last_changed": "2024-01-02" }, { - "pattern": "MantraAgent", + "pattern": "OpenISearch\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "MapoftheInternet.com", + "pattern": "OpenOffice\\/4\\.1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Marketwave Hit List", + "pattern": "Openstat\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Martini", + "pattern": "Opera\\/6\\.01", "last_changed": "2024-01-02" }, { - "pattern": "Marvin v0.3", + "pattern": "Oracle\\sUltra\\sSearch", "last_changed": "2024-01-02" }, { - "pattern": "MasterSeek", + "pattern": "OzySoftware\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Mata Hari/2.00", + "pattern": "PBrowse\\s1\\.4b", "last_changed": "2024-01-02" }, { - "pattern": "MediaSearch/0.1", + "pattern": "PEval\\s1\\.4b", "last_changed": "2024-01-02" }, { - "pattern": "Mediapartners-Google", - "last_changed": "2024-01-02", - "url": "https://support.google.com/webmasters/answer/1061943?hl=en", - "instances": [ - "Mediapartners-Google", - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" - ] + "pattern": "POE-Component-Client", + "last_changed": "2024-01-02" }, { - "pattern": "MegaSheep v1.0 (www.", + "pattern": "PeopleChat\\/Search_En", "last_changed": "2024-01-02" }, { - "pattern": "MeltwaterNews www.me", + "pattern": "PicoSearch\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Mercator-1.x", + "pattern": "Piffany_Web_Scraper_", "last_changed": "2024-01-02" }, { - "pattern": "Mercator-2.0", + "pattern": "PingALink\\sMonitoring", "last_changed": "2024-01-02" }, { - "pattern": "Mercator-Scrub-1.1", + "pattern": "Pita", "last_changed": "2024-01-02" }, { - "pattern": "Mercury/894 CFNetwor", + "pattern": "Poirot", "last_changed": "2024-01-02" }, { - "pattern": "MetaURI API/2.0 +met", + "pattern": "Pompos\\/1\\.x\\spompos@il", "last_changed": "2024-01-02" }, { - "pattern": "MicroBaz", + "pattern": "Popdexter\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Microsoft Data Acces", + "pattern": "Port\\sHuron\\sLabs", "last_changed": "2024-01-02" }, { - "pattern": "Microsoft Small Busi", + "pattern": "PostFavorites", "last_changed": "2024-01-02" }, { - "pattern": "Microsoft URL Contro", + "pattern": "PostRank\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Microsoft Windows Ne", + "pattern": "PrivacyFinder\\/1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "MiracleAlphaTest", + "pattern": "Program\\sShareware\\s1\\.", "last_changed": "2024-01-02" }, { - "pattern": "Missauga Locate 1.0.", + "pattern": "Progressive\\sDownload", "last_changed": "2024-01-02" }, { - "pattern": "Missigua Locator 1.9", + "pattern": "Project\\sXP5\\s\\[2\\.03\\.07", "last_changed": "2024-01-02" }, { - "pattern": "Missouri College Bro", + "pattern": "Pulsepoint\\sXT3\\sweb\\ss", "last_changed": "2024-01-02" }, { - "pattern": "Mister PiX version.d", + "pattern": "QPCreep\\sTest\\sRig", "last_changed": "2024-01-02" }, { - "pattern": "Mister Pix II 2.02a", + "pattern": "Qseero\\sv1\\.0\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Miva (AlgoFeedback@m", + "pattern": "QuepasaCreep\\sv0\\.9\\.1x", "last_changed": "2024-01-02" }, { - "pattern": "Mizzu Labs 2.2", + "pattern": "QueryN\\sMetasearch", "last_changed": "2024-01-02" }, { - "pattern": "MnogoSearch/3.2.xx", + "pattern": "R6_VoteReader", "last_changed": "2024-01-02" }, { - "pattern": "Mnogosearch-3.1.21", + "pattern": "RPT-HTTPClient\\/0\\.3-x", "last_changed": "2024-01-02" }, { - "pattern": "Mo College 1.9", + "pattern": "RepoMonkey\\sBait\\s&\\sTa", "last_changed": "2024-01-02" }, { - "pattern": "More yacy user agent", + "pattern": "Ruby", "last_changed": "2024-01-02" }, { - "pattern": "Mozi!", + "pattern": "Rumours-Agent", "last_changed": "2024-01-02" }, { - "pattern": "MultiText/0.1", + "pattern": "SMRF\\sURL\\sExpander", "last_changed": "2024-01-02" }, { - "pattern": "My WinHTTP Connectio", + "pattern": "ScoutAbout", "last_changed": "2024-01-02" }, { - "pattern": "MyNutchTest/Nutch-1.", + "pattern": "Search\\+", "last_changed": "2024-01-02" }, { - "pattern": "NASA Search 1.0", + "pattern": "Search-Engine-Studio", "last_changed": "2024-01-02" }, { - "pattern": "NEC Research Agent -", + "pattern": "SearchGuild\\/DMOZ\\/Exp", "last_changed": "2024-01-02" }, { - "pattern": "NEC-Hayek/1.0", + "pattern": "SearchGuild_DMOZ_Exp", "last_changed": "2024-01-02" }, { - "pattern": "NG/4.0.1229", + "pattern": "SearchSight\\.com", "last_changed": "2024-01-02" }, { - "pattern": "NICO/1.0", + "pattern": "SearchTone2\\.0\\s-\\sIDEA", "last_changed": "2024-01-02" }, { - "pattern": "NING/1.0", + "pattern": "Seeker\\.lookseek\\.com", "last_changed": "2024-01-02" }, { - "pattern": "NationalDirectoryAdd", + "pattern": "Silk\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "NativeHost", + "pattern": "SiteBar\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "NetLookout/2.24", + "pattern": "SiteRecon\\+", "last_changed": "2024-01-02" }, { - "pattern": "NetMechanic Vx.0", + "pattern": "SiteTruth\\.com\\ssite\\sr", "last_changed": "2024-01-02" }, { - "pattern": "NetSprint -- 2.0", + "pattern": "SiteXpert", "last_changed": "2024-01-02" }, { - "pattern": "NetZippy", + "pattern": "Skywalker\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Nocilla/1.0", + "pattern": "Slarp\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Nsauditor/1.x", + "pattern": "Slider_Search_v1-de", "last_changed": "2024-01-02" }, { - "pattern": "Nuzzel", - "last_changed": "2024-01-02", - "addition_date": "2017-11-02", - "instances": [ - "Nuzzel" - ] + "pattern": "SlySearch", + "last_changed": "2024-01-02" }, { - "pattern": "Nuzzel/876 CFNetwork", + "pattern": "SnoopRob\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Octora Beta - www.oc", + "pattern": "Snoopy\\sv1\\.xx", "last_changed": "2024-01-02" }, { - "pattern": "Oculus360/Nutch-1.7", + "pattern": "Snoopy_v0\\.xx", "last_changed": "2024-01-02" }, { - "pattern": "OliverPerry", + "pattern": "Sphere\\sScout&v4\\.0\\s-", "last_changed": "2024-01-02" }, { - "pattern": "OpenAcoon v4.0.x (ww", + "pattern": "Sphider", "last_changed": "2024-01-02" }, { - "pattern": "OpenISearch/1.x (www", + "pattern": "Spida\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "OpenOffice/4.1.1", + "pattern": "Spinne\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "OpenWebIndex/Nutch-1", + "pattern": "Spinne\\/2\\.0\\smed", "last_changed": "2024-01-02" }, { - "pattern": "Openstat/0.1", + "pattern": "Spinne\\/2\\.0\\smed_AH", "last_changed": "2024-01-02" }, { - "pattern": "Opera/6.01 (larbin@u", + "pattern": "SyncIT\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Oracle Ultra Search", + "pattern": "Szukacz\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "OzySoftware.com", + "pattern": "TE", "last_changed": "2024-01-02" }, { - "pattern": "PBrowse 1.4b", + "pattern": "TJvMultiHttpGrabber", "last_changed": "2024-01-02" }, { - "pattern": "PEval 1.4b", + "pattern": "Tagyu\\sAgent\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "PHP/3.x.xx", + "pattern": "TeamSoft\\sWinInet\\sCom", "last_changed": "2024-01-02" }, { - "pattern": "PHP/4.0.4pl1", + "pattern": "The\\sEuropean\\sLibrary", "last_changed": "2024-01-02" }, { - "pattern": "PHP/4.0.6", + "pattern": "Thinklab", "last_changed": "2024-01-02" }, { - "pattern": "PHP/4.1.1", + "pattern": "TinEye", + "last_changed": "2024-01-02", + "addition_date": "2017-10-14", + "url": "http://www.tineye.com/crawler.html", + "instances": [ + "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", + "TinEye/1.1 (http://tineye.com/crawler.html)" + ] + }, + { + "pattern": "Toutatis\\sx-xx\\.x", "last_changed": "2024-01-02" }, { - "pattern": "PHP/4.1.2", + "pattern": "Toutatis\\sx\\.x", "last_changed": "2024-01-02" }, { - "pattern": "POE-Component-Client", + "pattern": "Toutatis\\sx\\.x-x", "last_changed": "2024-01-02" }, { - "pattern": "Pcore-HTTP/v0.40.3", + "pattern": "Traackr\\.com", + "last_changed": "2024-01-02", + "addition_date": "2017-11-02", + "url": "Traackr.com", + "instances": [ + "Traackr.com" + ] + }, + { + "pattern": "Twisted\\sPageGetter", "last_changed": "2024-01-02" }, { - "pattern": "PeopleChat/Search_En", + "pattern": "Twitter\\/6\\.18\\.3\\sCFNet", "last_changed": "2024-01-02" }, { - "pattern": "PicoSearch/1.0", + "pattern": "Twitturly\\s\\/\\sv0\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Piffany_Web_Scraper_", + "pattern": "TygoProwler", "last_changed": "2024-01-02" }, { - "pattern": "PingALink Monitoring", + "pattern": "UCWEB\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Pita", + "pattern": "UDM", "last_changed": "2024-01-02" }, { - "pattern": "Poirot", + "pattern": "URLBase\\/6\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Pompos/1.x pompos@il", + "pattern": "URLBlaze", "last_changed": "2024-01-02" }, { - "pattern": "Popdexter/1.0", + "pattern": "UdmSearch\\/3\\.1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Port Huron Labs", + "pattern": "Under\\sthe\\sRainbow\\s2\\.", "last_changed": "2024-01-02" }, { - "pattern": "PostFavorites", + "pattern": "UofTDB_experiment", "last_changed": "2024-01-02" }, { - "pattern": "PostRank/2.0 (postra", + "pattern": "Upflow\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "PrivacyFinder/1.1", + "pattern": "VYU2", "last_changed": "2024-01-02" }, { - "pattern": "Program Shareware 1.", + "pattern": "Vagabondo\\/1\\.x\\sMT", "last_changed": "2024-01-02" }, { - "pattern": "Progressive Download", + "pattern": "Vagabondo\\/2\\.0\\sMT", "last_changed": "2024-01-02" }, { - "pattern": "Project XP5 [2.03.07", + "pattern": "Vagabondo\\/3\\.0", "last_changed": "2024-01-02" }, { - "pattern": "PubCrawl (pubcrawl.s", + "pattern": "VayalaCreep-v0\\.0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Pulsepoint XT3 web s", + "pattern": "Vayala\\|Creep-v0\\.0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.13.2", + "pattern": "VeryGoodSearch\\.com\\.D", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.15.5", + "pattern": "W3C-mobileOK\\/DDC-1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.16.4", + "pattern": "W3C_Unicorn\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.18.0", + "pattern": "WEP\\sSearch\\s00", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.18.2", + "pattern": "WFARC", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.19.0", + "pattern": "WIRE\\/0\\.22", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.19.3", + "pattern": "WST-Plagiarism", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.19.5", + "pattern": "WWWC\\/1\\.0x", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.19.7", + "pattern": "Wada\\.vn\\sVietnamese\\sS", "last_changed": "2024-01-02" }, { - "pattern": "PycURL/7.23.1", + "pattern": "Waypath\\sScout\\sv2\\.x\\s-", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/1.17", + "pattern": "WeSEE", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/1.1x", + "pattern": "WeSEE:Search", + "last_changed": "2024-01-02", + "addition_date": "2013-11-18", + "instances": [ + "WeSEE:Search", + "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" + ] + }, + { + "pattern": "Web\\sSnooper", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/2.0a1", + "pattern": "WebBug\\/5\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/2.5", + "pattern": "WebCompass\\s2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/2.6", + "pattern": "WebCorp\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/2.7", + "pattern": "WebSearch\\.COM\\.AU;", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.1", + "pattern": "WebStat\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.2", + "pattern": "WebTrends\\/3\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.3", + "pattern": "WebVac", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.4", + "pattern": "WebVal\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.5", + "pattern": "WebWatcherMonitor\\/2\\.", "last_changed": "2024-01-02" }, { - "pattern": "Python-urllib/3.6", + "pattern": "Webclipping\\.com", "last_changed": "2024-01-02" }, { - "pattern": "QPCreep Test Rig ( W", + "pattern": "Webdup\\/0\\.9", "last_changed": "2024-01-02" }, { - "pattern": "Qseero v1.0.0", + "pattern": "Weblog\\sAttitude\\sDiff", "last_changed": "2024-01-02" }, { - "pattern": "QuepasaCreep v0.9.1x", + "pattern": "WebsiteWorth\\sv1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "QueryN Metasearch", + "pattern": "Webspinne\\/1\\.0\\swebmas", "last_changed": "2024-01-02" }, { - "pattern": "Qwantify/1.0", + "pattern": "Webverzeichnis\\.de\\s-", "last_changed": "2024-01-02" }, { - "pattern": "R6_VoteReader(www.ra", + "pattern": "Wells\\sSearch\\sII", "last_changed": "2024-01-02" }, { - "pattern": "REBOL Core 2.x.x.x.x", + "pattern": "Whatsup\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "RPT-HTTPClient/0.3-x", + "pattern": "WhizBang!\\sLab", "last_changed": "2024-01-02" }, { - "pattern": "RepoMonkey Bait & Ta", + "pattern": "WinHTTP\\sExample\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "RoboCrawl (www.canad", + "pattern": "WinampMPEG\\/2\\.00", "last_changed": "2024-01-02" }, { - "pattern": "Robozilla/1.0", + "pattern": "WincerSong\\sAgent\\sv1\\.", "last_changed": "2024-01-02" }, { - "pattern": "Rotondo/3.1 libwww/5", + "pattern": "WorQmada\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Ruby", + "pattern": "WordupInfoSearch\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Rumours-Agent", + "pattern": "WorldLight", "last_changed": "2024-01-02" }, { - "pattern": "SMRF URL Expander", + "pattern": "Wusage\\/x\\.0@boutell\\.c", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.0.EU", + "pattern": "Wwlib\\/Linux", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.0.FS", + "pattern": "Xylix", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.0.HD", + "pattern": "Yanadoo", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.0.VNS", + "pattern": "Zearchit", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.0QI", + "pattern": "ZendHttpClient", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2", + "pattern": "Ziggy\\s--\\sThe\\sClown\\sF", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.BT", + "pattern": "Zippy\\sv2\\.0\\s-\\sZippyfi", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.DIL", + "pattern": "a\\.pr-cy\\.ru", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.EX", + "pattern": "about\\.ask\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.JT", + "pattern": "about\\.depspid\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.NIV", + "pattern": "accoona", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.SF0", + "pattern": "aihit\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.2.snippet", + "pattern": "amzn_assoc", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-3.3dev", + "pattern": "app\\.hypefactors\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-ARS-1.1", + "pattern": "archive\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-ARS-1.1-ih", + "pattern": "ask\\.24x\\.info", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-W3-1.0", + "pattern": "augurfind", "last_changed": "2024-01-02" }, { - "pattern": "Scooter-W3.1.2", + "pattern": "augurnfind\\sV-1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/1.0", + "pattern": "autohttp", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/1.0 scooter@", + "pattern": "autowebdir\\s1\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/1.1 (custom)", + "pattern": "axios\\/0\\.18\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/2.0 G.R.A.B.", + "pattern": "bCentral\\sBilling\\sPos", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/3.3", + "pattern": "bdcindexer_2\\.6\\.2", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/3.3.QA.pczuk", + "pattern": "blacklistdeclassified\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/3.3.vscooter", + "pattern": "bookmarkbase\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Scooter/3.3_SF", + "pattern": "bulkfeeds\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Scooter2_Mercator_x-", + "pattern": "bumblebee@relevare\\.c", "last_changed": "2024-01-02" }, { - "pattern": "Scooter_bh0-3.0.3", + "pattern": "cantor\\.ee\\.ucla\\.edu", "last_changed": "2024-01-02" }, { - "pattern": "Scooter_trk3-3.0.3", + "pattern": "carleson\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "ScoutAbout", + "pattern": "ccubee\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Search+", + "pattern": "cfetch\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Search-Engine-Studio", + "pattern": "cg-eye\\sinteractive", "last_changed": "2024-01-02" }, { - "pattern": "SearchGuild/DMOZ/Exp", + "pattern": "cobitsprobe", "last_changed": "2024-01-02" }, { - "pattern": "SearchGuild_DMOZ_Exp", + "pattern": "codegator", "last_changed": "2024-01-02" }, { - "pattern": "SearchSight.com", + "pattern": "collage\\.cgi\\/1\\.xx", "last_changed": "2024-01-02" }, { - "pattern": "SearchTone2.0 - IDEA", + "pattern": "computerorgs\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Seeker.lookseek.com", + "pattern": "content\\.easybib\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Silk/1.0", + "pattern": "coralcdn\\.org", "last_changed": "2024-01-02" }, { - "pattern": "SiteBar/x.x", + "pattern": "cosco\\.hiit\\.fi", "last_changed": "2024-01-02" }, { - "pattern": "SiteRecon+(xx)", + "pattern": "cs\\.daum\\.net", "last_changed": "2024-01-02" }, { - "pattern": "SiteTruth.com site r", + "pattern": "csci_b659\\/0\\.13", "last_changed": "2024-01-02" }, { - "pattern": "SiteXpert", + "pattern": "dds\\sexplorer\\sv1\\.0\\sbe", "last_changed": "2024-01-02" }, { - "pattern": "Skywalker/0.1 (Skywa", + "pattern": "deepak-USC\\/ISI", "last_changed": "2024-01-02" }, { - "pattern": "Slarp/0.1", + "pattern": "deusu\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Slider_Search_v1-de", + "pattern": "digg\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Slurpy Verifier/1.0", + "pattern": "dir\\.com", "last_changed": "2024-01-02" }, { - "pattern": "SlySearch (slysearch", + "pattern": "dznet\\.com", "last_changed": "2024-01-02" }, { - "pattern": "SnoopRob/x.x", + "pattern": "e-sense\\s1\\.0\\sea", "last_changed": "2024-01-02" }, { - "pattern": "Snoopy v1.xx", + "pattern": "eStyleSearch\\s4", "last_changed": "2024-01-02" }, { - "pattern": "Snoopy_v0.xx", + "pattern": "ec2linkfinder", + "last_changed": "2024-01-02", + "addition_date": "2012-03-22", + "instances": [ + "ec2linkfinder" + ] + }, + { + "pattern": "ecairn\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Sphere Scout&v4.0 - ", + "pattern": "ejupiter\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Sphider", + "pattern": "enter4u\\.eu", "last_changed": "2024-01-02" }, { - "pattern": "Spida/0.1", + "pattern": "exactseek\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Spinne/2.0", + "pattern": "example-masto-instance\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Spinne/2.0 med", + "pattern": "feed\\.moo\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "Spinne/2.0 med_AH", + "pattern": "feedafever\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "Sqworm/2.9.81-BETA (", + "pattern": "flipboard\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Sqworm/2.9.85-BETA (", + "pattern": "flunky", "last_changed": "2024-01-02" }, { - "pattern": "Sqworm/2.9.89-BETA (", + "pattern": "free-downloads\\.net\\sd", "last_changed": "2024-01-02" }, { - "pattern": "StackRambler/2.0", + "pattern": "gazz\\/x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "StackRambler/2.0 (MS", + "pattern": "geourl\\/2\\.0b2", "last_changed": "2024-01-02" }, { - "pattern": "StackRambler/x.x", + "pattern": "getpocket\\.com", "last_changed": "2024-01-02" }, { - "pattern": "SyncIT/x.x", + "pattern": "gigabaz\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Szukacz/1.x", + "pattern": "github\\.com", "last_changed": "2024-01-02" }, { - "pattern": "TE", + "pattern": "gossamer-threads\\.com", "last_changed": "2024-01-02" }, { - "pattern": "TJvMultiHttpGrabber ", + "pattern": "guess\\.scritch\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Tagyu Agent/1.0", + "pattern": "haste\\.kytoon\\.com", "last_changed": "2024-01-02" }, { - "pattern": "TeamSoft WinInet Com", + "pattern": "help\\.coccoc\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Teoma MP", + "pattern": "help\\.coccoc\\.vn", "last_changed": "2024-01-02" }, { - "pattern": "The European Library", + "pattern": "help\\.goo\\.ne\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "Thinklab (thinklab.c", + "pattern": "help\\.hepburnhome\\.ca", "last_changed": "2024-01-02" }, { - "pattern": "TinEye", - "last_changed": "2024-01-02", - "addition_date": "2017-10-14", - "url": "http://www.tineye.com/crawler.html", - "instances": [ - "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", - "TinEye/1.1 (http://tineye.com/crawler.html)" - ] + "pattern": "hgrepurl\\/1\\.0", + "last_changed": "2024-01-02" }, { - "pattern": "Toutatis x-xx.x (hop", + "pattern": "http:\\/\\/www\\.mabontlan", "last_changed": "2024-01-02" }, { - "pattern": "Toutatis x.x (hoppa.", + "pattern": "http:\\/\\/www\\.monogol\\.d", "last_changed": "2024-01-02" }, { - "pattern": "Toutatis x.x-x", + "pattern": "http:\\/\\/www\\.timelyweb", "last_changed": "2024-01-02" }, { - "pattern": "Traackr.com", + "pattern": "http_get", "last_changed": "2024-01-02", "addition_date": "2017-11-02", - "url": "Traackr.com", "instances": [ - "Traackr.com" + "http_get" ] }, { - "pattern": "Twisted PageGetter", + "pattern": "httpunit\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "Twitter/6.18.3 CFNet", + "pattern": "i\\.stanford\\.edu", "last_changed": "2024-01-02" }, { - "pattern": "Twitturly / v0.x", + "pattern": "iFeed\\.jp\\/2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Tycoon Agent/Nutch-1", + "pattern": "icerocket\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "TygoProwler", + "pattern": "ideare\\s-\\sSignSite\\/1\\.", "last_changed": "2024-01-02" }, { - "pattern": "UCWEB/2.0(Linux; U; ", + "pattern": "igde\\.ru", "last_changed": "2024-01-02" }, { - "pattern": "UDM", + "pattern": "innovationblog\\.com", "last_changed": "2024-01-02" }, { - "pattern": "URLBase/6.x", + "pattern": "integrity\\/1\\.6", "last_changed": "2024-01-02" }, { - "pattern": "URLBlaze", + "pattern": "integrity\\/5", "last_changed": "2024-01-02" }, { - "pattern": "UdmSearch/3.1.x", + "pattern": "ivia\\.ucr\\.edu", "last_changed": "2024-01-02" }, { - "pattern": "Ultraseek", + "pattern": "jack", "last_changed": "2024-01-02" }, { - "pattern": "Under the Rainbow 2.", + "pattern": "journster\\.com", "last_changed": "2024-01-02" }, { - "pattern": "UofTDB_experiment (l", + "pattern": "knil", "last_changed": "2024-01-02" }, { - "pattern": "Upflow/1.0", + "pattern": "kulturarw3\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "VYU2 (GNU; OpenRISC)", + "pattern": "law\\.di\\.unimi\\.it", "last_changed": "2024-01-02" }, { - "pattern": "Vagabondo/1.x MT (we", + "pattern": "lc\\/\\$ROADS::Version\\sl", "last_changed": "2024-01-02" }, { - "pattern": "Vagabondo/2.0 MT", + "pattern": "libWeb\\/clsHTTP\\s--\\shi", "last_changed": "2024-01-02" }, { - "pattern": "Vagabondo/2.0 MT (we", + "pattern": "linkdex\\.com\\/v2\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Vagabondo/3.0 (webag", + "pattern": "mailto:webcraft@bea\\.", "last_changed": "2024-01-02" }, { - "pattern": "Validator.nu/LV", + "pattern": "metatagsdir\\.com", "last_changed": "2024-01-02" }, { - "pattern": "VayalaCreep-v0.0.1 (", + "pattern": "miniflux\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Vayala|Creep-v0.0.1 ", + "pattern": "mogimogi\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "VeryGoodSearch.com.D", + "pattern": "moiNAG\\s0\\.02", "last_changed": "2024-01-02" }, { - "pattern": "W3C-mobileOK/DDC-1.0", + "pattern": "muckrack\\.com", "last_changed": "2024-01-02" }, { - "pattern": "W3C_Unicorn/1.0", + "pattern": "myDaemon", "last_changed": "2024-01-02" }, { - "pattern": "W3C_Validator/1.3", + "pattern": "net-promoter\\.com", "last_changed": "2024-01-02" }, { - "pattern": "W3C_Validator/1.xxx ", + "pattern": "netarkivet\\.dk", "last_changed": "2024-01-02" }, { - "pattern": "WDG_Validator/1.1", + "pattern": "newsearchengine", "last_changed": "2024-01-02" }, { - "pattern": "WEP Search 00", + "pattern": "newsme\\/1\\.0;\\sfeedback", "last_changed": "2024-01-02" }, { - "pattern": "WFARC", + "pattern": "newspaper\\/0\\.1\\.0\\.7", "last_changed": "2024-01-02" }, { - "pattern": "WIRE/0.22 (Linux; x8", + "pattern": "newspaper\\/0\\.2\\.5", "last_changed": "2024-01-02" }, { - "pattern": "WST-Plagiarism", + "pattern": "newspaper\\/0\\.2\\.6", "last_changed": "2024-01-02" }, { - "pattern": "WWWC/1.0x", + "pattern": "newstin\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Wada.vn Vietnamese S", + "pattern": "nlp\\.uned\\.es", "last_changed": "2024-01-02" }, { - "pattern": "Waypath Scout v2.x -", + "pattern": "noyona_0_1", "last_changed": "2024-01-02" }, { - "pattern": "WeSEE", + "pattern": "nymesis\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WeSEE:Search", - "last_changed": "2024-01-02", - "addition_date": "2013-11-18", - "instances": [ - "WeSEE:Search", - "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" - ] + "pattern": "oegp\\sv\\.\\s1\\.3\\.0", + "last_changed": "2024-01-02" }, { - "pattern": "Web Link Validator 1", + "pattern": "omgili\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Web Snooper", + "pattern": "ostermiller\\.org", "last_changed": "2024-01-02" }, { - "pattern": "WebBug/5.x", + "pattern": "owler-agent", "last_changed": "2024-01-02" }, { - "pattern": "WebCompass 2.0", + "pattern": "ozelot\\/2\\.7\\.3", "last_changed": "2024-01-02" }, { - "pattern": "WebCorp/1.0", + "pattern": "pandora\\.nla\\.gov\\.au", "last_changed": "2024-01-02" }, { - "pattern": "WebSearch.COM.AU;", + "pattern": "pauillac\\.inria\\.fr", "last_changed": "2024-01-02" }, { - "pattern": "WebStat/1.0 (Unix; b", + "pattern": "pd02_1\\.0\\.0\\spd02_1\\.0\\.", "last_changed": "2024-01-02" }, { - "pattern": "WebTrends/3.0 (WinNT", + "pattern": "people\\.eng\\.unimelb\\.edu\\.au", "last_changed": "2024-01-02" }, { - "pattern": "WebVac (webmaster@pi", + "pattern": "pompos\\.iliad\\.fr", "last_changed": "2024-01-02" }, { - "pattern": "WebVal/1.0", + "pattern": "psycheclone", "last_changed": "2024-01-02" }, { - "pattern": "WebVulnCrawl.unknown", + "pattern": "r\\.hatena\\.ne\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "WebWatcherMonitor/2.", + "pattern": "rabaz", "last_changed": "2024-01-02" }, { - "pattern": "Webclipping.com", + "pattern": "ranksonic\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Webdup/0.9", + "pattern": "rico\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "Weblog Attitude Diff", + "pattern": "rjpower\\.org", "last_changed": "2024-01-02" }, { - "pattern": "WebsiteWorth v1.0", + "pattern": "sagool\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "Webspinne/1.0 webmas", + "pattern": "scoutmaster", "last_changed": "2024-01-02" }, { - "pattern": "Webverzeichnis.de - ", + "pattern": "scrapy\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Wells Search II", + "pattern": "search\\.ch\\sV1\\.4", "last_changed": "2024-01-02" }, { - "pattern": "Wget/1.14 (linux-gnu", + "pattern": "search\\.denmex\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.15/i", + "pattern": "search\\.goo\\.ne\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.16/i", + "pattern": "seekda\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.17/i", + "pattern": "semanticdiscovery\\/0\\.", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.449 A", + "pattern": "sentry\\.io", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.453 A", + "pattern": "silk\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.510 A", + "pattern": "sitebar\\.org", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.540 A", + "pattern": "siteexplorer\\.info", + "last_changed": "2024-01-02", + "addition_date": "2013-05-01", + "instances": [ + "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", + "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" + ] + }, + { + "pattern": "smartwit\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.548 A", + "pattern": "sna-0\\.0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.555 A", + "pattern": "sna-0\\.0\\.1\\smikeelliot", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.12.556 A", + "pattern": "sohu\\sagent", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.16.1/i", + "pattern": "sohu-search", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.16.13 A", + "pattern": "sp\\.ask\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.16.2/i", + "pattern": "suchbaer\\.de", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.16.42 A", + "pattern": "support\\.embed\\.ly", "last_changed": "2024-01-02" }, { - "pattern": "WhatsApp/2.16.57 A", + "pattern": "surf", "last_changed": "2024-01-02" }, { - "pattern": "Whatsup/x.x", + "pattern": "suzuran", "last_changed": "2024-01-02" }, { - "pattern": "WhizBang! Lab", + "pattern": "szukaj\\.onet\\.pl", "last_changed": "2024-01-02" }, { - "pattern": "WinHTTP Example/1.0", + "pattern": "tags2dir\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WinampMPEG/2.00 (lar", + "pattern": "tagword\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WincerSong Agent v1.", + "pattern": "targetblaster\\.com\\/0\\.", "last_changed": "2024-01-02" }, { - "pattern": "WorQmada/1.0", + "pattern": "therarestwords\\.com", "last_changed": "2024-01-02" }, { - "pattern": "WordupInfoSearch/1.0", + "pattern": "tuezilla\\.de", "last_changed": "2024-01-02" }, { - "pattern": "WorldLight", + "pattern": "twikle\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Wusage/x.0@boutell.c", + "pattern": "twurly\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Wwlib/Linux", + "pattern": "unknownght\\.com\\sWeb\\sS", "last_changed": "2024-01-02" }, { - "pattern": "XMLSlurp/0.1 libwww-", + "pattern": "users\\.cs\\.cf\\.ac\\.uk", "last_changed": "2024-01-02" }, { - "pattern": "Xenu Link Sleuth 1.x", + "pattern": "versus\\.integis\\.ch", "last_changed": "2024-01-02" }, { - "pattern": "Xenu Link Sleuth/1.3", + "pattern": "vias\\.ncsa\\.uiuc\\.edu", "last_changed": "2024-01-02" }, { - "pattern": "Xenu's Link Sleuth 1", + "pattern": "visionutils\\/0\\.2", "last_changed": "2024-01-02" }, { - "pattern": "Xylix", + "pattern": "vk\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Y!J-SRD/1.0", + "pattern": "voyager-hc\\/1\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Yahoo Pipes 1.0", + "pattern": "web-bekannt", "last_changed": "2024-01-02" }, { - "pattern": "Yahoo! Mindset", + "pattern": "web2express\\.org", "last_changed": "2024-01-02" }, { - "pattern": "Yahoo-Test/4.0", + "pattern": "webGobbler\\/1\\.x\\.x", "last_changed": "2024-01-02" }, { - "pattern": "YahooCacheSystem", + "pattern": "webagent\\.wise-guys\\.nl", "last_changed": "2024-01-02" }, { - "pattern": "YahooCacheSystem; Ya", + "pattern": "webalgo\\.iit\\.cnr\\.it", "last_changed": "2024-01-02" }, { - "pattern": "YahooSeeker/CafeKels", + "pattern": "webbandit\\/4\\.xx\\.0", "last_changed": "2024-01-02" }, { - "pattern": "Yanadoo", + "pattern": "webdatastats\\.com", "last_changed": "2024-01-02" }, { - "pattern": "Yandex/1.01.001 (com", + "pattern": "webglimpse\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Zearchit", + "pattern": "webhack", "last_changed": "2024-01-02" }, { - "pattern": "ZendHttpClient", + "pattern": "websearchbench\\.cs\\.uni-dortmund\\.de", "last_changed": "2024-01-02" }, { - "pattern": "Zeus ThemeSite Viewe", + "pattern": "websitealert\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Zeus xxxxx Webster P", + "pattern": "webster-internet\\.de", "last_changed": "2024-01-02" }, { - "pattern": "Ziggy -- The Clown F", + "pattern": "webster\\.healeys\\.net", "last_changed": "2024-01-02" }, { - "pattern": "Zippy v2.0 - Zippyfi", + "pattern": "whatUseek_winona\\/3\\.0", "last_changed": "2024-01-02" }, { - "pattern": "a.pr-cy.ru", + "pattern": "wiki\\.creativecommons\\.org", "last_changed": "2024-01-02" }, { - "pattern": "about.ask.com", + "pattern": "wish\\.slis\\.tsukuba\\.ac\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "about.depspid.net", + "pattern": "wkd\\.iis\\.sinica\\.edu\\.tw", "last_changed": "2024-01-02" }, { - "pattern": "accoona", + "pattern": "wortschatz\\.uni-leipzig\\.de", "last_changed": "2024-01-02" }, { - "pattern": "aihit.com", + "pattern": "ws\\.daum\\.net", "last_changed": "2024-01-02" }, { - "pattern": "amzn_assoc", + "pattern": "www\\.Abolimba\\.de", "last_changed": "2024-01-02" }, { - "pattern": "app.hypefactors.com", + "pattern": "www\\.WISEnut\\.com", "last_changed": "2024-01-02" }, { - "pattern": "appie 1.1 (www.walhe", + "pattern": "www\\.abonti\\.com", "last_changed": "2024-01-02" }, { - "pattern": "archive.org", + "pattern": "www\\.ackerm\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ask.24x.info", + "pattern": "www\\.activtourist\\.com", "last_changed": "2024-01-02" }, { - "pattern": "asterias/2.0", + "pattern": "www\\.agada\\.de", "last_changed": "2024-01-02" }, { - "pattern": "augurfind", + "pattern": "www\\.ahmadsoftware\\.co", "last_changed": "2024-01-02" }, { - "pattern": "augurnfind V-1.x", + "pattern": "www\\.answerbus\\.com", "last_changed": "2024-01-02" }, { - "pattern": "autohttp", + "pattern": "www\\.ant\\.com", "last_changed": "2024-01-02" }, { - "pattern": "autowebdir 1.1 (www.", + "pattern": "www\\.aol\\.com", "last_changed": "2024-01-02" }, { - "pattern": "axios/0.18.0", + "pattern": "www\\.aramabeta\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "bCentral Billing Pos", + "pattern": "www\\.arianna\\.it", "last_changed": "2024-01-02" }, { - "pattern": "bdcindexer_2.6.2 (re", + "pattern": "www\\.arocom\\.de", "last_changed": "2024-01-02" }, { - "pattern": "binlar_2.6.3 binlar2", + "pattern": "www\\.asaha\\.com", "last_changed": "2024-01-02" }, { - "pattern": "binlar_2.6.3 binlar_", + "pattern": "www\\.axxus\\.de", "last_changed": "2024-01-02" }, { - "pattern": "binlar_2.6.3 larbin2", + "pattern": "www\\.ay-up\\.com", "last_changed": "2024-01-02" }, { - "pattern": "binlar_2.6.3 phanend", + "pattern": "www\\.ba\\.be", "last_changed": "2024-01-02" }, { - "pattern": "binlar_2.6.3 test@mg", + "pattern": "www\\.blaiz\\.net", "last_changed": "2024-01-02" }, { - "pattern": "blacklistdeclassified.net", + "pattern": "www\\.bortzmeyer\\.org", "last_changed": "2024-01-02" }, { - "pattern": "bookmarkbase.com", + "pattern": "www\\.brandverity\\.com", "last_changed": "2024-01-02" }, { - "pattern": "bulkfeeds.net", + "pattern": "www\\.buddybuzz\\.net", "last_changed": "2024-01-02" }, { - "pattern": "bumblebee@relevare.c", + "pattern": "www\\.buscaplus\\.com", "last_changed": "2024-01-02" }, { - "pattern": "cantor.ee.ucla.edu", + "pattern": "www\\.business-socket\\.", "last_changed": "2024-01-02" }, { - "pattern": "carleson/1.0", + "pattern": "www\\.canadiancontent\\.net", "last_changed": "2024-01-02" }, { - "pattern": "ccubee/x.x", + "pattern": "www\\.chepi\\.net", "last_changed": "2024-01-02" }, { - "pattern": "cfetch/1.0", + "pattern": "www\\.chilkatsoft\\.com", "last_changed": "2024-01-02" }, { - "pattern": "cg-eye interactive", + "pattern": "www\\.cloudflare\\.com", "last_changed": "2024-01-02" }, { - "pattern": "cobitsprobe", + "pattern": "www\\.cnet\\.com", "last_changed": "2024-01-02" }, { - "pattern": "coccoc/1.0 ()", + "pattern": "www\\.cougarsearch\\.com", "last_changed": "2024-01-02" }, { - "pattern": "code.google.com", + "pattern": "www\\.covario\\.com", "last_changed": "2024-01-02" }, { - "pattern": "codegator", + "pattern": "www\\.crim\\.ca", "last_changed": "2024-01-02" }, { - "pattern": "cohalinglit.wordpress.com", + "pattern": "www\\.cs\\.washington\\.edu", "last_changed": "2024-01-02" }, { - "pattern": "collage.cgi/1.xx", + "pattern": "www\\.cs\\.washington\\.edu;", "last_changed": "2024-01-02" }, { - "pattern": "combine.it.lth.se", + "pattern": "www\\.de\\.com", "last_changed": "2024-01-02" }, { - "pattern": "combine/0.0", + "pattern": "www\\.dead-links\\.com", "last_changed": "2024-01-02" }, { - "pattern": "computerorgs.com", + "pattern": "www\\.die-kraehe\\.de", "last_changed": "2024-01-02" }, { - "pattern": "content.easybib.com", + "pattern": "www\\.dnsdigger\\.com", "last_changed": "2024-01-02" }, { - "pattern": "coralcdn.org", + "pattern": "www\\.earthcom\\.info", "last_changed": "2024-01-02" }, { - "pattern": "cosco.hiit.fi", + "pattern": "www\\.eurekster\\.com", "last_changed": "2024-01-02" }, { - "pattern": "crawler", + "pattern": "www\\.europarchive\\.org", "last_changed": "2024-01-02" }, { - "pattern": "cs.daum.net", + "pattern": "www\\.eventax\\.de", "last_changed": "2024-01-02" }, { - "pattern": "csci_b659/0.13", + "pattern": "www\\.everatom\\.com", "last_changed": "2024-01-02" }, { - "pattern": "curl/7.18.2 (x86_64-", + "pattern": "www\\.evri\\.com", "last_changed": "2024-01-02" }, { - "pattern": "curl/7.19.7 (x86_64-", + "pattern": "www\\.exensa\\.com", "last_changed": "2024-01-02" }, { - "pattern": "curl/7.21.4 (univers", + "pattern": "www\\.exif-search\\.com", "last_changed": "2024-01-02" }, { - "pattern": "curl/7.37.1", + "pattern": "www\\.facebook\\.com", "last_changed": "2024-01-02" }, { - "pattern": "dcrawl/1.0", + "pattern": "www\\.favcollector\\.com", "last_changed": "2024-01-02" }, { - "pattern": "dds explorer v1.0 be", + "pattern": "www\\.feeddemon\\.com", "last_changed": "2024-01-02" }, { - "pattern": "deepak-USC/ISI", + "pattern": "www\\.feedhub\\.com", "last_changed": "2024-01-02" }, { - "pattern": "deusu.org", + "pattern": "www\\.feedly\\.com", "last_changed": "2024-01-02" }, { - "pattern": "developers.google.com", + "pattern": "www\\.feedmap\\.net", "last_changed": "2024-01-02" }, { - "pattern": "dhcibr.wordpress.com", + "pattern": "www\\.fileboost\\.net", "last_changed": "2024-01-02" }, { - "pattern": "digg.com", + "pattern": "www\\.findpal\\.com", "last_changed": "2024-01-02" }, { - "pattern": "dir.com", + "pattern": "www\\.floorstransformed\\.com", "last_changed": "2024-01-02" }, { - "pattern": "drupal.org", + "pattern": "www\\.flyindex\\.net", "last_changed": "2024-01-02" }, { - "pattern": "dznet.com", + "pattern": "www\\.followsite\\.com", "last_changed": "2024-01-02" }, { - "pattern": "e-sense 1.0 ea(www.v", + "pattern": "www\\.globalspec\\.com", "last_changed": "2024-01-02" }, { - "pattern": "eStyleSearch 4 (comp", + "pattern": "www\\.goguides\\.org", "last_changed": "2024-01-02" }, { - "pattern": "ec2linkfinder", - "last_changed": "2024-01-02", - "addition_date": "2012-03-22", - "instances": [ - "ec2linkfinder" - ] - }, - { - "pattern": "ecairn.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "ejupiter.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "enter4u.eu", - "last_changed": "2024-01-02" - }, - { - "pattern": "exactseek.com", + "pattern": "www\\.greaterera\\.com", "last_changed": "2024-01-02" }, { - "pattern": "example-masto-instance.org", + "pattern": "www\\.hermitsearch\\.com", "last_changed": "2024-01-02" }, { - "pattern": "exploringdh.wordpress.com", + "pattern": "www\\.hyperix\\.com", "last_changed": "2024-01-02" }, { - "pattern": "facebookexternalhit/", + "pattern": "www\\.ianett\\.com", "last_changed": "2024-01-02" }, { - "pattern": "feed.moo.jp", + "pattern": "www\\.icerocket\\.com", "last_changed": "2024-01-02" }, { - "pattern": "feedafever.com;", + "pattern": "www\\.illumit\\.com", "last_changed": "2024-01-02" }, { - "pattern": "flipboard.com", + "pattern": "www\\.iltrovatore\\.it", "last_changed": "2024-01-02" }, { - "pattern": "flunky", + "pattern": "www\\.inktomi\\.com", "last_changed": "2024-01-02" }, { - "pattern": "fly/6.01 libwww/4.0D", + "pattern": "www\\.intelliseek\\.com", "last_changed": "2024-01-02" }, { - "pattern": "free-downloads.net d", + "pattern": "www\\.ip2location\\.com", "last_changed": "2024-01-02" }, { - "pattern": "fulltext.seznam.cz", + "pattern": "www\\.jordomedia\\.com", "last_changed": "2024-01-02" }, { - "pattern": "gazz/x.x (gazz@nttrd", + "pattern": "www\\.kalooga\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "geourl/2.0b2", + "pattern": "www\\.kototoi\\.org", "last_changed": "2024-01-02" }, { - "pattern": "getpocket.com", + "pattern": "www\\.l3s\\.de", "last_changed": "2024-01-02" }, { - "pattern": "gigabaz.com", + "pattern": "www\\.lan4lano\\.net", "last_changed": "2024-01-02" }, { - "pattern": "github.com", + "pattern": "www\\.look\\.com", "last_changed": "2024-01-02" }, { - "pattern": "google-xrawler", - "last_changed": "2024-01-02", - "addition_date": "2018-09-05", - "instances": [ - "google-xrawler" - ], - "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for" - }, - { - "pattern": "googlebot", + "pattern": "www\\.lw01\\.com", "last_changed": "2024-01-02" }, { - "pattern": "gossamer-threads.com", + "pattern": "www\\.matuschek\\.net", "last_changed": "2024-01-02" }, { - "pattern": "grub-client", + "pattern": "www\\.megaindex\\.ru", "last_changed": "2024-01-02" }, { - "pattern": "grub.org", - "last_changed": "2024-01-02", - "instances": [ - "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" - ] - }, - { - "pattern": "guess.scritch.org", + "pattern": "www\\.meta-spinner\\.de", "last_changed": "2024-01-02" }, { - "pattern": "haste.kytoon.com", + "pattern": "www\\.mixdata\\.com", "last_changed": "2024-01-02" }, { - "pattern": "help.coccoc.com", + "pattern": "www\\.moreover\\.com", "last_changed": "2024-01-02" }, { - "pattern": "help.coccoc.vn", + "pattern": "www\\.neomo\\.de", "last_changed": "2024-01-02" }, { - "pattern": "help.goo.ne.jp", + "pattern": "www\\.newsgator\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "help.hepburnhome.ca", + "pattern": "www\\.ng-search\\.com", "last_changed": "2024-01-02" }, { - "pattern": "help.yahoo.co.jp", + "pattern": "www\\.page-store\\.com", "last_changed": "2024-01-02" }, { - "pattern": "help.yahoo.com", + "pattern": "www\\.pagebull\\.com", "last_changed": "2024-01-02" }, { - "pattern": "hgrepurl/1.0", + "pattern": "www\\.pingalink\\.com", "last_changed": "2024-01-02" }, { - "pattern": "holmes/3.9 (onet.pl)", + "pattern": "www\\.pingdom\\.com", "last_changed": "2024-01-02" }, { - "pattern": "holmes/3.9 (someurl.", + "pattern": "www\\.plinki\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "holmes/x.x", + "pattern": "www\\.powerset\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.5", + "pattern": "www\\.prairielandconsulting\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.5 (infosys", + "pattern": "www\\.pubblisito\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.5 (root@lo", + "pattern": "www\\.qango\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.5 (webmast", + "pattern": "www\\.quickiwiki\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.6 (mathieu", + "pattern": "www\\.qwant\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.6 (unconfi", + "pattern": "www\\.rawgrunt\\.com", "last_changed": "2024-01-02" }, { - "pattern": "htdig/3.1.x (root@lo", + "pattern": "www\\.relsoftware\\.com", "last_changed": "2024-01-02" }, { - "pattern": "http://www.mabontlan", + "pattern": "www\\.retrevo\\.com", "last_changed": "2024-01-02" }, { - "pattern": "http://www.monogol.d", + "pattern": "www\\.samsung\\.com", "last_changed": "2024-01-02" }, { - "pattern": "http://www.timelyweb", + "pattern": "www\\.scifihifi\\.com", "last_changed": "2024-01-02" }, { - "pattern": "http_get", - "last_changed": "2024-01-02", - "addition_date": "2017-11-02", - "instances": [ - "http_get" - ] - }, - { - "pattern": "httpunit/1.x", + "pattern": "www\\.scrubtheweb\\.com", "last_changed": "2024-01-02" }, { - "pattern": "i.stanford.edu", + "pattern": "www\\.searchme\\.com", "last_changed": "2024-01-02" }, { - "pattern": "iFeed.jp/2.0 (www.ps", + "pattern": "www\\.securecomputing\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver-web.arch", + "pattern": "www\\.semager\\.de", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/1.6", + "pattern": "www\\.shopwiki\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.0 (Win", + "pattern": "www\\.siigle\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.1 (Win", + "pattern": "www\\.simpy\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.2 (Win", + "pattern": "www\\.sitesearch\\.ca", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.8 (Win", + "pattern": "www\\.sitesell\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.9 (Lin", + "pattern": "www\\.sixxs\\.net", "last_changed": "2024-01-02" }, { - "pattern": "ia_archiver/8.9 (Win", + "pattern": "www\\.skaffe\\.com", "last_changed": "2024-01-02" }, { - "pattern": "icerocket.com;", + "pattern": "www\\.skyrocket\\.gr\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "ichiro/2.0 (ichiro@n", + "pattern": "www\\.sli-systems\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ichiro/x.0 (ichiro@n", + "pattern": "www\\.slider\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ideare - SignSite/1.", + "pattern": "www\\.slysearch\\.com", "last_changed": "2024-01-02" }, { - "pattern": "iearthworm/1.0, iear", + "pattern": "www\\.somusical\\.com", "last_changed": "2024-01-02" }, { - "pattern": "igde.ru", + "pattern": "www\\.suchen\\.de", "last_changed": "2024-01-02" }, { - "pattern": "ilial/Nutch-0.9-dev", + "pattern": "www\\.sygol\\.com", "last_changed": "2024-01-02" }, { - "pattern": "innovationblog.com", + "pattern": "www\\.sync2it\\.com", "last_changed": "2024-01-02" }, { - "pattern": "integrity/1.6", + "pattern": "www\\.targetgroups\\.net", "last_changed": "2024-01-02" }, { - "pattern": "integrity/5", + "pattern": "www\\.teradex\\.com", "last_changed": "2024-01-02" }, { - "pattern": "ivia.ucr.edu", + "pattern": "www\\.tkensaku\\.com", "last_changed": "2024-01-02" }, { - "pattern": "jack", + "pattern": "www\\.truveo\\.com", "last_changed": "2024-01-02" }, { - "pattern": "journster.com", + "pattern": "www\\.tumblr\\.com", "last_changed": "2024-01-02" }, { - "pattern": "knil", + "pattern": "www\\.ukwizz\\.com", "last_changed": "2024-01-02" }, { - "pattern": "kulturarw3/0.1", + "pattern": "www\\.unchaos\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin (samualt9@big", + "pattern": "www\\.updated\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.1.1 larbin2", + "pattern": "www\\.urltrends\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.2.0 (crawl@", + "pattern": "www\\.vakes\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.2.1_de_Vien", + "pattern": "www\\.versions-project\\.org", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.2.2 (sugaya", + "pattern": "www\\.verzamelgids\\.nl", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.2.2_guillau", + "pattern": "www\\.wavefire\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.5.0 (larbin", + "pattern": "www\\.webarchiv\\.cz", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.0 (larbin", + "pattern": "www\\.worio\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.1 (larbin", + "pattern": "www\\.wowlemmings\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.1 larbin2", + "pattern": "www\\.xirq\\.com;", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (hamasa", + "pattern": "www\\.xrayz\\.co\\.uk", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (larbin", + "pattern": "www\\.yama\\.info\\.waseda\\.ac\\.jp", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (liston", + "pattern": "www\\.yoono\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (pierre", + "pattern": "www\\.zedzo\\.com", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (pimena", + "pattern": "wwwster\\/1\\.x", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (tom@le", + "pattern": "xmlset_roodkcableoj2", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 (vitalb", + "pattern": "yarienavoir\\.net\\/0\\.2", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 dthunen", + "pattern": "yoogliFetchAgent\\/0\\.1", "last_changed": "2024-01-02" }, { - "pattern": "larbin_2.6.2 kalou@k", - "last_changed": "2024-01-02" + "pattern": "Mediapartners\\s\\\\", + "addition_date": "2017-08-08", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6.2 larbin2", - "last_changed": "2024-01-02" + "pattern": "\\[wW]get", + "instances": [ + "WGETbot/1.0 (+http://wget.alanreed.org)", + "Wget/1.14 (linux-gnu)", + "Wget/1.20.3 (linux-gnu)" + ] }, { - "pattern": "larbin_2.6.2 larbin@", - "last_changed": "2024-01-02" + "pattern": "aiohttp", + "addition_date": "2019-12-23", + "instances": [ + "Python/3.9 aiohttp/3.7.3", + "Python/3.8 aiohttp/3.7.2", + "Python/3.7 aiohttp/3.6.2a2" + ], + "url": "https://docs.aiohttp.org/en/stable/", + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6.2 listonA", - "last_changed": "2024-01-02" + "pattern": "httpunit", + "instances": [ + "httpunit/1.x" + ] }, { - "pattern": "larbin_2.6.2 pierre@", - "last_changed": "2024-01-02" + "pattern": "Go-http-client", + "addition_date": "2016-03-26", + "url": "https://golang.org/pkg/net/http/", + "instances": [ + "Go-http-client/1.1", + "Go-http-client/2.0" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6.2 vitalbo", - "last_changed": "2024-01-02" + "pattern": "Gigablast", + "instances": [ + "GigablastOpenSource/1.0" + ], + "url": "https://github.com/gigablast/open-source-search-engine" }, { - "pattern": "larbin_2.6.3 (larbin", - "last_changed": "2024-01-02" + "pattern": "netresearchserver", + "instances": [] }, { - "pattern": "larbin_2.6.3 (wgao@g", - "last_changed": "2024-01-02" + "pattern": "fluffy", + "instances": [] }, { - "pattern": "larbin_2.6.3 capveg@", - "last_changed": "2024-01-02" + "pattern": "findlink", + "instances": [ + "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", + "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" + ] }, { - "pattern": "larbin_2.6.3 ghary@s", - "last_changed": "2024-01-02" + "pattern": "ips-agent", + "instances": [ + "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", + "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" + ] }, { - "pattern": "larbin_2.6.3 gqnmgsp", - "last_changed": "2024-01-02" + "pattern": "yanga", + "instances": [ + "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" + ] }, { - "pattern": "larbin_2.6.3 larbin2", - "last_changed": "2024-01-02" + "pattern": "CyberPatrol", + "addition_date": "2010-02-11", + "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", + "instances": [ + "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6.3 tangyi8", - "last_changed": "2024-01-02" + "pattern": "postrank", + "addition_date": "2010-08-03", + "url": "http://www.postrank.com", + "instances": [ + "PostRank/2.0 (postrank.com)", + "PostRank/2.0 (postrank.com; 1 subscribers)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6.3 zumesun", - "last_changed": "2024-01-02" + "pattern": "linkdex", + "addition_date": "2011-01-06", + "url": "http://www.linkdex.com", + "instances": [ + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", + "linkdex.com/v2.0", + "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_2.6_basileoca", - "last_changed": "2024-01-02" + "pattern": "ezooms", + "addition_date": "2011-04-27", + "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", + "instances": [ + "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_test (nobody@", - "last_changed": "2024-01-02" + "pattern": "europarchive\\.org", + "addition_date": "2011-06-21", + "url": "", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_test nobody@a", - "last_changed": "2024-01-02" + "pattern": "Ahrefs", + "addition_date": "2011-08-28", + "instances": [ + "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "larbin_xy250 larbin2", - "last_changed": "2024-01-02" + "pattern": "mappydata", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "law.di.unimi.it", - "last_changed": "2024-01-02" + "pattern": "eright", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "lc/$ROADS::Version l", - "last_changed": "2024-01-02" + "pattern": "Apercite", + "addition_date": "2018-06-28", + "instances": [ + "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "letscrawl.com", - "last_changed": "2024-01-02" + "pattern": "Aboundex", + "addition_date": "2011-09-28", + "url": "http://www.aboundex.com/crawler/", + "instances": [ + "Aboundex/0.2 (http://www.aboundex.com/crawler/)", + "Aboundex/0.3 (http://www.aboundex.com/crawler/)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "libWeb/clsHTTP -- hi", - "last_changed": "2024-01-02" + "pattern": "facebookexternalhit", + "addition_date": "2012-05-07", + "instances": [ + "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", + "facebookexternalhit/1.1", + "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" + ], + "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", + "last_changed": "2024-01-10" }, { - "pattern": "libwww-perl/5.41", - "last_changed": "2024-01-02" + "pattern": "Yeti", + "addition_date": "2012-05-07", + "url": "http://naver.me/bot", + "instances": [ + "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "libwww-perl/5.45", - "last_changed": "2024-01-02" + "pattern": "RetrevoPageAnalyzer", + "addition_date": "2012-05-07", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "libwww-perl/5.48", - "last_changed": "2024-01-02" + "pattern": "wotbox", + "addition_date": "2012-06-12", + "url": "http://www.wotbox.com", + "instances": [ + "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", + "Wotbox/2.01 (+http://www.wotbox.com/bot/)" + ], + "last_changed": "2024-01-10" }, { - "pattern": "libwww-perl/5.50", - "last_changed": "2024-01-02" + "pattern": "drupact", + "addition_date": "2012-09-27", + "url": "http://www.arocom.de/drupact", + "instances": [ + "drupact/0.7; http://www.arocom.de/drupact" + ], + "last_changed": "2024-01-10" }, { - "pattern": "libwww-perl/5.52 FP/", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww-perl/5.53", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww-perl/5.63", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww-perl/5.64", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww-perl/5.65", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww-perl/5.800", - "last_changed": "2024-01-02" - }, - { - "pattern": "libwww/5.3.2", - "last_changed": "2024-01-02" - }, - { - "pattern": "linkdex.com/v2.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "ltx71.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "lunefran.wordpress.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.32", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.33", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.34", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.35", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.36", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.38", - "last_changed": "2024-01-02" - }, - { - "pattern": "lwp-trivial/1.41", - "last_changed": "2024-01-02" - }, - { - "pattern": "mailto:webcraft@bea.", - "last_changed": "2024-01-02" - }, - { - "pattern": "mbblog14.wordpress.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "metatagsdir.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "miniflux.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "misc.yahoo.com.cn", - "last_changed": "2024-01-02" - }, - { - "pattern": "moget/x.x (moget@goo", - "last_changed": "2024-01-02" - }, - { - "pattern": "mogimogi/1.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "moiNAG 0.02", - "last_changed": "2024-01-02" - }, - { - "pattern": "muckrack.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "myDaemon", - "last_changed": "2024-01-02" - }, - { - "pattern": "net-promoter.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "netarkivet.dk", - "last_changed": "2024-01-02" - }, - { - "pattern": "newsearchengine (Thi", - "last_changed": "2024-01-02" - }, - { - "pattern": "newsme/1.0; feedback", - "last_changed": "2024-01-02" - }, - { - "pattern": "newspaper/0.1.0.7", - "last_changed": "2024-01-02" - }, - { - "pattern": "newspaper/0.2.5", - "last_changed": "2024-01-02" - }, - { - "pattern": "newspaper/0.2.6", - "last_changed": "2024-01-02" - }, - { - "pattern": "newstin.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "nlp.uned.es", - "last_changed": "2024-01-02" - }, - { - "pattern": "noyona_0_1", - "last_changed": "2024-01-02" - }, - { - "pattern": "nsyght.com/Nutch-1.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "nsyght.com/Nutch-x.x", - "last_changed": "2024-01-02" - }, - { - "pattern": "nutch-1.4/Nutch-1.4", - "last_changed": "2024-01-02" - }, - { - "pattern": "nutchsearch/Nutch-0.", - "last_changed": "2024-01-02" - }, - { - "pattern": "nymesis.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "oegp v. 1.3.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "okhttp/2.5.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "okhttp/2.7.5", - "last_changed": "2024-01-02" - }, - { - "pattern": "okhttp/3.2.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "okhttp/3.5.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "omgili.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "ostermiller.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "owler-agent", - "last_changed": "2024-01-02" - }, - { - "pattern": "ozelot/2.7.3 (Search", - "last_changed": "2024-01-02" - }, - { - "pattern": "page2rss.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "pandora.nla.gov.au", - "last_changed": "2024-01-02" - }, - { - "pattern": "panscient.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "parsijoo.ir", - "last_changed": "2024-01-02" - }, - { - "pattern": "pauillac.inria.fr", - "last_changed": "2024-01-02" - }, - { - "pattern": "pd02_1.0.0 pd02_1.0.", - "last_changed": "2024-01-02" - }, - { - "pattern": "pear.php.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "people.eng.unimelb.edu.au", - "last_changed": "2024-01-02" - }, - { - "pattern": "phpcrawl", - "last_changed": "2024-01-02", - "addition_date": "2012-09/17", - "url": "http://phpcrawl.cuab.de/", - "instances": [ - "phpcrawl" - ] - }, - { - "pattern": "pompos.iliad.fr", - "last_changed": "2024-01-02" - }, - { - "pattern": "psycheclone", - "last_changed": "2024-01-02" - }, - { - "pattern": "publisher.yahoo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "python-requests/0.13", - "last_changed": "2024-01-02" - }, - { - "pattern": "python-requests/1.0.", - "last_changed": "2024-01-02" - }, - { - "pattern": "python-requests/2.18", - "last_changed": "2024-01-02" - }, - { - "pattern": "python-requests/2.3.", - "last_changed": "2024-01-02" - }, - { - "pattern": "r.hatena.ne.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "rabaz (rabaz at giga", - "last_changed": "2024-01-02" - }, - { - "pattern": "ranksonic.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "rico/0.1", - "last_changed": "2024-01-02" - }, - { - "pattern": "riddler.io", - "last_changed": "2024-01-02" - }, - { - "pattern": "rjpower.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "robot", - "last_changed": "2024-01-02" - }, - { - "pattern": "sagool.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "scooter-venus-3.0.vn", - "last_changed": "2024-01-02" - }, - { - "pattern": "scoutmaster", - "last_changed": "2024-01-02" - }, - { - "pattern": "scrapy.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "search.ch V1.4", - "last_changed": "2024-01-02" - }, - { - "pattern": "search.denmex.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "search.goo.ne.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "search.google.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "search.yahoo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "seekda.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "semanticdiscovery/0.", - "last_changed": "2024-01-02" - }, - { - "pattern": "sentry.io", - "last_changed": "2024-01-02" - }, - { - "pattern": "silk/1.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "sitebar.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "siteexplorer.info", - "last_changed": "2024-01-02", - "addition_date": "2013-05-01", - "instances": [ - "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", - "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" - ] - }, - { - "pattern": "smartwit.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "sna-0.0.1 (mikemuzio", - "last_changed": "2024-01-02" - }, - { - "pattern": "sna-0.0.1 mikeelliot", - "last_changed": "2024-01-02" - }, - { - "pattern": "sohu agent", - "last_changed": "2024-01-02" - }, - { - "pattern": "sohu-search", - "last_changed": "2024-01-02" - }, - { - "pattern": "sp.ask.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "suchbaer.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "summify.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "support.embed.ly", - "last_changed": "2024-01-02" - }, - { - "pattern": "surf", - "last_changed": "2024-01-02" - }, - { - "pattern": "suzuran", - "last_changed": "2024-01-02" - }, - { - "pattern": "szukaj.onet.pl", - "last_changed": "2024-01-02" - }, - { - "pattern": "tags2dir.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "tagword.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "targetblaster.com/0.", - "last_changed": "2024-01-02" - }, - { - "pattern": "teoma_agent1", - "last_changed": "2024-01-02" - }, - { - "pattern": "therarestwords.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "toCrawl/UrlDispatche", - "last_changed": "2024-01-02" - }, - { - "pattern": "tuezilla.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "twikle.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "twurly.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "unknownght.com Web S", - "last_changed": "2024-01-02" - }, - { - "pattern": "users.cs.cf.ac.uk", - "last_changed": "2024-01-02" - }, - { - "pattern": "versus.integis.ch", - "last_changed": "2024-01-02" - }, - { - "pattern": "vias.ncsa.uiuc.edu", - "last_changed": "2024-01-02" - }, - { - "pattern": "visionutils/0.2", - "last_changed": "2024-01-02" - }, - { - "pattern": "vk.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "voyager-hc/1.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "voyager/1.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "web-bekannt (Version", - "last_changed": "2024-01-02" - }, - { - "pattern": "web2express.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "webGobbler/1.x.x", - "last_changed": "2024-01-02" - }, - { - "pattern": "webagent.wise-guys.nl", - "last_changed": "2024-01-02" - }, - { - "pattern": "webalgo.iit.cnr.it", - "last_changed": "2024-01-02" - }, - { - "pattern": "webbandit/4.xx.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.114", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.117", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.125", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.129", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.93", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcollage/1.xx", - "last_changed": "2024-01-02" - }, - { - "pattern": "webcrawl.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "webdatastats.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "webglimpse.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "webhack", - "last_changed": "2024-01-02" - }, - { - "pattern": "websearchbench.cs.uni-dortmund.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "websitealert.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "webster-internet.de ", - "last_changed": "2024-01-02" - }, - { - "pattern": "webster.healeys.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "whatUseek_winona/3.0", - "last_changed": "2024-01-02" - }, - { - "pattern": "wiki.creativecommons.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "wish.slis.tsukuba.ac.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "wkd.iis.sinica.edu.tw", - "last_changed": "2024-01-02" - }, - { - "pattern": "wortschatz.uni-leipzig.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "ws.daum.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.Abolimba.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.WISEnut.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.abonti.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ackerm.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.activtourist.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.agada.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ahmadsoftware.co", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.answerbus.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ant.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.aol.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.aramabeta.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.arianna.it", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.arocom.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.asaha.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.axxus.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ay-up.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ba.be", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.blaiz.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.bloglines.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.bortzmeyer.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.brandverity.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.buddybuzz.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.buscaplus.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.business-socket.", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.canadiancontent.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.chepi.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.chilkatsoft.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.cloudflare.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.cnet.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.cougarsearch.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.covario.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.crim.ca", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.cs.washington.edu", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.cs.washington.edu;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.dataparksearch.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.de.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.dead-links.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.die-kraehe.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.dnsdigger.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.earthcom.info", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.eurekster.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.europarchive.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.eventax.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.everatom.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.evri.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.exensa.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.exif-search.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.facebook.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.favcollector.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.feeddemon.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.feedhub.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.feedly.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.feedmap.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.fileboost.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.findpal.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.floorstransformed.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.flyindex.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.followsite.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.genieo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.globalspec.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.goguides.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.google.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.greaterera.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.hermitsearch.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.htmlvalidator.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.hyperix.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ianett.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.icerocket.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.illumit.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.iltrovatore.it", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.inktomi.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.intelliseek.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ip2location.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.iskanie.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.jordomedia.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.kalooga.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.kototoi.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.l3s.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.lan4lano.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.lipperhey.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.look.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.lw01.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.matuschek.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.megaindex.ru", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.megite.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.meta-spinner.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.mixdata.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.moreover.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.neomo.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.netluchs.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.newsgator.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ng-search.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.page-store.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.pagebull.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.pingalink.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.pingdom.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.plinki.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.powerset.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.prairielandconsulting.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.proximic.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.pubblisito.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.qango.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.quickiwiki.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.qwant.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.rawgrunt.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.relsoftware.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.retrevo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.samsung.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.scifihifi.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.scoutjet.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.scrubtheweb.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.searchme.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.securecomputing.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.semager.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.shopwiki.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.siigle.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.simpy.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sitesearch.ca", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sitesell.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sixxs.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.skaffe.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.skyrocket.gr.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sli-systems.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.slider.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.slysearch.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.somusical.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.suchen.de", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sygol.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sync2it.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.sysomos.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.targetgroups.net", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.teradex.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.tkensaku.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.truveo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.tumblr.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.ukwizz.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.unchaos.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.updated.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.urltrends.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.vakes.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.versions-project.org", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.verzamelgids.nl", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.wavefire.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.webarchiv.cz", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.worio.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.wowlemmings.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.xirq.com;", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.xrayz.co.uk", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.yama.info.waseda.ac.jp", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.yoono.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www.zedzo.com", - "last_changed": "2024-01-02" - }, - { - "pattern": "www4mail/2.x libwww-", - "last_changed": "2024-01-02" - }, - { - "pattern": "wwwster/1.x (Beta- m", - "last_changed": "2024-01-02" - }, - { - "pattern": "xmlset_roodkcableoj2", - "last_changed": "2024-01-02" - }, - { - "pattern": "yacy (www.yacy.net; ", - "last_changed": "2024-01-02" - }, - { - "pattern": "yarienavoir.net/0.2", - "last_changed": "2024-01-02" - }, - { - "pattern": "yggdrasil/Nutch-0.9 ", - "last_changed": "2024-01-02" - }, - { - "pattern": "yoogliFetchAgent/0.1", - "last_changed": "2024-01-02" - }, - { - "pattern": "Googlebot\\/", - "url": "http://www.google.com/bot.html", - "instances": [ - "Googlebot/2.1 (+http://www.google.com/bot.html)", - "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36" - ] - }, - { - "pattern": "Googlebot-Mobile", - "instances": [ - "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", - "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", - "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" - ] - }, - { - "pattern": "Googlebot-Image", - "instances": [ - "Googlebot-Image/1.0" - ] - }, - { - "pattern": "Googlebot-News", - "instances": [ - "Googlebot-News" - ] - }, - { - "pattern": "Googlebot-Video", - "instances": [ - "Googlebot-Video/1.0" - ] - }, - { - "pattern": "AdsBot-Google([^-]|$)", - "url": "https://support.google.com/webmasters/answer/1061943?hl=en", - "instances": [ - "AdsBot-Google (+http://www.google.com/adsbot.html)" - ] - }, - { - "pattern": "AdsBot-Google-Mobile", - "addition_date": "2017-08-21", - "url": "https://support.google.com/adwords/answer/2404197", - "instances": [ - "AdsBot-Google-Mobile-Apps", - "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Feedfetcher-Google", - "addition_date": "2018-06-27", - "url": "https://support.google.com/webmasters/answer/178852", - "instances": [ - "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Mediapartners-Google", - "url": "https://support.google.com/webmasters/answer/1061943?hl=en", - "instances": [ - "Mediapartners-Google", - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" - ] - }, - { - "pattern": "Mediapartners \\(Googlebot\\)", - "addition_date": "2017-08-08", - "url": "https://support.google.com/webmasters/answer/1061943?hl=en", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "APIs-Google", - "addition_date": "2017-08-08", - "url": "https://support.google.com/webmasters/answer/1061943?hl=en", - "instances": [ - "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Google-InspectionTool", - "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", - "instances": [ - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", - "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" - ] - }, - { - "pattern": "Storebot-Google", - "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", - "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" - ] - }, - { - "pattern": "GoogleOther", - "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", - "instances": [ - "GoogleOther" - ] - }, - { - "pattern": "bingbot", - "url": "http://www.bing.com/bingbot.htm", - "instances": [ - "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36", - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36" - ] - }, - { - "pattern": "Slurp", - "url": "http://help.yahoo.com/help/us/ysearch/slurp", - "instances": [ - "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", - "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", - "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" - ] - }, - { - "pattern": "[wW]get", - "instances": [ - "WGETbot/1.0 (+http://wget.alanreed.org)", - "Wget/1.14 (linux-gnu)", - "Wget/1.20.3 (linux-gnu)" - ] - }, - { - "pattern": "LinkedInBot", - "instances": [ - "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", - "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", - "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)" - ] - }, - { - "pattern": "Python-urllib", - "instances": [ - "Python-urllib/1.17", - "Python-urllib/2.5", - "Python-urllib/2.6", - "Python-urllib/2.7", - "Python-urllib/3.1", - "Python-urllib/3.2", - "Python-urllib/3.3", - "Python-urllib/3.4", - "Python-urllib/3.5", - "Python-urllib/3.6", - "Python-urllib/3.7" - ] - }, - { - "pattern": "python-requests", - "addition_date": "2018-05-27", - "instances": [ - "python-requests/2.9.2", - "python-requests/2.11.1", - "python-requests/2.18.4", - "python-requests/2.19.1", - "python-requests/2.20.0", - "python-requests/2.21.0", - "python-requests/2.22.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "aiohttp", - "addition_date": "2019-12-23", - "instances": [ - "Python/3.9 aiohttp/3.7.3", - "Python/3.8 aiohttp/3.7.2", - "Python/3.7 aiohttp/3.6.2a2" - ], - "url": "https://docs.aiohttp.org/en/stable/", - "last_changed": "2024-01-10" - }, - { - "pattern": "httpx", - "addition_date": " 2019/12/23", - "instances": [ - "python-httpx/0.16.1", - "python-httpx/0.13.0.dev1" - ], - "url": "https://www.python-httpx.org" - }, - { - "pattern": "libwww-perl", - "instances": [ - "2Bone_LinkChecker/1.0 libwww-perl/6.03", - "2Bone_LinkChkr/1.0 libwww-perl/6.03", - "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" - ] - }, - { - "pattern": "httpunit", - "instances": [ - "httpunit/1.x" - ] - }, - { - "pattern": "nutch", - "instances": [ - "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", - "istellabot-nutch/Nutch-1.10" - ] - }, - { - "pattern": "Go-http-client", - "addition_date": "2016-03-26", - "url": "https://golang.org/pkg/net/http/", - "instances": [ - "Go-http-client/1.1", - "Go-http-client/2.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "phpcrawl", - "addition_date": "2012-09/17", - "url": "http://phpcrawl.cuab.de/", - "instances": [ - "phpcrawl" - ] - }, - { - "pattern": "msnbot", - "url": "http://search.msn.com/msnbot.htm", - "instances": [ - "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", - "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", - "librabot/1.0 (+http://search.msn.com/msnbot.htm)", - "librabot/2.0 (+http://search.msn.com/msnbot.htm)", - "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", - "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", - "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", - "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", - "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", - "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", - "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", - "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", - "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", - "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" - ] - }, - { - "pattern": "jyxobot", - "instances": [] - }, - { - "pattern": "FAST-WebCrawler", - "instances": [ - "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.8" - ] - }, - { - "pattern": "FAST Enterprise Crawler", - "instances": [ - "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", - "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" - ] - }, - { - "pattern": "BIGLOTRON", - "instances": [ - "BIGLOTRON (Beta 2;GNU/Linux)" - ] - }, - { - "pattern": "Teoma", - "instances": [ - "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", - "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" - ], - "url": "http://about.ask.com/en/docs/about/webmasters.shtml" - }, - { - "pattern": "convera", - "instances": [ - "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" - ], - "url": "http://ews.converasearch.com/crawl.htm" - }, - { - "pattern": "seekbot", - "instances": [ - "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" - ], - "url": "http://www.seekbot.net/bot.html" - }, - { - "pattern": "Gigabot", - "instances": [ - "Gigabot/1.0", - "Gigabot/2.0 (http://www.gigablast.com/spider.html)" - ], - "url": "http://www.gigablast.com/spider.html" - }, - { - "pattern": "Gigablast", - "instances": [ - "GigablastOpenSource/1.0" - ], - "url": "https://github.com/gigablast/open-source-search-engine" - }, - { - "pattern": "exabot", - "instances": [ - "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", - "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", - "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", - "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", - "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", - "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" - ] - }, - { - "pattern": "ia_archiver", - "instances": [ - "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", - "ia_archiver-web.archive.org" - ] - }, - { - "pattern": "GingerCrawler", - "instances": [ - "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" - ] - }, - { - "pattern": "webmon ", - "instances": [] - }, - { - "pattern": "HTTrack", - "instances": [ - "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" - ] - }, - { - "pattern": "grub.org", - "instances": [ - "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" - ] - }, - { - "pattern": "UsineNouvelleCrawler", - "instances": [] - }, - { - "pattern": "antibot", - "instances": [] - }, - { - "pattern": "netresearchserver", - "instances": [] - }, - { - "pattern": "speedy", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", - "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", - "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", - "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" - ] - }, - { - "pattern": "fluffy", - "instances": [] - }, - { - "pattern": "findlink", - "instances": [ - "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", - "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", - "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" - ] - }, - { - "pattern": "msrbot", - "instances": [] - }, - { - "pattern": "panscient", - "instances": [ - "panscient.com" - ] - }, - { - "pattern": "yacybot", - "instances": [ - "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", - "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", - "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", - "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html" - ] - }, - { - "pattern": "AISearchBot", - "instances": [] - }, - { - "pattern": "ips-agent", - "instances": [ - "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", - "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" - ] - }, - { - "pattern": "tagoobot", - "instances": [] - }, - { - "pattern": "MJ12bot", - "instances": [ - "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", - "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" - ] - }, - { - "pattern": "woriobot", - "instances": [ - "Mozilla/5.0 (compatible; woriobot +http://worio.com)", - "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" - ] - }, - { - "pattern": "yanga", - "instances": [ - "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" - ] - }, - { - "pattern": "buzzbot", - "instances": [ - "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" - ] - }, - { - "pattern": "mlbot", - "instances": [ - "MLBot (www.metadatalabs.com/mlbot)" - ] - }, - { - "pattern": "yandex\\.com\\/bots", - "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", - "instances": [ - "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots", - "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots yabs01)", - "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMetrika/4.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots", - "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMarket/2.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)", - "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)" - ], - "addition_date": "2015-04-14", - "last_changed": "2024-01-10" - }, - { - "pattern": "purebot", - "addition_date": "2010-01-19", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "Linguee Bot", - "addition_date": "2010-01-26", - "url": "http://www.linguee.com/bot", - "instances": [ - "Linguee Bot (http://www.linguee.com/bot)", - "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "CyberPatrol", - "addition_date": "2010-02-11", - "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", - "instances": [ - "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "voilabot", - "addition_date": "2010-05-18", - "instances": [ - "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Baiduspider", - "addition_date": "2010-07-15", - "url": "http://www.baidu.jp/spider/", - "instances": [ - "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", - "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "citeseerxbot", - "addition_date": "2010-07-17", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "spbot", - "addition_date": "2010-07-31", - "url": "http://www.seoprofiler.com/bot", - "instances": [ - "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", - "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", - "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", - "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "twengabot", - "addition_date": "2010-08-03", - "url": "http://www.twenga.com/bot.html", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "postrank", - "addition_date": "2010-08-03", - "url": "http://www.postrank.com", - "instances": [ - "PostRank/2.0 (postrank.com)", - "PostRank/2.0 (postrank.com; 1 subscribers)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Turnitin", - "addition_date": "2010-09-26", - "url": "http://www.turnitin.com", - "instances": [ - "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", - "Turnitin (https://bit.ly/2UvnfoQ)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "scribdbot", - "addition_date": "2010-09-28", - "url": "http://www.scribd.com", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "page2rss", - "addition_date": "2010-10-07", - "url": "http://www.page2rss.com", - "instances": [ - "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "sitebot", - "addition_date": "2010-12-15", - "url": "http://www.sitebot.org", - "instances": [ - "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "linkdex", - "addition_date": "2011-01-06", - "url": "http://www.linkdex.com", - "instances": [ - "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", - "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", - "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", - "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", - "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", - "linkdex.com/v2.0", - "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Adidxbot", - "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx", - "instances": [] - }, - { - "pattern": "ezooms", - "addition_date": "2011-04-27", - "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", - "instances": [ - "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "dotbot", - "addition_date": "2011-04-27", - "instances": [ - "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", - "dotbot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Mail.RU_Bot", - "addition_date": "2011-04-27", - "instances": [ - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", - "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/", - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "discobot", - "addition_date": "2011-05-03", - "url": "http://discoveryengine.com/discobot.html", - "instances": [ - "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", - "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", - "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "heritrix", - "addition_date": "2011-06-21", - "url": "https://github.com/internetarchive/heritrix3/wiki", - "instances": [ - "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", - "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", - "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", - "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", - "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", - "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", - "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", - "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", - "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", - "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", - "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", - "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", - "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", - "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", - "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", - "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", - "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", - "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", - "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", - "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", - "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "findthatfile", - "addition_date": "2011-06-21", - "url": "http://www.findthatfile.com/", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "europarchive.org", - "addition_date": "2011-06-21", - "url": "", - "instances": [ - "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "NerdByNature.Bot", - "addition_date": "2011-07-12", - "url": "http://www.nerdbynature.net/bot", - "instances": [ - "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "sistrix crawler", - "addition_date": "2011-08-02", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "Ahrefs(Bot|SiteAudit)", - "addition_date": "2011-08-28", - "instances": [ - "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", - "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "fuelbot", - "addition_date": "2018-06-28", - "instances": [ - "fuelbot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "CrunchBot", - "addition_date": "2018-06-28", - "instances": [ - "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "IndeedBot", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "mappydata", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "woobot", - "addition_date": "2018-06-28", - "instances": [ - "woobot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ZoominfoBot", - "addition_date": "2018-06-28", - "instances": [ - "ZoominfoBot (zoominfobot at zoominfo dot com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "PrivacyAwareBot", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Multiviewbot", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SWIMGBot", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Grobbot", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "eright", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Apercite", - "addition_date": "2018-06-28", - "instances": [ - "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "semanticbot", - "addition_date": "2018-06-28", - "instances": [ - "semanticbot", - "semanticbot (info@semanticaudience.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Aboundex", - "addition_date": "2011-09-28", - "url": "http://www.aboundex.com/crawler/", - "instances": [ - "Aboundex/0.2 (http://www.aboundex.com/crawler/)", - "Aboundex/0.3 (http://www.aboundex.com/crawler/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "domaincrawler", - "addition_date": "2011-10-21", - "instances": [ - "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "wbsearchbot", - "addition_date": "2011-12-21", - "url": "http://www.warebay.com/bot.html", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "summify", - "addition_date": "2012-01-04", - "url": "http://summify.com", - "instances": [ - "Summify (Summify/1.0.1; +http://summify.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "CCBot", - "addition_date": "2012-02-05", - "url": "http://www.commoncrawl.org/bot.html", - "instances": [ - "CCBot/2.0 (http://commoncrawl.org/faq/)", - "CCBot/2.0 (https://commoncrawl.org/faq/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "edisterbot", - "addition_date": "2012-02-25", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "SeznamBot", - "addition_date": "2012-03-14", - "instances": [ - "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", - "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", - "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", - "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", - "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)", - "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ec2linkfinder", - "addition_date": "2012-03-22", - "instances": [ - "ec2linkfinder" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "gslfbot", - "addition_date": "2012-04-03", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "aiHitBot", - "addition_date": "2012-04-16", - "instances": [ - "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "intelium_bot", - "addition_date": "2012-05-07", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "facebookexternalhit", - "addition_date": "2012-05-07", - "instances": [ - "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", - "facebookexternalhit/1.1", - "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" - ], - "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Yeti", - "addition_date": "2012-05-07", - "url": "http://naver.me/bot", - "instances": [ - "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "RetrevoPageAnalyzer", - "addition_date": "2012-05-07", - "instances": [ - "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "lb-spider", - "addition_date": "2012-05-07", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "Sogou", - "addition_date": "2012-05-13", - "url": "http://www.sogou.com/docs/help/webmasters.htm#07", - "instances": [ - "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", - "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", - "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "lssbot", - "addition_date": "2012-05-15", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "careerbot", - "addition_date": "2012-05-23", - "url": "http://www.career-x.de/bot.html", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "wotbox", - "addition_date": "2012-06-12", - "url": "http://www.wotbox.com", - "instances": [ - "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", - "Wotbox/2.01 (+http://www.wotbox.com/bot/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "wocbot", - "addition_date": "2012-07-25", - "url": "http://www.wocodi.com/crawler", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "ichiro", - "addition_date": "2012-08-28", - "url": "http://help.goo.ne.jp/help/article/1142", - "instances": [ - "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", - "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", - "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", - "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", - "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", - "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", - "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", - "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", - "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", - "ichiro/2.0 (ichiro@nttr.co.jp)", - "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", - "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", - "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", - "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", - "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "DuckDuckBot", - "addition_date": "2012-09-19", - "url": "http://duckduckgo.com/duckduckbot.html", - "instances": [ - "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", - "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", - "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)", - "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "lssrocketcrawler", - "addition_date": "2012-09-24", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "drupact", - "addition_date": "2012-09-27", - "url": "http://www.arocom.de/drupact", - "instances": [ - "drupact/0.7; http://www.arocom.de/drupact" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "webcompanycrawler", - "addition_date": "2012-10-03", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "acoonbot", - "addition_date": "2012-10-07", - "url": "http://www.acoon.de/robot.asp", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "openindexspider", - "addition_date": "2012-10-26", - "url": "http://www.openindex.io/en/webmasters/spider.html", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "gnam gnam spider", - "addition_date": "2012-10-31", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "web-archive-net.com.bot", - "instances": [] - }, - { - "pattern": "backlinkcrawler", - "addition_date": "2013-01-04", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "coccoc", - "addition_date": "2013-01-04", - "url": "http://help.coccoc.vn/", - "instances": [ - "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", - "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", - "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", - "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", - "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", - "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", - "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", - "coccoc", - "coccoc/1.0 ()", - "coccoc/1.0 (http://help.coccoc.com/)", - "coccoc/1.0 (http://help.coccoc.vn/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "integromedb", - "addition_date": "2013-01-10", - "url": "http://www.integromedb.org/Crawler", - "instances": [ - "www.integromedb.org/Crawler" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "content crawler spider", - "addition_date": "2013-01-11", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "toplistbot", - "addition_date": "2013-02-05", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "it2media-domain-crawler", - "addition_date": "2013-03-12", - "instances": [ - "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", - "it2media-domain-crawler/2.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ip-web-crawler.com", - "addition_date": "2013-03-22", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "siteexplorer.info", - "addition_date": "2013-05-01", - "instances": [ - "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", - "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "elisabot", - "addition_date": "2013-06-27", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "proximic", - "addition_date": "2013-09-12", - "url": "http://www.proximic.com/info/spider.php", - "instances": [ - "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", - "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "changedetection", - "addition_date": "2013-09-13", - "url": "http://www.changedetection.com/bot.html", - "instances": [ - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "arabot", - "addition_date": "2013-10-09", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "WeSEE:Search", - "addition_date": "2013-11-18", - "instances": [ - "WeSEE:Search", - "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "niki-bot", - "addition_date": "2014-01-01", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "CrystalSemanticsBot", - "addition_date": "2014-02-17", - "url": "http://www.crystalsemantics.com/user-agent/", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "rogerbot", - "addition_date": "2014-02-28", - "url": "http://moz.com/help/pro/what-is-rogerbot-", - "instances": [ - "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", - "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", - "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", - "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", - "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", - "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", - "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", - "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", - "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", - "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", - "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", - "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", - "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "360Spider", - "addition_date": "2014-03-14", - "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", - "instances": [ - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "psbot", - "addition_date": "2014-03-31", - "url": "http://www.picsearch.com/bot.html", - "instances": [ - "psbot-image (+http://www.picsearch.com/bot.html)", - "psbot-page (+http://www.picsearch.com/bot.html)", - "psbot/0.1 (+http://www.picsearch.com/bot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "InterfaxScanBot", - "addition_date": "2014-03-31", - "url": "http://scan-interfax.ru", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "CC Metadata Scaper", - "addition_date": "2014-04-01", - "url": "http://wiki.creativecommons.org/Metadata_Scraper", - "instances": [ - "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "g00g1e.net", - "addition_date": "2014-04-01", - "url": "http://www.g00g1e.net/", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "GrapeshotCrawler", - "addition_date": "2014-04-01", - "url": "http://www.grapeshot.co.uk/crawler.php", - "instances": [ - "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "urlappendbot", - "addition_date": "2014-05-10", - "url": "http://www.profound.net/urlappendbot.html", - "instances": [ - "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "brainobot", - "addition_date": "2014-06-24", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "fr-crawler", - "addition_date": "2014-07-31", - "instances": [ - "Mozilla/5.0 (compatible; fr-crawler/1.1)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "binlar", - "addition_date": "2014-09-12", - "instances": [ - "binlar_2.6.3 binlar2.6.3@unspecified.mail", - "binlar_2.6.3 binlar_2.6.3@unspecified.mail", - "binlar_2.6.3 larbin2.6.3@unspecified.mail", - "binlar_2.6.3 phanendra_kalapala@McAfee.com", - "binlar_2.6.3 test@mgmt.mic" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SimpleCrawler", - "addition_date": "2014-09-12", - "instances": [ - "SimpleCrawler/0.1" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Twitterbot", - "addition_date": "2014-09-12", - "url": "https://dev.twitter.com/cards/getting-started", - "instances": [ - "Twitterbot/0.1", - "Twitterbot/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "cXensebot", - "addition_date": "2014-10-05", - "instances": [ - "cXensebot/1.1a" - ], - "url": "http://www.cxense.com/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "smtbot", - "addition_date": "2014-10-04", - "instances": [ - "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", - "SMTBot (similartech.com/smtbot)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)" - ], - "url": "http://www.similartech.com/smtbot", - "last_changed": "2024-01-10" - }, - { - "pattern": "bnf.fr_bot", - "addition_date": "2014-11-18", - "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", - "instances": [ - "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", - "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "A6-Indexer", - "addition_date": "2014-12-05", - "url": "http://www.a6corp.com/a6-web-scraping-policy/", - "instances": [ - "A6-Indexer" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ADmantX", - "addition_date": "2014-12-05", - "url": "http://www.admantx.com", - "instances": [ - "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Facebot", - "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl", - "addition_date": "2014-12-30", - "instances": [ - "Facebot/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "OrangeBot\\/", - "instances": [ - "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" - ], - "addition_date": "2015-01-12", - "last_changed": "2024-01-10" - }, - { - "pattern": "memorybot", - "url": "http://mignify.com/bot.htm", - "instances": [ - "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" - ], - "addition_date": "2015-02-01", - "last_changed": "2024-01-10" - }, - { - "pattern": "AdvBot", - "url": "http://advbot.net/bot.html", - "instances": [ - "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" - ], - "addition_date": "2015-02-01", - "last_changed": "2024-01-10" - }, - { - "pattern": "MegaIndex", - "url": "https://www.megaindex.ru/?tab=linkAnalyze", - "instances": [ - "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", - "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" - ], - "addition_date": "2015-03-28", - "last_changed": "2024-01-10" - }, - { - "pattern": "SemanticScholarBot", - "url": "https://www.semanticscholar.org/crawler", - "instances": [ - "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", - "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" - ], - "addition_date": "2015-03-28", - "last_changed": "2024-01-10" - }, - { - "pattern": "ltx71", - "url": "http://ltx71.com/", - "instances": [ - "ltx71 - (http://ltx71.com/)" - ], - "addition_date": "2015-04-04", - "last_changed": "2024-01-10" - }, - { - "pattern": "nerdybot", - "url": "http://nerdybot.com/", - "instances": [ - "nerdybot" - ], - "addition_date": "2015-04-05", - "last_changed": "2024-01-10" - }, - { - "pattern": "xovibot", - "url": "http://www.xovibot.net/", - "instances": [ - "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" - ], - "addition_date": "2015-04-05", - "last_changed": "2024-01-10" - }, - { - "pattern": "BUbiNG", - "url": "http://law.di.unimi.it/BUbiNG.html", - "instances": [ - "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" - ], - "addition_date": "2015-04-06", - "last_changed": "2024-01-10" - }, - { - "pattern": "Qwantify", - "url": "https://www.qwant.com/", - "instances": [ - "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", - "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", - "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", - "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" - ], - "addition_date": "2015-04-06", - "last_changed": "2024-01-10" - }, - { - "pattern": "archive.org_bot", - "url": "http://www.archive.org/details/archive.org_bot", - "depends_on": [ - "heritrix" - ], - "instances": [ - "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", - "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", - "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", - "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", - "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" - ], - "addition_date": "2015-04-14", - "last_changed": "2024-01-10" - }, - { - "pattern": "Applebot", - "url": "http://www.apple.com/go/applebot", - "addition_date": "2015-04-15", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", - "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "TweetmemeBot", - "url": "http://datasift.com/bot.html", - "instances": [ - "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" - ], - "addition_date": "2015-04-15", - "last_changed": "2024-01-10" - }, - { - "pattern": "crawler4j", - "url": "https://github.com/yasserg/crawler4j", - "instances": [ - "crawler4j (http://code.google.com/p/crawler4j/)", - "crawler4j (https://github.com/yasserg/crawler4j/)" - ], - "addition_date": "2015-05-07", - "last_changed": "2024-01-10" - }, - { - "pattern": "findxbot", - "url": "http://www.findxbot.com", - "instances": [ - "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" - ], - "addition_date": "2015-05-07", - "last_changed": "2024-01-10" - }, - { - "pattern": "S[eE][mM]rushBot", - "url": "http://www.semrush.com/bot.html", - "instances": [ - "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", - "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", - "SEMrushBot" - ], - "addition_date": "2015-05-26", - "last_changed": "2024-01-10" - }, - { - "pattern": "yoozBot", - "url": "http://yooz.ir", - "instances": [ - "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" - ], - "addition_date": "2015-05-26", - "last_changed": "2024-01-10" - }, - { - "pattern": "lipperhey", - "url": "http://www.lipperhey.com/", - "instances": [ - "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", - "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", - "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", - "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" - ], - "addition_date": "2015-08-26", - "last_changed": "2024-01-10" - }, - { - "pattern": "Y!J", - "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", - "instances": [ - "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", - "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" - ], - "addition_date": "2015-05-26", - "last_changed": "2024-01-10" - }, - { - "pattern": "Domain Re-Animator Bot", - "url": "http://domainreanimator.com", - "instances": [ - "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" - ], - "addition_date": "2015-04-14", - "last_changed": "2024-01-10" - }, - { - "pattern": "AddThis", - "url": "https://www.addthis.com", - "instances": [ - "AddThis.com robot tech.support@clearspring.com" - ], - "addition_date": "2015-06-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "Screaming Frog SEO Spider", - "url": "http://www.screamingfrog.co.uk/seo-spider", - "instances": [ - "Screaming Frog SEO Spider/5.1" - ], - "addition_date": "2016-01-08", - "last_changed": "2024-01-10" - }, - { - "pattern": "MetaURI", - "url": "http://www.useragentstring.com/MetaURI_id_17683.php", - "instances": [ - "MetaURI API/2.0 +metauri.com" - ], - "addition_date": "2016-01-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "Scrapy", - "url": "http://scrapy.org/", - "instances": [ - "Scrapy/1.0.3 (+http://scrapy.org)" - ], - "addition_date": "2016-01-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "Livelap[bB]ot", - "url": "http://site.livelap.com/crawler", - "instances": [ - "LivelapBot/0.2 (http://site.livelap.com/crawler)", - "Livelapbot/0.1" - ], - "addition_date": "2016-01-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "OpenHoseBot", - "url": "http://www.openhose.org/bot.html", - "instances": [ - "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" - ], - "addition_date": "2016-01-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "CapsuleChecker", - "url": "http://www.capsulink.com/about", - "instances": [ - "CapsuleChecker (http://www.capsulink.com/)" - ], - "addition_date": "2016-01-02", - "last_changed": "2024-01-10" - }, - { - "pattern": "collection@infegy.com", - "url": "http://infegy.com/", - "instances": [ - "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" - ], - "addition_date": "2016-01-03", - "last_changed": "2024-01-10" - }, - { - "pattern": "IstellaBot", - "url": "http://www.tiscali.it/", - "instances": [ - "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" - ], - "addition_date": "2016-01-09", - "last_changed": "2024-01-10" - }, - { - "pattern": "DeuSu\\/", - "addition_date": "2016-01-23", - "url": "https://deusu.de/robot.html", - "instances": [ - "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", - "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "betaBot", - "addition_date": "2016-01-23", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "Cliqzbot\\/", - "addition_date": "2016-01-23", - "url": "http://cliqz.com/company/cliqzbot", - "instances": [ - "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", - "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", - "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", - "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", - "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "MojeekBot\\/", - "addition_date": "2016-01-23", - "url": "https://www.mojeek.com/bot.html", - "instances": [ - "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", - "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", - "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", - "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", - "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", - "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "netEstate NE Crawler", - "addition_date": "2016-01-23", - "url": "http://www.website-datenbank.de/", - "instances": [ - "netEstate NE Crawler (+http://www.sengine.info/)", - "netEstate NE Crawler (+http://www.website-datenbank.de/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SafeSearch microdata crawler", - "addition_date": "2016-01-23", - "url": "https://safesearch.avira.com", - "instances": [ - "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Gluten Free Crawler\\/", - "addition_date": "2016-01-23", - "url": "http://glutenfreepleasure.com/", - "instances": [ - "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Sonic", - "addition_date": "2016-02-08", - "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", - "instances": [ - "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", - "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", - "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Sysomos", - "addition_date": "2016-02-08", - "url": "http://www.sysomos.com", - "instances": [ - "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Trove", - "addition_date": "2016-02-08", - "url": "http://www.trove.com", - "instances": [], - "last_changed": "2024-01-10" - }, - { - "pattern": "deadlinkchecker", - "addition_date": "2016-02-08", - "url": "http://www.deadlinkchecker.com", - "instances": [ - "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", - "www.deadlinkchecker.com XMLHTTP/1.0", - "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Slack-ImgProxy", - "addition_date": "2016-04-25", - "url": "https://api.slack.com/robots", - "instances": [ - "Slack-ImgProxy (+https://api.slack.com/robots)", - "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", - "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", - "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", - "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", - "Slack-ImgProxy 149 (+https://api.slack.com/robots)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Embedly", - "addition_date": "2016-04-25", - "url": "http://support.embed.ly", - "instances": [ - "Embedly +support@embed.ly", - "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", - "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "RankActiveLinkBot", - "addition_date": "2016-06-20", - "url": "https://rankactive.com/resources/rankactive-linkbot", - "instances": [ - "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "iskanie", - "addition_date": "2016-09-02", - "url": "http://www.iskanie.com", - "instances": [ - "iskanie (+http://www.iskanie.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SafeDNSBot", - "addition_date": "2016-09-10", - "url": "https://www.safedns.com/searchbot", - "instances": [ - "SafeDNSBot (https://www.safedns.com/searchbot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SkypeUriPreview", - "addition_date": "2016-10-10", - "instances": [ - "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Veoozbot", - "addition_date": "2016-11-03", - "url": "http://www.veooz.com/veoozbot.html", - "instances": [ - "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Slackbot", - "addition_date": "2016-11-03", - "url": "https://api.slack.com/robots", - "instances": [ - "Slackbot-LinkExpanding (+https://api.slack.com/robots)", - "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", - "Slackbot 1.0 (+https://api.slack.com/robots)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "redditbot", - "addition_date": "2016-11-03", - "url": "http://www.reddit.com/feedback", - "instances": [ - "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "datagnionbot", - "addition_date": "2016-11-03", - "url": "http://www.datagnion.com/bot.html", - "instances": [ - "datagnionbot (+http://www.datagnion.com/bot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Google-Adwords-Instant", - "addition_date": "2016-11-03", - "url": "http://www.google.com/adsbot.html", - "instances": [ - "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "adbeat_bot", - "addition_date": "2016-11-04", - "instances": [ - "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", - "adbeat_bot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "WhatsApp", - "addition_date": "2016-11-15", - "url": "https://www.whatsapp.com/", - "instances": [ - "WhatsApp", - "WhatsApp/0.3.4479 N", - "WhatsApp/0.3.4679 N", - "WhatsApp/0.3.4941 N", - "WhatsApp/2.12.15/i", - "WhatsApp/2.12.16/i", - "WhatsApp/2.12.17/i", - "WhatsApp/2.12.449 A", - "WhatsApp/2.12.453 A", - "WhatsApp/2.12.510 A", - "WhatsApp/2.12.540 A", - "WhatsApp/2.12.548 A", - "WhatsApp/2.12.555 A", - "WhatsApp/2.12.556 A", - "WhatsApp/2.16.1/i", - "WhatsApp/2.16.13 A", - "WhatsApp/2.16.2/i", - "WhatsApp/2.16.42 A", - "WhatsApp/2.16.57 A", - "WhatsApp/2.19.92 i", - "WhatsApp/2.19.175 A", - "WhatsApp/2.19.244 A", - "WhatsApp/2.19.258 A", - "WhatsApp/2.19.308 A", - "WhatsApp/2.19.330 A" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "contxbot", - "addition_date": "2017-02-25", - "instances": [ - "Mozilla/5.0 (compatible;contxbot/1.0)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "pinterest.com.bot", - "addition_date": "2017-03-03", - "instances": [ - "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", - "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" - ], - "url": "http://www.pinterest.com/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "electricmonk", - "addition_date": "2017-03-04", - "instances": [ - "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" - ], - "url": "https://www.duedil.com/our-crawler/", - "last_changed": "2024-01-10" - }, - { - "pattern": "GarlikCrawler", - "addition_date": "2017-03-18", - "instances": [ - "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" - ], - "url": "http://garlik.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "BingPreview\\/", - "addition_date": "2017-04-23", - "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", - "instances": [ - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", - "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "vebidoobot", - "addition_date": "2017-05-08", - "instances": [ - "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" - ], - "url": "https://blog.vebidoo.de/vebidoobot/", - "last_changed": "2024-01-10" - }, - { - "pattern": "FemtosearchBot", - "addition_date": "2017-05-16", - "instances": [ - "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" - ], - "url": "http://femtosearch.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Yahoo Link Preview", - "addition_date": "2017-06-28", - "instances": [ - "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" - ], - "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "MetaJobBot", - "addition_date": "2017-08-16", - "instances": [ - "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" - ], - "url": "http://www.metajob.de/the/crawler", - "last_changed": "2024-01-10" - }, - { - "pattern": "DomainStatsBot", - "addition_date": "2017-08-16", - "instances": [ - "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" - ], - "url": "http://domainstats.io/our-bot", - "last_changed": "2024-01-10" - }, - { - "pattern": "mindUpBot", - "addition_date": "2017-08-16", - "instances": [ - "mindUpBot (datenbutler.de)" - ], - "url": "http://www.datenbutler.de/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Daum\\/", - "addition_date": "2017-08-16", - "instances": [ - "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" - ], - "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966", - "last_changed": "2024-01-10" - }, - { - "pattern": "Jugendschutzprogramm-Crawler", - "addition_date": "2017-08-16", - "instances": [ - "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" - ], - "url": "http://www.jugendschutzprogramm.de", - "last_changed": "2024-01-10" - }, - { - "pattern": "Xenu Link Sleuth", - "addition_date": "2017-08-19", - "instances": [ - "Xenu Link Sleuth/1.3.8" - ], - "url": "http://home.snafu.de/tilman/xenulink.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "Pcore-HTTP", - "addition_date": "2017-08-19", - "instances": [ - "Pcore-HTTP/v0.40.3", - "Pcore-HTTP/v0.44.0" - ], - "url": "https://bitbucket.org/softvisio/pcore/overview", - "last_changed": "2024-01-10" - }, - { - "pattern": "moatbot", - "addition_date": "2017-09-16", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" - ], - "url": "https://moat.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "KosmioBot", - "addition_date": "2017-09-16", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" - ], - "url": "http://kosm.io/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "[pP]ingdom", - "addition_date": "2017-09-16", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", - "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", - "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" - ], - "url": "http://www.pingdom.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "AppInsights", - "addition_date": "2019-03-09", - "instances": [ - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" - ], - "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview", - "last_changed": "2024-01-10" - }, - { - "pattern": "PhantomJS", - "addition_date": "2017-09-18", - "instances": [ - "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" - ], - "url": "http://phantomjs.org/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Gowikibot", - "addition_date": "2017-10-26", - "instances": [ - "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" - ], - "url": "http://www.gowikibot.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "PiplBot", - "addition_date": "2017-10-30", - "instances": [ - "PiplBot (+http://www.pipl.com/bot/)", - "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" - ], - "url": "http://www.pipl.com/bot/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Discordbot", - "addition_date": "2017-09-22", - "url": "https://discordapp.com", - "instances": [ - "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "TelegramBot", - "addition_date": "2017-10-01", - "instances": [ - "TelegramBot (like TwitterBot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Jetslide", - "addition_date": "2017-09-27", - "url": "http://jetsli.de/crawler", - "instances": [ - "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "newsharecounts", - "addition_date": "2017-09-30", - "url": "http://newsharecounts.com/crawler", - "instances": [ - "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "James BOT", - "addition_date": "2017-10-12", - "url": "http://cognitiveseo.com/bot.html", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Bark[rR]owler", - "addition_date": "2017-10-09", - "url": "http://www.exensa.com/crawl", - "instances": [ - "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", - "Barkrowler/0.7 (+http://www.exensa.com/crawl)", - "BarkRowler/0.7 (+http://www.exensa.com/crawling)", - "Barkrowler/0.9 (+http://www.exensa.com/crawl)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "TinEye", - "addition_date": "2017-10-14", - "url": "http://www.tineye.com/crawler.html", - "instances": [ - "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", - "TinEye/1.1 (http://tineye.com/crawler.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SocialRankIOBot", - "addition_date": "2017-10-19", - "url": "http://socialrank.io/about", - "instances": [ - "SocialRankIOBot; http://socialrank.io/about" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "trendictionbot", - "addition_date": "2017-10-30", - "url": "http://www.trendiction.de/bot", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Ocarinabot", - "addition_date": "2017-09-27", - "instances": [ - "Ocarinabot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "epicbot", - "addition_date": "2017-10-31", - "url": "http://www.epictions.com/epicbot", - "instances": [ - "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Primalbot", - "addition_date": "2017-09-27", - "url": "https://www.primal.com", - "instances": [ - "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "DuckDuckGo-Favicons-Bot", - "addition_date": "2017-10-06", - "url": "http://duckduckgo.com", - "instances": [ - "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "GnowitNewsbot", - "addition_date": "2017-10-30", - "url": "http://www.gnowit.com", - "instances": [ - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Leikibot", - "addition_date": "2017-09-24", - "url": "http://www.leiki.com", - "instances": [ - "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "LinkArchiver", - "addition_date": "2017-09-24", - "instances": [ - "@LinkArchiver twitter bot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "YaK\\/", - "addition_date": "2017-09-25", - "url": "http://linkfluence.com", - "instances": [ - "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "PaperLiBot", - "addition_date": "2017-09-25", - "url": "http://support.paper.li/entries/20023257-what-is-paper-li", - "instances": [ - "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", - "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Digg Deeper", - "addition_date": "2017-09-26", - "url": "http://digg.com/about", - "instances": [ - "Digg Deeper/v1 (http://digg.com/about)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "dcrawl", - "addition_date": "2017-09-22", - "instances": [ - "dcrawl/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Snacktory", - "addition_date": "2017-09-23", - "url": "https://github.com/karussell/snacktory", - "instances": [ - "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "AndersPinkBot", - "addition_date": "2017-09-24", - "url": "http://anderspink.com/bot.html", - "instances": [ - "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Fyrebot", - "addition_date": "2017-09-22", - "instances": [ - "Fyrebot/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "EveryoneSocialBot", - "addition_date": "2017-09-22", - "url": "http://everyonesocial.com", - "instances": [ - "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Mediatoolkitbot", - "addition_date": "2017-10-06", - "url": "http://mediatoolkit.com", - "instances": [ - "Mediatoolkitbot (complaints@mediatoolkit.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Luminator-robots", - "addition_date": "2017-09-22", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ExtLinksBot", - "addition_date": "2017-11-02", - "url": "https://extlinks.com/Bot.html", - "instances": [ - "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "SurveyBot", - "addition_date": "2017-11-02", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "NING\\/", - "addition_date": "2017-11-02", - "instances": [ - "NING/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "okhttp", - "addition_date": "2017-11-02", - "instances": [ - "okhttp/2.5.0", - "okhttp/2.7.5", - "okhttp/3.2.0", - "okhttp/3.5.0", - "okhttp/4.1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Nuzzel", - "addition_date": "2017-11-02", - "instances": [ - "Nuzzel" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "omgili", - "addition_date": "2017-11-02", - "url": "http://omgili.com", - "instances": [ - "omgili/0.5 +http://omgili.com" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "PocketParser", - "addition_date": "2017-11-02", - "url": "https://getpocket.com/pocketparser_ua", - "instances": [ - "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "YisouSpider", - "addition_date": "2017-11-02", - "instances": [ - "YisouSpider", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "um-LN", - "addition_date": "2017-11-02", - "instances": [ - "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ToutiaoSpider", - "addition_date": "2017-11-02", - "url": "http://web.toutiao.com/media_cooperation/", - "instances": [ - "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "MuckRack", - "addition_date": "2017-11-02", - "url": "http://muckrack.com", - "instances": [ - "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Jamie's Spider", - "addition_date": "2017-11-02", - "url": "http://jamiembrown.com/", - "instances": [ - "Jamie's Spider (http://jamiembrown.com/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "AHC\\/", - "addition_date": "2017-11-02", - "instances": [ - "AHC/2.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "NetcraftSurveyAgent", - "addition_date": "2017-11-02", - "instances": [ - "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Laserlikebot", - "addition_date": "2017-11-02", - "instances": [ - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "^Apache-HttpClient", - "addition_date": "2017-11-02", - "instances": [ - "Apache-HttpClient/4.2.3 (java 1.5)", - "Apache-HttpClient/4.2.5 (java 1.5)", - "Apache-HttpClient/4.3.1 (java 1.5)", - "Apache-HttpClient/4.3.3 (java 1.5)", - "Apache-HttpClient/4.3.5 (java 1.5)", - "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", - "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", - "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", - "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", - "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", - "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", - "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", - "Apache-HttpClient/4.5.7 (Java/11.0.3)", - "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "AppEngine-Google", - "addition_date": "2017-11-02", - "instances": [ - "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Jetty", - "addition_date": "2017-11-02", - "instances": [ - "Jetty/9.3.z-SNAPSHOT" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Upflow", - "addition_date": "2017-11-02", - "instances": [ - "Upflow/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Thinklab", - "addition_date": "2017-11-02", - "url": "thinklab.com", - "instances": [ - "Thinklab (thinklab.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Traackr.com", - "addition_date": "2017-11-02", - "url": "Traackr.com", - "instances": [ - "Traackr.com" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Twurly", - "addition_date": "2017-11-02", - "url": "http://twurly.org", - "instances": [ - "Ruby, Twurly v1.1 (http://twurly.org)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Mastodon", - "addition_date": "2017-11-02", - "instances": [ - "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "http_get", - "addition_date": "2017-11-02", - "instances": [ - "http_get" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "DnyzBot", - "addition_date": "2017-11-20", - "instances": [ - "Mozilla/5.0 (compatible; DnyzBot/1.0)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "botify", - "addition_date": "2018-02-01", - "instances": [ - "Mozilla/5.0 (compatible; botify; http://botify.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "007ac9 Crawler", - "addition_date": "2018-02-09", - "instances": [ - "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BehloolBot", - "addition_date": "2018-02-09", - "instances": [ - "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BrandVerity", - "addition_date": "2018-02-27", - "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" - ], - "url": "http://www.brandverity.com/why-is-brandverity-visiting-me", - "last_changed": "2024-01-10" - }, - { - "pattern": "check_http", - "addition_date": "2018-02-09", - "instances": [ - "check_http/v2.2.1 (nagios-plugins 2.2.1)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BDCbot", - "addition_date": "2018-02-09", - "instances": [ - "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ZumBot", - "addition_date": "2018-02-09", - "instances": [ - "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "EZID", - "addition_date": "2018-02-09", - "instances": [ - "EZID (EZID link checker; https://ezid.cdlib.org/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ICC-Crawler", - "addition_date": "2018-02-28", - "instances": [ - "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" - ], - "url": "http://ucri.nict.go.jp/en/icccrawler.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "ArchiveBot", - "addition_date": "2018-02-28", - "instances": [ - "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" - ], - "url": "https://github.com/ArchiveTeam/ArchiveBot", - "last_changed": "2024-01-10" - }, - { - "pattern": "^LCC ", - "addition_date": "2018-02-28", - "instances": [ - "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" - ], - "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "filterdb.iss.net\\/crawler", - "addition_date": "2018-03-16", - "instances": [ - "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" - ], - "url": "http://filterdb.iss.net/crawler/", - "last_changed": "2024-01-10" - }, - { - "pattern": "BLP_bbot", - "addition_date": "2018-03-27", - "instances": [ - "BLP_bbot/0.1" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BomboraBot", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" - ], - "url": "http://www.bombora.com/bot", - "last_changed": "2024-01-10" - }, - { - "pattern": "Buck\\/", - "addition_date": "2018-03-27", - "instances": [ - "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" - ], - "url": "https://app.hypefactors.com/media-monitoring/about.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "Companybook-Crawler", - "addition_date": "2018-03-27", - "instances": [ - "Companybook-Crawler (+https://www.companybooknetworking.com/)" - ], - "url": "https://www.companybooknetworking.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Genieo", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" - ], - "url": "http://www.genieo.com/webfilter.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "magpie-crawler", - "addition_date": "2018-03-27", - "instances": [ - "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" - ], - "url": "http://www.brandwatch.net", - "last_changed": "2024-01-10" - }, - { - "pattern": "MeltwaterNews", - "addition_date": "2018-03-27", - "instances": [ - "MeltwaterNews www.meltwater.com" - ], - "url": "http://www.meltwater.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Moreover", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" - ], - "url": "http://www.moreover.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "newspaper\\/", - "addition_date": "2018-03-27", - "instances": [ - "newspaper/0.1.0.7", - "newspaper/0.2.5", - "newspaper/0.2.6", - "newspaper/0.2.8" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "ScoutJet", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" - ], - "url": "http://www.scoutjet.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "(^| )sentry\\/", - "addition_date": "2018-03-27", - "instances": [ - "sentry/8.22.0 (https://sentry.io)" - ], - "url": "https://sentry.io", - "last_changed": "2024-01-10" - }, - { - "pattern": "StorygizeBot", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" - ], - "url": "http://www.storygize.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "UptimeRobot", - "addition_date": "2018-03-27", - "instances": [ - "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" - ], - "url": "http://www.uptimerobot.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "OutclicksBot", - "addition_date": "2018-04-21", - "instances": [ - "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", - "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", - "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", - "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" - ], - "url": "https://www.outclicks.net", - "last_changed": "2024-01-10" - }, - { - "pattern": "seoscanners", - "addition_date": "2018-05-27", - "instances": [ - "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" - ], - "url": "http://www.seoscanners.net/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Hatena", - "addition_date": "2018-05-29", - "instances": [ - "Hatena Antenna/0.3", - "Hatena::Russia::Crawler/0.01", - "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", - "Hatena::Scissors/0.01", - "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", - "Hatena::Fetcher/0.01 (master) Furl/3.13" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Google Web Preview", - "addition_date": "2018-05-31", - "instances": [ - "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "MauiBot", - "addition_date": "2018-06-06", - "instances": [ - "MauiBot (crawler.feedback+wc@gmail.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "AlphaBot", - "addition_date": "2018-05-27", - "instances": [ - "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" - ], - "url": "http://alphaseobot.com/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "SBL-BOT", - "addition_date": "2018-06-06", - "instances": [ - "SBL-BOT (http://sbl.net)" - ], - "url": "http://sbl.net", - "description": "Bot of SoftByte BlackWidow", - "last_changed": "2024-01-10" - }, - { - "pattern": "IAS crawler", - "addition_date": "2018-06-06", - "instances": [ - "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" - ], - "url": "http://integralads.com/site-indexing-policy/", - "description": "Bot of Integral Ad Science, Inc.", - "last_changed": "2024-01-10" - }, - { - "pattern": "adscanner", - "addition_date": "2018-06-24", - "instances": [ - "Mozilla/5.0 (compatible; adscanner/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Netvibes", - "addition_date": "2018-06-24", - "instances": [ - "Netvibes (crawler/bot; http://www.netvibes.com", - "Netvibes (crawler; http://www.netvibes.com)" - ], - "url": "http://www.netvibes.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "acapbot", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", - "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Baidu-YunGuanCe", - "addition_date": "2018-06-27", - "instances": [ - "Baidu-YunGuanCe-Bot(ce.baidu.com)", - "Baidu-YunGuanCe-SLABot(ce.baidu.com)", - "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", - "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", - "Baidu-YunGuanCe-VSBot(ce.baidu.com)" - ], - "url": "https://ce.baidu.com/topic/topic20150908", - "description": "Baidu Cloud Watch", - "last_changed": "2024-01-10" - }, - { - "pattern": "bitlybot", - "addition_date": "2018-06-27", - "instances": [ - "bitlybot/3.0 (+http://bit.ly/)", - "bitlybot/2.0", - "bitlybot" - ], - "url": "http://bit.ly/", - "last_changed": "2024-01-10" - }, - { - "pattern": "blogmuraBot", - "addition_date": "2018-06-27", - "instances": [ - "blogmuraBot (+http://www.blogmura.com)" - ], - "url": "http://www.blogmura.com", - "description": "A blog ranking site which links to blogs on just about every theme possible.", - "last_changed": "2024-01-10" - }, - { - "pattern": "Bot.AraTurka.com", - "addition_date": "2018-06-27", - "instances": [ - "Bot.AraTurka.com/0.0.1" - ], - "url": "http://www.araturka.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "bot-pge.chlooe.com", - "addition_date": "2018-06-27", - "instances": [ - "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BoxcarBot", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" - ], - "url": "https://boxcar.io/", - "last_changed": "2024-01-10" - }, - { - "pattern": "BTWebClient", - "addition_date": "2018-06-27", - "instances": [ - "BTWebClient/180B(9704)" - ], - "url": "http://www.utorrent.com/", - "description": "\u00b5Torrent BitTorrent Client", - "last_changed": "2024-01-10" - }, - { - "pattern": "ContextAd Bot", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", - "ContextAd Bot 1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Digincore bot", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" - ], - "url": "http://www.digincore.com/crawler.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "Disqus", - "addition_date": "2018-06-27", - "instances": [ - "Disqus/1.0" - ], - "url": "https://disqus.com/", - "description": "validate and quality check pages.", - "last_changed": "2024-01-10" - }, - { - "pattern": "Feedly", - "addition_date": "2018-06-27", - "instances": [ - "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", - "FeedlyBot/1.0 (http://feedly.com)" - ], - "url": "https://www.feedly.com/fetcher.html", - "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud.", - "last_changed": "2024-01-10" - }, - { - "pattern": "Fetch\\/", - "addition_date": "2018-06-27", - "instances": [ - "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Fever", - "addition_date": "2018-06-27", - "instances": [ - "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" - ], - "url": "http://feedafever.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Flamingo_SearchEngine", - "addition_date": "2018-06-27", - "instances": [ - "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "FlipboardProxy", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", - "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", - "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" - ], - "url": "https://about.flipboard.com/browserproxy/", - "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application", - "last_changed": "2024-01-10" - }, - { - "pattern": "g2reader-bot", - "addition_date": "2018-06-27", - "instances": [ - "g2reader-bot/1.0 (+http://www.g2reader.com/)" - ], - "url": "http://www.g2reader.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "G2 Web Services", - "addition_date": "2019-03-01", - "instances": [ - "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" - ], - "url": "https://www.g2webservices.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "imrbot", - "addition_date": "2018-06-27", - "instances": [ - "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" - ], - "url": "http://www.mignify.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "K7MLWCBot", - "addition_date": "2018-06-27", - "instances": [ - "K7MLWCBot/1.0 (+http://www.k7computing.com)" - ], - "url": "http://www.k7computing.com", - "description": "Virus scanner", - "last_changed": "2024-01-10" - }, - { - "pattern": "Kemvibot", - "addition_date": "2018-06-27", - "instances": [ - "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" - ], - "url": "http://kemvi.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Landau-Media-Spider", - "addition_date": "2018-06-27", - "instances": [ - "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" - ], - "url": "http://bots.landaumedia.de/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "linkapediabot", - "addition_date": "2018-06-27", - "instances": [ - "linkapediabot (+http://www.linkapedia.com)" - ], - "url": "http://www.linkapedia.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "vkShare", - "addition_date": "2018-07-02", - "instances": [ - "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" - ], - "url": "http://vk.com/dev/Share", - "last_changed": "2024-01-10" - }, - { - "pattern": "Siteimprove.com", - "addition_date": "2018-06-22", - "instances": [ - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", - "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", - "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "BLEXBot\\/", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" - ], - "url": "http://webmeup-crawler.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "DareBoost", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" - ], - "url": "https://www.dareboost.com/", - "description": "Bot to test, Analyze and Optimize website", - "last_changed": "2024-01-10" - }, - { - "pattern": "ZuperlistBot\\/", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Miniflux\\/", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", - "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", - "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", - "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", - "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", - "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", - "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", - "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", - "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" - ], - "url": "https://miniflux.net", - "description": "Miniflux is a minimalist and opinionated feed reader.", - "last_changed": "2024-01-10" - }, - { - "pattern": "Feedspot", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", - "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" - ], - "url": "http://www.feedspot.com/fs/bot", - "last_changed": "2024-01-10" - }, - { - "pattern": "Diffbot\\/", - "addition_date": "2018-07-07", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" - ], - "url": "http://www.diffbot.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "SEOkicks", - "addition_date": "2018-08-22", - "instances": [ - "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" - ], - "url": "https://www.seokicks.de/robot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "tracemyfile", - "addition_date": "2018-08-23", - "instances": [ - "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Nimbostratus-Bot", - "addition_date": "2018-08-29", - "instances": [ - "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "zgrab", - "addition_date": "2018-08-30", - "instances": [ - "Mozilla/5.0 zgrab/0.x" - ], - "url": "https://zmap.io/", - "last_changed": "2024-01-10" - }, - { - "pattern": "PR-CY.RU", - "addition_date": "2018-08-30", - "instances": [ - "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" - ], - "url": "https://a.pr-cy.ru/", - "last_changed": "2024-01-10" - }, - { - "pattern": "AdsTxtCrawler", - "addition_date": "2018-08-30", - "instances": [ - "AdsTxtCrawler/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Datafeedwatch", - "addition_date": "2018-09-05", - "instances": [ - "Datafeedwatch/2.1.x" - ], - "url": "https://www.datafeedwatch.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Zabbix", - "addition_date": "2018-09-05", - "instances": [ - "Zabbix" - ], - "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring", - "last_changed": "2024-01-10" - }, - { - "pattern": "TangibleeBot", - "addition_date": "2018-09-05", - "instances": [ - "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" - ], - "url": "http://tangiblee.com/bot", - "last_changed": "2024-01-10" - }, - { - "pattern": "google-xrawler", - "addition_date": "2018-09-05", - "instances": [ - "google-xrawler" - ], - "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for", - "last_changed": "2024-01-10" - }, - { - "pattern": "axios", - "addition_date": "2018-09-06", - "instances": [ - "axios/0.18.0", - "axios/0.19.0" - ], - "url": "https://github.com/axios/axios", - "last_changed": "2024-01-10" - }, - { - "pattern": "Amazon CloudFront", - "addition_date": "2018-09-07", - "instances": [ - "Amazon CloudFront" - ], - "url": "https://aws.amazon.com/cloudfront/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Pulsepoint", - "addition_date": "2018-09-24", - "instances": [ - "Pulsepoint XT3 web scraper" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "CloudFlare-AlwaysOnline", - "addition_date": "2018-09-27", - "instances": [ - "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", - "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" - ], - "url": "https://www.cloudflare.com/always-online/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Google-Structured-Data-Testing-Tool", - "addition_date": "2018-10-02", - "instances": [ - "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", - "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" - ], - "url": "https://search.google.com/structured-data/testing-tool", - "last_changed": "2024-01-10" - }, - { - "pattern": "WordupInfoSearch", - "addition_date": "2018-10-07", - "instances": [ - "WordupInfoSearch/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "WebDataStats", - "addition_date": "2018-10-08", - "instances": [ - "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" - ], - "url": "https://webdatastats.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "HttpUrlConnection", - "addition_date": "2018-10-08", - "instances": [ - "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Seekport Crawler", - "addition_date": "2018-10-08", - "instances": [ - "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" - ], - "url": "http://seekport.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "ZoomBot", - "addition_date": "2018-10-10", - "instances": [ - "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" - ], - "url": "http://suite.seozoom.it/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "VelenPublicWebCrawler", - "addition_date": "2018-10-09", - "instances": [ - "VelenPublicWebCrawler (velen.io)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "MoodleBot", - "addition_date": "2018-10-10", - "instances": [ - "MoodleBot/1.0" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "jpg-newsbot", - "addition_date": "2018-10-10", - "instances": [ - "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" - ], - "url": "https://vipnytt.no/bots/", - "last_changed": "2024-01-10" - }, - { - "pattern": "outbrain", - "addition_date": "2018-10-14", - "instances": [ - "Mozilla/5.0 (Java) outbrain" - ], - "url": "https://www.outbrain.com/help/advertisers/invalid-url/", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C_Validator", - "addition_date": "2018-10-14", - "instances": [ - "W3C_Validator/1.3" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "Validator\\.nu", - "addition_date": "2018-10-14", - "instances": [ - "Validator.nu/LV" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C-checklink", - "addition_date": "2018-10-14", - "depends_on": [ - "libwww-perl" - ], - "instances": [ - "W3C-checklink/2.90 libwww-perl/5.64", - "W3C-checklink/3.6.2.3 libwww-perl/5.64", - "W3C-checklink/4.2 [4.20] libwww-perl/5.803", - "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", - "W3C-checklink/4.3 [4.42] libwww-perl/5.805", - "W3C-checklink/4.3 [4.42] libwww-perl/5.808", - "W3C-checklink/4.3 [4.42] libwww-perl/5.820", - "W3C-checklink/4.5 [4.154] libwww-perl/5.823", - "W3C-checklink/4.5 [4.160] libwww-perl/5.823" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C-mobileOK", - "addition_date": "2018-10-14", - "instances": [ - "W3C-mobileOK/DDC-1.0" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C_I18n-Checker", - "addition_date": "2018-10-14", - "instances": [ - "W3C_I18n-Checker/1.0" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "FeedValidator", - "addition_date": "2018-10-14", - "instances": [ - "FeedValidator/1.3" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C_CSS_Validator", - "addition_date": "2018-10-14", - "instances": [ - "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "W3C_Unicorn", - "addition_date": "2018-10-14", - "instances": [ - "W3C_Unicorn/1.0" - ], - "url": "https://validator.w3.org/services", - "last_changed": "2024-01-10" - }, - { - "pattern": "Google-PhysicalWeb", - "addition_date": "2018-10-21", - "instances": [ - "Mozilla/5.0 (Google-PhysicalWeb)" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Blackboard", - "addition_date": "2018-10-28", - "instances": [ - "Blackboard Safeassign" - ], - "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign", - "last_changed": "2024-01-10" - }, - { - "pattern": "ICBot\\/", - "addition_date": "2018-10-23", - "instances": [ - "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" - ], - "url": "https://ideasandcode.xyz", - "last_changed": "2024-01-10" - }, - { - "pattern": "BazQux", - "addition_date": "2018-10-23", - "instances": [ - "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" - ], - "url": "https://bazqux.com/fetcher", - "last_changed": "2024-01-10" - }, - { - "pattern": "Twingly", - "addition_date": "2018-10-23", - "instances": [ - "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" - ], - "url": "https://twingly.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Rivva", - "addition_date": "2018-10-23", - "instances": [ - "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" - ], - "url": "http://rivva.de", - "last_changed": "2024-01-10" - }, - { - "pattern": "Experibot", - "addition_date": "2018-11-03", - "instances": [ - "Experibot-v2 http://goo.gl/ZAr8wX", - "Experibot-v3 http://goo.gl/ZAr8wX" - ], - "url": "https://amirkr.wixsite.com/experibot", - "last_changed": "2024-01-10" - }, - { - "pattern": "awesomecrawler", - "addition_date": "2018-11-24", - "instances": [ - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Dataprovider.com", - "addition_date": "2018-11-24", - "instances": [ - "Mozilla/5.0 (compatible; Dataprovider.com)" - ], - "url": "https://www.dataprovider.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "GroupHigh\\/", - "addition_date": "2018-11-24", - "instances": [ - "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" - ], - "url": "http://www.grouphigh.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "theoldreader.com", - "addition_date": "2018-12-02", - "instances": [ - "Mozilla/5.0 (compatible; theoldreader.com)" - ], - "url": "https://www.theoldreader.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "AnyEvent", - "addition_date": "2018-12-07", - "instances": [ - "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" - ], - "url": "http://software.schmorp.de/pkg/AnyEvent.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "Uptimebot\\.org", - "addition_date": "2019-01-17", - "instances": [ - "Uptimebot.org - Free website monitoring" - ], - "url": "http://uptimebot.org/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Nmap Scripting Engine", - "addition_date": "2019-02-04", - "instances": [ - "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" - ], - "url": "https://nmap.org/book/nse.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "2ip.ru", - "addition_date": "2019-02-12", - "instances": [ - "2ip.ru CMS Detector (https://2ip.ru/cms/)" - ], - "url": "https://2ip.ru/cms/", - "last_changed": "2024-01-10" - }, - { - "pattern": "Clickagy", - "addition_date": "2019-02-19", - "instances": [ - "Clickagy Intelligence Bot v2" - ], - "url": "https://www.clickagy.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "Caliperbot", - "addition_date": "2019-03-02", - "instances": [ - "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" - ], - "url": "http://www.conductor.com/caliperbot", - "last_changed": "2024-01-10" - }, - { - "pattern": "MBCrawler", - "addition_date": "2019-03-02", - "instances": [ - "MBCrawler/1.0 (https://monitorbacklinks.com)" - ], - "url": "https://monitorbacklinks.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "online-webceo-bot", - "addition_date": "2019-03-02", - "instances": [ - "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" - ], - "url": "http://online.webceo.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "B2B Bot", - "addition_date": "2019-03-02", - "instances": [ - "B2B Bot" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "AddSearchBot", - "addition_date": "2019-03-02", - "instances": [ - "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" - ], - "url": "http://www.addsearch.com/bot", - "last_changed": "2024-01-10" - }, - { - "pattern": "Google Favicon", - "addition_date": "2019-03-14", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "HubSpot", - "addition_date": "2019-04-15", - "instances": [ - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", - "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", - "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", - "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "Chrome-Lighthouse", - "addition_date": "2019-03-15", - "instances": [ - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" - ], - "url": "https://developers.google.com/speed/pagespeed/insights", - "last_changed": "2024-01-10" - }, - { - "pattern": "HeadlessChrome", - "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", - "addition_date": "2019-06-17", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" - ], - "last_changed": "2024-01-10" - }, - { - "pattern": "CheckMarkNetwork\\/", - "addition_date": "2019-06-30", - "instances": [ - "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" - ], - "url": "https://www.checkmarknetwork.com/", - "last_changed": "2024-01-10" - }, - { - "pattern": "www\\.uptime\\.com", - "addition_date": "2019-07-21", - "instances": [ - "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" - ], - "url": "http://www.uptime.com/uptimebot", - "last_changed": "2024-01-10" - }, - { - "pattern": "Streamline3Bot\\/", - "addition_date": "2019-07-21", - "instances": [ - "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" - ], - "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php", - "last_changed": "2024-01-10" - }, - { - "pattern": "serpstatbot\\/", - "addition_date": "2019-07-25", + "pattern": "coccoc", + "addition_date": "2013-01-04", + "url": "http://help.coccoc.vn/", "instances": [ - "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", - "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", + "coccoc", + "coccoc/1.0 ()", + "coccoc/1.0 (http://help.coccoc.com/)", + "coccoc/1.0 (http://help.coccoc.vn/)" ], - "url": "http://serpstatbot.com", "last_changed": "2024-01-10" }, { - "pattern": "MixnodeCache\\/", - "addition_date": "2019-08-04", + "pattern": "integromedb", + "addition_date": "2013-01-10", + "url": "http://www.integromedb.org/Crawler", "instances": [ - "MixnodeCache/1.8(+https://cache.mixnode.com/)" + "www.integromedb.org/Crawler" ], - "url": "https://cache.mixnode.com/", "last_changed": "2024-01-10" }, { - "pattern": "^curl", - "addition_date": "2019-08-15", + "pattern": "changedetection", + "addition_date": "2013-09-13", + "url": "http://www.changedetection.com/bot.html", "instances": [ - "curl", - "curl/7.29.0", - "curl/7.47.0", - "curl/7.54.0", - "curl/7.55.1", - "curl/7.64.0", - "curl/7.64.1", - "curl/7.65.3" + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" ], - "url": "https://curl.haxx.se/", "last_changed": "2024-01-10" }, { - "pattern": "SimpleScraper", - "addition_date": "2019-08-16", + "pattern": "CC\\sMetadata\\sScaper", + "addition_date": "2014-04-01", + "url": "http://wiki.creativecommons.org/Metadata_Scraper", "instances": [ - "Mozilla/5.0 (compatible; SimpleScraper)" + "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" ], - "url": "https://github.com/ramonkcom/simple-scraper/", "last_changed": "2024-01-10" }, { - "pattern": "RSSingBot", - "addition_date": "2019-09-15", - "instances": [ - "RSSingBot (http://www.rssing.com)" - ], - "url": "http://www.rssing.com", + "pattern": "g00g1e\\.net", + "addition_date": "2014-04-01", + "url": "http://www.g00g1e.net/", + "instances": [], "last_changed": "2024-01-10" }, { - "pattern": "Jooblebot", - "addition_date": "2019-09-25", + "pattern": "MegaIndex", + "url": "https://www.megaindex.ru/?tab=linkAnalyze", "instances": [ - "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" ], - "url": "http://jooble.org/jooble-bot", + "addition_date": "2015-03-28", "last_changed": "2024-01-10" }, { - "pattern": "fedoraplanet", - "addition_date": "2019-09-28", + "pattern": "MetaURI", + "url": "http://www.useragentstring.com/MetaURI_id_17683.php", "instances": [ - "venus/fedoraplanet" + "MetaURI API/2.0 +metauri.com" ], - "url": "http://fedoraplanet.org/", + "addition_date": "2016-01-02", "last_changed": "2024-01-10" }, { - "pattern": "Friendica", - "addition_date": "2019-09-28", + "pattern": "Scrapy", + "url": "http://scrapy.org/", "instances": [ - "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" + "Scrapy/1.0.3 (+http://scrapy.org)" ], - "url": "https://hoyer.xyz", + "addition_date": "2016-01-02", "last_changed": "2024-01-10" }, { - "pattern": "NextCloud", - "addition_date": "2019-09-30", + "pattern": "Livelap\\[bB]ot", + "url": "http://site.livelap.com/crawler", "instances": [ - "NextCloud-News/1.0" + "LivelapBot/0.2 (http://site.livelap.com/crawler)", + "Livelapbot/0.1" ], - "url": "https://nextcloud.com/", + "addition_date": "2016-01-02", "last_changed": "2024-01-10" }, { - "pattern": "Tiny Tiny RSS", - "addition_date": "2019-10-04", + "pattern": "CapsuleChecker", + "url": "http://www.capsulink.com/about", "instances": [ - "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", - "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", - "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)", - "Tiny Tiny RSS/19.8 (http://tt-rss.org/)" + "CapsuleChecker (http://www.capsulink.com/)" ], - "url": "http://tt-rss.org/", + "addition_date": "2016-01-02", "last_changed": "2024-01-10" }, { - "pattern": "RegionStuttgartBot", - "addition_date": "2019-10-17", + "pattern": "collection@infegy\\.com", + "url": "http://infegy.com/", "instances": [ - "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" + "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" ], - "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/", + "addition_date": "2016-01-03", "last_changed": "2024-01-10" }, { - "pattern": "Bytespider", - "addition_date": "2019-11-11", + "pattern": "DeuSu\\\\/", + "addition_date": "2016-01-23", + "url": "https://deusu.de/robot.html", "instances": [ - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com", - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider", - "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider" + "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", + "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" ], - "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent", "last_changed": "2024-01-10" }, { - "pattern": "Datanyze", - "addition_date": "2019-11-17", + "pattern": "Sonic", + "addition_date": "2016-02-08", + "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", "instances": [ - "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" + "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", + "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", + "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" ], - "url": "https://www.datanyze.com/dnyzbot/", "last_changed": "2024-01-10" }, { - "pattern": "Google-Site-Verification", - "addition_date": "2019-12-11", + "pattern": "deadlinkchecker", + "addition_date": "2016-02-08", + "url": "http://www.deadlinkchecker.com", "instances": [ - "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" + "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", + "www.deadlinkchecker.com XMLHTTP/1.0", + "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" ], - "url": "https://support.google.com/webmasters/answer/9008080", "last_changed": "2024-01-10" }, { - "pattern": "TrendsmapResolver", - "addition_date": "2020-02-24", + "pattern": "Slack-ImgProxy", + "addition_date": "2016-04-25", + "url": "https://api.slack.com/robots", "instances": [ - "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" + "Slack-ImgProxy (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", + "Slack-ImgProxy 149 (+https://api.slack.com/robots)" ], - "url": "https://www.trendsmap.com/", "last_changed": "2024-01-10" }, { - "pattern": "tweetedtimes", - "addition_date": "2020-02-24", + "pattern": "electricmonk", + "addition_date": "2017-03-04", "instances": [ - "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" + "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" ], - "url": "https://tweetedtimes.com/", + "url": "https://www.duedil.com/our-crawler/", "last_changed": "2024-01-10" }, { - "pattern": "NTENTbot", - "addition_date": "2020-02-24", + "pattern": "Daum\\\\/", + "addition_date": "2017-08-16", "instances": [ - "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" + "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" ], - "url": "https://ntent.com/ntentbot/", + "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966", "last_changed": "2024-01-10" }, { - "pattern": "Gwene", - "addition_date": "2020-02-24", + "pattern": "\\[pP]ingdom", + "addition_date": "2017-09-16", "instances": [ - "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", + "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", + "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" ], - "url": "https://gwene.org", + "url": "http://www.pingdom.com", "last_changed": "2024-01-10" }, { - "pattern": "SimplePie", - "addition_date": "2020-02-24", + "pattern": "AppInsights", + "addition_date": "2019-03-09", "instances": [ - "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" ], - "url": "http://simplepie.org", + "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview", "last_changed": "2024-01-10" }, { - "pattern": "SearchAtlas", - "addition_date": "2020-03-02", + "pattern": "PhantomJS", + "addition_date": "2017-09-18", "instances": [ - "SearchAtlas.com SEO Crawler" + "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" ], - "url": "http://SearchAtlas.com", + "url": "http://phantomjs.org/", "last_changed": "2024-01-10" }, { - "pattern": "Superfeedr", - "addition_date": "2020-03-02", + "pattern": "Jetslide", + "addition_date": "2017-09-27", + "url": "http://jetsli.de/crawler", "instances": [ - "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" + "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" ], - "url": "http://superfeedr.com", "last_changed": "2024-01-10" }, { - "pattern": "feedbot", - "addition_date": "2020-03-02", + "pattern": "newsharecounts", + "addition_date": "2017-09-30", + "url": "http://newsharecounts.com/crawler", "instances": [ - "wp.com feedbot/1.0 (+https://wp.com)" + "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" ], - "url": "http://wp.com", "last_changed": "2024-01-10" }, { - "pattern": "UT-Dorkbot", - "addition_date": "2020-03-02", + "pattern": "Bark\\[rR]owler", + "addition_date": "2017-10-09", + "url": "http://www.exensa.com/crawl", "instances": [ - "UT-Dorkbot/1.0" + "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", + "Barkrowler/0.7 (+http://www.exensa.com/crawl)", + "BarkRowler/0.7 (+http://www.exensa.com/crawling)", + "Barkrowler/0.9 (+http://www.exensa.com/crawl)" ], - "url": "https://security.utexas.edu/dorkbot", "last_changed": "2024-01-10" }, { - "pattern": "Amazonbot", - "addition_date": "2020-03-02", + "pattern": "LinkArchiver", + "addition_date": "2017-09-24", "instances": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" + "@LinkArchiver twitter bot" ], - "url": "https://developer.amazon.com/support/amazonbot", "last_changed": "2024-01-10" }, { - "pattern": "SerendeputyBot", - "addition_date": "2020-03-02", + "pattern": "YaK\\\\/", + "addition_date": "2017-09-25", + "url": "http://linkfluence.com", "instances": [ - "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" + "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" ], - "url": "http://serendeputy.com/about/serendeputy-bot", "last_changed": "2024-01-10" }, { - "pattern": "Eyeotabot", - "addition_date": "2020-03-02", + "pattern": "Digg\\sDeeper", + "addition_date": "2017-09-26", + "url": "http://digg.com/about", "instances": [ - "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" + "Digg Deeper/v1 (http://digg.com/about)" ], - "url": "http://www.eyeota.com", "last_changed": "2024-01-10" }, { - "pattern": "officestorebot", - "addition_date": "2020-03-02", + "pattern": "Snacktory", + "addition_date": "2017-09-23", + "url": "https://github.com/karussell/snacktory", "instances": [ - "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" + "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" ], - "url": "https://aka.ms/officestorebot", "last_changed": "2024-01-10" }, { - "pattern": "Neticle Crawler", - "addition_date": "2020-03-02", + "pattern": "NING\\\\/", + "addition_date": "2017-11-02", "instances": [ - "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" + "NING/1.0" ], - "url": "https://neticle.com/bot/en/", "last_changed": "2024-01-10" }, { - "pattern": "SurdotlyBot", - "addition_date": "2020-03-02", + "pattern": "omgili", + "addition_date": "2017-11-02", + "url": "http://omgili.com", "instances": [ - "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" + "omgili/0.5 +http://omgili.com" ], - "url": "http://sur.ly/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "LinkisBot", - "addition_date": "2020-03-02", + "pattern": "PocketParser", + "addition_date": "2017-11-02", + "url": "https://getpocket.com/pocketparser_ua", "instances": [ - "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" + "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" ], "last_changed": "2024-01-10" }, { - "pattern": "AwarioSmartBot", - "addition_date": "2020-03-02", + "pattern": "um-LN", + "addition_date": "2017-11-02", "instances": [ - "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" + "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" ], - "url": "https://awario.com/bots.html", "last_changed": "2024-01-10" }, { - "pattern": "AwarioRssBot", - "addition_date": "2020-03-02", + "pattern": "MuckRack", + "addition_date": "2017-11-02", + "url": "http://muckrack.com", "instances": [ - "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" + "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" ], - "url": "https://awario.com/bots.html", "last_changed": "2024-01-10" }, { - "pattern": "RyteBot", - "addition_date": "2020-03-02", + "pattern": "AHC\\\\/", + "addition_date": "2017-11-02", "instances": [ - "RyteBot/1.0.0 (+https://bot.ryte.com/)" + "AHC/2.0" ], - "url": "https://bot.ryte.com/", "last_changed": "2024-01-10" }, { - "pattern": "FreeWebMonitoring SiteChecker", - "addition_date": "2020-03-02", + "pattern": "\\^Apache-HttpClient", + "addition_date": "2017-11-02", "instances": [ - "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" + "Apache-HttpClient/4.2.3 (java 1.5)", + "Apache-HttpClient/4.2.5 (java 1.5)", + "Apache-HttpClient/4.3.1 (java 1.5)", + "Apache-HttpClient/4.3.3 (java 1.5)", + "Apache-HttpClient/4.3.5 (java 1.5)", + "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", + "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", + "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", + "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", + "Apache-HttpClient/4.5.7 (Java/11.0.3)", + "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" ], - "url": "https://www.freewebmonitoring.com/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "AspiegelBot", - "addition_date": "2020-03-16", + "pattern": "Jetty", + "addition_date": "2017-11-02", "instances": [ - "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" + "Jetty/9.3.z-SNAPSHOT" ], - "url": "https://aspiegel.com", "last_changed": "2024-01-10" }, { - "pattern": "NAVER Blog Rssbot", - "addition_date": "2020-03-16", + "pattern": "Upflow", + "addition_date": "2017-11-02", "instances": [ - "NAVER Blog Rssbot" + "Upflow/1.0" ], - "url": "http://www.naver.com", "last_changed": "2024-01-10" }, { - "pattern": "zenback bot", - "addition_date": "2020-03-16", + "pattern": "Twurly", + "addition_date": "2017-11-02", + "url": "http://twurly.org", "instances": [ - "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" + "Ruby, Twurly v1.1 (http://twurly.org)" ], - "url": "http://corp.logly.co.jp/", "last_changed": "2024-01-10" }, { - "pattern": "SentiBot", - "addition_date": "2020-03-16", + "pattern": "Mastodon", + "addition_date": "2017-11-02", "instances": [ - "SentiBot www.sentibot.eu (compatible with Googlebot)" + "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" ], - "url": "https://www.sentibot.eu", "last_changed": "2024-01-10" }, { - "pattern": "Domains Project\\/", - "addition_date": "2020-03-16", + "pattern": "BrandVerity", + "addition_date": "2018-02-27", "instances": [ - "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" ], - "url": "https://github.com/tb0hdan/domains", + "url": "http://www.brandverity.com/why-is-brandverity-visiting-me", "last_changed": "2024-01-10" }, { - "pattern": "Pandalytics", - "addition_date": "2020-03-16", + "pattern": "check_http", + "addition_date": "2018-02-09", "instances": [ - "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" + "check_http/v2.2.1 (nagios-plugins 2.2.1)" ], - "url": "https://domainsbot.com/pandalytics/", "last_changed": "2024-01-10" }, { - "pattern": "VKRobot", - "addition_date": "2020-03-16", + "pattern": "EZID", + "addition_date": "2018-02-09", "instances": [ - "Mozilla/5.0 (compatible; VKRobot/1.0)" + "EZID (EZID link checker; https://ezid.cdlib.org/)" ], "last_changed": "2024-01-10" }, { - "pattern": "bidswitchbot", - "addition_date": "2020-03-16", + "pattern": "\\^LCC", + "addition_date": "2018-02-28", "instances": [ - "bidswitchbot/1.0" + "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" ], - "url": "https://www.bidswitch.com/about-us/", + "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html", "last_changed": "2024-01-10" }, { - "pattern": "tigerbot", - "addition_date": "2020-03-16", + "pattern": "Buck\\\\/", + "addition_date": "2018-03-27", "instances": [ - "tigerbot" + "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" ], + "url": "https://app.hypefactors.com/media-monitoring/about.html", "last_changed": "2024-01-10" }, { - "pattern": "NIXStatsbot", - "addition_date": "2020-03-16", + "pattern": "MeltwaterNews", + "addition_date": "2018-03-27", "instances": [ - "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" + "MeltwaterNews www.meltwater.com" ], - "url": "http://www.nixstats.com/bot.html", + "url": "http://www.meltwater.com", "last_changed": "2024-01-10" }, { - "pattern": "Atom Feed Robot", - "addition_date": "2020-03-16", + "pattern": "Moreover", + "addition_date": "2018-03-27", "instances": [ - "RSSMicro.com RSS/Atom Feed Robot" + "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" ], - "url": "https://rssmicro.com", + "url": "http://www.moreover.com", "last_changed": "2024-01-10" }, { - "pattern": "[Cc]urebot", - "addition_date": "2020-03-16", + "pattern": "newspaper\\\\/", + "addition_date": "2018-03-27", "instances": [ - "Curebot/1.0", - "curebot-feed-fetcher" + "newspaper/0.1.0.7", + "newspaper/0.2.5", + "newspaper/0.2.6", + "newspaper/0.2.8" ], "last_changed": "2024-01-10" }, { - "pattern": "PagePeeker\\/", - "addition_date": "2020-03-16", + "pattern": "", + "addition_date": "2018-03-27", "instances": [ - "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" + "sentry/8.22.0 (https://sentry.io)" ], - "url": "https://pagepeeker.com/robots/", + "url": "https://sentry.io", "last_changed": "2024-01-10" }, { - "pattern": "Vigil\\/", - "addition_date": "2020-03-16", + "pattern": "seoscanners", + "addition_date": "2018-05-27", "instances": [ - "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" + "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" ], - "url": "http://vigil-app.com/bot.html", + "url": "http://www.seoscanners.net/", "last_changed": "2024-01-10" }, { - "pattern": "rssbot\\/", - "addition_date": "2020-03-16", + "pattern": "Hatena", + "addition_date": "2018-05-29", "instances": [ - "rssbot/1.4.3 (+https://t.me/RustRssBot)" + "Hatena Antenna/0.3", + "Hatena::Russia::Crawler/0.01", + "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", + "Hatena::Scissors/0.01", + "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", + "Hatena::Fetcher/0.01 (master) Furl/3.13" ], - "url": "https://t.me/RustRssBot", "last_changed": "2024-01-10" }, { - "pattern": "startmebot\\/", - "addition_date": "2020-03-16", + "pattern": "adscanner", + "addition_date": "2018-06-24", "instances": [ - "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" + "Mozilla/5.0 (compatible; adscanner/)" ], - "url": "https://start.me/bot", "last_changed": "2024-01-10" }, { - "pattern": "JobboerseBot", - "addition_date": "2020-03-16", + "pattern": "Netvibes", + "addition_date": "2018-06-24", "instances": [ - "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" + "Netvibes (crawler/bot; http://www.netvibes.com", + "Netvibes (crawler; http://www.netvibes.com)" ], - "url": "http://www.jobboerse.com/bot.htm", + "url": "http://www.netvibes.com", "last_changed": "2024-01-10" }, { - "pattern": "seewithkids", - "addition_date": "2020-03-16", + "pattern": "BTWebClient", + "addition_date": "2018-06-27", "instances": [ - "http://seewithkids.com/bot" + "BTWebClient/180B(9704)" ], - "url": "http://seewithkids.com/bot", + "url": "http://www.utorrent.com/", + "description": "\u00b5Torrent BitTorrent Client", "last_changed": "2024-01-10" }, { - "pattern": "NINJA bot", - "addition_date": "2020-03-16", + "pattern": "Disqus", + "addition_date": "2018-06-27", "instances": [ - "NINJA bot" + "Disqus/1.0" ], + "url": "https://disqus.com/", + "description": "validate and quality check pages.", "last_changed": "2024-01-10" }, { - "pattern": "Cutbot", - "addition_date": "2020-03-16", + "pattern": "Feedly", + "addition_date": "2018-06-27", "instances": [ - "Cutbot; 1.5; http://cutbot.net/" + "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", + "FeedlyBot/1.0 (http://feedly.com)" ], - "url": "http://cutbot.net/", + "url": "https://www.feedly.com/fetcher.html", + "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud.", "last_changed": "2024-01-10" }, { - "pattern": "BublupBot", - "addition_date": "2020-03-16", + "pattern": "Fetch\\\\/", + "addition_date": "2018-06-27", "instances": [ - "BublupBot (+https://www.bublup.com/bublup-bot.html)" + "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" ], - "url": "https://www.bublup.com/bublup-bot.html", "last_changed": "2024-01-10" }, { - "pattern": "BrandONbot", - "addition_date": "2020-03-16", + "pattern": "Fever", + "addition_date": "2018-06-27", "instances": [ - "BrandONbot (http://brandonmedia.net)" + "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" ], - "url": "http://brandonmedia.net", + "url": "http://feedafever.com", "last_changed": "2024-01-10" }, { - "pattern": "RidderBot", - "addition_date": "2020-03-16", + "pattern": "Flamingo_SearchEngine", + "addition_date": "2018-06-27", "instances": [ - "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", - "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" + "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" ], - "url": "http://brandonmedia.net", "last_changed": "2024-01-10" }, { - "pattern": "Taboolabot", - "addition_date": "2020-03-16", + "pattern": "FlipboardProxy", + "addition_date": "2018-06-27", "instances": [ - "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" + "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" ], - "url": "http://www.taboola.com", + "url": "https://about.flipboard.com/browserproxy/", + "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application", "last_changed": "2024-01-10" }, { - "pattern": "Dubbotbot", - "addition_date": "2020-03-16", + "pattern": "G2\\sWeb\\sServices", + "addition_date": "2019-03-01", "instances": [ - "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" + "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" ], - "url": "http://dubbot.com", + "url": "https://www.g2webservices.com/", "last_changed": "2024-01-10" }, { - "pattern": "FindITAnswersbot", - "addition_date": "2020-03-16", + "pattern": "vkShare", + "addition_date": "2018-07-02", "instances": [ - "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" + "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" ], - "url": "http://search.it-influentials.com/bot.htm", + "url": "http://vk.com/dev/Share", "last_changed": "2024-01-10" }, { - "pattern": "infoobot", - "addition_date": "2020-03-16", + "pattern": "Siteimprove\\.com", + "addition_date": "2018-06-22", "instances": [ - "infoobot/0.1 (https://www.infoo.nl/bot.html)" + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" ], - "url": "https://www.infoo.nl/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "Refindbot", - "addition_date": "2020-03-16", + "pattern": "DareBoost", + "addition_date": "2018-07-07", "instances": [ - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" ], - "url": "https://refind.com/about", + "url": "https://www.dareboost.com/", + "description": "Bot to test, Analyze and Optimize website", "last_changed": "2024-01-10" }, { - "pattern": "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher", - "addition_date": "2020-03-16", + "pattern": "Miniflux\\\\/", + "addition_date": "2018-07-07", "instances": [ - "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" + "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", + "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" ], - "url": "http://www.blogtraffic.de/rss-bot.html", + "url": "https://miniflux.net", + "description": "Miniflux is a minimalist and opinionated feed reader.", "last_changed": "2024-01-10" }, { - "pattern": "SeobilityBot", - "addition_date": "2020-03-16", + "pattern": "Feedspot", + "addition_date": "2018-07-07", "instances": [ - "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" + "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", + "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" ], - "url": "https://www.seobility.net/sites/bot.html", + "url": "http://www.feedspot.com/fs/bot", "last_changed": "2024-01-10" }, { - "pattern": "Cincraw", - "addition_date": "2020-03-16", + "pattern": "SEOkicks", + "addition_date": "2018-08-22", "instances": [ - "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" + "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" ], - "url": "http://cincrawdata.net/bot/", + "url": "https://www.seokicks.de/robot.html", "last_changed": "2024-01-10" }, { - "pattern": "Dragonbot", - "addition_date": "2020-03-16", + "pattern": "tracemyfile", + "addition_date": "2018-08-23", "instances": [ - "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" + "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" ], - "url": "http://www.dragonmetrics.com", "last_changed": "2024-01-10" }, { - "pattern": "VoluumDSP-content-bot", - "addition_date": "2020-03-16", + "pattern": "zgrab", + "addition_date": "2018-08-30", "instances": [ - "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" + "Mozilla/5.0 zgrab/0.x" ], - "url": "https://codewise.com", + "url": "https://zmap.io/", "last_changed": "2024-01-10" }, { - "pattern": "FreshRSS", - "addition_date": "2020-03-16", + "pattern": "PR-CY\\.RU", + "addition_date": "2018-08-30", "instances": [ - "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" + "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" ], - "url": "https://freshrss.org", + "url": "https://a.pr-cy.ru/", "last_changed": "2024-01-10" }, { - "pattern": "BitBot", - "addition_date": "2020-03-16", + "pattern": "Datafeedwatch", + "addition_date": "2018-09-05", "instances": [ - "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" + "Datafeedwatch/2.1.x" ], - "url": "https://bitbot.dev", + "url": "https://www.datafeedwatch.com/", "last_changed": "2024-01-10" }, { - "pattern": "^PHP-Curl-Class", - "addition_date": "2020-12-10", + "pattern": "axios", + "addition_date": "2018-09-06", "instances": [ - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", - "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" + "axios/0.18.0", + "axios/0.19.0" ], - "url": "https://github.com/php-curl-class/php-curl-class", + "url": "https://github.com/axios/axios", "last_changed": "2024-01-10" }, { - "pattern": "Google-Certificates-Bridge", - "addition_date": "2020-12-23", + "pattern": "Pulsepoint", + "addition_date": "2018-09-24", "instances": [ - "Google-Certificates-Bridge" + "Pulsepoint XT3 web scraper" ], "last_changed": "2024-01-10" }, { - "pattern": "centurybot", - "addition_date": "2022-04-26", + "pattern": "CloudFlare-AlwaysOnline", + "addition_date": "2018-09-27", "instances": [ - "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" ], - "url": "unknown", + "url": "https://www.cloudflare.com/always-online/", "last_changed": "2024-01-10" }, { - "pattern": "Viber", - "addition_date": "2021-04-27", + "pattern": "WordupInfoSearch", + "addition_date": "2018-10-07", "instances": [ - "Viber" + "WordupInfoSearch/1.0" ], - "url": "https://www.viber.com/", "last_changed": "2024-01-10" }, { - "pattern": "e\\.ventures Investment Crawler", - "addition_date": "2021-06-05", + "pattern": "WebDataStats", + "addition_date": "2018-10-08", "instances": [ - "e.ventures Investment Crawler (eventures.vc)" + "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" ], + "url": "https://webdatastats.com/", "last_changed": "2024-01-10" }, { - "pattern": "evc-batch", - "addition_date": "2021-06-07", + "pattern": "HttpUrlConnection", + "addition_date": "2018-10-08", "instances": [ - "Mozilla/5.0 (compatible; evc-batch/2.0)" + "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" ], "last_changed": "2024-01-10" }, { - "pattern": "PetalBot", - "addition_date": "2021-06-07", + "pattern": "outbrain", + "addition_date": "2018-10-14", "instances": [ - "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", - "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" + "Mozilla/5.0 (Java) outbrain" ], - "url": "https://webmaster.petalsearch.com/site/petalbot", + "url": "https://www.outbrain.com/help/advertisers/invalid-url/", "last_changed": "2024-01-10" }, { - "pattern": "virustotal", - "addition_date": "2021-09-22", + "pattern": "W3C-mobileOK", + "addition_date": "2018-10-14", "instances": [ - "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", - "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" + "W3C-mobileOK/DDC-1.0" ], - "url": "https://www.virustotal.com/gui/home/url", + "url": "https://validator.w3.org/services", "last_changed": "2024-01-10" }, { - "pattern": "(^| )PTST\\/", - "addition_date": "2021-12-05", + "pattern": "W3C_I18n-Checker", + "addition_date": "2018-10-14", "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" + "W3C_I18n-Checker/1.0" ], - "url": "https://www.webpagetest.org", + "url": "https://validator.w3.org/services", "last_changed": "2024-01-10" }, { - "pattern": "minicrawler", - "addition_date": "2022-01-12", + "pattern": "W3C_Unicorn", + "addition_date": "2018-10-14", "instances": [ - "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" + "W3C_Unicorn/1.0" ], - "url": "https://www.testomato.com/bot", + "url": "https://validator.w3.org/services", "last_changed": "2024-01-10" }, { - "pattern": "Cookiebot", - "addition_date": "2022-01-23", - "url": "https://www.cookiebot.com/", + "pattern": "Blackboard", + "addition_date": "2018-10-28", "instances": [ - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" + "Blackboard Safeassign" ], + "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign", "last_changed": "2024-01-10" }, { - "pattern": "trovitBot", - "addition_date": "2022-06-08", - "url": "http://www.trovit.com/bot.html", + "pattern": "BazQux", + "addition_date": "2018-10-23", "instances": [ - "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" + "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" ], + "url": "https://bazqux.com/fetcher", "last_changed": "2024-01-10" }, { - "pattern": "seostar\\.co", - "addition_date": "2022-08-04", - "url": "https://seostar.co/robot/", + "pattern": "Twingly", + "addition_date": "2018-10-23", "instances": [ - "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" + "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" ], + "url": "https://twingly.com", "last_changed": "2024-01-10" }, { - "pattern": "IonCrawl", - "addition_date": "2022-08-04", - "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", + "pattern": "Rivva", + "addition_date": "2018-10-23", "instances": [ - "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" + "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" ], + "url": "http://rivva.de", "last_changed": "2024-01-10" }, { - "pattern": "Uptime-Kuma", - "addition_date": "2022-10-17", - "url": "https://uptime.kuma.pet/", + "pattern": "Dataprovider\\.com", + "addition_date": "2018-11-24", "instances": [ - "Uptime-Kuma/1.18.0" + "Mozilla/5.0 (compatible; Dataprovider.com)" ], + "url": "https://www.dataprovider.com/", "last_changed": "2024-01-10" }, { - "pattern": "SeekportBot", - "addition_date": "2022-10-17", - "url": "https://bot.seekport.com", + "pattern": "GroupHigh\\\\/", + "addition_date": "2018-11-24", "instances": [ - "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)" + "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" ], + "url": "http://www.grouphigh.com/", "last_changed": "2024-01-10" }, { - "pattern": "FreshpingBot", - "addition_date": "2022-10-17", - "url": "https://www.freshworks.com/website-monitoring/", + "pattern": "theoldreader\\.com", + "addition_date": "2018-12-02", "instances": [ - "FreshpingBot/1.0 (+https://freshping.io/)" + "Mozilla/5.0 (compatible; theoldreader.com)" ], + "url": "https://www.theoldreader.com/", "last_changed": "2024-01-10" }, { - "pattern": "Feedbin", - "addition_date": "2022-11-05", - "url": "https://feedbin.com/", + "pattern": "AnyEvent", + "addition_date": "2018-12-07", "instances": [ - "Feedbin feed-id:2005098 - 2 subscribers" + "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" ], + "url": "http://software.schmorp.de/pkg/AnyEvent.html", "last_changed": "2024-01-10" }, { - "pattern": "CriteoBot", - "addition_date": "2022-11-13", - "url": "https://www.criteo.com/", + "pattern": "Nmap\\sScripting\\sEngine", + "addition_date": "2019-02-04", "instances": [ - "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" + "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" ], + "url": "https://nmap.org/book/nse.html", "last_changed": "2024-01-10" }, { - "pattern": "Snap URL Preview Service", - "addition_date": "2022-11-13", - "url": "https://snap.com/", + "pattern": "2ip\\.ru", + "addition_date": "2019-02-12", "instances": [ - "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" + "2ip.ru CMS Detector (https://2ip.ru/cms/)" ], + "url": "https://2ip.ru/cms/", "last_changed": "2024-01-10" }, { - "pattern": "Better Uptime Bot", - "addition_date": "2022-11-13", - "url": "https://betteruptime.com/", + "pattern": "Clickagy", + "addition_date": "2019-02-19", "instances": [ - "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" + "Clickagy Intelligence Bot v2" ], + "url": "https://www.clickagy.com", "last_changed": "2024-01-10" }, { - "pattern": "RuxitSynthetic", - "addition_date": "2023-02-16", - "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", + "pattern": "HubSpot", + "addition_date": "2019-04-15", "instances": [ - "RuxitSynthetic/1.0" + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", + "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", + "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" ], "last_changed": "2024-01-10" }, { - "pattern": "Google-Read-Aloud", - "addition_date": "2023-02-16", - "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "pattern": "Chrome-Lighthouse", + "addition_date": "2019-03-15", "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", - "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" ], + "url": "https://developers.google.com/speed/pagespeed/insights", "last_changed": "2024-01-10" }, { - "pattern": "Valve\\/Steam", - "addition_date": "2023-05-24", + "pattern": "CheckMarkNetwork\\\\/", + "addition_date": "2019-06-30", "instances": [ - "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" + "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" ], + "url": "https://www.checkmarknetwork.com/", "last_changed": "2024-01-10" }, { - "pattern": "OdklBot\\/", - "addition_date": "2023-05-24", + "pattern": "www\\\\\\.uptime\\\\\\.com", + "addition_date": "2019-07-21", "instances": [ - "OdklBot/1.0 (share@odnoklassniki.ru)", - "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" + "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" ], - "url": "https://odnoklassniki.ru/", + "url": "http://www.uptime.com/uptimebot", "last_changed": "2024-01-10" }, { - "pattern": "GPTBot", - "addition_date": "2023-08-09", + "pattern": "MixnodeCache\\\\/", + "addition_date": "2019-08-04", "instances": [ - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" + "MixnodeCache/1.8(+https://cache.mixnode.com/)" ], - "url": "https://platform.openai.com/docs/gptbot", + "url": "https://cache.mixnode.com/", "last_changed": "2024-01-10" }, { - "pattern": "YandexRenderResourcesBot\\/", - "addition_date": "2023-08-16", + "pattern": "\\^curl", + "addition_date": "2019-08-15", "instances": [ - "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" + "curl", + "curl/7.29.0", + "curl/7.47.0", + "curl/7.54.0", + "curl/7.55.1", + "curl/7.64.0", + "curl/7.64.1", + "curl/7.65.3" ], - "url": "http://yandex.com/bots", + "url": "https://curl.haxx.se/", "last_changed": "2024-01-10" }, { - "pattern": "LightspeedSystemsCrawler", - "addition_date": "2023-08-16", + "pattern": "SimpleScraper", + "addition_date": "2019-08-16", "instances": [ - "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" + "Mozilla/5.0 (compatible; SimpleScraper)" ], + "url": "https://github.com/ramonkcom/simple-scraper/", "last_changed": "2024-01-10" }, { - "pattern": "ev-crawler\\/", - "addition_date": "2023-08-16", + "pattern": "fedoraplanet", + "addition_date": "2019-09-28", "instances": [ - "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" + "venus/fedoraplanet" ], - "url": "https://headline.com/legal/crawler", + "url": "http://fedoraplanet.org/", "last_changed": "2024-01-10" }, { - "pattern": "BitSightBot\\/", - "addition_date": "2023-08-16", + "pattern": "Friendica", + "addition_date": "2019-09-28", "instances": [ - "Mozilla/5.0 (compatible; BitSightBot/1.0)" + "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" ], - "url": "https://www.bitsight.com", + "url": "https://hoyer.xyz", "last_changed": "2024-01-10" }, { - "pattern": "woorankreview\\/", - "addition_date": "2023-08-16", + "pattern": "NextCloud", + "addition_date": "2019-09-30", "instances": [ - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", - "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" + "NextCloud-News/1.0" ], - "url": "https://www.woorank.com/", + "url": "https://nextcloud.com/", "last_changed": "2024-01-10" }, { - "pattern": "Google-Safety", - "addition_date": "2023-08-17", + "pattern": "Datanyze", + "addition_date": "2019-11-17", "instances": [ - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", - "Google-Safety" + "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" ], - "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "url": "https://www.datanyze.com/dnyzbot/", "last_changed": "2024-01-10" }, { - "pattern": "AwarioBot", - "addition_date": "2023-08-23", + "pattern": "TrendsmapResolver", + "addition_date": "2020-02-24", "instances": [ - "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" + "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" ], - "url": "https://awario.com/bots.html", + "url": "https://www.trendsmap.com/", "last_changed": "2024-01-10" }, { - "pattern": "DataForSeoBot", - "addition_date": "2023-08-23", + "pattern": "tweetedtimes", + "addition_date": "2020-02-24", "instances": [ - "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" + "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" ], - "url": "https://dataforseo.com/dataforseo-bot", + "url": "https://tweetedtimes.com/", "last_changed": "2024-01-10" }, { - "pattern": "Linespider", - "addition_date": "2023-08-24", + "pattern": "Gwene", + "addition_date": "2020-02-24", "instances": [ - "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" + "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" ], - "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en", + "url": "https://gwene.org", "last_changed": "2024-01-10" }, { - "pattern": "WellKnownBot", - "addition_date": "2023-08-29", + "pattern": "SimplePie", + "addition_date": "2020-02-24", "instances": [ - "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" + "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" ], - "url": "https://well-known.dev/about/#bot)", + "url": "http://simplepie.org", "last_changed": "2024-01-10" }, { - "pattern": "A Patent Crawler", - "addition_date": "2023-08-29", + "pattern": "SearchAtlas", + "addition_date": "2020-03-02", "instances": [ - "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" + "SearchAtlas.com SEO Crawler" ], - "url": "http://scitas.epfl.ch/", + "url": "http://SearchAtlas.com", "last_changed": "2024-01-10" }, { - "pattern": "StractBot", - "addition_date": "2023-09-06", + "pattern": "Superfeedr", + "addition_date": "2020-03-02", "instances": [ - "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" + "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" ], - "url": "https://trystract.com/webmasters", + "url": "http://superfeedr.com", "last_changed": "2024-01-10" }, { - "pattern": "search\\.marginalia\\.nu", - "addition_date": "2023-09-08", + "pattern": "Domains\\sProject\\\\/", + "addition_date": "2020-03-16", "instances": [ - "search.marginalia.nu" + "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" ], - "url": "https://search.marginalia.nu", + "url": "https://github.com/tb0hdan/domains", "last_changed": "2024-01-10" }, { - "pattern": "YouBot", - "addition_date": "2023-09-08", + "pattern": "Pandalytics", + "addition_date": "2020-03-16", "instances": [ - "YouBot (+http://www.you.com)" + "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" ], - "url": "https://you.com/", + "url": "https://domainsbot.com/pandalytics/", "last_changed": "2024-01-10" }, { - "pattern": "Nicecrawler", - "addition_date": "2023-09-08", + "pattern": "PagePeeker\\\\/", + "addition_date": "2020-03-16", "instances": [ - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" ], - "url": "http://www.nicecrawler.com/", + "url": "https://pagepeeker.com/robots/", "last_changed": "2024-01-10" }, { - "pattern": "Neevabot", - "addition_date": "2023-09-08", + "pattern": "Vigil\\\\/", + "addition_date": "2020-03-16", "instances": [ - "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" + "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" ], - "url": "https://neeva.com/neevabot", + "url": "http://vigil-app.com/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "BrightEdge Crawler", - "addition_date": "2023-09-08", + "pattern": "seewithkids", + "addition_date": "2020-03-16", "instances": [ - "BrightEdge Crawler/1.0 (crawler@brightedge.com)" + "http://seewithkids.com/bot" ], - "url": "https://www.brightedge.com/", + "url": "http://seewithkids.com/bot", "last_changed": "2024-01-10" }, { - "pattern": "SiteCheckerBotCrawler", - "addition_date": "2023-09-08", + "pattern": "BlogTraffic\\\\/\\\\d\\\\\\.\\\\d\\+\\sFeed-Fetcher", + "addition_date": "2020-03-16", "instances": [ - "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" + "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" ], - "url": "http://sitechecker.pro", + "url": "http://www.blogtraffic.de/rss-bot.html", "last_changed": "2024-01-10" }, { - "pattern": "TombaPublicWebCrawler", - "addition_date": "2023-09-08", + "pattern": "Cincraw", + "addition_date": "2020-03-16", "instances": [ - "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" + "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" ], - "url": "https://tombascraper.com", + "url": "http://cincrawdata.net/bot/", "last_changed": "2024-01-10" }, { - "pattern": "CrawlyProjectCrawler", - "addition_date": "2023-09-08", + "pattern": "\\^PHP-Curl-Class", + "addition_date": "2020-12-10", "instances": [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", + "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" ], - "url": "https://crawlyproject.digitaldragon.dev/", + "url": "https://github.com/php-curl-class/php-curl-class", "last_changed": "2024-01-10" }, { - "pattern": "KomodiaBot", - "addition_date": "2023-09-08", + "pattern": "Viber", + "addition_date": "2021-04-27", "instances": [ - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" + "Viber" ], - "url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler", + "url": "https://www.viber.com/", "last_changed": "2024-01-10" }, { - "pattern": "KStandBot", - "addition_date": "2023-09-08", + "pattern": "evc-batch", + "addition_date": "2021-06-07", "instances": [ - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" + "Mozilla/5.0 (compatible; evc-batch/2.0)" ], - "url": "http://url-classification.io", "last_changed": "2024-01-10" }, { - "pattern": "CISPA Webcrawler", - "addition_date": "2023-09-08", + "pattern": "virustotal", + "addition_date": "2021-09-22", "instances": [ - "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" + "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", + "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" ], - "url": "https://vuln-notify-checker.cispa.saarland", + "url": "https://www.virustotal.com/gui/home/url", "last_changed": "2024-01-10" }, { - "pattern": "MTRobot", - "addition_date": "2023-09-08", + "pattern": "seostar\\\\\\.co", + "addition_date": "2022-08-04", + "url": "https://seostar.co/robot/", "instances": [ - "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" + "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" ], - "url": "https://metrics-tools.de/robot.html", "last_changed": "2024-01-10" }, { - "pattern": "hyscore.io", - "addition_date": "2023-09-08", + "pattern": "Uptime-Kuma", + "addition_date": "2022-10-17", + "url": "https://uptime.kuma.pet/", "instances": [ - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" + "Uptime-Kuma/1.18.0" ], - "url": "https://hyscore.io/crawler/", "last_changed": "2024-01-10" }, { - "pattern": "AlexandriaOrgBot", - "addition_date": "2023-09-08", + "pattern": "Snap\\sURL\\sPreview\\sService", + "addition_date": "2022-11-13", + "url": "https://snap.com/", "instances": [ - "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" + "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" ], - "url": "https://www.alexandria.org/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "2ip bot", - "addition_date": "2023-09-08", + "pattern": "RuxitSynthetic", + "addition_date": "2023-02-16", + "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", "instances": [ - "2ip bot/1.1 (+http://2ip.io)" + "RuxitSynthetic/1.0" ], - "url": "http://2ip.io", "last_changed": "2024-01-10" }, { - "pattern": "Yellowbrandprotectionbot", - "addition_date": "2023-09-08", + "pattern": "Valve\\\\/Steam", + "addition_date": "2023-05-24", "instances": [ - "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" + "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" ], - "url": "https://www.yellowbp.com/bot.html", "last_changed": "2024-01-10" }, { - "pattern": "SEOlizer", - "addition_date": "2023-09-08", + "pattern": "woorankreview\\\\/", + "addition_date": "2023-08-16", "instances": [ - "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", + "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" ], - "url": "https://www.seolizer.de/bot.html", + "url": "https://www.woorank.com/", "last_changed": "2024-01-10" }, { - "pattern": "vuhuvBot", + "pattern": "search\\\\\\.marginalia\\\\\\.nu", "addition_date": "2023-09-08", "instances": [ - "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" + "search.marginalia.nu" ], - "url": "http://vuhuv.com/bot.html", + "url": "https://search.marginalia.nu", "last_changed": "2024-01-10" }, { - "pattern": "INETDEX-BOT", + "pattern": "SEOlizer", "addition_date": "2023-09-08", "instances": [ - "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" + "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" ], - "url": "https://inetdex.com/bot.html", + "url": "https://www.seolizer.de/bot.html", "last_changed": "2024-01-10" }, { @@ -10926,15 +5889,6 @@ "url": "https://github.com/matrix-org/synapse", "last_changed": "2024-01-10" }, - { - "pattern": "t3versionsBot", - "addition_date": "2023-09-08", - "instances": [ - "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" - ], - "url": "https://www.t3versions.com/bot", - "last_changed": "2024-01-10" - }, { "pattern": "deepnoc", "addition_date": "2023-09-08", @@ -10944,15 +5898,6 @@ "url": "https://deepnoc.com/bot", "last_changed": "2024-01-10" }, - { - "pattern": "Cocolyzebot", - "addition_date": "2023-09-08", - "instances": [ - "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" - ], - "url": "https://cocolyze.com/bot", - "last_changed": "2024-01-10" - }, { "pattern": "hypestat", "addition_date": "2023-09-08", @@ -10963,16 +5908,7 @@ "last_changed": "2024-01-10" }, { - "pattern": "ReverseEngineeringBot", - "addition_date": "2023-09-08", - "instances": [ - "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" - ], - "url": "https://torus.company/bot.html", - "last_changed": "2024-01-10" - }, - { - "pattern": "sempi.tech", + "pattern": "sempi\\.tech", "addition_date": "2023-09-08", "instances": [ "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" @@ -10989,15 +5925,6 @@ "url": "https://iframely.com/docs/about", "last_changed": "2024-01-10" }, - { - "pattern": "MetaInspector", - "addition_date": "2023-09-08", - "instances": [ - "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" - ], - "url": "https://github.com/jaimeiniesta/metainspector", - "last_changed": "2024-01-10" - }, { "pattern": "node-fetch", "addition_date": "2023-09-08", @@ -11016,15 +5943,6 @@ "url": "https://github.com/LeakIX/l9explore", "last_changed": "2024-01-10" }, - { - "pattern": "python-opengraph", - "addition_date": "2023-09-08", - "instances": [ - "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" - ], - "url": "https://github.com/jaywink/python-opengraph", - "last_changed": "2024-01-10" - }, { "pattern": "OpenGraphCheck", "addition_date": "2023-09-08", @@ -11034,34 +5952,6 @@ "url": "https://opengraphcheck.com", "last_changed": "2024-01-10" }, - { - "pattern": "developers.google.com\\/\\+\\/web\\/snippet", - "addition_date": "2023-09-08", - "instances": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" - ], - "url": "https://developers.google.com/+/web/snippet", - "last_changed": "2024-01-10" - }, - { - "pattern": "SenutoBot", - "addition_date": "2023-09-08", - "instances": [ - "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" - ], - "url": "https://www.senuto.com", - "last_changed": "2024-01-10" - }, - { - "pattern": "MaCoCu", - "addition_date": "2023-09-08", - "instances": [ - "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" - ], - "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data", - "last_changed": "2024-01-10" - }, { "pattern": "NewsBlur", "addition_date": "2023-09-08", @@ -11097,14 +5987,5 @@ ], "url": "http://pagething.com", "last_changed": "2024-01-10" - }, - { - "pattern": "WordPress\\/", - "addition_date": "2023-10-24", - "instances": [ - "WordPress/X.X.X; https://example.com" - ], - "url": "https://wordpress.org", - "last_changed": "2024-01-10" } ] \ No newline at end of file diff --git a/generated/COUNTER_Robots_list.txt b/generated/COUNTER_Robots_list.txt index b1c7c86..d14dad5 100644 --- a/generated/COUNTER_Robots_list.txt +++ b/generated/COUNTER_Robots_list.txt @@ -320,1638 +320,861 @@ zeus zyborg 7siters &as_qdr=all -*/Nutch-0.9-dev -192.comAgent -222.122.15.190 -2Bone_LinkChkr/1.0 l +192\.comAgent +222\.122\.15\.190 +2Bone_LinkChkr\/1\.0\sl 7331230CB97FADECF052 -8484 Boston Project +8484\sBoston\sProject 8LEGS -A-Online Search -ADmantX Platform Sem -AHC/2.0 -ASPSeek/1.2.5 -ASPSeek/1.2.x -ASPSeek/1.2.xa -ASPSeek/1.2.xxpre -ASPseek/1.2.9d -ASPseek/1.2.xx -ASSORT/0.10 -AV Fetch 1.0 -AVSearch-1.0(peter.t -AVSearch-2.0-fusionI -AVSearch-3.0(AltaVis -About/0.1libwww-perl -Accoona-AI-Agent/1.1 -Acorn/Nutch-0.9 (Non -ActiveBookmark 1.x -AdobeUxTechC4-Async/ -AgentName/0.1 libwww -AideRSS/1.0 (aiderss -Aladin/3.324 -Allesklar/0.1 libwww -Allrati/1.1 (+) -AltaVista Intranet V -Amazon CloudFront -AmiTCP Miami (AmigaO -Anonymizer/1.1 -AnswerChase PROve x. -AnswerChase x.0 -AnzwersCrawl/2.0 (an -Apache (internal dum +A-Online\sSearch +AHC\/2\.0 +ASPSeek\/1\.2\.5 +ASPSeek\/1\.2\.x +ASPseek\/1\.2\.9d +ASSORT\/0\.10 +AV\sFetch\s1\.0 +AVSearch-1\.0 +AVSearch-2\.0-fusionI +AVSearch-3\.0 +Accoona-AI-Agent\/1\.1 +ActiveBookmark\s1\.x +AdobeUxTechC4-Async\/ +Aladin\/3\.324 +Allrati\/1\.1 +AltaVista\sIntranet\sV +Amazon\sCloudFront +AmiTCP\sMiami +Anonymizer\/1\.1 +AnswerChase\sPROve\sx\. +AnswerChase\sx\.0 +Apache Apache-HttpAsyncClie -Apache-HttpClient/4. -Apache-HttpClient/UN +Apache-HttpClient\/4\. +Apache-HttpClient\/UN Aport -Arachnoidea (arachno -Asahina-Antenna/1.x -Asahina-Antenna/1.x -Ask.24x.Info -Atomic_Email_Hunter/ -Atomz/1.0 -BIGLOTRON (Beta 2;GN +Arachnoidea +Asahina-Antenna\/1\.x +Ask\.24x\.Info +Atomic_Email_Hunter\/ BMCLIENT BMLAUNCHER -BPImageWalker/2.0 (w -BSDSeek/1.0 -BStop.BravoBrian.it -BTWebClient/180B(970 -BeamMachine/0.5 (dea -BeebwareDirectory/v0 -Big Fish v1.0 -BigBrother/1.6e -Bilbo/2.3b-UNIX -BlackBerry9000/4.6.0 +BPImageWalker\/2\.0 +BSDSeek\/1\.0 +BStop\.BravoBrian\.it +BTWebClient\/180B +BeamMachine\/0\.5 +BeebwareDirectory\/v0 +Big\sFish\sv1\.0 +BigBrother\/1\.6e +Bilbo\/2\.3b-UNIX +BlackBerry9000\/4\.6\.0 BlackWidow -Blackboard Safeassig -BlogPulseLive (suppo -Blogpulse (info@blog -Bobby/4.0.x RPT-HTTP -Bookdog/x.x -BravoBrian bstop.bra +Blackboard\sSafeassig +Bobby\/4\.0\.x\sRPT-HTTP +Bookdog\/x\.x +BravoBrian\sbstop\.bra BullsEye BunnySlippers -CB/Nutch-1.7 -CFNetwork/x.x -CHttpClient by Open -CLIPish Pro2 10.21 ( -COAST WebMaster Pro/ -CamelHttpStream/1.0 -Cancer Information a -Chat Catcher/1.0 -China Local Browse 2 -Chitika ContentHit 1 -Claymont.com -ColdFusion (Bookmark -Combine/x.0 +CHttpClient\sby\sOpen +CLIPish\sPro2\s10\.21 +COAST\sWebMaster\sPro\/ +CamelHttpStream\/1\.0 +Cancer\sInformation\sa +Chat\sCatcher\/1\.0 +China\sLocal\sBrowse\s2 +Chitika\sContentHit\s1 +Claymont\.com Contact -Cookie%20Stumbler/22 -CrawlConvera0.1 (Cra -Custo x.x (www.netwu -CyberSpyder Link Tes -DBrowse 1.4b -DBrowse 1.4d +Cookie%20Stumbler\/22 +CyberSpyder\sLink\sTes +DBrowse\s1\.4b +DBrowse\s1\.4d DTAAgent -Dart Communications -DataCha0s/2.0 -DataFountains/DMOZ D -Datafeedwatch/2.1.x -Deep Link Calculator +Dart\sCommunications +DataFountains\/DMOZ\sD +Datafeedwatch\/2\.1\.x +Deep\sLink\sCalculator DeepIndex -DeepIndex (www.en.de -DeepIndexer.ca -DevComponents.com Ht -Diamond/x.0 +DeepIndexer\.ca +DevComponents\.com\sHt +Diamond\/x\.0 DigOut4U -Digger/1.0 JDK/1.3.0 -Disqus/1.0 +Digger\/1\.0\sJDK\/1\.3\.0 +Disqus\/1\.0 DittoSpyder -DoCoMo/1.0/Nxxxi/c10 -DoCoMo/2.0 P900iV(c1 -EARTHCOM.info/1.x [w -EARTHCOM.info/1.xbet -EBrowse 1.4b -EasyDL/3.xx -EchO!/2.0 -Educate Search VxB -EldoS TimelyWeb/3.x -EmailWolf 1.00 -Embedly +support@emb -Enfish Tracker -Enterprise_Search/1. -Evernote Clip Resolv -Exalead NG/MimeLive -Expired Domain Sleut +EARTHCOM\.info\/1\.x\s\[w +EARTHCOM\.info\/1\.xbet +EBrowse\s1\.4b +EchO!\/2\.0 +Educate\sSearch\sVxB +EldoS\sTimelyWeb\/3\.x +Enfish\sTracker +Enterprise_Search\/1\. +Evernote\sClip\sResolv +Exalead\sNG\/MimeLive +Expired\sDomain\sSleut ExtractorPro -EyeCatcher (Download -FARK.com link verifi +EyeCatcher +FARK\.com\slink\sverifi FLATARTS_FAVICO -FaEdit/2.0.x -FavOrg -Favorites Sweeper v. -Feed24.com -FeedForAll rss2html. -FeedValidator/1.3 -Filtrbox/1.0 -FindAnISP.com_ISP_Fi -Firefly/1.0 -Firefox (kastaneta03 -Firefox_1.0.6 (kaspa -FirstGov.gov Search -Flexum/2.0 -FocusedSampler/1.0 -Franklin Locator 1.8 -FuseBulb.Com -GigablastOpenSource/ -Go 1.1 package http -Go-http-client/1.1 -Go-http-client/2.0 -GoForIt.com -Goofer/0.2 -GrapeFX/0.3 libwww/5 -Gromit/1.0 -Guestbook Auto Submi -Gulliver/1.3 -Guzzle/4.2.3 curl/7. +FaEdit\/2\.0\.x +Favorites\sSweeper\sv\. +Feed24\.com +Filtrbox\/1\.0 +FindAnISP\.com_ISP_Fi +Firefly\/1\.0 +Firefox +Firefox_1\.0\.6 +FirstGov\.gov\sSearch +Flexum\/2\.0 +FocusedSampler\/1\.0 +Franklin\sLocator\s1\.8 +FuseBulb\.Com +Go\s1\.1\spackage\shttp +Go-http-client\/1\.1 +Go-http-client\/2\.0 +GoForIt\.com +Goofer\/0\.2 +Gromit\/1\.0 +Guestbook\sAuto\sSubmi HLoader -HPL/Nutch-0.9 - Haansoft -Harvest-NG/1.0.2 -Hatena Antenna/0.3 -Hatena Pagetitle Age -Hatena::Bookmark/2.0 -Hippias/0.9 Beta +Hatena\sAntenna\/0\.3 +Hatena\sPagetitle\sAge +Hatena::Bookmark\/2\.0 +Hippias\/0\.9\sBeta HitList Hivemind -HomePageSearch(hpsea -Html Link Validator -Hybrid/1.2 [en] (OS -IAArchiver-1.0 -IBM WebExplorer /v0. +HomePageSearch +Hybrid\/1\.2\s\[en] +IAArchiver-1\.0 +IBM\sWebExplorer\s\/v0\. IEFav172Free -INFOMINE/8.0 Adders -INFOMINE/8.0 RemoteS -IP*Works! V5 HTTP/S -ISC Systems iRc Sear -IWAgent/ 1.0 - www.b -Icarus6 - (contact: -ImageWalker/2.0 (www -Incutio HttpClient v -IncyWincy data gathe -Industry Program 1.0 -Inet library -InetURL/1.0 -InfoLink/1.x -InfoSeek Sidewinder/ -Infoaxe./Nutch-0.9 -Infoseek SideWinder/ -Inktomi Search -Insitor.com search a +INFOMINE\/8\.0\sAdders +INFOMINE\/8\.0\sRemoteS +IP\*Works!\sV5\sHTTP\/S +ISC\sSystems\siRc\sSear +IWAgent\/\s1\.0\s-\swww\.b +Icarus6\s- +ImageWalker\/2\.0 +Incutio\sHttpClient\sv +IncyWincy\sdata\sgathe +Industry\sProgram\s1\.0 +Inet\slibrary +InetURL\/1\.0 +InfoLink\/1\.x +InfoSeek\sSidewinder\/ +Infoseek\sSideWinder\/ +Inktomi\sSearch +Insitor\.com\ssearch\sa Insitornaut -Internet Ninja x.0 -InternetLinkAgent/3. -InternetSeer.com -IrssiUrlLog/0.2 -JDK/1.1 +Internet\sNinja\sx\.0 +InternetLinkAgent\/3\. +JDK\/1\.1 Jack -Jakarta Commons-Http -Java 1.1 -Java Browser -Java/1.4.1_01 -Java/1.4.1_04 -Java/1.5.0_16 -Java/1.6.0_04 -Java/1.6.0_07 -Java/1.6.0_18 -Java/1.6.0_22 -Java/1.6.0_24 -Java/1.6.0_25 -Java/1.6.0_26 -Java/1.6.0_27 -Java/1.6.0_29 -Java/1.6.0_30 -Java/1.6.0_31 -Java/1.6.0_32 -Java/1.6.0_33 -Java/1.6.0_34 -Java/1.6.0_35 -Java/1.6.0_39 -Java/1.6.0_45 -Java/1.6.0_51 -Java/1.7.0_03 -Java/1.7.0_09 -Java/1.7.0_15 -Java/1.7.0_17 -Java/1.7.0_21 -Java/1.7.0_25 -Java/1.7.0_40 -Java/1.7.0_45 -Java/1.7.0_51 -Java/1.7.0_55 -Java/1.7.0_60 -Java/1.7.0_65 -Java/1.7.0_71 -Java/1.7.0_75 -Java/1.7.0_80 -Java/1.8.0_20 -Java/1.8.0_25 -Java/1.8.0_31 -Java/1.8.0_40 -Java/1.8.0_45 -Java/1.8.0_51 -Java1.0.21.0 -Java1.1.xx.x -Java1.3.0rc1 -Java1.3.x -Java1.4.0 -Jersey/2.25.1 (HttpU -Jetty/9.3.z-SNAPSHOT -Jigsaw/2.2.x W3C_CSS -Jigsaw/2.3.0 W3C_CSS -Junkie/1.0 -KE_1.0/2.0 libwww/5. -KIT-Fireball/2.0 -KnowItAll(knowitall@ -Knowledge.com/0.x -KummHttp/1.1 (compat -LARBIN-EXPERIMENTAL -LEIA/2.90 -LWP::Simple/5.22 -LWP::Simple/5.36 -LWP::Simple/5.48 -LWP::Simple/5.50 -LWP::Simple/5.51 -LWP::Simple/5.53 -LWP::Simple/5.63 -LWP::Simple/5.803 +Jakarta\sCommons-Http +Java1\.0\.21\.0 +Java1\.1\.xx\.x +Java1\.3\.0rc1 +Java1\.3\.x +Java1\.4\.0 +Jersey\/2\.25\.1 +Jetty\/9\.3\.z-SNAPSHOT +Jigsaw\/2\.2\.x\sW3C_CSS +Jigsaw\/2\.3\.0\sW3C_CSS +Junkie\/1\.0 +KIT-Fireball\/2\.0 +KnowItAll +Knowledge\.com\/0\.x +KummHttp\/1\.1 +LEIA\/2\.90 Lachesis -Libby_1.1/libwww-per -Lincoln State Web Br -Link Valet Online 1. -LinkAlarm/2.x -LinkPimpin v1.0 -LinkProver 2.1 -LinkSonar/1.35 -LinkSweeper/1.x -LinkWalker -LinkWalker/2.0 -Look.com -Lynx/2.8.4rel.1 libw +Lincoln\sState\sWeb\sBr +Link\sValet\sOnline\s1\. +LinkAlarm\/2\.x +LinkPimpin\sv1\.0 +LinkProver\s2\.1 +LinkSonar\/1\.35 +LinkSweeper\/1\.x +Look\.com +Lynx\/2\.8\.4rel\.1\slibw MARTINI -MFC Foundation Class +MFC\sFoundation\sClass MFC_Tear_Sample -MSIE-5.13 (larbin@un -MSNPTC/1.0 +MSIE-5\.13 +MSNPTC\/1\.0 MVAClient -Mac Finder 1.0.xx +Mac\sFinder\s1\.0\.xx Mag-Net -Mail.Ru/1.0 MantraAgent -MapoftheInternet.com -Marketwave Hit List +MapoftheInternet\.com +Marketwave\sHit\sList Martini -Marvin v0.3 +Marvin\sv0\.3 MasterSeek -Mata Hari/2.00 -MediaSearch/0.1 -Mediapartners-Google -MegaSheep v1.0 (www. -MeltwaterNews www.me -Mercator-1.x -Mercator-2.0 -Mercator-Scrub-1.1 -Mercury/894 CFNetwor -MetaURI API/2.0 +met +Mata\sHari\/2\.00 +MediaSearch\/0\.1 +MegaSheep\sv1\.0 +MeltwaterNews\swww\.me +Mercator-1\.x +Mercator-2\.0 +Mercator-Scrub-1\.1 +Mercury\/894\sCFNetwor +MetaURI\sAPI\/2\.0\s\+met MicroBaz -Microsoft Data Acces -Microsoft Small Busi -Microsoft URL Contro -Microsoft Windows Ne +Microsoft\sData\sAcces +Microsoft\sSmall\sBusi +Microsoft\sURL\sContro +Microsoft\sWindows\sNe MiracleAlphaTest -Missauga Locate 1.0. -Missigua Locator 1.9 -Missouri College Bro -Mister PiX version.d -Mister Pix II 2.02a -Miva (AlgoFeedback@m -Mizzu Labs 2.2 -MnogoSearch/3.2.xx -Mnogosearch-3.1.21 -Mo College 1.9 -More yacy user agent +Missauga\sLocate\s1\.0\. +Missigua\sLocator\s1\.9 +Missouri\sCollege\sBro +Mister\sPiX\sversion\.d +Mister\sPix\sII\s2\.02a +Miva +Mizzu\sLabs\s2\.2 +Mo\sCollege\s1\.9 Mozi! -MultiText/0.1 -My WinHTTP Connectio -MyNutchTest/Nutch-1. -NASA Search 1.0 -NEC Research Agent - -NEC-Hayek/1.0 -NG/4.0.1229 -NICO/1.0 -NING/1.0 +MultiText\/0\.1 +My\sWinHTTP\sConnectio +NASA\sSearch\s1\.0 +NEC\sResearch\sAgent\s- +NEC-Hayek\/1\.0 +NG\/4\.0\.1229 +NICO\/1\.0 +NING\/1\.0 NationalDirectoryAdd NativeHost -NetLookout/2.24 -NetMechanic Vx.0 -NetSprint -- 2.0 +NetLookout\/2\.24 +NetMechanic\sVx\.0 +NetSprint\s--\s2\.0 NetZippy -Nocilla/1.0 -Nsauditor/1.x +Nocilla\/1\.0 +Nsauditor\/1\.x Nuzzel -Nuzzel/876 CFNetwork -Octora Beta - www.oc -Oculus360/Nutch-1.7 +Octora\sBeta\s-\swww\.oc OliverPerry -OpenAcoon v4.0.x (ww -OpenISearch/1.x (www -OpenOffice/4.1.1 -OpenWebIndex/Nutch-1 -Openstat/0.1 -Opera/6.01 (larbin@u -Oracle Ultra Search -OzySoftware.com -PBrowse 1.4b -PEval 1.4b -PHP/3.x.xx -PHP/4.0.4pl1 -PHP/4.0.6 -PHP/4.1.1 -PHP/4.1.2 +OpenAcoon\sv4\.0\.x +OpenISearch\/1\.x +OpenOffice\/4\.1\.1 +Openstat\/0\.1 +Opera\/6\.01 +Oracle\sUltra\sSearch +OzySoftware\.com +PBrowse\s1\.4b +PEval\s1\.4b POE-Component-Client -Pcore-HTTP/v0.40.3 -PeopleChat/Search_En -PicoSearch/1.0 +PeopleChat\/Search_En +PicoSearch\/1\.0 Piffany_Web_Scraper_ -PingALink Monitoring +PingALink\sMonitoring Pita Poirot -Pompos/1.x pompos@il -Popdexter/1.0 -Port Huron Labs +Pompos\/1\.x\spompos@il +Popdexter\/1\.0 +Port\sHuron\sLabs PostFavorites -PostRank/2.0 (postra -PrivacyFinder/1.1 -Program Shareware 1. -Progressive Download -Project XP5 [2.03.07 -PubCrawl (pubcrawl.s -Pulsepoint XT3 web s -PycURL/7.13.2 -PycURL/7.15.5 -PycURL/7.16.4 -PycURL/7.18.0 -PycURL/7.18.2 -PycURL/7.19.0 -PycURL/7.19.3 -PycURL/7.19.5 -PycURL/7.19.7 -PycURL/7.23.1 -Python-urllib/1.17 -Python-urllib/1.1x -Python-urllib/2.0a1 -Python-urllib/2.5 -Python-urllib/2.6 -Python-urllib/2.7 -Python-urllib/3.1 -Python-urllib/3.2 -Python-urllib/3.3 -Python-urllib/3.4 -Python-urllib/3.5 -Python-urllib/3.6 -QPCreep Test Rig ( W -Qseero v1.0.0 -QuepasaCreep v0.9.1x -QueryN Metasearch -Qwantify/1.0 -R6_VoteReader(www.ra -REBOL Core 2.x.x.x.x -RPT-HTTPClient/0.3-x -RepoMonkey Bait & Ta -RoboCrawl (www.canad -Robozilla/1.0 -Rotondo/3.1 libwww/5 +PostRank\/2\.0 +PrivacyFinder\/1\.1 +Program\sShareware\s1\. +Progressive\sDownload +Project\sXP5\s\[2\.03\.07 +Pulsepoint\sXT3\sweb\ss +QPCreep\sTest\sRig +Qseero\sv1\.0\.0 +QuepasaCreep\sv0\.9\.1x +QueryN\sMetasearch +R6_VoteReader +RPT-HTTPClient\/0\.3-x +RepoMonkey\sBait\s&\sTa Ruby Rumours-Agent -SMRF URL Expander -Scooter-3.0.EU -Scooter-3.0.FS -Scooter-3.0.HD -Scooter-3.0.VNS -Scooter-3.0QI -Scooter-3.2 -Scooter-3.2.BT -Scooter-3.2.DIL -Scooter-3.2.EX -Scooter-3.2.JT -Scooter-3.2.NIV -Scooter-3.2.SF0 -Scooter-3.2.snippet -Scooter-3.3dev -Scooter-ARS-1.1 -Scooter-ARS-1.1-ih -Scooter-W3-1.0 -Scooter-W3.1.2 -Scooter/1.0 -Scooter/1.0 scooter@ -Scooter/1.1 (custom) -Scooter/2.0 G.R.A.B. -Scooter/3.3 -Scooter/3.3.QA.pczuk -Scooter/3.3.vscooter -Scooter/3.3_SF -Scooter2_Mercator_x- -Scooter_bh0-3.0.3 -Scooter_trk3-3.0.3 +SMRF\sURL\sExpander ScoutAbout -Search+ +Search\+ Search-Engine-Studio -SearchGuild/DMOZ/Exp +SearchGuild\/DMOZ\/Exp SearchGuild_DMOZ_Exp -SearchSight.com -SearchTone2.0 - IDEA -Seeker.lookseek.com -Silk/1.0 -SiteBar/x.x -SiteRecon+(xx) -SiteTruth.com site r +SearchSight\.com +SearchTone2\.0\s-\sIDEA +Seeker\.lookseek\.com +Silk\/1\.0 +SiteBar\/x\.x +SiteRecon\+ +SiteTruth\.com\ssite\sr SiteXpert -Skywalker/0.1 (Skywa -Slarp/0.1 +Skywalker\/0\.1 +Slarp\/0\.1 Slider_Search_v1-de -Slurpy Verifier/1.0 -SlySearch (slysearch -SnoopRob/x.x -Snoopy v1.xx -Snoopy_v0.xx -Sphere Scout&v4.0 - +SlySearch +SnoopRob\/x\.x +Snoopy\sv1\.xx +Snoopy_v0\.xx +Sphere\sScout&v4\.0\s- Sphider -Spida/0.1 -Spinne/2.0 -Spinne/2.0 med -Spinne/2.0 med_AH -Sqworm/2.9.81-BETA ( -Sqworm/2.9.85-BETA ( -Sqworm/2.9.89-BETA ( -StackRambler/2.0 -StackRambler/2.0 (MS -StackRambler/x.x -SyncIT/x.x -Szukacz/1.x +Spida\/0\.1 +Spinne\/2\.0 +Spinne\/2\.0\smed +Spinne\/2\.0\smed_AH +SyncIT\/x\.x +Szukacz\/1\.x TE -TJvMultiHttpGrabber -Tagyu Agent/1.0 -TeamSoft WinInet Com -Teoma MP -The European Library -Thinklab (thinklab.c +TJvMultiHttpGrabber +Tagyu\sAgent\/1\.0 +TeamSoft\sWinInet\sCom +The\sEuropean\sLibrary +Thinklab TinEye -Toutatis x-xx.x (hop -Toutatis x.x (hoppa. -Toutatis x.x-x -Traackr.com -Twisted PageGetter -Twitter/6.18.3 CFNet -Twitturly / v0.x -Tycoon Agent/Nutch-1 +Toutatis\sx-xx\.x +Toutatis\sx\.x +Toutatis\sx\.x-x +Traackr\.com +Twisted\sPageGetter +Twitter\/6\.18\.3\sCFNet +Twitturly\s\/\sv0\.x TygoProwler -UCWEB/2.0(Linux; U; +UCWEB\/2\.0 UDM -URLBase/6.x +URLBase\/6\.x URLBlaze -UdmSearch/3.1.x -Ultraseek -Under the Rainbow 2. -UofTDB_experiment (l -Upflow/1.0 -VYU2 (GNU; OpenRISC) -Vagabondo/1.x MT (we -Vagabondo/2.0 MT -Vagabondo/2.0 MT (we -Vagabondo/3.0 (webag -Validator.nu/LV -VayalaCreep-v0.0.1 ( -Vayala|Creep-v0.0.1 -VeryGoodSearch.com.D -W3C-mobileOK/DDC-1.0 -W3C_Unicorn/1.0 -W3C_Validator/1.3 -W3C_Validator/1.xxx -WDG_Validator/1.1 -WEP Search 00 +UdmSearch\/3\.1\.x +Under\sthe\sRainbow\s2\. +UofTDB_experiment +Upflow\/1\.0 +VYU2 +Vagabondo\/1\.x\sMT +Vagabondo\/2\.0\sMT +Vagabondo\/3\.0 +VayalaCreep-v0\.0\.1 +Vayala\|Creep-v0\.0\.1 +VeryGoodSearch\.com\.D +W3C-mobileOK\/DDC-1\.0 +W3C_Unicorn\/1\.0 +WEP\sSearch\s00 WFARC -WIRE/0.22 (Linux; x8 +WIRE\/0\.22 WST-Plagiarism -WWWC/1.0x -Wada.vn Vietnamese S -Waypath Scout v2.x - +WWWC\/1\.0x +Wada\.vn\sVietnamese\sS +Waypath\sScout\sv2\.x\s- WeSEE WeSEE:Search -Web Link Validator 1 -Web Snooper -WebBug/5.x -WebCompass 2.0 -WebCorp/1.0 -WebSearch.COM.AU; -WebStat/1.0 (Unix; b -WebTrends/3.0 (WinNT -WebVac (webmaster@pi -WebVal/1.0 -WebVulnCrawl.unknown -WebWatcherMonitor/2. -Webclipping.com -Webdup/0.9 -Weblog Attitude Diff -WebsiteWorth v1.0 -Webspinne/1.0 webmas -Webverzeichnis.de - -Wells Search II -Wget/1.14 (linux-gnu -WhatsApp/2.12.15/i -WhatsApp/2.12.16/i -WhatsApp/2.12.17/i -WhatsApp/2.12.449 A -WhatsApp/2.12.453 A -WhatsApp/2.12.510 A -WhatsApp/2.12.540 A -WhatsApp/2.12.548 A -WhatsApp/2.12.555 A -WhatsApp/2.12.556 A -WhatsApp/2.16.1/i -WhatsApp/2.16.13 A -WhatsApp/2.16.2/i -WhatsApp/2.16.42 A -WhatsApp/2.16.57 A -Whatsup/x.x -WhizBang! Lab -WinHTTP Example/1.0 -WinampMPEG/2.00 (lar -WincerSong Agent v1. -WorQmada/1.0 -WordupInfoSearch/1.0 +Web\sSnooper +WebBug\/5\.x +WebCompass\s2\.0 +WebCorp\/1\.0 +WebSearch\.COM\.AU; +WebStat\/1\.0 +WebTrends\/3\.0 +WebVac +WebVal\/1\.0 +WebWatcherMonitor\/2\. +Webclipping\.com +Webdup\/0\.9 +Weblog\sAttitude\sDiff +WebsiteWorth\sv1\.0 +Webspinne\/1\.0\swebmas +Webverzeichnis\.de\s- +Wells\sSearch\sII +Whatsup\/x\.x +WhizBang!\sLab +WinHTTP\sExample\/1\.0 +WinampMPEG\/2\.00 +WincerSong\sAgent\sv1\. +WorQmada\/1\.0 +WordupInfoSearch\/1\.0 WorldLight -Wusage/x.0@boutell.c -Wwlib/Linux -XMLSlurp/0.1 libwww- -Xenu Link Sleuth 1.x -Xenu Link Sleuth/1.3 -Xenu's Link Sleuth 1 +Wusage\/x\.0@boutell\.c +Wwlib\/Linux Xylix -Y!J-SRD/1.0 -Yahoo Pipes 1.0 -Yahoo! Mindset -Yahoo-Test/4.0 -YahooCacheSystem -YahooCacheSystem; Ya -YahooSeeker/CafeKels Yanadoo -Yandex/1.01.001 (com Zearchit ZendHttpClient -Zeus ThemeSite Viewe -Zeus xxxxx Webster P -Ziggy -- The Clown F -Zippy v2.0 - Zippyfi -a.pr-cy.ru -about.ask.com -about.depspid.net +Ziggy\s--\sThe\sClown\sF +Zippy\sv2\.0\s-\sZippyfi +a\.pr-cy\.ru +about\.ask\.com +about\.depspid\.net accoona -aihit.com +aihit\.com amzn_assoc -app.hypefactors.com -appie 1.1 (www.walhe -archive.org -ask.24x.info -asterias/2.0 +app\.hypefactors\.com +archive\.org +ask\.24x\.info augurfind -augurnfind V-1.x +augurnfind\sV-1\.x autohttp -autowebdir 1.1 (www. -axios/0.18.0 -bCentral Billing Pos -bdcindexer_2.6.2 (re -binlar_2.6.3 binlar2 -binlar_2.6.3 binlar_ -binlar_2.6.3 larbin2 -binlar_2.6.3 phanend -binlar_2.6.3 test@mg -blacklistdeclassified.net -bookmarkbase.com -bulkfeeds.net -bumblebee@relevare.c -cantor.ee.ucla.edu -carleson/1.0 -ccubee/x.x -cfetch/1.0 -cg-eye interactive +autowebdir\s1\.1 +axios\/0\.18\.0 +bCentral\sBilling\sPos +bdcindexer_2\.6\.2 +blacklistdeclassified\.net +bookmarkbase\.com +bulkfeeds\.net +bumblebee@relevare\.c +cantor\.ee\.ucla\.edu +carleson\/1\.0 +ccubee\/x\.x +cfetch\/1\.0 +cg-eye\sinteractive cobitsprobe -coccoc/1.0 () -code.google.com codegator -cohalinglit.wordpress.com -collage.cgi/1.xx -combine.it.lth.se -combine/0.0 -computerorgs.com -content.easybib.com -coralcdn.org -cosco.hiit.fi -crawler -cs.daum.net -csci_b659/0.13 -curl/7.18.2 (x86_64- -curl/7.19.7 (x86_64- -curl/7.21.4 (univers -curl/7.37.1 -dcrawl/1.0 -dds explorer v1.0 be -deepak-USC/ISI -deusu.org -developers.google.com -dhcibr.wordpress.com -digg.com -dir.com -drupal.org -dznet.com -e-sense 1.0 ea(www.v -eStyleSearch 4 (comp +collage\.cgi\/1\.xx +computerorgs\.com +content\.easybib\.com +coralcdn\.org +cosco\.hiit\.fi +cs\.daum\.net +csci_b659\/0\.13 +dds\sexplorer\sv1\.0\sbe +deepak-USC\/ISI +deusu\.org +digg\.com +dir\.com +dznet\.com +e-sense\s1\.0\sea +eStyleSearch\s4 ec2linkfinder -ecairn.com -ejupiter.com -enter4u.eu -exactseek.com -example-masto-instance.org -exploringdh.wordpress.com -facebookexternalhit/ -feed.moo.jp -feedafever.com; -flipboard.com +ecairn\.com +ejupiter\.com +enter4u\.eu +exactseek\.com +example-masto-instance\.org +feed\.moo\.jp +feedafever\.com; +flipboard\.com flunky -fly/6.01 libwww/4.0D -free-downloads.net d -fulltext.seznam.cz -gazz/x.x (gazz@nttrd -geourl/2.0b2 -getpocket.com -gigabaz.com -github.com -google-xrawler -googlebot -gossamer-threads.com -grub-client -grub.org -guess.scritch.org -haste.kytoon.com -help.coccoc.com -help.coccoc.vn -help.goo.ne.jp -help.hepburnhome.ca -help.yahoo.co.jp -help.yahoo.com -hgrepurl/1.0 -holmes/3.9 (onet.pl) -holmes/3.9 (someurl. -holmes/x.x -htdig/3.1.5 -htdig/3.1.5 (infosys -htdig/3.1.5 (root@lo -htdig/3.1.5 (webmast -htdig/3.1.6 (mathieu -htdig/3.1.6 (unconfi -htdig/3.1.x (root@lo -http://www.mabontlan -http://www.monogol.d -http://www.timelyweb +free-downloads\.net\sd +gazz\/x\.x +geourl\/2\.0b2 +getpocket\.com +gigabaz\.com +github\.com +gossamer-threads\.com +guess\.scritch\.org +haste\.kytoon\.com +help\.coccoc\.com +help\.coccoc\.vn +help\.goo\.ne\.jp +help\.hepburnhome\.ca +hgrepurl\/1\.0 +http:\/\/www\.mabontlan +http:\/\/www\.monogol\.d +http:\/\/www\.timelyweb http_get -httpunit/1.x -i.stanford.edu -iFeed.jp/2.0 (www.ps -ia_archiver-web.arch -ia_archiver/1.6 -ia_archiver/8.0 (Win -ia_archiver/8.1 (Win -ia_archiver/8.2 (Win -ia_archiver/8.8 (Win -ia_archiver/8.9 (Lin -ia_archiver/8.9 (Win -icerocket.com; -ichiro/2.0 (ichiro@n -ichiro/x.0 (ichiro@n -ideare - SignSite/1. -iearthworm/1.0, iear -igde.ru -ilial/Nutch-0.9-dev -innovationblog.com -integrity/1.6 -integrity/5 -ivia.ucr.edu +httpunit\/1\.x +i\.stanford\.edu +iFeed\.jp\/2\.0 +icerocket\.com; +ideare\s-\sSignSite\/1\. +igde\.ru +innovationblog\.com +integrity\/1\.6 +integrity\/5 +ivia\.ucr\.edu jack -journster.com +journster\.com knil -kulturarw3/0.1 -larbin (samualt9@big -larbin_2.1.1 larbin2 -larbin_2.2.0 (crawl@ -larbin_2.2.1_de_Vien -larbin_2.2.2 (sugaya -larbin_2.2.2_guillau -larbin_2.5.0 (larbin -larbin_2.6.0 (larbin -larbin_2.6.1 (larbin -larbin_2.6.1 larbin2 -larbin_2.6.2 (hamasa -larbin_2.6.2 (larbin -larbin_2.6.2 (liston -larbin_2.6.2 (pierre -larbin_2.6.2 (pimena -larbin_2.6.2 (tom@le -larbin_2.6.2 (vitalb -larbin_2.6.2 dthunen -larbin_2.6.2 kalou@k -larbin_2.6.2 larbin2 -larbin_2.6.2 larbin@ -larbin_2.6.2 listonA -larbin_2.6.2 pierre@ -larbin_2.6.2 vitalbo -larbin_2.6.3 (larbin -larbin_2.6.3 (wgao@g -larbin_2.6.3 capveg@ -larbin_2.6.3 ghary@s -larbin_2.6.3 gqnmgsp -larbin_2.6.3 larbin2 -larbin_2.6.3 tangyi8 -larbin_2.6.3 zumesun -larbin_2.6_basileoca -larbin_test (nobody@ -larbin_test nobody@a -larbin_xy250 larbin2 -law.di.unimi.it -lc/$ROADS::Version l -letscrawl.com -libWeb/clsHTTP -- hi -libwww-perl/5.41 -libwww-perl/5.45 -libwww-perl/5.48 -libwww-perl/5.50 -libwww-perl/5.52 FP/ -libwww-perl/5.53 -libwww-perl/5.63 -libwww-perl/5.64 -libwww-perl/5.65 -libwww-perl/5.800 -libwww/5.3.2 -linkdex.com/v2.0 -ltx71.com -lunefran.wordpress.com -lwp-trivial/1.32 -lwp-trivial/1.33 -lwp-trivial/1.34 -lwp-trivial/1.35 -lwp-trivial/1.36 -lwp-trivial/1.38 -lwp-trivial/1.41 -mailto:webcraft@bea. -mbblog14.wordpress.com -metatagsdir.com -miniflux.net -misc.yahoo.com.cn -moget/x.x (moget@goo -mogimogi/1.0 -moiNAG 0.02 -muckrack.com +kulturarw3\/0\.1 +law\.di\.unimi\.it +lc\/\$ROADS::Version\sl +libWeb\/clsHTTP\s--\shi +linkdex\.com\/v2\.0 +mailto:webcraft@bea\. +metatagsdir\.com +miniflux\.net +mogimogi\/1\.0 +moiNAG\s0\.02 +muckrack\.com myDaemon -net-promoter.com -netarkivet.dk -newsearchengine (Thi -newsme/1.0; feedback -newspaper/0.1.0.7 -newspaper/0.2.5 -newspaper/0.2.6 -newstin.com -nlp.uned.es +net-promoter\.com +netarkivet\.dk +newsearchengine +newsme\/1\.0;\sfeedback +newspaper\/0\.1\.0\.7 +newspaper\/0\.2\.5 +newspaper\/0\.2\.6 +newstin\.com +nlp\.uned\.es noyona_0_1 -nsyght.com/Nutch-1.0 -nsyght.com/Nutch-x.x -nutch-1.4/Nutch-1.4 -nutchsearch/Nutch-0. -nymesis.com -oegp v. 1.3.0 -okhttp/2.5.0 -okhttp/2.7.5 -okhttp/3.2.0 -okhttp/3.5.0 -omgili.com -ostermiller.org +nymesis\.com +oegp\sv\.\s1\.3\.0 +omgili\.com +ostermiller\.org owler-agent -ozelot/2.7.3 (Search -page2rss.com -pandora.nla.gov.au -panscient.com -parsijoo.ir -pauillac.inria.fr -pd02_1.0.0 pd02_1.0. -pear.php.net -people.eng.unimelb.edu.au -phpcrawl -pompos.iliad.fr +ozelot\/2\.7\.3 +pandora\.nla\.gov\.au +pauillac\.inria\.fr +pd02_1\.0\.0\spd02_1\.0\. +people\.eng\.unimelb\.edu\.au +pompos\.iliad\.fr psycheclone -publisher.yahoo.com -python-requests/0.13 -python-requests/1.0. -python-requests/2.18 -python-requests/2.3. -r.hatena.ne.jp -rabaz (rabaz at giga -ranksonic.com -rico/0.1 -riddler.io -rjpower.org -robot -sagool.jp -scooter-venus-3.0.vn +r\.hatena\.ne\.jp +rabaz +ranksonic\.com +rico\/0\.1 +rjpower\.org +sagool\.jp scoutmaster -scrapy.org -search.ch V1.4 -search.denmex.com -search.goo.ne.jp -search.google.com -search.yahoo.com -seekda.com -semanticdiscovery/0. -sentry.io -silk/1.0 -sitebar.org -siteexplorer.info -smartwit.com -sna-0.0.1 (mikemuzio -sna-0.0.1 mikeelliot -sohu agent +scrapy\.org +search\.ch\sV1\.4 +search\.denmex\.com +search\.goo\.ne\.jp +seekda\.com +semanticdiscovery\/0\. +sentry\.io +silk\/1\.0 +sitebar\.org +siteexplorer\.info +smartwit\.com +sna-0\.0\.1 +sna-0\.0\.1\smikeelliot +sohu\sagent sohu-search -sp.ask.com -suchbaer.de -summify.com -support.embed.ly +sp\.ask\.com +suchbaer\.de +support\.embed\.ly surf suzuran -szukaj.onet.pl -tags2dir.com -tagword.com -targetblaster.com/0. -teoma_agent1 -therarestwords.com -toCrawl/UrlDispatche -tuezilla.de -twikle.com -twurly.org -unknownght.com Web S -users.cs.cf.ac.uk -versus.integis.ch -vias.ncsa.uiuc.edu -visionutils/0.2 -vk.com -voyager-hc/1.0 -voyager/1.0 -web-bekannt (Version -web2express.org -webGobbler/1.x.x -webagent.wise-guys.nl -webalgo.iit.cnr.it -webbandit/4.xx.0 -webcollage/1.114 -webcollage/1.117 -webcollage/1.125 -webcollage/1.129 -webcollage/1.93 -webcollage/1.xx -webcrawl.net -webdatastats.com -webglimpse.net +szukaj\.onet\.pl +tags2dir\.com +tagword\.com +targetblaster\.com\/0\. +therarestwords\.com +tuezilla\.de +twikle\.com +twurly\.org +unknownght\.com\sWeb\sS +users\.cs\.cf\.ac\.uk +versus\.integis\.ch +vias\.ncsa\.uiuc\.edu +visionutils\/0\.2 +vk\.com +voyager-hc\/1\.0 +web-bekannt +web2express\.org +webGobbler\/1\.x\.x +webagent\.wise-guys\.nl +webalgo\.iit\.cnr\.it +webbandit\/4\.xx\.0 +webdatastats\.com +webglimpse\.net webhack -websearchbench.cs.uni-dortmund.de -websitealert.net -webster-internet.de -webster.healeys.net -whatUseek_winona/3.0 -wiki.creativecommons.org -wish.slis.tsukuba.ac.jp -wkd.iis.sinica.edu.tw -wortschatz.uni-leipzig.de -ws.daum.net -www.Abolimba.de -www.WISEnut.com -www.abonti.com -www.ackerm.com -www.activtourist.com -www.agada.de -www.ahmadsoftware.co -www.answerbus.com -www.ant.com -www.aol.com -www.aramabeta.com; -www.arianna.it -www.arocom.de -www.asaha.com -www.axxus.de -www.ay-up.com -www.ba.be -www.blaiz.net -www.bloglines.com -www.bortzmeyer.org -www.brandverity.com -www.buddybuzz.net -www.buscaplus.com -www.business-socket. -www.canadiancontent.net -www.chepi.net -www.chilkatsoft.com -www.cloudflare.com -www.cnet.com -www.cougarsearch.com -www.covario.com -www.crim.ca -www.cs.washington.edu -www.cs.washington.edu; -www.dataparksearch.org -www.de.com -www.dead-links.com -www.die-kraehe.de -www.dnsdigger.com -www.earthcom.info -www.eurekster.com -www.europarchive.org -www.eventax.de -www.everatom.com -www.evri.com -www.exensa.com -www.exif-search.com -www.facebook.com -www.favcollector.com -www.feeddemon.com -www.feedhub.com -www.feedly.com -www.feedmap.net -www.fileboost.net -www.findpal.com -www.floorstransformed.com -www.flyindex.net -www.followsite.com -www.genieo.com -www.globalspec.com -www.goguides.org -www.google.com -www.greaterera.com -www.hermitsearch.com -www.htmlvalidator.com -www.hyperix.com -www.ianett.com -www.icerocket.com -www.illumit.com -www.iltrovatore.it -www.inktomi.com -www.intelliseek.com -www.ip2location.com -www.iskanie.com -www.jordomedia.com -www.kalooga.com; -www.kototoi.org -www.l3s.de -www.lan4lano.net -www.lipperhey.com -www.look.com -www.lw01.com -www.matuschek.net -www.megaindex.ru -www.megite.com -www.meta-spinner.de -www.mixdata.com -www.moreover.com -www.neomo.de -www.netluchs.de -www.newsgator.com; -www.ng-search.com -www.page-store.com -www.pagebull.com -www.pingalink.com -www.pingdom.com -www.plinki.com; -www.powerset.com -www.prairielandconsulting.com -www.proximic.com -www.pubblisito.com -www.qango.com -www.quickiwiki.com; -www.qwant.com -www.rawgrunt.com -www.relsoftware.com -www.retrevo.com -www.samsung.com -www.scifihifi.com -www.scoutjet.com -www.scrubtheweb.com -www.searchme.com -www.securecomputing.com -www.semager.de -www.shopwiki.com -www.siigle.com -www.simpy.com -www.sitesearch.ca -www.sitesell.com -www.sixxs.net -www.skaffe.com -www.skyrocket.gr.jp -www.sli-systems.com -www.slider.com -www.slysearch.com -www.somusical.com -www.suchen.de -www.sygol.com -www.sync2it.com -www.sysomos.com -www.targetgroups.net -www.teradex.com -www.tkensaku.com -www.truveo.com -www.tumblr.com -www.ukwizz.com -www.unchaos.com -www.updated.com; -www.urltrends.com -www.vakes.com -www.versions-project.org -www.verzamelgids.nl -www.wavefire.com; -www.webarchiv.cz -www.worio.com -www.wowlemmings.com -www.xirq.com; -www.xrayz.co.uk -www.yama.info.waseda.ac.jp -www.yoono.com -www.zedzo.com -www4mail/2.x libwww- -wwwster/1.x (Beta- m +websearchbench\.cs\.uni-dortmund\.de +websitealert\.net +webster-internet\.de +webster\.healeys\.net +whatUseek_winona\/3\.0 +wiki\.creativecommons\.org +wish\.slis\.tsukuba\.ac\.jp +wkd\.iis\.sinica\.edu\.tw +wortschatz\.uni-leipzig\.de +ws\.daum\.net +www\.Abolimba\.de +www\.WISEnut\.com +www\.abonti\.com +www\.ackerm\.com +www\.activtourist\.com +www\.agada\.de +www\.ahmadsoftware\.co +www\.answerbus\.com +www\.ant\.com +www\.aol\.com +www\.aramabeta\.com; +www\.arianna\.it +www\.arocom\.de +www\.asaha\.com +www\.axxus\.de +www\.ay-up\.com +www\.ba\.be +www\.blaiz\.net +www\.bortzmeyer\.org +www\.brandverity\.com +www\.buddybuzz\.net +www\.buscaplus\.com +www\.business-socket\. +www\.canadiancontent\.net +www\.chepi\.net +www\.chilkatsoft\.com +www\.cloudflare\.com +www\.cnet\.com +www\.cougarsearch\.com +www\.covario\.com +www\.crim\.ca +www\.cs\.washington\.edu +www\.cs\.washington\.edu; +www\.de\.com +www\.dead-links\.com +www\.die-kraehe\.de +www\.dnsdigger\.com +www\.earthcom\.info +www\.eurekster\.com +www\.europarchive\.org +www\.eventax\.de +www\.everatom\.com +www\.evri\.com +www\.exensa\.com +www\.exif-search\.com +www\.facebook\.com +www\.favcollector\.com +www\.feeddemon\.com +www\.feedhub\.com +www\.feedly\.com +www\.feedmap\.net +www\.fileboost\.net +www\.findpal\.com +www\.floorstransformed\.com +www\.flyindex\.net +www\.followsite\.com +www\.globalspec\.com +www\.goguides\.org +www\.greaterera\.com +www\.hermitsearch\.com +www\.hyperix\.com +www\.ianett\.com +www\.icerocket\.com +www\.illumit\.com +www\.iltrovatore\.it +www\.inktomi\.com +www\.intelliseek\.com +www\.ip2location\.com +www\.jordomedia\.com +www\.kalooga\.com; +www\.kototoi\.org +www\.l3s\.de +www\.lan4lano\.net +www\.look\.com +www\.lw01\.com +www\.matuschek\.net +www\.megaindex\.ru +www\.meta-spinner\.de +www\.mixdata\.com +www\.moreover\.com +www\.neomo\.de +www\.newsgator\.com; +www\.ng-search\.com +www\.page-store\.com +www\.pagebull\.com +www\.pingalink\.com +www\.pingdom\.com +www\.plinki\.com; +www\.powerset\.com +www\.prairielandconsulting\.com +www\.pubblisito\.com +www\.qango\.com +www\.quickiwiki\.com; +www\.qwant\.com +www\.rawgrunt\.com +www\.relsoftware\.com +www\.retrevo\.com +www\.samsung\.com +www\.scifihifi\.com +www\.scrubtheweb\.com +www\.searchme\.com +www\.securecomputing\.com +www\.semager\.de +www\.shopwiki\.com +www\.siigle\.com +www\.simpy\.com +www\.sitesearch\.ca +www\.sitesell\.com +www\.sixxs\.net +www\.skaffe\.com +www\.skyrocket\.gr\.jp +www\.sli-systems\.com +www\.slider\.com +www\.slysearch\.com +www\.somusical\.com +www\.suchen\.de +www\.sygol\.com +www\.sync2it\.com +www\.targetgroups\.net +www\.teradex\.com +www\.tkensaku\.com +www\.truveo\.com +www\.tumblr\.com +www\.ukwizz\.com +www\.unchaos\.com +www\.updated\.com; +www\.urltrends\.com +www\.vakes\.com +www\.versions-project\.org +www\.verzamelgids\.nl +www\.wavefire\.com; +www\.webarchiv\.cz +www\.worio\.com +www\.wowlemmings\.com +www\.xirq\.com; +www\.xrayz\.co\.uk +www\.yama\.info\.waseda\.ac\.jp +www\.yoono\.com +www\.zedzo\.com +wwwster\/1\.x xmlset_roodkcableoj2 -yacy (www.yacy.net; -yarienavoir.net/0.2 -yggdrasil/Nutch-0.9 -yoogliFetchAgent/0.1 -Googlebot\/ -Googlebot-Mobile -Googlebot-Image -Googlebot-News -Googlebot-Video -AdsBot-Google([^-]|$) -AdsBot-Google-Mobile -Feedfetcher-Google -Mediapartners-Google -Mediapartners \(Googlebot\) -APIs-Google -Google-InspectionTool -Storebot-Google -GoogleOther -bingbot -Slurp -[wW]get -LinkedInBot -Python-urllib -python-requests +yarienavoir\.net\/0\.2 +yoogliFetchAgent\/0\.1 +Mediapartners\s\\ +\[wW]get aiohttp -httpx -libwww-perl httpunit -nutch Go-http-client -phpcrawl -msnbot -jyxobot -FAST-WebCrawler -FAST Enterprise Crawler -BIGLOTRON -Teoma -convera -seekbot -Gigabot Gigablast -exabot -ia_archiver -GingerCrawler -webmon -HTTrack -grub.org -UsineNouvelleCrawler -antibot netresearchserver -speedy fluffy findlink -msrbot -panscient -yacybot -AISearchBot ips-agent -tagoobot -MJ12bot -woriobot yanga -buzzbot -mlbot -yandex\.com\/bots -purebot -Linguee Bot CyberPatrol -voilabot -Baiduspider -citeseerxbot -spbot -twengabot postrank -Turnitin -scribdbot -page2rss -sitebot linkdex -Adidxbot ezooms -dotbot -Mail.RU_Bot -discobot -heritrix -findthatfile -europarchive.org -NerdByNature.Bot -sistrix crawler -Ahrefs(Bot|SiteAudit) -fuelbot -CrunchBot -IndeedBot +europarchive\.org +Ahrefs mappydata -woobot -ZoominfoBot -PrivacyAwareBot -Multiviewbot -SWIMGBot -Grobbot eright Apercite -semanticbot Aboundex -domaincrawler -wbsearchbot -summify -CCBot -edisterbot -SeznamBot -ec2linkfinder -gslfbot -aiHitBot -intelium_bot facebookexternalhit Yeti RetrevoPageAnalyzer -lb-spider -Sogou -lssbot -careerbot wotbox -wocbot -ichiro -DuckDuckBot -lssrocketcrawler drupact -webcompanycrawler -acoonbot -openindexspider -gnam gnam spider -web-archive-net.com.bot -backlinkcrawler coccoc integromedb -content crawler spider -toplistbot -it2media-domain-crawler -ip-web-crawler.com -siteexplorer.info -elisabot -proximic changedetection -arabot -WeSEE:Search -niki-bot -CrystalSemanticsBot -rogerbot -360Spider -psbot -InterfaxScanBot -CC Metadata Scaper -g00g1e.net -GrapeshotCrawler -urlappendbot -brainobot -fr-crawler -binlar -SimpleCrawler -Twitterbot -cXensebot -smtbot -bnf.fr_bot -A6-Indexer -ADmantX -Facebot -OrangeBot\/ -memorybot -AdvBot +CC\sMetadata\sScaper +g00g1e\.net MegaIndex -SemanticScholarBot -ltx71 -nerdybot -xovibot -BUbiNG -Qwantify -archive.org_bot -Applebot -TweetmemeBot -crawler4j -findxbot -S[eE][mM]rushBot -yoozBot -lipperhey -Y!J -Domain Re-Animator Bot -AddThis -Screaming Frog SEO Spider MetaURI Scrapy -Livelap[bB]ot -OpenHoseBot +Livelap\[bB]ot CapsuleChecker -collection@infegy.com -IstellaBot -DeuSu\/ -betaBot -Cliqzbot\/ -MojeekBot\/ -netEstate NE Crawler -SafeSearch microdata crawler -Gluten Free Crawler\/ +collection@infegy\.com +DeuSu\\/ Sonic -Sysomos -Trove deadlinkchecker Slack-ImgProxy -Embedly -RankActiveLinkBot -iskanie -SafeDNSBot -SkypeUriPreview -Veoozbot -Slackbot -redditbot -datagnionbot -Google-Adwords-Instant -adbeat_bot -WhatsApp -contxbot -pinterest.com.bot electricmonk -GarlikCrawler -BingPreview\/ -vebidoobot -FemtosearchBot -Yahoo Link Preview -MetaJobBot -DomainStatsBot -mindUpBot -Daum\/ -Jugendschutzprogramm-Crawler -Xenu Link Sleuth -Pcore-HTTP -moatbot -KosmioBot -[pP]ingdom +Daum\\/ +\[pP]ingdom AppInsights PhantomJS -Gowikibot -PiplBot -Discordbot -TelegramBot Jetslide newsharecounts -James BOT -Bark[rR]owler -TinEye -SocialRankIOBot -trendictionbot -Ocarinabot -epicbot -Primalbot -DuckDuckGo-Favicons-Bot -GnowitNewsbot -Leikibot +Bark\[rR]owler LinkArchiver -YaK\/ -PaperLiBot -Digg Deeper -dcrawl +YaK\\/ +Digg\sDeeper Snacktory -AndersPinkBot -Fyrebot -EveryoneSocialBot -Mediatoolkitbot -Luminator-robots -ExtLinksBot -SurveyBot -NING\/ -okhttp -Nuzzel +NING\\/ omgili PocketParser -YisouSpider um-LN -ToutiaoSpider MuckRack -Jamie's Spider -AHC\/ -NetcraftSurveyAgent -Laserlikebot -^Apache-HttpClient -AppEngine-Google +AHC\\/ +\^Apache-HttpClient Jetty Upflow -Thinklab -Traackr.com Twurly Mastodon -http_get -DnyzBot -botify -007ac9 Crawler -BehloolBot BrandVerity check_http -BDCbot -ZumBot EZID -ICC-Crawler -ArchiveBot -^LCC -filterdb.iss.net\/crawler -BLP_bbot -BomboraBot -Buck\/ -Companybook-Crawler -Genieo -magpie-crawler +\^LCC +Buck\\/ MeltwaterNews Moreover -newspaper\/ -ScoutJet -(^| )sentry\/ -StorygizeBot -UptimeRobot -OutclicksBot +newspaper\\/ + seoscanners Hatena -Google Web Preview -MauiBot -AlphaBot -SBL-BOT -IAS crawler adscanner Netvibes -acapbot -Baidu-YunGuanCe -bitlybot -blogmuraBot -Bot.AraTurka.com -bot-pge.chlooe.com -BoxcarBot BTWebClient -ContextAd Bot -Digincore bot Disqus Feedly -Fetch\/ +Fetch\\/ Fever Flamingo_SearchEngine FlipboardProxy -g2reader-bot -G2 Web Services -imrbot -K7MLWCBot -Kemvibot -Landau-Media-Spider -linkapediabot +G2\sWeb\sServices vkShare -Siteimprove.com -BLEXBot\/ +Siteimprove\.com DareBoost -ZuperlistBot\/ -Miniflux\/ +Miniflux\\/ Feedspot -Diffbot\/ SEOkicks tracemyfile -Nimbostratus-Bot zgrab -PR-CY.RU -AdsTxtCrawler +PR-CY\.RU Datafeedwatch -Zabbix -TangibleeBot -google-xrawler axios -Amazon CloudFront Pulsepoint CloudFlare-AlwaysOnline -Google-Structured-Data-Testing-Tool WordupInfoSearch WebDataStats HttpUrlConnection -Seekport Crawler -ZoomBot -VelenPublicWebCrawler -MoodleBot -jpg-newsbot outbrain -W3C_Validator -Validator\.nu -W3C-checklink W3C-mobileOK W3C_I18n-Checker -FeedValidator -W3C_CSS_Validator W3C_Unicorn -Google-PhysicalWeb Blackboard -ICBot\/ BazQux Twingly Rivva -Experibot -awesomecrawler -Dataprovider.com -GroupHigh\/ -theoldreader.com +Dataprovider\.com +GroupHigh\\/ +theoldreader\.com AnyEvent -Uptimebot\.org -Nmap Scripting Engine -2ip.ru +Nmap\sScripting\sEngine +2ip\.ru Clickagy -Caliperbot -MBCrawler -online-webceo-bot -B2B Bot -AddSearchBot -Google Favicon HubSpot Chrome-Lighthouse -HeadlessChrome -CheckMarkNetwork\/ -www\.uptime\.com -Streamline3Bot\/ -serpstatbot\/ -MixnodeCache\/ -^curl +CheckMarkNetwork\\/ +www\\\.uptime\\\.com +MixnodeCache\\/ +\^curl SimpleScraper -RSSingBot -Jooblebot fedoraplanet Friendica NextCloud -Tiny Tiny RSS -RegionStuttgartBot -Bytespider Datanyze -Google-Site-Verification TrendsmapResolver tweetedtimes -NTENTbot Gwene SimplePie SearchAtlas Superfeedr -feedbot -UT-Dorkbot -Amazonbot -SerendeputyBot -Eyeotabot -officestorebot -Neticle Crawler -SurdotlyBot -LinkisBot -AwarioSmartBot -AwarioRssBot -RyteBot -FreeWebMonitoring SiteChecker -AspiegelBot -NAVER Blog Rssbot -zenback bot -SentiBot -Domains Project\/ +Domains\sProject\\/ Pandalytics -VKRobot -bidswitchbot -tigerbot -NIXStatsbot -Atom Feed Robot -[Cc]urebot -PagePeeker\/ -Vigil\/ -rssbot\/ -startmebot\/ -JobboerseBot +PagePeeker\\/ +Vigil\\/ seewithkids -NINJA bot -Cutbot -BublupBot -BrandONbot -RidderBot -Taboolabot -Dubbotbot -FindITAnswersbot -infoobot -Refindbot -BlogTraffic\/\d\.\d+ Feed-Fetcher -SeobilityBot +BlogTraffic\\/\\d\\\.\\d\+\sFeed-Fetcher Cincraw -Dragonbot -VoluumDSP-content-bot -FreshRSS -BitBot -^PHP-Curl-Class -Google-Certificates-Bridge -centurybot +\^PHP-Curl-Class Viber -e\.ventures Investment Crawler evc-batch -PetalBot virustotal -(^| )PTST\/ -minicrawler -Cookiebot -trovitBot -seostar\.co -IonCrawl +seostar\\\.co Uptime-Kuma -SeekportBot -FreshpingBot -Feedbin -CriteoBot -Snap URL Preview Service -Better Uptime Bot +Snap\sURL\sPreview\sService RuxitSynthetic -Google-Read-Aloud -Valve\/Steam -OdklBot\/ -GPTBot -YandexRenderResourcesBot\/ -LightspeedSystemsCrawler -ev-crawler\/ -BitSightBot\/ -woorankreview\/ -Google-Safety -AwarioBot -DataForSeoBot -Linespider -WellKnownBot -A Patent Crawler -StractBot -search\.marginalia\.nu -YouBot -Nicecrawler -Neevabot -BrightEdge Crawler -SiteCheckerBotCrawler -TombaPublicWebCrawler -CrawlyProjectCrawler -KomodiaBot -KStandBot -CISPA Webcrawler -MTRobot -hyscore.io -AlexandriaOrgBot -2ip bot -Yellowbrandprotectionbot +Valve\\/Steam +woorankreview\\/ +search\\\.marginalia\\\.nu SEOlizer -vuhuvBot -INETDEX-BOT Synapse -t3versionsBot deepnoc -Cocolyzebot hypestat -ReverseEngineeringBot -sempi.tech +sempi\.tech Iframely -MetaInspector node-fetch lkxscan -python-opengraph OpenGraphCheck -developers.google.com\/\+\/web\/snippet -SenutoBot -MaCoCu NewsBlur inoreader NetSystemsResearch PageThing -WordPress\/