Use html.Pattern in K6 with Examples

crawler-user-agents.go

Source:crawler-user-agents.go

2import (3	"encoding/json"4	"sync"5)6type CrawlerPattern struct {7	Pattern      string   `json:"pattern"`8	AdditionDate string   `json:"addition_date"`9	Instances    []string `json:"instances"`10	Url          string   `json:"url"`11}12var parseOnce sync.Once13var crawlerPatterns []CrawlerPattern14func getCrawlerPatterns() []CrawlerPattern {15	parseOnce.Do(func() {16		crawlerPatterns = parseCrawlerPatterns()17	})18	return crawlerPatterns19}20func parseCrawlerPatterns() []CrawlerPattern {21	crawlers := `22		[23		  {24			"pattern": "Googlebot\\/",25			"url": "http://www.google.com/bot.html",26			"instances": [27			  "Googlebot/2.1 (+http://www.google.com/bot.html)",28			  "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",29			  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",30			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",31			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",32			  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",33			  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",34			  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36"35			]36		  }37		,38		  {39			"pattern": "Googlebot-Mobile",40			"instances": [41			  "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",42			  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",43			  "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",44			  "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",45			  "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"46			]47		  }48		,49		  {50			"pattern": "Googlebot-Image",51			"instances": [52			  "Googlebot-Image/1.0"53			]54		  }55		,56		  {57			"pattern": "Googlebot-News",58			"instances": [59			  "Googlebot-News"60			]61		  }62		,63		  {64			"pattern": "Googlebot-Video",65			"instances": [66			  "Googlebot-Video/1.0"67			]68		  }69		,70		  {71			"pattern": "AdsBot-Google([^-]|$)",72			"url": "https://support.google.com/webmasters/answer/1061943?hl=en",73			"instances": [74			  "AdsBot-Google (+http://www.google.com/adsbot.html)"75			]76		  }77		,78		  {79			"pattern": "AdsBot-Google-Mobile",80			"addition_date": "2017/08/21",81			"url": "https://support.google.com/adwords/answer/2404197",82			"instances": [83			  "AdsBot-Google-Mobile-Apps",84			  "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",85			  "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"86			]87		  }88		,89		  {90			"pattern": "Feedfetcher-Google",91			"addition_date": "2018/06/27",92			"url": "https://support.google.com/webmasters/answer/178852",93			"instances": [94			  "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)"95			]96		  }97		,98		  {99			"pattern": "Mediapartners-Google",100			"url": "https://support.google.com/webmasters/answer/1061943?hl=en",101			"instances": [102			  "Mediapartners-Google",103			  "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)",104			  "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",105			  "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"106			]107		  }108		,109		  {110			"pattern": "Mediapartners \\(Googlebot\\)",111			"addition_date": "2017/08/08",112			"url": "https://support.google.com/webmasters/answer/1061943?hl=en",113			"instances": []114		  }115		,116		  {117			"pattern": "APIs-Google",118			"addition_date": "2017/08/08",119			"url": "https://support.google.com/webmasters/answer/1061943?hl=en",120			"instances": [121			  "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"122			]123		  }124		,125		  {126			"pattern": "bingbot",127			"url": "http://www.bing.com/bingbot.htm",128			"instances": [129			  "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",130			  "Mozilla/5.0 (compatible; adidxbot/2.0;  http://www.bing.com/bingbot.htm)",131			  "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",132			  "Mozilla/5.0 (compatible; bingbot/2.0;  http://www.bing.com/bingbot.htm)",133			  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm",134			  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",135			  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe",136			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0;  http://www.bing.com/bingbot.htm)",137			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",138			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0;  http://www.bing.com/bingbot.htm)",139			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",140			  "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",141			  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36"142			]143		  }144		,145		  {146			"pattern": "Slurp",147			"url": "http://help.yahoo.com/help/us/ysearch/slurp",148			"instances": [149			  "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",150			  "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",151			  "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"152			]153		  }154		,155		  {156			"pattern": "[wW]get",157			"instances": [158			  "WGETbot/1.0 (+http://wget.alanreed.org)",159			  "Wget/1.14 (linux-gnu)",160			  "Wget/1.20.3 (linux-gnu)"161			]162		  }163		,164		  {165			"pattern": "LinkedInBot",166			"instances": [167			  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",168			  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)",169			  "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)"170			]171		  }172		,173		  {174			"pattern": "Python-urllib",175			"instances": [176			  "Python-urllib/1.17",177			  "Python-urllib/2.5",178			  "Python-urllib/2.6",179			  "Python-urllib/2.7",180			  "Python-urllib/3.1",181			  "Python-urllib/3.2",182			  "Python-urllib/3.3",183			  "Python-urllib/3.4",184			  "Python-urllib/3.5",185			  "Python-urllib/3.6",186			  "Python-urllib/3.7"187			]188		  }189		,190		  {191			"pattern": "python-requests",192			"addition_date": "2018/05/27",193			"instances": [194			  "python-requests/2.9.2",195			  "python-requests/2.11.1",196			  "python-requests/2.18.4",197			  "python-requests/2.19.1",198			  "python-requests/2.20.0",199			  "python-requests/2.21.0",200			  "python-requests/2.22.0"201			]202		  }203		,204		  {205			"pattern": "libwww-perl",206			"instances": [207			  "2Bone_LinkChecker/1.0 libwww-perl/6.03",208			  "2Bone_LinkChkr/1.0 libwww-perl/6.03",209			  "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"210			]211		  }212		,213		  {214			"pattern": "httpunit",215			"instances": [216			  "httpunit/1.x"217			]218		  }219		,220		  {221			"pattern": "nutch",222			"instances": [223			  "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",224			  "istellabot-nutch/Nutch-1.10"225			]226		  }227		,228		  {229			"pattern": "Go-http-client",230			"addition_date": "2016/03/26",231			"url": "https://golang.org/pkg/net/http/",232			"instances": [233			  "Go-http-client/1.1",234			  "Go-http-client/2.0"235			]236		  }237		,238		  {239			"pattern": "phpcrawl",240			"addition_date": "2012-09/17",241			"url": "http://phpcrawl.cuab.de/",242			"instances": [243			  "phpcrawl"244			]245		  }246		,247		  {248			"pattern": "msnbot",249			"url": "http://search.msn.com/msnbot.htm",250			"instances": [251			  "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)",252			  "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)",253			  "librabot/1.0 (+http://search.msn.com/msnbot.htm)",254			  "librabot/2.0 (+http://search.msn.com/msnbot.htm)",255			  "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)",256			  "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)",257			  "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",258			  "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)",259			  "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)",260			  "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",261			  "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",262			  "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",263			  "msnbot/2.0b (+http://search.msn.com/msnbot.htm).",264			  "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"265			]266		  }267		,268		  {269			"pattern": "jyxobot",270			"instances": []271		  }272		,273		  {274			"pattern": "FAST-WebCrawler",275			"instances": [276			  "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",277			  "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",278			  "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",279			  "FAST-WebCrawler/3.8"280			]281		  }282		,283		  {284			"pattern": "FAST Enterprise Crawler",285			"instances": [286			  "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",287			  "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"288			]289		  }290		,291		  {292			"pattern": "BIGLOTRON",293			"instances": [294			  "BIGLOTRON (Beta 2;GNU/Linux)"295			]296		  }297		,298		  {299			"pattern": "Teoma",300			"instances": [301			  "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",302			  "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"303			],304			"url": "http://about.ask.com/en/docs/about/webmasters.shtml"305		  }306		,307		  {308			"pattern": "convera",309			"instances": [310			  "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"311			],312			"url": "http://ews.converasearch.com/crawl.htm"313		  }314		,315		  {316			"pattern": "seekbot",317			"instances": [318			  "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"319			],320			"url": "http://www.seekbot.net/bot.html"321		  }322		,323		  {324			"pattern": "Gigabot",325			"instances": [326			  "Gigabot/1.0",327			  "Gigabot/2.0 (http://www.gigablast.com/spider.html)"328			],329			"url": "http://www.gigablast.com/spider.html"330		  }331		,332		  {333			"pattern": "Gigablast",334			"instances": [335			  "GigablastOpenSource/1.0"336			],337			"url": "https://github.com/gigablast/open-source-search-engine"338		  }339		,340		  {341			"pattern": "exabot",342			"instances": [343			  "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)",344			  "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)",345			  "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)",346			  "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",347			  "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",348			  "Mozilla/5.0 (compatible; Exabot/3.0;  http://www.exabot.com/go/robot)"349			]350		  }351		,352		  {353			"pattern": "ia_archiver",354			"instances": [355			  "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",356			  "ia_archiver-web.archive.org"357			]358		  }359		,360		  {361			"pattern": "GingerCrawler",362			"instances": [363			  "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"364			]365		  }366		,367		  {368			"pattern": "webmon ",369			"instances": []370		  }371		,372		  {373			"pattern": "HTTrack",374			"instances": [375			  "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"376			]377		  }378		,379		  {380			"pattern": "grub.org",381			"instances": [382			  "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)",383			  "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)",384			  "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)",385			  "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)",386			  "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)",387			  "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)",388			  "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)",389			  "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)",390			  "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)",391			  "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",392			  "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"393			]394		  }395		,396		  {397			"pattern": "UsineNouvelleCrawler",398			"instances": []399		  }400		,401		  {402			"pattern": "antibot",403			"instances": []404		  }405		,406		  {407			"pattern": "netresearchserver",408			"instances": []409		  }410		,411		  {412			"pattern": "speedy",413			"instances": [414			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)",415			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)",416			  "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)",417			  "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",418			  "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"419			]420		  }421		,422		  {423			"pattern": "fluffy",424			"instances": []425		  }426		,427		  {428			"pattern": "findlink",429			"instances": [430			  "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",431			  "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)",432			  "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)",433			  "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)",434			  "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)",435			  "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)",436			  "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)",437			  "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)",438			  "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)",439			  "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)",440			  "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)",441			  "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",442			  "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",443			  "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",444			  "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)",445			  "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",446			  "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)",447			  "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",448			  "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)",449			  "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",450			  "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",451			  "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",452			  "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"453			]454		  }455		,456		  {457			"pattern": "msrbot",458			"instances": []459		  }460		,461		  {462			"pattern": "panscient",463			"instances": [464			  "panscient.com"465			]466		  }467		,468		  {469			"pattern": "yacybot",470			"instances": [471			  "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html",472			  "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html",473			  "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",474			  "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html",475			  "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html",476			  "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html",477			  "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",478			  "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",479			  "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html",480			  "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",481			  "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",482			  "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",483			  "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html",484			  "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html",485			  "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",486			  "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",487			  "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",488			  "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",489			  "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",490			  "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html",491			  "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html",492			  "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",493			  "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",494			  "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",495			  "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",496			  "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",497			  "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",498			  "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html",499			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html",500			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html",501			  "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",502			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",503			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",504			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",505			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html",506			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",507			  "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html",508			  "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",509			  "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html",510			  "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",511			  "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html",512			  "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html",513			  "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",514			  "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",515			  "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html",516			  "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html",517			  "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html"518			]519		  }520		,521		  {522			"pattern": "AISearchBot",523			"instances": []524		  }525		,526		  {527			"pattern": "ips-agent",528			"instances": [529			  "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent",530			  "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7",531			  "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4  Firefox/3.5.3",532			  "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",533			  "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"534			]535		  }536		,537		  {538			"pattern": "tagoobot",539			"instances": []540		  }541		,542		  {543			"pattern": "MJ12bot",544			"instances": [545			  "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)",546			  "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)",547			  "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)",548			  "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)",549			  "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)",550			  "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)",551			  "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)",552			  "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)",553			  "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)",554			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)",555			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)",556			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)",557			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)",558			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)",559			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",560			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",561			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)",562			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)",563			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)",564			  "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"565			]566		  }567		,568		  {569			"pattern": "woriobot",570			"instances": [571			  "Mozilla/5.0 (compatible; woriobot +http://worio.com)",572			  "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"573			]574		  }575		,576		  {577			"pattern": "yanga",578			"instances": [579			  "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"580			]581		  }582		,583		  {584			"pattern": "buzzbot",585			"instances": [586			  "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"587			]588		  }589		,590		  {591			"pattern": "mlbot",592			"instances": [593			  "MLBot (www.metadatalabs.com/mlbot)"594			]595		  }596		,597		  {598			"pattern": "YandexBot",599			"url": "http://yandex.com/bots",600			"instances": [601			  "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"602			],603			"addition_date": "2015/04/14"604		  }605		,606		  {607			"pattern": "YandexImages",608			"url": "http://yandex.com/bots",609			"instances": [610			  "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"611			],612			"addition_date": "2015/04/14"613		  }614		,615		  {616			"pattern": "YandexAccessibilityBot",617			"url": "http://yandex.com/bots",618			"instances": [619			  "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots"620			],621			"addition_date": "2019/03/01"622		  }623		,624		  {625			"pattern": "YandexMobileBot",626			"url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs",627			"instances": [628			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"629			],630			"addition_date": "2016/12/01"631		  }632		,633		  {634			"pattern": "purebot",635			"addition_date": "2010/01/19",636			"instances": []637		  }638		,639		  {640			"pattern": "Linguee Bot",641			"addition_date": "2010/01/26",642			"url": "http://www.linguee.com/bot",643			"instances": [644			  "Linguee Bot (http://www.linguee.com/bot)",645			  "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"646			]647		  }648		,649		  {650			"pattern": "CyberPatrol",651			"addition_date": "2010/02/11",652			"url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp",653			"instances": [654			  "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"655			]656		  }657		,658		  {659			"pattern": "voilabot",660			"addition_date": "2010/05/18",661			"instances": [662			  "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",663			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)"664			]665		  }666		,667		  {668			"pattern": "Baiduspider",669			"addition_date": "2010/07/15",670			"url": "http://www.baidu.jp/spider/",671			"instances": [672			  "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",673			  "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)"674			]675		  }676		,677		  {678			"pattern": "citeseerxbot",679			"addition_date": "2010/07/17",680			"instances": []681		  }682		,683		  {684			"pattern": "spbot",685			"addition_date": "2010/07/31",686			"url": "http://www.seoprofiler.com/bot",687			"instances": [688			  "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )",689			  "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )",690			  "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )",691			  "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )",692			  "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )",693			  "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )",694			  "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )",695			  "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )",696			  "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )",697			  "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )",698			  "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )",699			  "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )",700			  "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )",701			  "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )",702			  "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )",703			  "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )",704			  "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )",705			  "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )",706			  "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )",707			  "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )",708			  "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )",709			  "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )",710			  "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )",711			  "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )",712			  "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )",713			  "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )",714			  "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )",715			  "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )",716			  "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )",717			  "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )",718			  "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )",719			  "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )",720			  "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",721			  "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"722			]723		  }724		,725		  {726			"pattern": "twengabot",727			"addition_date": "2010/08/03",728			"url": "http://www.twenga.com/bot.html",729			"instances": []730		  }731		,732		  {733			"pattern": "postrank",734			"addition_date": "2010/08/03",735			"url": "http://www.postrank.com",736			"instances": [737			  "PostRank/2.0 (postrank.com)",738			  "PostRank/2.0 (postrank.com; 1 subscribers)"739			]740		  }741		,742		  {743			"pattern": "TurnitinBot",744			"addition_date": "2010/09/26",745			"url": "http://www.turnitin.com",746			"instances": [747			  "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)"748			]749		  }750		,751		  {752			"pattern": "scribdbot",753			"addition_date": "2010/09/28",754			"url": "http://www.scribd.com",755			"instances": []756		  }757		,758		  {759			"pattern": "page2rss",760			"addition_date": "2010/10/07",761			"url": "http://www.page2rss.com",762			"instances": [763			  "Mozilla/5.0 (compatible;  Page2RSS/0.7; +http://page2rss.com/)"764			]765		  }766		,767		  {768			"pattern": "sitebot",769			"addition_date": "2010/12/15",770			"url": "http://www.sitebot.org",771			"instances": [772			  "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"773			]774		  }775		,776		  {777			"pattern": "linkdex",778			"addition_date": "2011/01/06",779			"url": "http://www.linkdex.com",780			"instances": [781			  "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)",782			  "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",783			  "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)",784			  "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)",785			  "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)",786			  "linkdex.com/v2.0",787			  "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"788			]789		  }790		,791		  {792			"pattern": "Adidxbot",793			"url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",794			"instances": []795		  }796		,797		  {798			"pattern": "ezooms",799			"addition_date": "2011/04/27",800			"url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289",801			"instances": [802			  "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"803			]804		  }805		,806		  {807			"pattern": "dotbot",808			"addition_date": "2011/04/27",809			"instances": [810			  "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",811			  "dotbot"812			]813		  }814		,815		  {816			"pattern": "Mail.RU_Bot",817			"addition_date": "2011/04/27",818			"instances": [819			  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)",820			  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/",821			  "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/",822			  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)"823			]824		  }825		,826		  {827			"pattern": "discobot",828			"addition_date": "2011/05/03",829			"url": "http://discoveryengine.com/discobot.html",830			"instances": [831			  "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)",832			  "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",833			  "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"834			]835		  }836		,837		  {838			"pattern": "heritrix",839			"addition_date": "2011/06/21",840			"url": "https://github.com/internetarchive/heritrix3/wiki",841			"instances": [842			  "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)",843			  "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)",844			  "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)",845			  "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)",846			  "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)",847			  "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)",848			  "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)",849			  "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)",850			  "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)",851			  "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)",852			  "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)",853			  "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)",854			  "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)",855			  "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)",856			  "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)",857			  "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)",858			  "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",859			  "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)",860			  "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)",861			  "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)",862			  "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",863			  "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"864			]865		  }866		,867		  {868			"pattern": "findthatfile",869			"addition_date": "2011/06/21",870			"url": "http://www.findthatfile.com/",871			"instances": []872		  }873		,874		  {875			"pattern": "europarchive.org",876			"addition_date": "2011/06/21",877			"url": "",878			"instances": [879			  "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"880			]881		  }882		,883		  {884			"pattern": "NerdByNature.Bot",885			"addition_date": "2011/07/12",886			"url": "http://www.nerdbynature.net/bot",887			"instances": [888			  "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"889			]890		  }891		,892		  {893			"pattern": "sistrix crawler",894			"addition_date": "2011/08/02",895			"instances": []896		  }897		,898		  {899			"pattern": "Ahrefs(Bot|SiteAudit)",900			"addition_date": "2011/08/28",901			"instances": [902			  "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)",903			  "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)",904			  "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)",905			  "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)",906			  "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)",907			  "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)"908			]909		  }910		,911		  {912			"pattern": "fuelbot",913			"addition_date": "2018/06/28",914			"instances": [915			  "fuelbot"916			]917		  }918		,919		  {920			"pattern": "CrunchBot",921			"addition_date": "2018/06/28",922			"instances": [923			  "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)"924			]925		  }926		,927		  {928			"pattern": "IndeedBot",929			"addition_date": "2018/06/28",930			"instances": [931			  "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)"932			]933		  }934		,935		  {936			"pattern": "mappydata",937			"addition_date": "2018/06/28",938			"instances": [939			  "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)"940			]941		  }942		,943		  {944			"pattern": "woobot",945			"addition_date": "2018/06/28",946			"instances": [947			  "woobot"948			]949		  }950		,951		  {952			"pattern": "ZoominfoBot",953			"addition_date": "2018/06/28",954			"instances": [955			  "ZoominfoBot (zoominfobot at zoominfo dot com)"956			]957		  }958		,959		  {960			"pattern": "PrivacyAwareBot",961			"addition_date": "2018/06/28",962			"instances": [963			  "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)"964			]965		  }966		,967		  {968			"pattern": "Multiviewbot",969			"addition_date": "2018/06/28",970			"instances": [971			  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot"972			]973		  }974		,975		  {976			"pattern": "SWIMGBot",977			"addition_date": "2018/06/28",978			"instances": [979			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot"980			]981		  }982		,983		  {984			"pattern": "Grobbot",985			"addition_date": "2018/06/28",986			"instances": [987			  "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)"988			]989		  }990		,991		  {992			"pattern": "eright",993			"addition_date": "2018/06/28",994			"instances": [995			  "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)"996			]997		  }998		,999		  {1000			"pattern": "Apercite",1001			"addition_date": "2018/06/28",1002			"instances": [1003			  "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)"1004			]1005		  }1006		,1007		  {1008			"pattern": "semanticbot",1009			"addition_date": "2018/06/28",1010			"instances": [1011			  "semanticbot",1012			  "semanticbot (info@semanticaudience.com)"1013			]1014		  }1015		,1016		  {1017			"pattern": "Aboundex",1018			"addition_date": "2011/09/28",1019			"url": "http://www.aboundex.com/crawler/",1020			"instances": [1021			  "Aboundex/0.2 (http://www.aboundex.com/crawler/)",1022			  "Aboundex/0.3 (http://www.aboundex.com/crawler/)"1023			]1024		  }1025		,1026		  {1027			"pattern": "domaincrawler",1028			"addition_date": "2011/10/21",1029			"instances": [1030			  "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"1031			]1032		  }1033		,1034		  {1035			"pattern": "wbsearchbot",1036			"addition_date": "2011/12/21",1037			"url": "http://www.warebay.com/bot.html",1038			"instances": []1039		  }1040		,1041		  {1042			"pattern": "summify",1043			"addition_date": "2012/01/04",1044			"url": "http://summify.com",1045			"instances": [1046			  "Summify (Summify/1.0.1; +http://summify.com)"1047			]1048		  }1049		,1050		  {1051			"pattern": "CCBot",1052			"addition_date": "2012/02/05",1053			"url": "http://www.commoncrawl.org/bot.html",1054			"instances": [1055			  "CCBot/2.0 (http://commoncrawl.org/faq/)",1056			  "CCBot/2.0 (https://commoncrawl.org/faq/)"1057			]1058		  }1059		,1060		  {1061			"pattern": "edisterbot",1062			"addition_date": "2012/02/25",1063			"instances": []1064		  }1065		,1066		  {1067			"pattern": "seznambot",1068			"addition_date": "2012/03/14",1069			"instances": [1070			  "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)",1071			  "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)",1072			  "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)",1073			  "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)",1074			  "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"1075			]1076		  }1077		,1078		  {1079			"pattern": "ec2linkfinder",1080			"addition_date": "2012/03/22",1081			"instances": [1082			  "ec2linkfinder"1083			]1084		  }1085		,1086		  {1087			"pattern": "gslfbot",1088			"addition_date": "2012/04/03",1089			"instances": []1090		  }1091		,1092		  {1093			"pattern": "aiHitBot",1094			"addition_date": "2012/04/16",1095			"instances": [1096			  "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)"1097			]1098		  }1099		,1100		  {1101			"pattern": "intelium_bot",1102			"addition_date": "2012/05/07",1103			"instances": []1104		  }1105		,1106		  {1107			"pattern": "facebookexternalhit",1108			"addition_date": "2012/05/07",1109			"instances": [1110			  "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",1111			  "facebookexternalhit/1.1",1112			  "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"1113			],1114			"url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/"1115		  }1116		,1117		  {1118			"pattern": "Yeti",1119			"addition_date": "2012/05/07",1120			"url": "http://naver.me/bot",1121			"instances": [1122			  "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"1123			]1124		  }1125		,1126		  {1127			"pattern": "RetrevoPageAnalyzer",1128			"addition_date": "2012/05/07",1129			"instances": [1130			  "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"1131			]1132		  }1133		,1134		  {1135			"pattern": "lb-spider",1136			"addition_date": "2012/05/07",1137			"instances": []1138		  }1139		,1140		  {1141			"pattern": "Sogou",1142			"addition_date": "2012/05/13",1143			"url": "http://www.sogou.com/docs/help/webmasters.htm#07",1144			"instances": [1145			  "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",1146			  "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",1147			  "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"1148			]1149		  }1150		,1151		  {1152			"pattern": "lssbot",1153			"addition_date": "2012/05/15",1154			"instances": []1155		  }1156		,1157		  {1158			"pattern": "careerbot",1159			"addition_date": "2012/05/23",1160			"url": "http://www.career-x.de/bot.html",1161			"instances": []1162		  }1163		,1164		  {1165			"pattern": "wotbox",1166			"addition_date": "2012/06/12",1167			"url": "http://www.wotbox.com",1168			"instances": [1169			  "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",1170			  "Wotbox/2.01 (+http://www.wotbox.com/bot/)"1171			]1172		  }1173		,1174		  {1175			"pattern": "wocbot",1176			"addition_date": "2012/07/25",1177			"url": "http://www.wocodi.com/crawler",1178			"instances": []1179		  }1180		,1181		  {1182			"pattern": "ichiro",1183			"addition_date": "2012/08/28",1184			"url": "http://help.goo.ne.jp/help/article/1142",1185			"instances": [1186			  "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",1187			  "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1188			  "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1189			  "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)",1190			  "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)",1191			  "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",1192			  "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1193			  "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1194			  "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)",1195			  "ichiro/2.0 (ichiro@nttr.co.jp)",1196			  "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)",1197			  "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)",1198			  "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1199			  "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",1200			  "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"1201			]1202		  }1203		,1204		  {1205			"pattern": "DuckDuckBot",1206			"addition_date": "2012/09/19",1207			"url": "http://duckduckgo.com/duckduckbot.html",1208			"instances": [1209			  "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",1210			  "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",1211			  "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)",1212			  "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'"1213			]1214		  }1215		,1216		  {1217			"pattern": "lssrocketcrawler",1218			"addition_date": "2012/09/24",1219			"instances": []1220		  }1221		,1222		  {1223			"pattern": "drupact",1224			"addition_date": "2012/09/27",1225			"url": "http://www.arocom.de/drupact",1226			"instances": [1227			  "drupact/0.7; http://www.arocom.de/drupact"1228			]1229		  }1230		,1231		  {1232			"pattern": "webcompanycrawler",1233			"addition_date": "2012/10/03",1234			"instances": []1235		  }1236		,1237		  {1238			"pattern": "acoonbot",1239			"addition_date": "2012/10/07",1240			"url": "http://www.acoon.de/robot.asp",1241			"instances": []1242		  }1243		,1244		  {1245			"pattern": "openindexspider",1246			"addition_date": "2012/10/26",1247			"url": "http://www.openindex.io/en/webmasters/spider.html",1248			"instances": []1249		  }1250		,1251		  {1252			"pattern": "gnam gnam spider",1253			"addition_date": "2012/10/31",1254			"instances": []1255		  }1256		,1257		  {1258			"pattern": "web-archive-net.com.bot",1259			"instances": []1260		  }1261		,1262		  {1263			"pattern": "backlinkcrawler",1264			"addition_date": "2013/01/04",1265			"instances": []1266		  }1267		,1268		  {1269			"pattern": "coccoc",1270			"addition_date": "2013/01/04",1271			"url": "http://help.coccoc.vn/",1272			"instances": [1273			  "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)",1274			  "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)",1275			  "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)",1276			  "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",1277			  "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)",1278			  "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)",1279			  "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)",1280			  "coccoc",1281			  "coccoc/1.0 ()",1282			  "coccoc/1.0 (http://help.coccoc.com/)",1283			  "coccoc/1.0 (http://help.coccoc.vn/)"1284			]1285		  }1286		,1287		  {1288			"pattern": "integromedb",1289			"addition_date": "2013/01/10",1290			"url": "http://www.integromedb.org/Crawler",1291			"instances": [1292			  "www.integromedb.org/Crawler"1293			]1294		  }1295		,1296		  {1297			"pattern": "content crawler spider",1298			"addition_date": "2013/01/11",1299			"instances": []1300		  }1301		,1302		  {1303			"pattern": "toplistbot",1304			"addition_date": "2013/02/05",1305			"instances": []1306		  }1307		,1308		  {1309			"pattern": "it2media-domain-crawler",1310			"addition_date": "2013/03/12",1311			"instances": [1312			  "it2media-domain-crawler/1.0 on crawler-prod.it2media.de",1313			  "it2media-domain-crawler/2.0"1314			]1315		  }1316		,1317		  {1318			"pattern": "ip-web-crawler.com",1319			"addition_date": "2013/03/22",1320			"instances": []1321		  }1322		,1323		  {1324			"pattern": "siteexplorer.info",1325			"addition_date": "2013/05/01",1326			"instances": [1327			  "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",1328			  "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"1329			]1330		  }1331		,1332		  {1333			"pattern": "elisabot",1334			"addition_date": "2013/06/27",1335			"instances": []1336		  }1337		,1338		  {1339			"pattern": "proximic",1340			"addition_date": "2013/09/12",1341			"url": "http://www.proximic.com/info/spider.php",1342			"instances": [1343			  "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",1344			  "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"1345			]1346		  }1347		,1348		  {1349			"pattern": "changedetection",1350			"addition_date": "2013/09/13",1351			"url": "http://www.changedetection.com/bot.html",1352			"instances": [1353			  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;  http://www.changedetection.com/bot.html )"1354			]1355		  }1356		,1357		  {1358			"pattern": "arabot",1359			"addition_date": "2013/10/09",1360			"instances": []1361		  }1362		,1363		  {1364			"pattern": "WeSEE:Search",1365			"addition_date": "2013/11/18",1366			"instances": [1367			  "WeSEE:Search",1368			  "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"1369			]1370		  }1371		,1372		  {1373			"pattern": "niki-bot",1374			"addition_date": "2014/01/01",1375			"instances": []1376		  }1377		,1378		  {1379			"pattern": "CrystalSemanticsBot",1380			"addition_date": "2014/02/17",1381			"url": "http://www.crystalsemantics.com/user-agent/",1382			"instances": []1383		  }1384		,1385		  {1386			"pattern": "rogerbot",1387			"addition_date": "2014/02/28",1388			"url": "http://moz.com/help/pro/what-is-rogerbot-",1389			"instances": [1390			  "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",1391			  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)",1392			  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)",1393			  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com",1394			  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)",1395			  "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)",1396			  "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)",1397			  "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)",1398			  "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)",1399			  "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)",1400			  "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)",1401			  "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",1402			  "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"1403			]1404		  }1405		,1406		  {1407			"pattern": "360Spider",1408			"addition_date": "2014/03/14",1409			"url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html",1410			"instances": [1411			  "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider",1412			  "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",1413			  "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider",1414			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; )  Firefox/1.5.0.11; 360Spider",1415			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11)  Firefox/1.5.0.11; 360Spider",1416			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;",1417			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider",1418			  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider",1419			  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",1420			  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider"1421			]1422		  }1423		,1424		  {1425			"pattern": "psbot",1426			"addition_date": "2014/03/31",1427			"url": "http://www.picsearch.com/bot.html",1428			"instances": [1429			  "psbot-image (+http://www.picsearch.com/bot.html)",1430			  "psbot-page (+http://www.picsearch.com/bot.html)",1431			  "psbot/0.1 (+http://www.picsearch.com/bot.html)"1432			]1433		  }1434		,1435		  {1436			"pattern": "InterfaxScanBot",1437			"addition_date": "2014/03/31",1438			"url": "http://scan-interfax.ru",1439			"instances": []1440		  }1441		,1442		  {1443			"pattern": "CC Metadata Scaper",1444			"addition_date": "2014/04/01",1445			"url": "http://wiki.creativecommons.org/Metadata_Scraper",1446			"instances": [1447			  "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"1448			]1449		  }1450		,1451		  {1452			"pattern": "g00g1e.net",1453			"addition_date": "2014/04/01",1454			"url": "http://www.g00g1e.net/",1455			"instances": []1456		  }1457		,1458		  {1459			"pattern": "GrapeshotCrawler",1460			"addition_date": "2014/04/01",1461			"url": "http://www.grapeshot.co.uk/crawler.php",1462			"instances": [1463			  "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"1464			]1465		  }1466		,1467		  {1468			"pattern": "urlappendbot",1469			"addition_date": "2014/05/10",1470			"url": "http://www.profound.net/urlappendbot.html",1471			"instances": [1472			  "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"1473			]1474		  }1475		,1476		  {1477			"pattern": "brainobot",1478			"addition_date": "2014/06/24",1479			"instances": []1480		  }1481		,1482		  {1483			"pattern": "fr-crawler",1484			"addition_date": "2014/07/31",1485			"instances": [1486			  "Mozilla/5.0 (compatible; fr-crawler/1.1)"1487			]1488		  }1489		,1490		  {1491			"pattern": "binlar",1492			"addition_date": "2014/09/12",1493			"instances": [1494			  "binlar_2.6.3 binlar2.6.3@unspecified.mail",1495			  "binlar_2.6.3 binlar_2.6.3@unspecified.mail",1496			  "binlar_2.6.3 larbin2.6.3@unspecified.mail",1497			  "binlar_2.6.3 phanendra_kalapala@McAfee.com",1498			  "binlar_2.6.3 test@mgmt.mic"1499			]1500		  }1501		,1502		  {1503			"pattern": "SimpleCrawler",1504			"addition_date": "2014/09/12",1505			"instances": [1506			  "SimpleCrawler/0.1"1507			]1508		  }1509		,1510		  {1511			"pattern": "Twitterbot",1512			"addition_date": "2014/09/12",1513			"url": "https://dev.twitter.com/cards/getting-started",1514			"instances": [1515			  "Twitterbot/0.1",1516			  "Twitterbot/1.0"1517			]1518		  }1519		,1520		  {1521			"pattern": "cXensebot",1522			"addition_date": "2014/10/05",1523			"instances": [1524			  "cXensebot/1.1a"1525			],1526			"url": "http://www.cxense.com/bot.html"1527		  }1528		,1529		  {1530			"pattern": "smtbot",1531			"addition_date": "2014/10/04",1532			"instances": [1533			  "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1534			  "SMTBot (similartech.com/smtbot)",1535			  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko)                 Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1536			  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1537			  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)"1538			],1539			"url": "http://www.similartech.com/smtbot"1540		  }1541		,1542		  {1543			"pattern": "bnf.fr_bot",1544			"addition_date": "2014/11/18",1545			"url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html",1546			"instances": [1547			  "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)",1548			  "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"1549			]1550		  }1551		,1552		  {1553			"pattern": "A6-Indexer",1554			"addition_date": "2014/12/05",1555			"url": "http://www.a6corp.com/a6-web-scraping-policy/",1556			"instances": [1557			  "A6-Indexer"1558			]1559		  }1560		,1561		  {1562			"pattern": "ADmantX",1563			"addition_date": "2014/12/05",1564			"url": "http://www.admantx.com",1565			"instances": [1566			  "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"1567			]1568		  }1569		,1570		  {1571			"pattern": "Facebot",1572			"url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",1573			"addition_date": "2014/12/30",1574			"instances": [1575			  "Facebot/1.0"1576			]1577		  }1578		,1579		  {1580			"pattern": "OrangeBot\\/",1581			"instances": [1582			  "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"1583			],1584			"addition_date": "2015/01/12"1585		  }1586		,1587		  {1588			"pattern": "memorybot",1589			"url": "http://mignify.com/bot.htm",1590			"instances": [1591			  "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"1592			],1593			"addition_date": "2015/02/01"1594		  }1595		,1596		  {1597			"pattern": "AdvBot",1598			"url": "http://advbot.net/bot.html",1599			"instances": [1600			  "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"1601			],1602			"addition_date": "2015/02/01"1603		  }1604		,1605		  {1606			"pattern": "MegaIndex",1607			"url": "https://www.megaindex.ru/?tab=linkAnalyze",1608			"instances": [1609			  "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)",1610			  "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)"1611			],1612			"addition_date": "2015/03/28"1613		  }1614		,1615		  {1616			"pattern": "SemanticScholarBot",1617			"url": "https://www.semanticscholar.org/crawler",1618			"instances": [1619			  "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)",1620			  "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)"1621			],1622			"addition_date": "2015/03/28"1623		  }1624		,1625		  {1626			"pattern": "ltx71",1627			"url": "http://ltx71.com/",1628			"instances": [1629			  "ltx71 - (http://ltx71.com/)"1630			],1631			"addition_date": "2015/04/04"1632		  }1633		,1634		  {1635			"pattern": "nerdybot",1636			"url": "http://nerdybot.com/",1637			"instances": [1638			  "nerdybot"1639			],1640			"addition_date": "2015/04/05"1641		  }1642		,1643		  {1644			"pattern": "xovibot",1645			"url": "http://www.xovibot.net/",1646			"instances": [1647			  "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"1648			],1649			"addition_date": "2015/04/05"1650		  }1651		,1652		  {1653			"pattern": "BUbiNG",1654			"url": "http://law.di.unimi.it/BUbiNG.html",1655			"instances": [1656			  "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"1657			],1658			"addition_date": "2015/04/06"1659		  }1660		,1661		  {1662			"pattern": "Qwantify",1663			"url": "https://www.qwant.com/",1664			"instances": [1665			  "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*",1666			  "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w",1667			  "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)",1668			  "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)"1669			],1670			"addition_date": "2015/04/06"1671		  }1672		,1673		  {1674			"pattern": "archive.org_bot",1675			"url": "http://www.archive.org/details/archive.org_bot",1676			"depends_on": ["heritrix"],1677			"instances": [1678			  "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)",1679			  "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)",1680			  "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)",1681			  "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)",1682			  "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",1683			  "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)"1684			],1685			"addition_date": "2015/04/14"1686		  }1687		,1688		  {1689			"pattern": "Applebot",1690			"url": "http://www.apple.com/go/applebot",1691			"addition_date": "2015/04/15",1692			"instances": [1693			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)",1694			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)",1695			  "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",1696			  "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",1697			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"1698			]1699		  }1700		,1701		  {1702			"pattern": "TweetmemeBot",1703			"url": "http://datasift.com/bot.html",1704			"instances": [1705			  "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"1706			],1707			"addition_date": "2015/04/15"1708		  }1709		,1710		  {1711			"pattern": "crawler4j",1712			"url": "https://github.com/yasserg/crawler4j",1713			"instances": [1714			  "crawler4j (http://code.google.com/p/crawler4j/)",1715			  "crawler4j (https://github.com/yasserg/crawler4j/)"1716			],1717			"addition_date": "2015/05/07"1718		  }1719		,1720		  {1721			"pattern": "findxbot",1722			"url": "http://www.findxbot.com",1723			"instances": [1724			  "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"1725			],1726			"addition_date": "2015/05/07"1727		  }1728		,1729		  {1730			"pattern": "S[eE][mM]rushBot",1731			"url": "http://www.semrush.com/bot.html",1732			"instances": [1733			  "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)",1734			  "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)",1735			  "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)",1736			  "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",1737			  "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)",1738			  "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)",1739			  "SEMrushBot"1740			],1741			"addition_date": "2015/05/26"1742		  }1743		,1744		  {1745			"pattern": "yoozBot",1746			"url": "http://yooz.ir",1747			"instances": [1748			  "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"1749			],1750			"addition_date": "2015/05/26"1751		  }1752		,1753		  {1754			"pattern": "lipperhey",1755			"url": "http://www.lipperhey.com/",1756			"instances": [1757			  "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)",1758			  "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)",1759			  "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)",1760			  "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"1761			],1762			"addition_date": "2015/08/26"1763		  }1764		,1765		  {1766			"pattern": "Y!J",1767			"url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6",1768			"instances": [1769			  "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)",1770			  "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1771			  "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1772			  "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1773			  "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1774			  "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))"1775			],1776			"addition_date": "2015/05/26"1777		  }1778		,1779		  {1780			"pattern": "Domain Re-Animator Bot",1781			"url": "http://domainreanimator.com",1782			"instances": [1783			  "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"1784			],1785			"addition_date": "2015/04/14"1786		  }1787		,1788		  {1789			"pattern": "AddThis",1790			"url": "https://www.addthis.com",1791			"instances": [1792			  "AddThis.com robot tech.support@clearspring.com"1793			],1794			"addition_date": "2015/06/02"1795		  }1796		,1797		  {1798			"pattern": "Screaming Frog SEO Spider",1799			"url": "http://www.screamingfrog.co.uk/seo-spider",1800			"instances": [1801			  "Screaming Frog SEO Spider/5.1"1802			],1803			"addition_date": "2016/01/08"1804		  }1805		,1806		  {1807			"pattern": "MetaURI",1808			"url": "http://www.useragentstring.com/MetaURI_id_17683.php",1809			"instances": [1810			  "MetaURI API/2.0 +metauri.com"1811			],1812			"addition_date": "2016/01/02"1813		  }1814		,1815		  {1816			"pattern": "Scrapy",1817			"url": "http://scrapy.org/",1818			"instances": [1819			  "Scrapy/1.0.3 (+http://scrapy.org)"1820			],1821			"addition_date": "2016/01/02"1822		  }1823		,1824		  {1825			"pattern": "Livelap[bB]ot",1826			"url": "http://site.livelap.com/crawler",1827			"instances": [1828			  "LivelapBot/0.2 (http://site.livelap.com/crawler)",1829			  "Livelapbot/0.1"1830			],1831			"addition_date": "2016/01/02"1832		  }1833		,1834		  {1835			"pattern": "OpenHoseBot",1836			"url": "http://www.openhose.org/bot.html",1837			"instances": [1838			  "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"1839			],1840			"addition_date": "2016/01/02"1841		  }1842		,1843		  {1844			"pattern": "CapsuleChecker",1845			"url": "http://www.capsulink.com/about",1846			"instances": [1847			  "CapsuleChecker (http://www.capsulink.com/)"1848			],1849			"addition_date": "2016/01/02"1850		  }1851		,1852		  {1853			"pattern": "collection@infegy.com",1854			"url": "http://infegy.com/",1855			"instances": [1856			  "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"1857			],1858			"addition_date": "2016/01/03"1859		  }1860		,1861		  {1862			"pattern": "IstellaBot",1863			"url": "http://www.tiscali.it/",1864			"instances": [1865			  "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"1866			],1867			"addition_date": "2016/01/09"1868		  }1869		,1870		  {1871			"pattern": "DeuSu\\/",1872			"addition_date": "2016/01/23",1873			"url": "https://deusu.de/robot.html",1874			"instances": [1875			  "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",1876			  "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"1877			]1878		  }1879		,1880		  {1881			"pattern": "betaBot",1882			"addition_date": "2016/01/23",1883			"instances": []1884		  }1885		,1886		  {1887			"pattern": "Cliqzbot\\/",1888			"addition_date": "2016/01/23",1889			"url": "http://cliqz.com/company/cliqzbot",1890			"instances": [1891			  "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)",1892			  "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)",1893			  "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",1894			  "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",1895			  "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"1896			]1897		  }1898		,1899		  {1900			"pattern": "MojeekBot\\/",1901			"addition_date": "2016/01/23",1902			"url": "https://www.mojeek.com/bot.html",1903			"instances": [1904			  "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)",1905			  "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)",1906			  "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)",1907			  "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)",1908			  "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",1909			  "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"1910			]1911		  }1912		,1913		  {1914			"pattern": "netEstate NE Crawler",1915			"addition_date": "2016/01/23",1916			"url": "+http://www.website-datenbank.de/",1917			"instances": [1918			  "netEstate NE Crawler (+http://www.sengine.info/)",1919			  "netEstate NE Crawler (+http://www.website-datenbank.de/)"1920			]1921		  }1922		,1923		  {1924			"pattern": "SafeSearch microdata crawler",1925			"addition_date": "2016/01/23",1926			"url": "https://safesearch.avira.com",1927			"instances": [1928			  "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"1929			]1930		  }1931		,1932		  {1933			"pattern": "Gluten Free Crawler\\/",1934			"addition_date": "2016/01/23",1935			"url": "http://glutenfreepleasure.com/",1936			"instances": [1937			  "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"1938			]1939		  }1940		,1941		  {1942			"pattern": "Sonic",1943			"addition_date": "2016/02/08",1944			"url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html",1945			"instances": [1946			  "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)",1947			  "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",1948			  "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"1949			]1950		  }1951		,1952		  {1953			"pattern": "Sysomos",1954			"addition_date": "2016/02/08",1955			"url": "http://www.sysomos.com",1956			"instances": [1957			  "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"1958			]1959		  }1960		,1961		  {1962			"pattern": "Trove",1963			"addition_date": "2016/02/08",1964			"url": "http://www.trove.com",1965			"instances": []1966		  }1967		,1968		  {1969			"pattern": "deadlinkchecker",1970			"addition_date": "2016/02/08",1971			"url": "http://www.deadlinkchecker.com",1972			"instances": [1973			  "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36",1974			  "www.deadlinkchecker.com XMLHTTP/1.0",1975			  "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"1976			]1977		  }1978		,1979		  {1980			"pattern": "Slack-ImgProxy",1981			"addition_date": "2016/04/25",1982			"url": "https://api.slack.com/robots",1983			"instances": [1984			  "Slack-ImgProxy (+https://api.slack.com/robots)",1985			  "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)",1986			  "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)",1987			  "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)",1988			  "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",1989			  "Slack-ImgProxy 149 (+https://api.slack.com/robots)"1990			]1991		  }1992		,1993		  {1994			"pattern": "Embedly",1995			"addition_date": "2016/04/25",1996			"url": "http://support.embed.ly",1997			"instances": [1998			  "Embedly +support@embed.ly",1999			  "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",2000			  "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"2001			]2002		  }2003		,2004		  {2005			"pattern": "RankActiveLinkBot",2006			"addition_date": "2016/06/20",2007			"url": "https://rankactive.com/resources/rankactive-linkbot",2008			"instances": [2009			  "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"2010			]2011		  }2012		,2013		  {2014			"pattern": "iskanie",2015			"addition_date": "2016/09/02",2016			"url": "http://www.iskanie.com",2017			"instances": [2018			  "iskanie (+http://www.iskanie.com)"2019			]2020		  }2021		,2022		  {2023			"pattern": "SafeDNSBot",2024			"addition_date": "2016/09/10",2025			"url": "https://www.safedns.com/searchbot",2026			"instances": [2027			  "SafeDNSBot (https://www.safedns.com/searchbot)"2028			]2029		  }2030		,2031		  {2032			"pattern": "SkypeUriPreview",2033			"addition_date": "2016/10/10",2034			"instances": [2035			  "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"2036			]2037		  }2038		,2039		  {2040			"pattern": "Veoozbot",2041			"addition_date": "2016/11/03",2042			"url": "http://www.veooz.com/veoozbot.html",2043			"instances": [2044			  "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"2045			]2046		  }2047		,2048		  {2049			"pattern": "Slackbot",2050			"addition_date": "2016/11/03",2051			"url": "https://api.slack.com/robots",2052			"instances": [2053			  "Slackbot-LinkExpanding (+https://api.slack.com/robots)",2054			  "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",2055			  "Slackbot 1.0 (+https://api.slack.com/robots)"2056			]2057		  }2058		,2059		  {2060			"pattern": "redditbot",2061			"addition_date": "2016/11/03",2062			"url": "http://www.reddit.com/feedback",2063			"instances": [2064			  "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"2065			]2066		  }2067		,2068		  {2069			"pattern": "datagnionbot",2070			"addition_date": "2016/11/03",2071			"url": "http://www.datagnion.com/bot.html",2072			"instances": [2073			  "datagnionbot (+http://www.datagnion.com/bot.html)"2074			]2075		  }2076		,2077		  {2078			"pattern": "Google-Adwords-Instant",2079			"addition_date": "2016/11/03",2080			"url": "http://www.google.com/adsbot.html",2081			"instances": [2082			  "Google-Adwords-Instant (+http://www.google.com/adsbot.html)"2083			]2084		  }2085		,2086		  {2087			"pattern": "adbeat_bot",2088			"addition_date": "2016/11/04",2089			"instances": [2090			  "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",2091			  "adbeat_bot"2092			]2093		  }2094		,2095		  {2096			"pattern": "WhatsApp",2097			"addition_date": "2016/11/15",2098			"url": "https://www.whatsapp.com/",2099			"instances": [2100			  "WhatsApp",2101			  "WhatsApp/0.3.4479 N",2102			  "WhatsApp/0.3.4679 N",2103			  "WhatsApp/0.3.4941 N",2104			  "WhatsApp/2.12.15/i",2105			  "WhatsApp/2.12.16/i",2106			  "WhatsApp/2.12.17/i",2107			  "WhatsApp/2.12.449 A",2108			  "WhatsApp/2.12.453 A",2109			  "WhatsApp/2.12.510 A",2110			  "WhatsApp/2.12.540 A",2111			  "WhatsApp/2.12.548 A",2112			  "WhatsApp/2.12.555 A",2113			  "WhatsApp/2.12.556 A",2114			  "WhatsApp/2.16.1/i",2115			  "WhatsApp/2.16.13 A",2116			  "WhatsApp/2.16.2/i",2117			  "WhatsApp/2.16.42 A",2118			  "WhatsApp/2.16.57 A",2119			  "WhatsApp/2.19.92 i",2120			  "WhatsApp/2.19.175 A",2121			  "WhatsApp/2.19.244 A",2122			  "WhatsApp/2.19.258 A",2123			  "WhatsApp/2.19.308 A",2124			  "WhatsApp/2.19.330 A"2125			]2126		  }2127		,2128		  {2129			"pattern": "contxbot",2130			"addition_date": "2017/02/25",2131			"instances": [2132			  "Mozilla/5.0 (compatible;contxbot/1.0)"2133			]2134		  }2135		,2136		  {2137			"pattern": "pinterest.com.bot",2138			"addition_date": "2017/03/03",2139			"instances": [2140			  "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)",2141			  "Pinterest/0.2 (+http://www.pinterest.com/bot.html)"2142			],2143			"url": "http://www.pinterest.com/bot.html"2144		  }2145		,2146		  {2147			"pattern": "electricmonk",2148			"addition_date": "2017/03/04",2149			"instances": [2150			  "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"2151			],2152			"url": "https://www.duedil.com/our-crawler/"2153		  }2154		,2155		  {2156			"pattern": "GarlikCrawler",2157			"addition_date": "2017/03/18",2158			"instances": [2159			  "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"2160			],2161			"url": "http://garlik.com/"2162		  }2163		,2164		  {2165			"pattern": "BingPreview\\/",2166			"addition_date": "2017/04/23",2167			"url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0",2168			"instances": [2169			  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",2170			  "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko",2171			  "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0;  WOW64;  Trident/6.0;  BingPreview/1.0b)",2172			  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;  WOW64;  Trident/5.0;  BingPreview/1.0b)",2173			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"2174			]2175		  }2176		,2177		  {2178			"pattern": "vebidoobot",2179			"addition_date": "2017/05/08",2180			"instances": [2181			  "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"2182			],2183			"url": "https://blog.vebidoo.de/vebidoobot/"2184		  }2185		,2186		  {2187			"pattern": "FemtosearchBot",2188			"addition_date": "2017/05/16",2189			"instances": [2190			  "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"2191			],2192			"url": "http://femtosearch.com"2193		  }2194		,2195		  {2196			"pattern": "Yahoo Link Preview",2197			"addition_date": "2017/06/28",2198			"instances": [2199			  "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"2200			],2201			"url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"2202		  }2203		,2204		  {2205			"pattern": "MetaJobBot",2206			"addition_date": "2017/08/16",2207			"instances": [2208			  "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"2209			],2210			"url": "http://www.metajob.de/the/crawler"2211		  }2212		,2213		  {2214			"pattern": "DomainStatsBot",2215			"addition_date": "2017/08/16",2216			"instances": [2217			  "DomainStatsBot/1.0 (http://domainstats.io/our-bot)"2218			],2219			"url": "http://domainstats.io/our-bot"2220		  }2221		,2222		  {2223			"pattern": "mindUpBot",2224			"addition_date": "2017/08/16",2225			"instances": [2226			  "mindUpBot (datenbutler.de)"2227			],2228			"url": "http://www.datenbutler.de/"2229		  }2230		,2231		  {2232			"pattern": "Daum\\/",2233			"addition_date": "2017/08/16",2234			"instances": [2235			  "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"2236			],2237			"url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"2238		  }2239		,2240		  {2241			"pattern": "Jugendschutzprogramm-Crawler",2242			"addition_date": "2017/08/16",2243			"instances": [2244			  "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"2245			],2246			"url": "http://www.jugendschutzprogramm.de"2247		  }2248		,2249		  {2250			"pattern": "Xenu Link Sleuth",2251			"addition_date": "2017/08/19",2252			"instances": [2253			  "Xenu Link Sleuth/1.3.8"2254			],2255			"url": "http://home.snafu.de/tilman/xenulink.html"2256		  }2257		,2258		  {2259			"pattern": "Pcore-HTTP",2260			"addition_date": "2017/08/19",2261			"instances": [2262			  "Pcore-HTTP/v0.40.3",2263			  "Pcore-HTTP/v0.44.0"2264			],2265			"url": "https://bitbucket.org/softvisio/pcore/overview"2266		  }2267		,2268		  {2269			"pattern": "moatbot",2270			"addition_date": "2017/09/16",2271			"instances": [2272			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot",2273			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"2274			],2275			"url": "https://moat.com"2276		  }2277		,2278		  {2279			"pattern": "KosmioBot",2280			"addition_date": "2017/09/16",2281			"instances": [2282			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"2283			],2284			"url": "http://kosm.io/bot.html"2285		  }2286		,2287		  {2288			"pattern": "pingdom",2289			"addition_date": "2017/09/16",2290			"instances": [2291			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)",2292			  "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)"2293			],2294			"url": "http://www.pingdom.com"2295		  }2296		,2297		  {2298			"pattern": "AppInsights",2299			"addition_date": "2019/03/09",2300			"instances": [2301			  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)"2302			],2303			"url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview"2304		  }2305		,2306		  {2307			"pattern": "PhantomJS",2308			"addition_date": "2017/09/18",2309			"instances": [2310			  "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"2311			],2312			"url": "http://phantomjs.org/"2313		  }2314		,2315		  {2316			"pattern": "Gowikibot",2317			"addition_date": "2017/10/26",2318			"instances": [2319			  "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"2320			],2321			"url": "http://www.gowikibot.com"2322		  }2323		,2324		  {2325			"pattern": "PiplBot",2326			"addition_date": "2017/10/30",2327			"instances": [2328			  "PiplBot (+http://www.pipl.com/bot/)",2329			  "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"2330			],2331			"url": "http://www.pipl.com/bot/"2332		  }2333		,2334		  {2335			"pattern": "Discordbot",2336			"addition_date": "2017/09/22",2337			"url": "https://discordapp.com",2338			"instances": [2339			  "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"2340			]2341		  }2342		,2343		  {2344			"pattern": "TelegramBot",2345			"addition_date": "2017/10/01",2346			"instances": [2347			  "TelegramBot (like TwitterBot)"2348			]2349		  }2350		,2351		  {2352			"pattern": "Jetslide",2353			"addition_date": "2017/09/27",2354			"url": "http://jetsli.de/crawler",2355			"instances": [2356			  "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"2357			]2358		  }2359		,2360		  {2361			"pattern": "newsharecounts",2362			"addition_date": "2017/09/30",2363			"url": "http://newsharecounts.com/crawler",2364			"instances": [2365			  "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"2366			]2367		  }2368		,2369		  {2370			"pattern": "James BOT",2371			"addition_date": "2017/10/12",2372			"url": "http://cognitiveseo.com/bot.html",2373			"instances": [2374			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"2375			]2376		  }2377		,2378		  {2379			"pattern": "Bark[rR]owler",2380			"addition_date": "2017/10/09",2381			"url": "http://www.exensa.com/crawl",2382			"instances": [2383			  "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG",2384			  "Barkrowler/0.7 (+http://www.exensa.com/crawl)",2385			  "BarkRowler/0.7 (+http://www.exensa.com/crawling)",2386			  "Barkrowler/0.9 (+http://www.exensa.com/crawl)"2387			]2388		  }2389		,2390		  {2391			"pattern": "TinEye",2392			"addition_date": "2017/10/14",2393			"url": "http://www.tineye.com/crawler.html",2394			"instances": [2395			  "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)",2396			  "TinEye/1.1 (http://tineye.com/crawler.html)"2397			]2398		  }2399		,2400		  {2401			"pattern": "SocialRankIOBot",2402			"addition_date": "2017/10/19",2403			"url": "http://socialrank.io/about",2404			"instances": [2405			  "SocialRankIOBot; http://socialrank.io/about"2406			]2407		  }2408		,2409		  {2410			"pattern": "trendictionbot",2411			"addition_date": "2017/10/30",2412			"url": "http://www.trendiction.de/bot",2413			"instances": [2414			  "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",2415			  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0"2416			]2417		  }2418		,2419		  {2420			"pattern": "Ocarinabot",2421			"addition_date": "2017/09/27",2422			"instances": [2423			  "Ocarinabot"2424			]2425		  }2426		,2427		  {2428			"pattern": "epicbot",2429			"addition_date": "2017/10/31",2430			"url": "http://www.epictions.com/epicbot",2431			"instances": [2432			  "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"2433			]2434		  }2435		,2436		  {2437			"pattern": "Primalbot",2438			"addition_date": "2017/09/27",2439			"url": "https://www.primal.com",2440			"instances": [2441			  "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"2442			]2443		  }2444		,2445		  {2446			"pattern": "DuckDuckGo-Favicons-Bot",2447			"addition_date": "2017/10/06",2448			"url": "http://duckduckgo.com",2449			"instances": [2450			  "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"2451			]2452		  }2453		,2454		  {2455			"pattern": "GnowitNewsbot",2456			"addition_date": "2017/10/30",2457			"url": "http://www.gnowit.com",2458			"instances": [2459			  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"2460			]2461		  }2462		,2463		  {2464			"pattern": "Leikibot",2465			"addition_date": "2017/09/24",2466			"url": "http://www.leiki.com",2467			"instances": [2468			  "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"2469			]2470		  }2471		,2472		  {2473			"pattern": "LinkArchiver",2474			"addition_date": "2017/09/24",2475			"instances": [2476			  "@LinkArchiver twitter bot"2477			]2478		  }2479		,2480		  {2481			"pattern": "YaK\\/",2482			"addition_date": "2017/09/25",2483			"url": "http://linkfluence.com",2484			"instances": [2485			  "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"2486			]2487		  }2488		,2489		  {2490			"pattern": "PaperLiBot",2491			"addition_date": "2017/09/25",2492			"url": "http://support.paper.li/entries/20023257-what-is-paper-li",2493			"instances": [2494			  "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",2495			  "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"2496		2497			]2498		  }2499		,2500		  {2501			"pattern": "Digg Deeper",2502			"addition_date": "2017/09/26",2503			"url": "http://digg.com/about",2504			"instances": [2505			  "Digg Deeper/v1 (http://digg.com/about)"2506			]2507		  }2508		,2509		  {2510			"pattern": "dcrawl",2511			"addition_date": "2017/09/22",2512			"instances": [2513			  "dcrawl/1.0"2514			]2515		  }2516		,2517		  {2518			"pattern": "Snacktory",2519			"addition_date": "2017/09/23",2520			"url": "https://github.com/karussell/snacktory",2521			"instances": [2522			  "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"2523			]2524		  }2525		,2526		  {2527			"pattern": "AndersPinkBot",2528			"addition_date": "2017/09/24",2529			"url": "http://anderspink.com/bot.html",2530			"instances": [2531			  "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"2532			]2533		  }2534		,2535		  {2536			"pattern": "Fyrebot",2537			"addition_date": "2017/09/22",2538			"instances": [2539			  "Fyrebot/1.0"2540			]2541		  }2542		,2543		  {2544			"pattern": "EveryoneSocialBot",2545			"addition_date": "2017/09/22",2546			"url": "http://everyonesocial.com",2547			"instances": [2548			  "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"2549			]2550		  }2551		,2552		  {2553			"pattern": "Mediatoolkitbot",2554			"addition_date": "2017/10/06",2555			"url": "http://mediatoolkit.com",2556			"instances": [2557			  "Mediatoolkitbot (complaints@mediatoolkit.com)"2558			]2559		  }2560		,2561		  {2562			"pattern": "Luminator-robots",2563			"addition_date": "2017/09/22",2564			"instances": [2565			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"2566			]2567		  }2568		,2569		  {2570			"pattern": "ExtLinksBot",2571			"addition_date": "2017/11/02",2572			"url": "https://extlinks.com/Bot.html",2573			"instances": [2574			  "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"2575			]2576		  }2577		,2578		  {2579			"pattern": "SurveyBot",2580			"addition_date": "2017/11/02",2581			"instances": [2582			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"2583			]2584		  }2585		,2586		  {2587			"pattern": "NING\\/",2588			"addition_date": "2017/11/02",2589			"instances": [2590			  "NING/1.0"2591			]2592		  }2593		,2594		  {2595			"pattern": "okhttp",2596			"addition_date": "2017/11/02",2597			"instances": [2598			  "okhttp/2.5.0",2599			  "okhttp/2.7.5",2600			  "okhttp/3.2.0",2601			  "okhttp/3.5.0",2602			  "okhttp/4.1.0"2603			]2604		  }2605		,2606		  {2607			"pattern": "Nuzzel",2608			"addition_date": "2017/11/02",2609			"instances": [2610			  "Nuzzel"2611			]2612		  }2613		,2614		  {2615			"pattern": "omgili",2616			"addition_date": "2017/11/02",2617			"url": "http://omgili.com",2618			"instances": [2619			  "omgili/0.5 +http://omgili.com"2620			]2621		  }2622		,2623		  {2624			"pattern": "PocketParser",2625			"addition_date": "2017/11/02",2626			"url": "https://getpocket.com/pocketparser_ua",2627			"instances": [2628			  "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"2629			]2630		  }2631		,2632		  {2633			"pattern": "YisouSpider",2634			"addition_date": "2017/11/02",2635			"instances": [2636			  "YisouSpider",2637			  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36"2638			]2639		  }2640		,2641		  {2642			"pattern": "um-LN",2643			"addition_date": "2017/11/02",2644			"instances": [2645			  "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"2646			]2647		  }2648		,2649		  {2650			"pattern": "ToutiaoSpider",2651			"addition_date": "2017/11/02",2652			"url": "http://web.toutiao.com/media_cooperation/",2653			"instances": [2654			  "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"2655			]2656		  }2657		,2658		  {2659			"pattern": "MuckRack",2660			"addition_date": "2017/11/02",2661			"url": "http://muckrack.com",2662			"instances": [2663			  "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"2664			]2665		  }2666		,2667		  {2668			"pattern": "Jamie's Spider",2669			"addition_date": "2017/11/02",2670			"url": "http://jamiembrown.com/",2671			"instances": [2672			  "Jamie's Spider (http://jamiembrown.com/)"2673			]2674		  }2675		,2676		  {2677			"pattern": "AHC\\/",2678			"addition_date": "2017/11/02",2679			"instances": [2680			  "AHC/2.0"2681			]2682		  }2683		,2684		  {2685			"pattern": "NetcraftSurveyAgent",2686			"addition_date": "2017/11/02",2687			"instances": [2688			  "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"2689			]2690		  }2691		,2692		  {2693			"pattern": "Laserlikebot",2694			"addition_date": "2017/11/02",2695			"instances": [2696			  "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"2697			]2698		  }2699		,2700		  {2701			"pattern": "^Apache-HttpClient",2702			"addition_date": "2017/11/02",2703			"instances": [2704			  "Apache-HttpClient/4.2.3 (java 1.5)",2705			  "Apache-HttpClient/4.2.5 (java 1.5)",2706			  "Apache-HttpClient/4.3.1 (java 1.5)",2707			  "Apache-HttpClient/4.3.3 (java 1.5)",2708			  "Apache-HttpClient/4.3.5 (java 1.5)",2709			  "Apache-HttpClient/4.4.1 (Java/1.8.0_65)",2710			  "Apache-HttpClient/4.5.2 (Java/1.8.0_65)",2711			  "Apache-HttpClient/4.5.2 (Java/1.8.0_151)",2712			  "Apache-HttpClient/4.5.2 (Java/1.8.0_161)",2713			  "Apache-HttpClient/4.5.2 (Java/1.8.0_181)",2714			  "Apache-HttpClient/4.5.3 (Java/1.8.0_121)",2715			  "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)",2716			  "Apache-HttpClient/4.5.7 (Java/11.0.3)",2717			  "Apache-HttpClient/4.5.10 (Java/1.8.0_201)"2718			]2719		  }2720		,2721		  {2722			"pattern": "AppEngine-Google",2723			"addition_date": "2017/11/02",2724			"instances": [2725			  "AppEngine-Google; (+http://code.google.com/appengine; appid: example)",2726			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)"2727			]2728		  }2729		,2730		  {2731			"pattern": "Jetty",2732			"addition_date": "2017/11/02",2733			"instances": [2734			  "Jetty/9.3.z-SNAPSHOT"2735			]2736		  }2737		,2738		  {2739			"pattern": "Upflow",2740			"addition_date": "2017/11/02",2741			"instances": [2742			  "Upflow/1.0"2743			]2744		  }2745		,2746		  {2747			"pattern": "Thinklab",2748			"addition_date": "2017/11/02",2749			"url": "thinklab.com",2750			"instances": [2751			  "Thinklab (thinklab.com)"2752			]2753		  }2754		,2755		  {2756			"pattern": "Traackr.com",2757			"addition_date": "2017/11/02",2758			"url": "Traackr.com",2759			"instances": [2760			  "Traackr.com"2761			]2762		  }2763		,2764		  {2765			"pattern": "Twurly",2766			"addition_date": "2017/11/02",2767			"url": "http://twurly.org",2768			"instances": [2769			  "Ruby, Twurly v1.1 (http://twurly.org)"2770			]2771		  }2772		,2773		  {2774			"pattern": "Mastodon",2775			"addition_date": "2017/11/02",2776			"instances": [2777			  "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"2778			]2779		  }2780		,2781		  {2782			"pattern": "http_get",2783			"addition_date": "2017/11/02",2784			"instances": [2785			  "http_get"2786			]2787		  }2788		,2789		  {2790			"pattern": "DnyzBot",2791			"addition_date": "2017/11/20",2792			"instances": [2793			  "Mozilla/5.0 (compatible; DnyzBot/1.0)"2794			]2795		  }2796		,2797		  {2798			"pattern": "botify",2799			"addition_date": "2018/02/01",2800			"instances": [2801			  "Mozilla/5.0 (compatible; botify; http://botify.com)"2802			]2803		  }2804		,2805		  {2806			"pattern": "007ac9 Crawler",2807			"addition_date": "2018/02/09",2808			"instances": [2809			  "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)"2810			]2811		  }2812		,2813		  {2814			"pattern": "BehloolBot",2815			"addition_date": "2018/02/09",2816			"instances": [2817			  "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)"2818			]2819		  }2820		,2821		  {2822			"pattern": "BrandVerity",2823			"addition_date": "2018/02/27",2824			"instances": [2825			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)",2826			  "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)"2827			],2828			"url": "http://www.brandverity.com/why-is-brandverity-visiting-me"2829		  }2830		,2831		  {2832			"pattern": "check_http",2833			"addition_date": "2018/02/09",2834			"instances": [2835			  "check_http/v2.2.1 (nagios-plugins 2.2.1)"2836			]2837		  }2838		,2839		  {2840			"pattern": "BDCbot",2841			"addition_date": "2018/02/09",2842			"instances": [2843			  "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36",2844			  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"2845			]2846		  }2847		,2848		  {2849			"pattern": "ZumBot",2850			"addition_date": "2018/02/09",2851			"instances": [2852			  "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)"2853			]2854		  }2855		,2856		  {2857			"pattern": "EZID",2858			"addition_date": "2018/02/09",2859			"instances": [2860			  "EZID (EZID link checker; https://ezid.cdlib.org/)"2861			]2862		  }2863		,2864		  {2865			"pattern": "ICC-Crawler",2866			"addition_date": "2018/02/28",2867			"instances": [2868			  "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)"2869			],2870			"url": "http://ucri.nict.go.jp/en/icccrawler.html"2871		  }2872		,2873		  {2874			"pattern": "ArchiveBot",2875			"addition_date": "2018/02/28",2876			"instances": [2877			  "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)"2878			],2879			"url": "https://github.com/ArchiveTeam/ArchiveBot"2880		  }2881		,2882		  {2883			"pattern": "^LCC ",2884			"addition_date": "2018/02/28",2885			"instances": [2886			  "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)"2887			],2888			"url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html"2889		  }2890		,2891		  {2892			"pattern": "filterdb.iss.net\\/crawler",2893			"addition_date": "2018/03/16",2894			"instances": [2895			  "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)"2896			],2897			"url": "http://filterdb.iss.net/crawler/"2898		  }2899		,2900		  {2901			"pattern": "BLP_bbot",2902			"addition_date": "2018/03/27",2903			"instances": [2904			  "BLP_bbot/0.1"2905			]2906		  }2907		,2908		  {2909			"pattern": "BomboraBot",2910			"addition_date": "2018/03/27",2911			"instances": [2912			  "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)"2913			],2914			"url": "http://www.bombora.com/bot"2915		  }2916		,2917		  {2918			"pattern": "Buck\\/",2919			"addition_date": "2018/03/27",2920			"instances": [2921			  "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)"2922			],2923			"url": "https://app.hypefactors.com/media-monitoring/about.html"2924		  }2925		,2926		  {2927			"pattern": "Companybook-Crawler",2928			"addition_date": "2018/03/27",2929			"instances": [2930			  "Companybook-Crawler (+https://www.companybooknetworking.com/)"2931			],2932			"url": "https://www.companybooknetworking.com/"2933		  }2934		,2935		  {2936			"pattern": "Genieo",2937			"addition_date": "2018/03/27",2938			"instances": [2939			  "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)"2940			],2941			"url": "http://www.genieo.com/webfilter.html"2942		  }2943		,2944		  {2945			"pattern": "magpie-crawler",2946			"addition_date": "2018/03/27",2947			"instances": [2948			  "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)"2949			],2950			"url": "http://www.brandwatch.net"2951		  }2952		,2953		  {2954			"pattern": "MeltwaterNews",2955			"addition_date": "2018/03/27",2956			"instances": [2957			  "MeltwaterNews www.meltwater.com"2958			],2959			"url": "http://www.meltwater.com"2960		  }2961		,2962		  {2963			"pattern": "Moreover",2964			"addition_date": "2018/03/27",2965			"instances": [2966			  "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)"2967			],2968			"url": "http://www.moreover.com"2969		  }2970		,2971		  {2972			"pattern": "newspaper\\/",2973			"addition_date": "2018/03/27",2974			"instances": [2975			  "newspaper/0.1.0.7",2976			  "newspaper/0.2.5",2977			  "newspaper/0.2.6",2978			  "newspaper/0.2.8"2979			]2980		  }2981		,2982		  {2983			"pattern": "ScoutJet",2984			"addition_date": "2018/03/27",2985			"instances": [2986			  "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)"2987			],2988			"url": "http://www.scoutjet.com/"2989		  }2990		,2991		  {2992			"pattern": "(^| )sentry\\/",2993			"addition_date": "2018/03/27",2994			"instances": [2995			  "sentry/8.22.0 (https://sentry.io)"2996			],2997			"url": "https://sentry.io"2998		  }2999		,3000		  {3001			"pattern": "StorygizeBot",3002			"addition_date": "2018/03/27",3003			"instances": [3004			  "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)"3005			],3006			"url": "http://www.storygize.com"3007		  }3008		,3009		  {3010			"pattern": "UptimeRobot",3011			"addition_date": "2018/03/27",3012			"instances": [3013			  "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"3014			],3015			"url": "http://www.uptimerobot.com/"3016		  }3017		,3018		  {3019			"pattern": "OutclicksBot",3020			"addition_date": "2018/04/21",3021			"instances": [3022			  "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR",3023			  "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe",3024			  "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J",3025			  "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h"3026			],3027			"url": "https://www.outclicks.net"3028		  }3029		,3030		  {3031			"pattern": "seoscanners",3032			"addition_date": "2018/05/27",3033			"instances": [3034			  "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)"3035			],3036			"url": "http://www.seoscanners.net/"3037		  }3038		,3039		  {3040			"pattern": "Hatena",3041			"addition_date": "2018/05/29",3042			"instances": [3043			  "Hatena Antenna/0.3",3044			  "Hatena::Russia::Crawler/0.01",3045			  "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)",3046			  "Hatena::Scissors/0.01",3047			  "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)",3048			  "Hatena::Fetcher/0.01 (master) Furl/3.13"3049			]3050		  }3051		,3052		  {3053			"pattern": "Google Web Preview",3054			"addition_date": "2018/05/31",3055			"instances": [3056			  "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36",3057			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36"3058			]3059		  }3060		,3061		  {3062			"pattern": "MauiBot",3063			"addition_date": "2018/06/06",3064			"instances": [3065			  "MauiBot (crawler.feedback+wc@gmail.com)"3066			]3067		  }3068		,3069		  {3070			"pattern": "AlphaBot",3071			"addition_date": "2018/05/27",3072			"instances": [3073			  "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)"3074			],3075			"url": "http://alphaseobot.com/bot.html"3076		  }3077		,3078		  {3079			"pattern": "SBL-BOT",3080			"addition_date": "2018/06/06",3081			"instances": [3082			  "SBL-BOT (http://sbl.net)"3083			],3084			"url": "http://sbl.net",3085			"description" : "Bot of SoftByte BlackWidow"3086		  }3087		,3088		  {3089			"pattern": "IAS crawler",3090			"addition_date": "2018/06/06",3091			"instances": [3092			  "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)"3093			],3094			"url": "http://integralads.com/site-indexing-policy/",3095			"description" : "Bot of Integral Ad Science, Inc."3096		  }3097		,3098		  {3099			"pattern": "adscanner",3100			"addition_date": "2018/06/24",3101			"instances": [3102			  "Mozilla/5.0 (compatible; adscanner/)"3103			]3104		  }3105		,3106		  {3107			"pattern": "Netvibes",3108			"addition_date": "2018/06/24",3109			"instances": [3110			  "Netvibes (crawler/bot; http://www.netvibes.com",3111			  "Netvibes (crawler; http://www.netvibes.com)"3112			],3113			"url": "http://www.netvibes.com"3114		  }3115		,3116		  {3117			"pattern": "acapbot",3118			"addition_date": "2018/06/27",3119			"instances": [3120			  "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",3121			  "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)"3122			]3123		  }3124		,3125		  {3126			"pattern": "Baidu-YunGuanCe",3127			"addition_date": "2018/06/27",3128			"instances": [3129			  "Baidu-YunGuanCe-Bot(ce.baidu.com)",3130			  "Baidu-YunGuanCe-SLABot(ce.baidu.com)",3131			  "Baidu-YunGuanCe-ScanBot(ce.baidu.com)",3132			  "Baidu-YunGuanCe-PerfBot(ce.baidu.com)",3133			  "Baidu-YunGuanCe-VSBot(ce.baidu.com)"3134			],3135			"url": "https://ce.baidu.com/topic/topic20150908",3136			"description": "Baidu Cloud Watch"3137		  }3138		,3139		  {3140			"pattern": "bitlybot",3141			"addition_date": "2018/06/27",3142			"instances": [3143			  "bitlybot/3.0 (+http://bit.ly/)",3144			  "bitlybot/2.0",3145			  "bitlybot"3146			],3147			"url": "http://bit.ly/"3148		  }3149		,3150		  {3151			"pattern": "blogmuraBot",3152			"addition_date": "2018/06/27",3153			"instances": [3154			  "blogmuraBot (+http://www.blogmura.com)"3155			],3156			"url": "http://www.blogmura.com",3157			"description": "A blog ranking site which links to blogs on just about every theme possible."3158		  }3159		,3160		  {3161			"pattern": "Bot.AraTurka.com",3162			"addition_date": "2018/06/27",3163			"instances": [3164			  "Bot.AraTurka.com/0.0.1"3165			],3166			"url": "http://www.araturka.com"3167		  }3168		,3169		  {3170			"pattern": "bot-pge.chlooe.com",3171			"addition_date": "2018/06/27",3172			"instances": [3173			  "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)"3174			]3175		  }3176		,3177		  {3178			"pattern": "BoxcarBot",3179			"addition_date": "2018/06/27",3180			"instances": [3181			  "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)"3182			],3183			"url": "https://boxcar.io/"3184		  }3185		,3186		  {3187			"pattern": "BTWebClient",3188			"addition_date": "2018/06/27",3189			"instances": [3190			  "BTWebClient/180B(9704)"3191			],3192			"url": "http://www.utorrent.com/",3193			"description": "ÂµTorrent BitTorrent Client"3194		  }3195		,3196		  {3197			"pattern": "ContextAd Bot",3198			"addition_date": "2018/06/27",3199			"instances": [3200			  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)",3201			  "ContextAd Bot 1.0"3202			]3203		  }3204		,3205		  {3206			"pattern": "Digincore bot",3207			"addition_date": "2018/06/27",3208			"instances": [3209			  "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)"3210			],3211			"url": "http://www.digincore.com/crawler.html"3212		  }3213		,3214		  {3215			"pattern": "Disqus",3216			"addition_date": "2018/06/27",3217			"instances": [3218			  "Disqus/1.0"3219			],3220			"url": "https://disqus.com/",3221			"description": "validate and quality check pages."3222		  }3223		,3224		  {3225			"pattern": "Feedly",3226			"addition_date": "2018/06/27",3227			"instances": [3228			  "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)",3229			  "FeedlyBot/1.0 (http://feedly.com)"3230			],3231			"url": "https://www.feedly.com/fetcher.html",3232			"description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud."3233		  }3234		,3235		  {3236			"pattern": "Fetch\\/",3237			"addition_date": "2018/06/27",3238			"instances": [3239			  "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)"3240			]3241		  }3242		,3243		  {3244			"pattern": "Fever",3245			"addition_date": "2018/06/27",3246			"instances": [3247			  "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)"3248			],3249			"url": "http://feedafever.com"3250		  }3251		,3252		  {3253			"pattern": "Flamingo_SearchEngine",3254			"addition_date": "2018/06/27",3255			"instances": [3256			  "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)"3257			]3258		  }3259		,3260		  {3261			"pattern": "FlipboardProxy",3262			"addition_date": "2018/06/27",3263			"instances": [3264			  "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3265			  "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)",3266			  "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3267			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3268			  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)"3269			],3270			"url": "https://about.flipboard.com/browserproxy/",3271			"description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application"3272		  }3273		,3274		  {3275			"pattern": "g2reader-bot",3276			"addition_date": "2018/06/27",3277			"instances": [3278			  "g2reader-bot/1.0 (+http://www.g2reader.com/)"3279			],3280			"url": "http://www.g2reader.com/"3281		  }3282		,3283		  {3284			"pattern": "G2 Web Services",3285			"addition_date": "2019/03/01",3286			"instances": [3287			  "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)"3288			],3289			"url": "https://www.g2webservices.com/"3290		  }3291		,3292		  {3293			"pattern": "imrbot",3294			"addition_date": "2018/06/27",3295			"instances": [3296			  "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)"3297			],3298			"url": "http://www.mignify.com"3299		  }3300		,3301		  {3302			"pattern": "K7MLWCBot",3303			"addition_date": "2018/06/27",3304			"instances": [3305			  "K7MLWCBot/1.0 (+http://www.k7computing.com)"3306			],3307			"url": "http://www.k7computing.com",3308			"description": "Virus scanner"3309		  }3310		,3311		  {3312			"pattern": "Kemvibot",3313			"addition_date": "2018/06/27",3314			"instances": [3315			  "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)"3316			],3317			"url": "http://kemvi.com"3318		  }3319		,3320		  {3321			"pattern": "Landau-Media-Spider",3322			"addition_date": "2018/06/27",3323			"instances": [3324			  "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)"3325			],3326			"url": "http://bots.landaumedia.de/bot.html"3327		  }3328		,3329		  {3330			"pattern": "linkapediabot",3331			"addition_date": "2018/06/27",3332			"instances": [3333			  "linkapediabot (+http://www.linkapedia.com)"3334			],3335			"url": "http://www.linkapedia.com"3336		  }3337		,3338		  {3339			"pattern": "vkShare",3340			"addition_date": "2018/07/02",3341			"instances": [3342			  "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)"3343			],3344			"url": "http://vk.com/dev/Share"3345		  }3346		,3347		  {3348			"pattern": "Siteimprove.com",3349			"addition_date": "2018/06/22",3350			"instances": [3351			  "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com",3352			  "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com",3353			  "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com",3354			  "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com"3355			]3356		  }3357		,3358		  {3359			"pattern": "BLEXBot\\/",3360			"addition_date": "2018/07/07",3361			"instances": [3362			  "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"3363			],3364			"url": "http://webmeup-crawler.com"3365		  }3366		,3367		  {3368			"pattern": "DareBoost",3369			"addition_date": "2018/07/07",3370			"instances": [3371			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost"3372			],3373			"url": "https://www.dareboost.com/",3374			"description": "Bot to test, Analyze and Optimize website"3375		  }3376		,3377		  {3378			"pattern": "ZuperlistBot\\/",3379			"addition_date": "2018/07/07",3380			"instances": [3381			  "Mozilla/5.0 (compatible; ZuperlistBot/1.0)"3382			]3383		  }3384		,3385		  {3386			"pattern": "Miniflux\\/",3387			"addition_date": "2018/07/07",3388			"instances": [3389			  "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)",3390			  "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)",3391			  "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)",3392			  "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)",3393			  "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)",3394			  "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)",3395			  "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)",3396			  "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)",3397			  "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)"3398			],3399			"url": "https://miniflux.net",3400			"description": "Miniflux is a minimalist and opinionated feed reader."3401		  }3402		,3403		  {3404			"pattern": "Feedspot",3405			"addition_date": "2018/07/07",3406			"instances": [3407			  "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)",3408			  "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"3409			],3410			"url": "http://www.feedspot.com/fs/bot"3411		  }3412		,3413		  {3414			"pattern": "Diffbot\\/",3415			"addition_date": "2018/07/07",3416			"instances": [3417			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)"3418			],3419			"url": "http://www.diffbot.com"3420		  }3421		,3422		  {3423			"pattern": "SEOkicks",3424			"addition_date": "2018/08/22",3425			"instances": [3426			  "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)"3427			],3428			"url": "https://www.seokicks.de/robot.html"3429		  }3430		,3431		  {3432			"pattern": "tracemyfile",3433			"addition_date": "2018/08/23",3434			"instances": [3435			  "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)"3436			]3437		  }3438		,3439		  {3440			"pattern": "Nimbostratus-Bot",3441			"addition_date": "2018/08/29",3442			"instances": [3443			  "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)"3444			]3445		  }3446		,3447		  {3448			"pattern": "zgrab",3449			"addition_date": "2018/08/30",3450			"instances": [3451			  "Mozilla/5.0 zgrab/0.x"3452			],3453			"url": "https://zmap.io/"3454		  }3455		,3456		  {3457			"pattern": "PR-CY.RU",3458			"addition_date": "2018/08/30",3459			"instances": [3460			  "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)"3461			],3462			"url": "https://a.pr-cy.ru/"3463		  }3464		,3465		  {3466			"pattern": "AdsTxtCrawler",3467			"addition_date": "2018/08/30",3468			"instances": [3469			  "AdsTxtCrawler/1.0"3470			]3471		  },3472		  {3473			"pattern": "Datafeedwatch",3474			"addition_date": "2018/09/05",3475			"instances": [3476			  "Datafeedwatch/2.1.x"3477			],3478			"url": "https://www.datafeedwatch.com/"3479		  }3480		,3481		  {3482			"pattern": "Zabbix",3483			"addition_date": "2018/09/05",3484			"instances": [3485			  "Zabbix"3486			],3487			"url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring"3488		  }3489		,3490		  {3491			"pattern": "TangibleeBot",3492			"addition_date": "2018/09/05",3493			"instances": [3494			  "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)"3495			],3496			"url": "http://tangiblee.com/bot"3497		  }3498		,3499		  {3500			"pattern": "google-xrawler",3501			"addition_date": "2018/09/05",3502			"instances": [3503			  "google-xrawler"3504			],3505			"url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for"3506		  }3507		,3508		  {3509			"pattern": "axios",3510			"addition_date": "2018/09/06",3511			"instances": [3512			  "axios/0.18.0",3513			  "axios/0.19.0"3514			],3515			"url": "https://github.com/axios/axios"3516		  }3517		,3518		  {3519			"pattern": "Amazon CloudFront",3520			"addition_date": "2018/09/07",3521			"instances": [3522			  "Amazon CloudFront"3523			],3524			"url": "https://aws.amazon.com/cloudfront/"3525		  }3526		,3527		  {3528			"pattern": "Pulsepoint",3529			"addition_date": "2018/09/24",3530			"instances": [3531			  "Pulsepoint XT3 web scraper"3532			]3533		  }3534		,3535		  {3536			"pattern": "CloudFlare-AlwaysOnline",3537			"addition_date": "2018/09/27",3538			"instances": [3539			  "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34",3540			  "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34"3541			],3542			"url" : "https://www.cloudflare.com/always-online/"3543		  }3544		,3545		  {3546			"pattern": "Google-Structured-Data-Testing-Tool",3547			"addition_date": "2018/10/02",3548			"instances": [3549			  "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)",3550			  "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)"3551			],3552			"url": "https://search.google.com/structured-data/testing-tool"3553		  }3554		,3555		  {3556			"pattern": "WordupInfoSearch",3557			"addition_date": "2018/10/07",3558			"instances": [3559			  "WordupInfoSearch/1.0"3560			]3561		  }3562		,3563		  {3564			"pattern": "WebDataStats",3565			"addition_date": "2018/10/08",3566			"instances": [3567			  "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)"3568			],3569			"url": "https://webdatastats.com/"3570		  }3571		,3572		  {3573			"pattern": "HttpUrlConnection",3574			"addition_date": "2018/10/08",3575			"instances": [3576			  "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)"3577			]3578		  }3579		,3580		  {3581			"pattern": "Seekport Crawler",3582			"addition_date": "2018/10/08",3583			"instances": [3584			  "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)"3585			],3586			"url": "http://seekport.com/"3587		  }3588		,3589		  {3590			"pattern": "ZoomBot",3591			"addition_date": "2018/10/10",3592			"instances": [3593			  "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)"3594			],3595			"url": "http://suite.seozoom.it/bot.html"3596		  }3597		,3598		  {3599			"pattern": "VelenPublicWebCrawler",3600			"addition_date": "2018/10/09",3601			"instances": [3602			  "VelenPublicWebCrawler (velen.io)"3603			]3604		  }3605		,3606		  {3607			"pattern": "MoodleBot",3608			"addition_date": "2018/10/10",3609			"instances": [3610			  "MoodleBot/1.0"3611			]3612		  }3613		,3614		  {3615			"pattern": "jpg-newsbot",3616			"addition_date": "2018/10/10",3617			"instances": [3618			  "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)"3619			],3620			"url": "https://vipnytt.no/bots/"3621		  }3622		,3623		  {3624			"pattern": "outbrain",3625			"addition_date": "2018/10/14",3626			"instances": [3627			  "Mozilla/5.0 (Java) outbrain"3628			],3629			"url": "https://www.outbrain.com/help/advertisers/invalid-url/"3630		  }3631		,3632		  {3633			"pattern": "W3C_Validator",3634			"addition_date": "2018/10/14",3635			"instances": [3636			  "W3C_Validator/1.3"3637			],3638			"url": "https://validator.w3.org/services"3639		  }3640		,3641		  {3642			"pattern": "Validator\\.nu",3643			"addition_date": "2018/10/14",3644			"instances": [3645			  "Validator.nu/LV"3646			],3647			"url": "https://validator.w3.org/services"3648		  }3649		,3650		  {3651			"pattern": "W3C-checklink",3652			"addition_date": "2018/10/14",3653			"depends_on": ["libwww-perl"],3654			"instances": [3655			  "W3C-checklink/2.90 libwww-perl/5.64",3656			  "W3C-checklink/3.6.2.3 libwww-perl/5.64",3657			  "W3C-checklink/4.2 [4.20] libwww-perl/5.803",3658			  "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803",3659			  "W3C-checklink/4.3 [4.42] libwww-perl/5.805",3660			  "W3C-checklink/4.3 [4.42] libwww-perl/5.808",3661			  "W3C-checklink/4.3 [4.42] libwww-perl/5.820",3662			  "W3C-checklink/4.5 [4.154] libwww-perl/5.823",3663			  "W3C-checklink/4.5 [4.160] libwww-perl/5.823"3664			],3665			"url": "https://validator.w3.org/services"3666		  }3667		,3668		  {3669			"pattern": "W3C-mobileOK",3670			"addition_date": "2018/10/14",3671			"instances": [3672			  "W3C-mobileOK/DDC-1.0"3673			],3674			"url": "https://validator.w3.org/services"3675		  }3676		,3677		  {3678			"pattern": "W3C_I18n-Checker",3679			"addition_date": "2018/10/14",3680			"instances": [3681			  "W3C_I18n-Checker/1.0"3682			],3683			"url": "https://validator.w3.org/services"3684		  }3685		,3686		  {3687			"pattern": "FeedValidator",3688			"addition_date": "2018/10/14",3689			"instances": [3690			  "FeedValidator/1.3"3691			],3692			"url": "https://validator.w3.org/services"3693		  }3694		,3695		  {3696			"pattern": "W3C_CSS_Validator",3697			"addition_date": "2018/10/14",3698			"instances": [3699			  "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0"3700			],3701			"url": "https://validator.w3.org/services"3702		  }3703		,3704		  {3705			"pattern": "W3C_Unicorn",3706			"addition_date": "2018/10/14",3707			"instances": [3708			  "W3C_Unicorn/1.0"3709			],3710			"url": "https://validator.w3.org/services"3711		  }3712		,3713		  {3714			"pattern": "Google-PhysicalWeb",3715			"addition_date": "2018/10/21",3716			"instances": [3717			  "Mozilla/5.0 (Google-PhysicalWeb)"3718			]3719		  }3720		,3721		  {3722			"pattern": "Blackboard",3723			"addition_date": "2018/10/28",3724			"instances": [3725			  "Blackboard Safeassign"3726			],3727			"url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign"3728		  },3729		  {3730			"pattern": "ICBot\\/",3731			"addition_date": "2018/10/23",3732			"instances": [3733			  "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz"3734			],3735			"url": "https://ideasandcode.xyz"3736		  },3737		  {3738			"pattern": "BazQux",3739			"addition_date": "2018/10/23",3740			"instances": [3741			  "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)"3742			],3743			"url": "https://bazqux.com/fetcher"3744		  },3745		  {3746			"pattern": "Twingly",3747			"addition_date": "2018/10/23",3748			"instances": [3749			  "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"3750			],3751			"url": "https://twingly.com"3752		  },3753		  {3754			"pattern": "Rivva",3755			"addition_date": "2018/10/23",3756			"instances": [3757			  "Mozilla/5.0 (compatible; Rivva; http://rivva.de)"3758			],3759			"url": "http://rivva.de"3760		  },3761		  {3762			"pattern": "Experibot",3763			"addition_date": "2018/11/03",3764			"instances": [3765			  "Experibot-v2 http://goo.gl/ZAr8wX",3766			  "Experibot-v3 http://goo.gl/ZAr8wX"3767			],3768			"url": "https://amirkr.wixsite.com/experibot"3769		  },3770		  {3771			"pattern": "awesomecrawler",3772			"addition_date": "2018/11/24",3773			"instances": [3774			  "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler"3775			]3776		  },3777		  {3778			"pattern": "Dataprovider.com",3779			"addition_date": "2018/11/24",3780			"instances": [3781			  "Mozilla/5.0 (compatible; Dataprovider.com)"3782			],3783			"url": "https://www.dataprovider.com/"3784		  },3785		  {3786			"pattern": "GroupHigh\\/",3787			"addition_date": "2018/11/24",3788			"instances": [3789			  "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/"3790			],3791			"url": "http://www.grouphigh.com/"3792		  },3793		  {3794			"pattern": "theoldreader.com",3795			"addition_date": "2018/12/02",3796			"instances": [3797			  "Mozilla/5.0 (compatible; theoldreader.com)"3798			],3799			"url": "https://www.theoldreader.com/"3800		  }3801		,3802		  {3803			"pattern": "AnyEvent",3804			"addition_date": "2018/12/07",3805			"instances": [3806			  "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)"3807			],3808			"url": "http://software.schmorp.de/pkg/AnyEvent.html"3809		  }3810		,3811		  {3812			"pattern": "Uptimebot\\.org",3813			"addition_date": "2019/01/17",3814			"instances": [3815			  "Uptimebot.org - Free website monitoring"3816			],3817			"url": "http://uptimebot.org/"3818		  }3819		,3820		  {3821			"pattern": "Nmap Scripting Engine",3822			"addition_date": "2019/02/04",3823			"instances": [3824			  "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"3825			],3826			"url": "https://nmap.org/book/nse.html"3827		  }3828		,3829		  {3830			"pattern": "2ip.ru",3831			"addition_date": "2019/02/12",3832			"instances": [3833			  "2ip.ru CMS Detector (https://2ip.ru/cms/)"3834			],3835			"url": "https://2ip.ru/cms/"3836		  },3837		  {3838			"pattern": "Clickagy",3839			"addition_date": "2019/02/19",3840			"instances": [3841			  "Clickagy Intelligence Bot v2"3842			],3843			"url": "https://www.clickagy.com"3844		  },3845		  {3846			"pattern": "Caliperbot",3847			"addition_date": "2019/03/02",3848			"instances": [3849			  "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)"3850			],3851			"url": "http://www.conductor.com/caliperbot"3852		  },3853		  {3854			"pattern": "MBCrawler",3855			"addition_date": "2019/03/02",3856			"instances": [3857			  "MBCrawler/1.0 (https://monitorbacklinks.com)"3858			],3859			"url": "https://monitorbacklinks.com"3860		  },3861		  {3862			"pattern": "online-webceo-bot",3863			"addition_date": "2019/03/02",3864			"instances": [3865			  "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)"3866			],3867			"url": "http://online.webceo.com"3868		  },3869		  {3870			"pattern": "B2B Bot",3871			"addition_date": "2019/03/02",3872			"instances": [3873			  "B2B Bot"3874			]3875		  },3876		  {3877			"pattern": "AddSearchBot",3878			"addition_date": "2019/03/02",3879			"instances": [3880			  "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)"3881			],3882			"url": "http://www.addsearch.com/bot"3883		  },3884		  {3885			"pattern": "Google Favicon",3886			"addition_date": "2019/03/14",3887			"instances": [3888			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon"3889			]3890		  },3891		  {3892			"pattern": "HubSpot",3893			"addition_date": "2019/04/15",3894			"instances": [3895			  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com",3896			  "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",3897			  "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)",3898			  "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit"3899			]3900		  },3901		  {3902			"pattern": "Chrome-Lighthouse",3903			"addition_date": "2019/03/15",3904			"instances": [3905			  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse",3906			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse",3907			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse",3908			  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse"3909			],3910			"url": "https://developers.google.com/speed/pagespeed/insights"3911		  },3912		  {3913			"pattern": "HeadlessChrome",3914			"url": "https://developers.google.com/web/updates/2017/04/headless-chrome",3915			"addition_date": "2019/06/17",3916			"instances": [3917			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36",3918			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36",3919			  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36"3920			]3921		  },3922		  {3923			"pattern": "CheckMarkNetwork\\/",3924			"addition_date": "2019/06/30",3925			"instances": [3926			  "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)"3927			],3928			"url": "https://www.checkmarknetwork.com/"3929		  },3930		  {3931			"pattern": "www\\.uptime\\.com",3932			"addition_date": "2019/07/21",3933			"instances": [3934			  "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)"3935			],3936			"url": "http://www.uptime.com/uptimebot"3937		  }3938		,3939		  {3940			"pattern": "Streamline3Bot\\/",3941			"addition_date": "2019/07/21",3942			"instances": [3943			  "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0",3944			  "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0"3945			],3946			"url": "https://www.ubtsupport.com/legal/Streamline3Bot.php"3947		  }3948		,3949		  {3950			"pattern": "serpstatbot\\/",3951			"addition_date": "2019/07/25",3952			"instances": [3953			  "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)",3954			  "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)"3955			],3956			"url": "http://serpstatbot.com"3957		  }3958		,3959		  {3960			"pattern": "MixnodeCache\\/",3961			"addition_date": "2019/08/04",3962			"instances": [3963			  "MixnodeCache/1.8(+https://cache.mixnode.com/)"3964			],3965			"url": "https://cache.mixnode.com/"3966		  }3967		,3968		  {3969			"pattern": "^curl",3970			"addition_date": "2019/08/15",3971			"instances": [3972			  "curl",3973			  "curl/7.29.0",3974			  "curl/7.47.0",3975			  "curl/7.54.0",3976			  "curl/7.55.1",3977			  "curl/7.64.0",3978			  "curl/7.64.1",3979			  "curl/7.65.3"3980			],3981			"url": "https://curl.haxx.se/"3982		  }3983		,3984		  {3985			"pattern": "SimpleScraper",3986			"addition_date": "2019/08/16",3987			"instances": [3988			  "Mozilla/5.0 (compatible; SimpleScraper)"3989			],3990			"url": "https://github.com/ramonkcom/simple-scraper/"3991		  }3992		,3993		  {3994			"pattern": "RSSingBot",3995			"addition_date": "2019/09/15",3996			"instances": [3997			  "RSSingBot (http://www.rssing.com)"3998			],3999			"url": "http://www.rssing.com"4000		  }4001		,4002		  {4003			"pattern": "Jooblebot",4004			"addition_date": "2019/09/25",4005			"instances": [4006			  "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36"4007			],4008			"url": "http://jooble.org/jooble-bot"4009		  }4010		,4011		  {4012			"pattern": "fedoraplanet",4013			"addition_date": "2019/09/28",4014			"instances": [4015			  "venus/fedoraplanet"4016			],4017			"url": "http://fedoraplanet.org/"4018		  }4019		,4020		  {4021			"pattern": "Friendica",4022			"addition_date": "2019/09/28",4023			"instances": [4024			  "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz"4025			],4026			"url": "https://hoyer.xyz"4027		  }4028		,4029		  {4030			"pattern": "NextCloud",4031			"addition_date": "2019/09/30",4032			"instances": [4033			  "NextCloud-News/1.0"4034			],4035			"url": "https://nextcloud.com/"4036		  }4037		,4038		  {4039			"pattern": "Tiny Tiny RSS",4040			"addition_date": "2019/10/04",4041			"instances": [4042			  "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)",4043			  "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)",4044			  "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)",4045			  "Tiny Tiny RSS/19.8 (http://tt-rss.org/)"4046			],4047			"url": "http://tt-rss.org/"4048		  }4049		,4050		  {4051			"pattern": "RegionStuttgartBot",4052			"addition_date": "2019/10/17",4053			"instances": [4054			  "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)"4055			],4056			"url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/"4057		  }4058		,4059		  {4060			"pattern": "Bytespider",4061			"addition_date": "2019/11/11",4062			"instances": [4063			  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider",4064			  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider",4065			  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com",4066				  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider",4067				  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider",4068				  "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider",4069				  "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider",4070				  "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider",4071				  "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider",4072				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider",4073				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider",4074				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider",4075				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider",4076				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider",4077				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider",4078				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider",4079				  "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider",4080				  "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider",4081				  "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider"4082		  	],4083			"url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent"4084		  }4085		,4086		  {4087			"pattern": "Datanyze",4088			"addition_date": "2019/11/17",4089			"instances": [4090			  "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"4091			],4092			"url": "https://www.datanyze.com/dnyzbot/"4093		  }4094		,4095		  {4096			"pattern": "Google-Site-Verification",4097			"addition_date": "2019/12/11",4098			"instances": [4099			  "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)"4100			],4101			"url": "https://support.google.com/webmasters/answer/9008080"4102		  }4103		,4104		  {4105			"pattern": "TrendsmapResolver",4106			"addition_date": "2020/02/24",4107			"instances": [4108			  "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)"4109			],4110			"url": "https://www.trendsmap.com/"4111		  }4112		,4113		  {4114			"pattern": "tweetedtimes",4115			"addition_date": "2020/02/24",4116			"instances": [4117			  "Mozilla/5.0 (compatible; +http://tweetedtimes.com)"4118			],4119			"url": "https://tweetedtimes.com/"4120		  }4121		]4122	`4123	var patterns []CrawlerPattern4124	err := json.Unmarshal([]byte(crawlers), &patterns)4125	if err != nil {4126		panic(err)4127	}4128	return patterns4129}...

tmpl_new_proj_adm_conf_secure.go

Source:tmpl_new_proj_adm_conf_secure.go

1// Copyright 2014 The goyy Authors.  All rights reserved.2// Use of this source code is governed by a MIT-style3// license that can be found in the LICENSE file.45package main67var tmplNewProjAdmConfSecure = `<?xml version="1.0" encoding="UTF-8" ?>8<!DOCTYPE env PUBLIC "-//GOYY//DTD ENV 1.0//EN" "http://goyy.org/dtd/env-1.0.dtd">9<configuration>10	<environments default="development">11		<environment id="development">12			<secure name="<%.NewProjName%>">13				<enable>true</enable>14				<login-url>/login.html</login-url>15				<forbid-url>/err/403.html</forbid-url>16				<success-url>/home.html</success-url>17				<filters>18					<intercept-url pattern="/**.(css|js|map)" access="anon"/>19					<intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>20					<intercept-url pattern="/login.html" access="anon"/>21					<intercept-url pattern="/login" access="anon"/>22					<intercept-url pattern="/signin" access="anon"/>23					<intercept-url pattern="/logout" access="anon"/>24					<intercept-url pattern="/captcha/build" access="anon"/>25					<intercept-url pattern="/err/**" access="anon"/>26					<intercept-url pattern="/**" access="authc"/>27				</filters>28			</secure>29		</environment>30		<environment id="test">31			<secure name="<%.NewProjName%>">32				<enable>true</enable>33				<login-url>/login.html</login-url>34				<forbid-url>/err/403.html</forbid-url>35				<success-url>/home.html</success-url>36				<filters>37					<intercept-url pattern="/**.(css|js|map)" access="anon"/>38					<intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>39					<intercept-url pattern="/login.html" access="anon"/>40					<intercept-url pattern="/login" access="anon"/>41					<intercept-url pattern="/signin" access="anon"/>42					<intercept-url pattern="/logout" access="anon"/>43					<intercept-url pattern="/captcha/build" access="anon"/>44					<intercept-url pattern="/err/**" access="anon"/>45					<intercept-url pattern="/**" access="authc"/>46				</filters>47			</secure>48		</environment>49		<environment id="production">50			<secure name="<%.NewProjName%>">51				<enable>true</enable>52				<login-url>/login.html</login-url>53				<forbid-url>/err/403.html</forbid-url>54				<success-url>/home.html</success-url>55				<filters>56					<intercept-url pattern="/**.(css|js|map)" access="anon"/>57					<intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>58					<intercept-url pattern="/login.html" access="anon"/>59					<intercept-url pattern="/login" access="anon"/>60					<intercept-url pattern="/signin" access="anon"/>61					<intercept-url pattern="/logout" access="anon"/>62					<intercept-url pattern="/captcha/build" access="anon"/>63					<intercept-url pattern="/err/**" access="anon"/>64					<intercept-url pattern="/**" access="authc"/>65				</filters>66			</secure>67		</environment>68	</environments>69</configuration>70`
...

tmpl_new_proj_web_conf_secure.go

Source:tmpl_new_proj_web_conf_secure.go

1// Copyright 2014 The goyy Authors.  All rights reserved.2// Use of this source code is governed by a MIT-style3// license that can be found in the LICENSE file.45package main67var tmplNewProjWebConfSecure = `<?xml version="1.0" encoding="UTF-8" ?>8<!DOCTYPE env PUBLIC "-//GOYY//DTD ENV 1.0//EN" "http://goyy.org/dtd/env-1.0.dtd">9<configuration>10	<environments default="development">11		<environment id="development">12			<secure name="<%.NewProjName%>">13				<enable>true</enable>14				<login-url>/login.html</login-url>15				<forbid-url>/err/403.html</forbid-url>16				<success-url>/home.html</success-url>17				<filters>18					<intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>19					<intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>20					<intercept-url pattern="/apis/sys/menu/**" access="forbid"/>21					<intercept-url pattern="/apis/sys/post/**" access="forbid"/>22					<intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>23					<intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>24					<intercept-url pattern="/**" access="anon"/>25				</filters>26			</secure>27		</environment>28		<environment id="test">29			<secure name="<%.NewProjName%>">30				<enable>true</enable>31				<login-url>/login.html</login-url>32				<forbid-url>/err/403.html</forbid-url>33				<success-url>/home.html</success-url>34				<filters>35					<intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>36					<intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>37					<intercept-url pattern="/apis/sys/menu/**" access="forbid"/>38					<intercept-url pattern="/apis/sys/post/**" access="forbid"/>39					<intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>40					<intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>41					<intercept-url pattern="/**" access="anon"/>42				</filters>43			</secure>44		</environment>45		<environment id="production">46			<secure name="<%.NewProjName%>">47				<enable>true</enable>48				<login-url>/login.html</login-url>49				<forbid-url>/err/403.html</forbid-url>50				<success-url>/home.html</success-url>51				<filters>52					<intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>53					<intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>54					<intercept-url pattern="/apis/sys/menu/**" access="forbid"/>55					<intercept-url pattern="/apis/sys/post/**" access="forbid"/>56					<intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>57					<intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>58					<intercept-url pattern="/**" access="anon"/>59				</filters>60			</secure>61		</environment>62	</environments>63</configuration>64`
...

Pattern

Using AI Code Generation

1import (2func main() {3	doc, err := html.Parse(os.Stdin)4	if err != nil {5		fmt.Fprintf(os.Stderr, "findlinks1: %v\n", err)6		os.Exit(1)7	}8	for _, link := range visit(nil, doc) {9		fmt.Println(link)10	}11}12func visit(links []string, n *html.Node) []string {13	if n.Type == html.ElementNode && n.Data == "a" {14		for _, a := range n.Attr {15			if a.Key == "href" {16				links = append(links, a.Val)17			}18		}19	}20	for c := n.FirstChild; c != nil; c = c.NextSibling {21		links = visit(links, c)22	}23}

Pattern

Using AI Code Generation

1import (2func main() {3	doc, err := html.Parse(os.Stdin)4	if err != nil {5		fmt.Fprintf(os.Stderr, "findlinks1: %v\n", err)6		os.Exit(1)7	}8	for _, link := range visit(nil, doc) {9		fmt.Println(link)10	}11}12func visit(links []string, n *html.Node) []string {13	if n.Type == html.ElementNode && n.Data == "a" {14		for _, a := range n.Attr {15			if a.Key == "href" {16				links = append(links, a.Val)17			}18		}19	}20	for c := n.FirstChild; c != nil; c = c.NextSibling {21		links = visit(links, c)22	}23}

Pattern

Using AI Code Generation

1import (2func main() {3	doc, err := html.Parse(os.Stdin)4	if err != nil {5		fmt.Fprintf(os.Stderr, "findlinks:%v\n", err)6		os.Exit(1)7	}8	for _, link := range visit(nil, doc) {9		fmt.Println(link))10}

Pattern

Using AI Code Generation

1import (2func main() {3	fmt.Println(html.EscapeString("This is <b>HTML</b>"))4	}5}6func visit(links []string, n *html.Node) []string {7			}8		}9	}10	}11}

Pattern

Using AI Code Generation

1import (2func main() {3	http.HandleFunc("/", index)4	http.HandleFunc("/process", process)5	http.Handle("/favicon.ico", http.NotFoundHandler())6	http.ListenAndServe(":8080", nil)7}8func index(w http.ResponseWriter, req *http.Request) {9	fmt.Fprintln(w, `<form method="POST" action="/process">10}11func process(w http.ResponseWriter, req *http.Request) {12	v := req.FormValue("q")13	v = strings.ToLower(v)14	fmt.Fprintln(w, "Method:", m)15	fmt.Fprintln(w, "You searched for:", v)16}

Pattern

Using AI Code Generation

1import (2func main() {3	fmt.Print("Enter the url: ")4	fmt.Scan(&url)5		log.Fatal("Invalid url")6	}7	resp, err := http.Get(url)8	if err != nil {9		log.Fatal(err)10	}11	if resp.StatusCode != http.StatusOK {12		resp.Body.Close()13		log.Fatal("Invalid url")14	}15	doc, err := html.Parse(resp.Body)16	resp.Body.Close()17	if err != nil {18		log.Fatal(err)19	}20	links := make(map[string]bool)21	getLinks(links, doc)22	for link := range links {23		fmt.Println(link)24	}25}26}

Pattern

Using AI Code Generation

1import (2func main() {3	for _, url := range os.Args[1:] {4		links, err := getLinks(url)5		if err != nil {6			fmt.Fprintf(os.Stderr, "getlinks: %v\n", err)7		}8		for _, link := range links {9			fmt.Println(link)10		}11	}12}13func getLinks(url string) ([]string, error) {14	resp, err := http.Get(url)15	if err != nil {16		return nil, fmt.Errorf("getlinks: %v\n", err)17	}18	if resp.StatusCode != http.StatusOK {19		resp.Body.Close()20		return nil, fmt.Errorf("getlinks: %s\n", resp.Status)21	}22	doc, err := html.Parse(resp.Body)23	resp.Body.Close()24	if err != nil {25		return nil, fmt.Errorf("getlinks: parsing %s as HTML: %v\n", url, err)26	}27	return visit(nil, doc), nil28}29func visit(links []string, n *html.Node) []string {30	if n.Type == html.ElementNode && n.Data == "a" {31		for _, a := range n.Attr {32			if a.Key == "href" {33				links = append(links, a.Val)34			}35		}36	}37	for c := n.FirstChild; c != nil; c = c.NextSibling {38		links = visit(links, c)39	}40func getLinks(links map[string]bool, n *html.Node) {41	if n.Type == html.ElementNode {42		if n.Data == "a" {43			for _, a := range n.Attr {44				if a.Key == "href" {45				}46			}47		}48	}49	for c := n.FirstChild; c != nil; c = c.NextSibling {50		getLinks(links, c)51	}52}

Pattern

Using AI Code Generation

1import (2func main() {3	res, err := http.Get(url)4	if err != nil {5		panic(err)6	}7	defer res.Body.Close()8	robots, err := ioutil.ReadAll(res.Body)9	if err != nil {10		panic(err)11	}12	str := string(robots)13	fmt.Println(strings.Replace(str, "Go", "golang", -1))14}

Pattern

Using AI Code Generation

1import (2func main() {3	r := strings.NewReader(`4	doc, err := html.Parse(r)5	if err != nil {6		log.Fatal(err)7	}8	matches := htmlutils.FindByID(doc, "hello")9	if len(matches) > 0 {10		fmt.Printf("%+v\n", matches[0])11	}12}

Pattern

Using AI Code Generation

1import (2func main() {3	r := strings.NewReader(`4	doc, err := html.Parse(r)5	if err != nil {6		log.Fatal(err)7	}8	matches := htmlutils.FindByID(doc, "hello")9	if len(matches) > 0 {10		fmt.Printf("%+v\n", matches[0])11	}12}

Pattern

Using AI Code Generation

1import (2func main() {3	resp, err := http.Get(url)4	if err != nil {5		log.Fatal(err)6	}7	doc, err := html.Parse(resp.Body)8	resp.Body.Close()9	if err != nil {10		log.Fatal(err)

Pattern

Using AI Code Generation

1import (2func main() {3	fmt.Println(html}4	links := visit(nil, doc)5	for _, link := range links {6		fmt.Println(link)7	}8}9func visit(links []string, n *html.Node) []string {10	if n.Type == html.ElementNode && n.Data == "a" {11		for _, a := range n.Attr {12			if a.Key == "href" {13				links = append(links, a.Val)14			}15		}16	}17	for c := n.FirstChild; c != nil; c = c.NextSibling {18		links = visit(links, c)19	}20}