Add HTTPUnit bot - very common bot (closes PR #509)

NielsLeenheer · NielsLeenheer · commit 8f8acf4575aa · 2021-01-04T13:35:41.000+01:00
diff --git a/data/applications-bots.php b/data/applications-bots.php
@@ -377,6 +377,7 @@
     [ 'name' => 'Borland Delphi',               'id'    => 'indy',      'regexp' => '/Indy Library/u' ],
     [ 'name' => 'Go Http Client',               'id'    => 'go',      'regexp' => '/Go-http-client\/([0-9.]*)/u' ],
     [ 'name' => 'Go Http Client',               'id'    => 'go',      'regexp' => '/go-httpclient v([0-9.]*)/u' ],
+    [ 'name' => 'HttpUnit',                     'id'    => 'httpunit',      'regexp' => '/httpunit\/([0-9.]*)/u' ],
     [ 'name' => 'OkHttp',                       'id'    => 'okhttp',      'regexp' => '/okhttp\/([0-9.]*)/u' ],
     [ 'name' => 'Node Fetch',                   'id'    => 'node',      'regexp' => '/node-fetch\/([0-9.]*)/u' ],
     [ 'name' => 'Wget',                         'id'    => 'wget',      'regexp' => '/Wget\/([0-9.]*)/u' ],
diff --git a/data/regexes/applications-bots.php b/data/regexes/applications-bots.php
@@ -2,4 +2,4 @@
 
 namespace WhichBrowser\Data;
 
-Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|adsbot|ahrefs|archiver|altavista|curious|cloudfront|amorank|apple|archive|jeeves|astra|backlink|baidu|barkrowler|bazqux|bingbot|msnbot|msmobot|bing|blex|blogbridge|blogger|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|lddc|browsershots|bubing|buck|builtwith|butterfly|buzzsumo|bytespider|heritrix|clarabot|cliqz|cloudflare|coccoc|comodo|commafeed|cbot|cresearch|watchdog|datanyze|datasearch|dataprovider|daum|daumoa|digg|domaincrawl|domainstats|discord|domain|dmca|duckduck|embed|evc|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gofeed|gomez|ichiro|gooblog|goorss|gowiki|grammarly|grub|hatena|capture|heureka|hootsuite|htdig|httpmon|hubpages|hubspot|spider|indeed|iodc|istellabot|crawler|kakao|kouio|larbin|letsearch|line|linkedin|linkpad|linkdex|livedoor|seeker|ltx71|magpie|mj12bot|mapion|mail\.ru|medium|megaindex|socialstreams|mixi|minds|mnogo|mojeek|monitor|dotbot|rogerbot|yeti|mxtoolbox|netcraft|netestate|netvibes|newsblur|newsgator|nimbostratus|odklbot|orange|pages|petal|aspiegel|psbot|pingdom|pinterest|plurk|pocket|postrank|comment|reddit|rssbar|hunter|quora|safesearch|scrapy|searchatlas|seekport|semrush|serpstat|seokicks|seznam|shopwiki|reader|simplepie|site24|skype|slack|sogou|sophora|soso|spaziodati|spdycheck|spinn|sputnik|detector|summify|telegram|twisted|twitter|tiny|tineye|tlsprober|trustedsite|typhoeus|vagabondo|velen|vkshare|voat|voila|vocus|mcrawler|vuhuv|w3bot|jigsaw|w3c|wappalyzer|wayback|webindexer|wordpress|mechanize|xenu|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|yioop|yisou|zmeu|httpclient|nutch|synapse|indy|go|okhttp|node|wget|curl|package|java|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i';
+Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|adsbot|ahrefs|archiver|altavista|curious|cloudfront|amorank|apple|archive|jeeves|astra|backlink|baidu|barkrowler|bazqux|bingbot|msnbot|msmobot|bing|blex|blogbridge|blogger|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|lddc|browsershots|bubing|buck|builtwith|butterfly|buzzsumo|bytespider|heritrix|clarabot|cliqz|cloudflare|coccoc|comodo|commafeed|cbot|cresearch|watchdog|datanyze|datasearch|dataprovider|daum|daumoa|digg|domaincrawl|domainstats|discord|domain|dmca|duckduck|embed|evc|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gofeed|gomez|ichiro|gooblog|goorss|gowiki|grammarly|grub|hatena|capture|heureka|hootsuite|htdig|httpmon|hubpages|hubspot|spider|indeed|iodc|istellabot|crawler|kakao|kouio|larbin|letsearch|line|linkedin|linkpad|linkdex|livedoor|seeker|ltx71|magpie|mj12bot|mapion|mail\.ru|medium|megaindex|socialstreams|mixi|minds|mnogo|mojeek|monitor|dotbot|rogerbot|yeti|mxtoolbox|netcraft|netestate|netvibes|newsblur|newsgator|nimbostratus|odklbot|orange|pages|petal|aspiegel|psbot|pingdom|pinterest|plurk|pocket|postrank|comment|reddit|rssbar|hunter|quora|safesearch|scrapy|searchatlas|seekport|semrush|serpstat|seokicks|seznam|shopwiki|reader|simplepie|site24|skype|slack|sogou|sophora|soso|spaziodati|spdycheck|spinn|sputnik|detector|summify|telegram|twisted|twitter|tiny|tineye|tlsprober|trustedsite|typhoeus|vagabondo|velen|vkshare|voat|voila|vocus|mcrawler|vuhuv|w3bot|jigsaw|w3c|wappalyzer|wayback|webindexer|wordpress|mechanize|xenu|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|yioop|yisou|zmeu|httpclient|nutch|synapse|indy|go|httpunit|okhttp|node|wget|curl|package|java|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i';
diff --git a/tests/data/bots/generic.yaml b/tests/data/bots/generic.yaml
@@ -985,4 +985,8 @@
 -
     headers: 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.0 Safari/537.36 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)'
     readable: 'Line Bot 1.1'
-    result: { browser: { name: 'Line Bot', version: '1.1' }, device: { type: bot } }
+    result: { browser: { name: 'Line Bot', version: '1.1' }, device: { type: bot } }
+-
+    headers: 'User-Agent: httpunit/1.5'
+    readable: 'HttpUnit 1.5'
+    result: { browser: { name: HttpUnit, version: '1.5' }, device: { type: bot } }

Original file line number	Diff line number	Diff line change
`@@ -2,4 +2,4 @@`
`2`	`2`
`3`	`3`	`namespace WhichBrowser\Data;`
`4`	`4`
`5`		-Applications::$BOTS_REGEX = '/(008\|360\|a6\|abound\|muncher\|adaxas\|addthis\|admantx\|adsbot\|ahrefs\|archiver\|altavista\|curious\|cloudfront\|amorank\|apple\|archive\|jeeves\|astra\|backlink\|baidu\|barkrowler\|bazqux\|bingbot\|msnbot\|msmobot\|bing\|blex\|blogbridge\|blogger\|bloglines\|bloglovin\|blogpulse\|blogram\|blogtrot\|blogshares\|boardreader\|lddc\|browsershots\|bubing\|buck\|builtwith\|butterfly\|buzzsumo\|bytespider\|heritrix\|clarabot\|cliqz\|cloudflare\|coccoc\|comodo\|commafeed\|cbot\|cresearch\|watchdog\|datanyze\|datasearch\|dataprovider\|daum\|daumoa\|digg\|domaincrawl\|domainstats\|discord\|domain\|dmca\|duckduck\|embed\|evc\|exabot\|exactseek\|ezooms\|facebook\|fast\|flamingo\|fastladder\|feed\|wrangler\|validator\|fever\|florienzh\|friendica\|robot\|genieo\|fetchor\|google\|gofeed\|gomez\|ichiro\|gooblog\|goorss\|gowiki\|grammarly\|grub\|hatena\|capture\|heureka\|hootsuite\|htdig\|httpmon\|hubpages\|hubspot\|spider\|indeed\|iodc\|istellabot\|crawler\|kakao\|kouio\|larbin\|letsearch\|line\|linkedin\|linkpad\|linkdex\|livedoor\|seeker\|ltx71\|magpie\|mj12bot\|mapion\|mail\.ru\|medium\|megaindex\|socialstreams\|mixi\|minds\|mnogo\|mojeek\|monitor\|dotbot\|rogerbot\|yeti\|mxtoolbox\|netcraft\|netestate\|netvibes\|newsblur\|newsgator\|nimbostratus\|odklbot\|orange\|pages\|petal\|aspiegel\|psbot\|pingdom\|pinterest\|plurk\|pocket\|postrank\|comment\|reddit\|rssbar\|hunter\|quora\|safesearch\|scrapy\|searchatlas\|seekport\|semrush\|serpstat\|seokicks\|seznam\|shopwiki\|reader\|simplepie\|site24\|skype\|slack\|sogou\|sophora\|soso\|spaziodati\|spdycheck\|spinn\|sputnik\|detector\|summify\|telegram\|twisted\|twitter\|tiny\|tineye\|tlsprober\|trustedsite\|typhoeus\|vagabondo\|velen\|vkshare\|voat\|voila\|vocus\|mcrawler\|vuhuv\|w3bot\|jigsaw\|w3c\|wappalyzer\|wayback\|webindexer\|wordpress\|mechanize\|xenu\|xerka\|sitemap\|yacy\|yandex\|yahoo\|y\!j\|slurp\|yelp\|yioop\|yisou\|zmeu\|httpclient\|nutch\|synapse\|indy\|go\|okhttp\|node\|wget\|curl\|package\|java\|simple\|libwww\|lwp\|urllib\|requests\|php\|pear\|zend\|ruby)/i';
	`5`	+Applications::$BOTS_REGEX = '/(008\|360\|a6\|abound\|muncher\|adaxas\|addthis\|admantx\|adsbot\|ahrefs\|archiver\|altavista\|curious\|cloudfront\|amorank\|apple\|archive\|jeeves\|astra\|backlink\|baidu\|barkrowler\|bazqux\|bingbot\|msnbot\|msmobot\|bing\|blex\|blogbridge\|blogger\|bloglines\|bloglovin\|blogpulse\|blogram\|blogtrot\|blogshares\|boardreader\|lddc\|browsershots\|bubing\|buck\|builtwith\|butterfly\|buzzsumo\|bytespider\|heritrix\|clarabot\|cliqz\|cloudflare\|coccoc\|comodo\|commafeed\|cbot\|cresearch\|watchdog\|datanyze\|datasearch\|dataprovider\|daum\|daumoa\|digg\|domaincrawl\|domainstats\|discord\|domain\|dmca\|duckduck\|embed\|evc\|exabot\|exactseek\|ezooms\|facebook\|fast\|flamingo\|fastladder\|feed\|wrangler\|validator\|fever\|florienzh\|friendica\|robot\|genieo\|fetchor\|google\|gofeed\|gomez\|ichiro\|gooblog\|goorss\|gowiki\|grammarly\|grub\|hatena\|capture\|heureka\|hootsuite\|htdig\|httpmon\|hubpages\|hubspot\|spider\|indeed\|iodc\|istellabot\|crawler\|kakao\|kouio\|larbin\|letsearch\|line\|linkedin\|linkpad\|linkdex\|livedoor\|seeker\|ltx71\|magpie\|mj12bot\|mapion\|mail\.ru\|medium\|megaindex\|socialstreams\|mixi\|minds\|mnogo\|mojeek\|monitor\|dotbot\|rogerbot\|yeti\|mxtoolbox\|netcraft\|netestate\|netvibes\|newsblur\|newsgator\|nimbostratus\|odklbot\|orange\|pages\|petal\|aspiegel\|psbot\|pingdom\|pinterest\|plurk\|pocket\|postrank\|comment\|reddit\|rssbar\|hunter\|quora\|safesearch\|scrapy\|searchatlas\|seekport\|semrush\|serpstat\|seokicks\|seznam\|shopwiki\|reader\|simplepie\|site24\|skype\|slack\|sogou\|sophora\|soso\|spaziodati\|spdycheck\|spinn\|sputnik\|detector\|summify\|telegram\|twisted\|twitter\|tiny\|tineye\|tlsprober\|trustedsite\|typhoeus\|vagabondo\|velen\|vkshare\|voat\|voila\|vocus\|mcrawler\|vuhuv\|w3bot\|jigsaw\|w3c\|wappalyzer\|wayback\|webindexer\|wordpress\|mechanize\|xenu\|xerka\|sitemap\|yacy\|yandex\|yahoo\|y\!j\|slurp\|yelp\|yioop\|yisou\|zmeu\|httpclient\|nutch\|synapse\|indy\|go\|httpunit\|okhttp\|node\|wget\|curl\|package\|java\|simple\|libwww\|lwp\|urllib\|requests\|php\|pear\|zend\|ruby)/i';