diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..9773c18 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -83,6 +83,7 @@ class Crawlers extends AbstractProvider '192\.comAgent', '2GDPR\/', '2ip\.ru', + '360JK', '404enemy', '7Siters', '80legs', @@ -121,6 +122,7 @@ class Crawlers extends AbstractProvider 'akka-http\/', 'akula\/', 'alertra', + 'AlertSite', 'alexa site audit', 'Alibaba\.Security\.Heimdall', 'Alligator', @@ -128,6 +130,7 @@ class Crawlers extends AbstractProvider 'AllSubmitter', 'alyze\.info', 'amagit', + 'Amazon Route 53 Health Check', 'amazon-Quick-on-behalf-of', 'amphp\/http-client', 'Anarchie', @@ -151,6 +154,7 @@ class Crawlers extends AbstractProvider 'AportWorm\/', 'AppBeat\/', 'AppEngine-Google', + 'AppInsights', 'AppleSyndication', 'Aprc\/[0-9]', 'Arachmo', @@ -175,6 +179,7 @@ class Crawlers extends AbstractProvider 'AWS Security Scanner', 'axios\/', 'a\.pr-cy\.ru', + 'azure-logic-apps', 'B-l-i-t-z-B-O-T', 'Backlink-Ceck', 'BacklinkHttpStatus', @@ -192,6 +197,7 @@ class Crawlers extends AbstractProvider 'baypup\/', 'BazQux', 'BBBike', + 'BBC-Forge-URL-Monitor', 'BCKLINKS', 'BDFetch', 'BegunAdvertising', @@ -200,6 +206,7 @@ class Crawlers extends AbstractProvider 'BigBozz', 'Bigfoot', 'biglotron', + 'BinaryCanary', 'BingLocalSearch', 'BingPreview', 'binlar', @@ -211,6 +218,7 @@ class Crawlers extends AbstractProvider 'Black Hole', 'Blackboard Safeassign', 'blackboardally', + 'Blackbox Exporter', 'BlackWidow', 'BlockNote\.Net', 'BlogBridge', @@ -246,6 +254,7 @@ class Crawlers extends AbstractProvider 'Camoufox', 'Canary%20Mail', 'CaretNail', + 'Catchpoint', 'catexplorador', 'CC Metadata Scaper', 'Cegbfeieh', @@ -268,6 +277,7 @@ class Crawlers extends AbstractProvider 'chkme\.com', 'Chlooe', 'Chromaxa', + 'Chrome Privacy Preserving Prefetch Proxy', 'CirrusExplorer', 'CISPA Vulnerability Notification', 'CISPA Web Analyser', @@ -279,14 +289,18 @@ class Crawlers extends AbstractProvider 'clips\.ua\.ac\.be', 'Cloud mapping', 'CloudEndure', + 'Cloudflare Custom Hostname Verification', + 'Cloudflare-?Diagnostics', 'CloudFlare-AlwaysOnline', 'Cloudflare-Healthchecks', + 'Cloudflare-Smart-Transit', 'Cloudflare-SSLDetector', 'Cloudinary', 'cmcm\.com', 'coccoc', 'cognitiveseo', 'ColdFusion', + 'collectd\/', 'colly -', 'CommaFeed', 'Commons-HttpClient', @@ -321,6 +335,8 @@ class Crawlers extends AbstractProvider 'DareBoost', 'DatabaseDriverMysqli', 'DataCha0s', + 'Datadog Agent', + 'Datadog\/Synthetics', 'DatadogSynthetics', 'Datafeedwatch', 'Datanyze', @@ -335,6 +351,7 @@ class Crawlers extends AbstractProvider 'delve\.ai', 'Demon', 'DeuSu', + 'developers\.cloudflare\.com\/security-center', 'developers\.google\.com\/\+\/web\/snippet\/', 'Devil', 'Digg', @@ -377,6 +394,7 @@ class Crawlers extends AbstractProvider 'ECCP', 'eContext\/', 'Ecxi', + 'EdgeWatch', 'EirGrabber', 'ElectricMonk', 'elefent', @@ -387,6 +405,7 @@ class Crawlers extends AbstractProvider 'Embed PHP Library', 'Embedly', 'endo\/', + 'Episode Duration Lookup', 'europarchive\.org', 'evc-batch', 'EventMachine HttpClient', @@ -402,6 +421,7 @@ class Crawlers extends AbstractProvider 'Express WebPictures', 'Extreme Picture Finder', 'EyeNetIE', + 'Ezgif\/', 'ezooms', 'facebookcatalog', 'facebookexternalhit', @@ -545,6 +565,7 @@ class Crawlers extends AbstractProvider 'GoZilla', 'grabify', 'GrabNet', + 'Grafana\/', 'Grafula', 'Grammarly', 'GrapeFX', @@ -619,6 +640,7 @@ class Crawlers extends AbstractProvider 'HubSpot ', 'HubSpot-Link-Resolver', 'Humanlinks', + 'Hydrozen\.io', 'i2kconnect\/', 'IAB-Tech-Lab', 'Iblog', @@ -657,6 +679,7 @@ class Crawlers extends AbstractProvider 'integromedb', 'Intelliseek', 'InterGET', + 'internal dummy connection', 'Internet Ninja', 'InternetMeasurement', 'InternetSeer', @@ -712,6 +735,7 @@ class Crawlers extends AbstractProvider 'khttp\/', 'KickFire', 'KimonoLabs\/', + 'kiwitcms-gitops', 'Kml-Google', 'knows\.is', 'KOCMOHABT', @@ -785,8 +809,10 @@ class Crawlers extends AbstractProvider 'MailChimp', 'Mail\.Ru', 'Mailpit\/', + 'MainWP', 'Majestic12', 'makecontact\/', + 'ManageWP', 'Mandrill', 'MapperCmd', 'marketinggrader', @@ -853,6 +879,7 @@ class Crawlers extends AbstractProvider 'mShots', 'MuckRack\/', 'muhstik-scan', + 'munin\/', 'MVAClient', 'MxToolbox\/', 'myseosnapshot', @@ -881,7 +908,9 @@ class Crawlers extends AbstractProvider 'NetSystemsResearch', 'nettle', 'NetTrack', + 'Netumo', 'Netvibes', + 'NETVIGIE', 'NetworkingExtension', 'NetZIP', 'Neustar WPM', @@ -930,6 +959,7 @@ class Crawlers extends AbstractProvider 'Offline Navigator', 'odin\.com', 'OgScrper', + 'Oh Dear', 'okhttp', 'omgili', 'OMSC', @@ -983,6 +1013,7 @@ class Crawlers extends AbstractProvider 'PhantomJS\/', 'Photon\/', 'php-requests', + 'phpMyAdmin', 'phpservermon', 'Pi-Monster', 'Picscout', @@ -993,6 +1024,7 @@ class Crawlers extends AbstractProvider 'PingAdmin\.Ru', 'Pingdom', 'Pingoscope', + 'pingping\.io', 'PingSpot', 'ping\.blo\.gs', 'pinterest\.com', @@ -1030,6 +1062,7 @@ class Crawlers extends AbstractProvider 'probethenet', 'Project ?25499', 'Project-Resonance', + 'Prometheus\/', 'prospectb2b', 'Protopage', 'ProWebWalker', @@ -1084,6 +1117,7 @@ class Crawlers extends AbstractProvider 'Rival IQ', 'Robosourcer', 'Robozilla', + 'rocketmonitor', 'ROI Hunter', 'RPT-HTTPClient', 'RSiteAuditor', @@ -1121,6 +1155,7 @@ class Crawlers extends AbstractProvider 'SearchSight', 'SearchWP', 'search\.thunderstone', + 'Sectigo DCV', 'Seeker', 'semanticdiscovery', 'semanticjuice', @@ -1146,6 +1181,8 @@ class Crawlers extends AbstractProvider 'SEOsearch', 'seositecheckup', 'SEOstats', + 'SerpReputationManagementAgent', + 'Server ?Density', 'servernfo', 'sexsearcher', 'Seznam', @@ -1184,6 +1221,7 @@ class Crawlers extends AbstractProvider 'SiteTruth', 'Sitevigil', 'sitexy\.com', + 'sixy\.ch', 'SkypeUriPreview', 'Slack\/', 'sli-systems\.com', @@ -1202,6 +1240,7 @@ class Crawlers extends AbstractProvider 'Snoopy', 'SnowHaze Search', 'sogou web', + 'solarwinds', 'SortSite', 'Sottopop', 'sovereign\.ai', @@ -1276,6 +1315,7 @@ class Crawlers extends AbstractProvider 'theinternetrules', 'TheNomad', 'Thinklab', + 'ThousandEyes', 'Thumbor', 'Thumbshots', 'ThumbSniper', @@ -1296,6 +1336,7 @@ class Crawlers extends AbstractProvider 'truwoGPS', 'TryJsoup', 'TulipChain', + 'TuneIn-DirMon', 'Turingos', 'Turnitin', 'tweetedtimes', @@ -1323,6 +1364,7 @@ class Crawlers extends AbstractProvider 'updown\.io daemon', 'Upflow', 'Uptimia', + 'uptrends', 'URL Verifier', 'Urlcheckr', 'URLitor', @@ -1335,6 +1377,7 @@ class Crawlers extends AbstractProvider 'URL\/Emacs', 'Vacuum', 'Vagabondo', + 'Valimail\/', 'VB Project', 'vBSEO', 'VCI', @@ -1400,6 +1443,7 @@ class Crawlers extends AbstractProvider 'WebLeacher', 'webmastercoffee', 'webmon ', + 'WebMon\/', 'WebPix', 'WebReaper', 'WebSauger', @@ -1431,6 +1475,7 @@ class Crawlers extends AbstractProvider 'WhatsMyIP', 'WhatWeb', 'WhereGoes\?', + 'wheresitup', 'Whibse', 'WhoAPI\/', 'WhoRunsCoinHive', @@ -1455,6 +1500,7 @@ class Crawlers extends AbstractProvider 'wpif', 'wprecon\.com survey', 'WPScan', + 'WPUmbrella', 'wscheck', 'Wtrace', 'WWW-Collector-E', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..3557323 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,54 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx +Apache/2.2.2 (Fedora) (internal dummy connection) +Castro 2, Episode Duration Lookup +Chrome Privacy Preserving Prefetch Proxy +Cloudflare Custom Hostname Verification +Mozilla/5.0 (compatible; Ezgif/1.0; +https://ezgif.com/about) +kiwitcms-gitops/0.1 +kiwitcms-gitops/1 +azure-logic-apps/1.0 (workflow xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; version xxxxxxxxxxxxxxxxxxxx) microsoft-flow/1.0 +Prometheus/2.40.5 +Sectigo DCV +Mozilla/5.0 (compatible; SerpReputationManagementAgent/1.0) +TuneIn-DirMon/1.0 +Valimail/1.0 (domain scanner) +phpMyAdmin +phpMyAdmin/reporting-script +Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322) 360JK yunjiankong 427691 +BBC-Forge-URL-Monitor-Twisted +Blackbox Exporter/0.24.0 +Blackbox Exporter/0.20.0-rc.0 +Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; Catchpoint) like Gecko +Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 CloudflareDiagnostics/1.0 +Cloudflare-diagnostics +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/) +Mozilla/5.0 (compatible; Cloudflare-Smart-Transit/1.0; +https://www.cloudflare.com/ +collectd/5.5.1 +Datadog Agent/5.10.1 +Datadog/Synthetics +Grafana/11.1.0 +munin/http_loadtime +Oh Dear Sitemap check https://ohdear.app +rocketmonitor 1.0 +sixy.ch/1.0 +ThousandEyes-Dragonfly-x1 +WebMon/2.0.0 +Mozilla/5.0 (compatible; AlertSite; +https://www.alertsite.com/) +Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights) +BinaryCanary/1.0 (+https://www.binarycanary.com) +Mozilla/5.0 (compatible; EdgeWatch/1.1; https://about.edgewatch.com/) +Hydrozen.io/1.0 +Mozilla/5.0 (compatible; MainWP/4.2.7.1; +http://mainwp.com) +ManageWP +NETVIGIE +Netumo/0.2 (+https://www.netumo.app/) +Mozilla/5.0 (compatible; ServerDensity; +https://www.serverdensity.com/) +WPUmbrella +pingping.io/1.0 +solarwinds/1.0 ( www.solarwinds.com/solarwinds-observability) +uptrends +wheresitup.com/1.1 diff --git a/tests/data/user_agent/devices.txt b/tests/data/user_agent/devices.txt index a0d9607..e49992c 100644 --- a/tests/data/user_agent/devices.txt +++ b/tests/data/user_agent/devices.txt @@ -99824,7 +99824,6 @@ Mozilla/5.0 (Linux; Android 7.0; SM-A520F Build/NRD90M) AppleWebKit/537.36 (KHTM Mozilla/5.0 (Linux; Android 7.0) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Focus/1.0 Chrome/59.0.3071.125 Mobile Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Mobile/14E304 (4299244784) Mozilla/5.0 (Linux; Android 7.1.1; Wileyfox Swift Build/N6F26Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36 -Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299214304) Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; 2014818 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 UCBrowser/11.5.0.1015 Mobile Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299207328)