Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions src/Fixtures/Crawlers.php
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ class Crawlers extends AbstractProvider
'192\.comAgent',
'2GDPR\/',
'2ip\.ru',
'360JK',
'404enemy',
'7Siters',
'80legs',
Expand Down Expand Up @@ -121,13 +122,15 @@ class Crawlers extends AbstractProvider
'akka-http\/',
'akula\/',
'alertra',
'AlertSite',
'alexa site audit',
'Alibaba\.Security\.Heimdall',
'Alligator',
'allloadin',
'AllSubmitter',
'alyze\.info',
'amagit',
'Amazon Route 53 Health Check',
'amazon-Quick-on-behalf-of',
'amphp\/http-client',
'Anarchie',
Expand All @@ -151,6 +154,7 @@ class Crawlers extends AbstractProvider
'AportWorm\/',
'AppBeat\/',
'AppEngine-Google',
'AppInsights',
'AppleSyndication',
'Aprc\/[0-9]',
'Arachmo',
Expand All @@ -175,6 +179,7 @@ class Crawlers extends AbstractProvider
'AWS Security Scanner',
'axios\/',
'a\.pr-cy\.ru',
'azure-logic-apps',
'B-l-i-t-z-B-O-T',
'Backlink-Ceck',
'BacklinkHttpStatus',
Expand All @@ -192,6 +197,7 @@ class Crawlers extends AbstractProvider
'baypup\/',
'BazQux',
'BBBike',
'BBC-Forge-URL-Monitor',
'BCKLINKS',
'BDFetch',
'BegunAdvertising',
Expand All @@ -200,6 +206,7 @@ class Crawlers extends AbstractProvider
'BigBozz',
'Bigfoot',
'biglotron',
'BinaryCanary',
'BingLocalSearch',
'BingPreview',
'binlar',
Expand All @@ -211,6 +218,7 @@ class Crawlers extends AbstractProvider
'Black Hole',
'Blackboard Safeassign',
'blackboardally',
'Blackbox Exporter',
'BlackWidow',
'BlockNote\.Net',
'BlogBridge',
Expand Down Expand Up @@ -246,6 +254,7 @@ class Crawlers extends AbstractProvider
'Camoufox',
'Canary%20Mail',
'CaretNail',
'Catchpoint',
'catexplorador',
'CC Metadata Scaper',
'Cegbfeieh',
Expand All @@ -268,6 +277,7 @@ class Crawlers extends AbstractProvider
'chkme\.com',
'Chlooe',
'Chromaxa',
'Chrome Privacy Preserving Prefetch Proxy',
'CirrusExplorer',
'CISPA Vulnerability Notification',
'CISPA Web Analyser',
Expand All @@ -279,14 +289,18 @@ class Crawlers extends AbstractProvider
'clips\.ua\.ac\.be',
'Cloud mapping',
'CloudEndure',
'Cloudflare Custom Hostname Verification',
'Cloudflare-?Diagnostics',
'CloudFlare-AlwaysOnline',
'Cloudflare-Healthchecks',
'Cloudflare-Smart-Transit',
'Cloudflare-SSLDetector',
'Cloudinary',
'cmcm\.com',
'coccoc',
'cognitiveseo',
'ColdFusion',
'collectd\/',
'colly -',
'CommaFeed',
'Commons-HttpClient',
Expand Down Expand Up @@ -321,6 +335,8 @@ class Crawlers extends AbstractProvider
'DareBoost',
'DatabaseDriverMysqli',
'DataCha0s',
'Datadog Agent',
'Datadog\/Synthetics',
'DatadogSynthetics',
'Datafeedwatch',
'Datanyze',
Expand All @@ -335,6 +351,7 @@ class Crawlers extends AbstractProvider
'delve\.ai',
'Demon',
'DeuSu',
'developers\.cloudflare\.com\/security-center',
'developers\.google\.com\/\+\/web\/snippet\/',
'Devil',
'Digg',
Expand Down Expand Up @@ -377,6 +394,7 @@ class Crawlers extends AbstractProvider
'ECCP',
'eContext\/',
'Ecxi',
'EdgeWatch',
'EirGrabber',
'ElectricMonk',
'elefent',
Expand All @@ -387,6 +405,7 @@ class Crawlers extends AbstractProvider
'Embed PHP Library',
'Embedly',
'endo\/',
'Episode Duration Lookup',
'europarchive\.org',
'evc-batch',
'EventMachine HttpClient',
Expand All @@ -402,6 +421,7 @@ class Crawlers extends AbstractProvider
'Express WebPictures',
'Extreme Picture Finder',
'EyeNetIE',
'Ezgif\/',
'ezooms',
'facebookcatalog',
'facebookexternalhit',
Expand Down Expand Up @@ -545,6 +565,7 @@ class Crawlers extends AbstractProvider
'GoZilla',
'grabify',
'GrabNet',
'Grafana\/',
'Grafula',
'Grammarly',
'GrapeFX',
Expand Down Expand Up @@ -619,6 +640,7 @@ class Crawlers extends AbstractProvider
'HubSpot ',
'HubSpot-Link-Resolver',
'Humanlinks',
'Hydrozen\.io',
'i2kconnect\/',
'IAB-Tech-Lab',
'Iblog',
Expand Down Expand Up @@ -657,6 +679,7 @@ class Crawlers extends AbstractProvider
'integromedb',
'Intelliseek',
'InterGET',
'internal dummy connection',
'Internet Ninja',
'InternetMeasurement',
'InternetSeer',
Expand Down Expand Up @@ -712,6 +735,7 @@ class Crawlers extends AbstractProvider
'khttp\/',
'KickFire',
'KimonoLabs\/',
'kiwitcms-gitops',
'Kml-Google',
'knows\.is',
'KOCMOHABT',
Expand Down Expand Up @@ -785,8 +809,10 @@ class Crawlers extends AbstractProvider
'MailChimp',
'Mail\.Ru',
'Mailpit\/',
'MainWP',
'Majestic12',
'makecontact\/',
'ManageWP',
'Mandrill',
'MapperCmd',
'marketinggrader',
Expand Down Expand Up @@ -853,6 +879,7 @@ class Crawlers extends AbstractProvider
'mShots',
'MuckRack\/',
'muhstik-scan',
'munin\/',
'MVAClient',
'MxToolbox\/',
'myseosnapshot',
Expand Down Expand Up @@ -881,7 +908,9 @@ class Crawlers extends AbstractProvider
'NetSystemsResearch',
'nettle',
'NetTrack',
'Netumo',
'Netvibes',
'NETVIGIE',
'NetworkingExtension',
'NetZIP',
'Neustar WPM',
Expand Down Expand Up @@ -930,6 +959,7 @@ class Crawlers extends AbstractProvider
'Offline Navigator',
'odin\.com',
'OgScrper',
'Oh Dear',
'okhttp',
'omgili',
'OMSC',
Expand Down Expand Up @@ -983,6 +1013,7 @@ class Crawlers extends AbstractProvider
'PhantomJS\/',
'Photon\/',
'php-requests',
'phpMyAdmin',
'phpservermon',
'Pi-Monster',
'Picscout',
Expand All @@ -993,6 +1024,7 @@ class Crawlers extends AbstractProvider
'PingAdmin\.Ru',
'Pingdom',
'Pingoscope',
'pingping\.io',
'PingSpot',
'ping\.blo\.gs',
'pinterest\.com',
Expand Down Expand Up @@ -1030,6 +1062,7 @@ class Crawlers extends AbstractProvider
'probethenet',
'Project ?25499',
'Project-Resonance',
'Prometheus\/',
'prospectb2b',
'Protopage',
'ProWebWalker',
Expand Down Expand Up @@ -1084,6 +1117,7 @@ class Crawlers extends AbstractProvider
'Rival IQ',
'Robosourcer',
'Robozilla',
'rocketmonitor',
'ROI Hunter',
'RPT-HTTPClient',
'RSiteAuditor',
Expand Down Expand Up @@ -1121,6 +1155,7 @@ class Crawlers extends AbstractProvider
'SearchSight',
'SearchWP',
'search\.thunderstone',
'Sectigo DCV',
'Seeker',
'semanticdiscovery',
'semanticjuice',
Expand All @@ -1146,6 +1181,8 @@ class Crawlers extends AbstractProvider
'SEOsearch',
'seositecheckup',
'SEOstats',
'SerpReputationManagementAgent',
'Server ?Density',
'servernfo',
'sexsearcher',
'Seznam',
Expand Down Expand Up @@ -1184,6 +1221,7 @@ class Crawlers extends AbstractProvider
'SiteTruth',
'Sitevigil',
'sitexy\.com',
'sixy\.ch',
'SkypeUriPreview',
'Slack\/',
'sli-systems\.com',
Expand All @@ -1202,6 +1240,7 @@ class Crawlers extends AbstractProvider
'Snoopy',
'SnowHaze Search',
'sogou web',
'solarwinds',
'SortSite',
'Sottopop',
'sovereign\.ai',
Expand Down Expand Up @@ -1276,6 +1315,7 @@ class Crawlers extends AbstractProvider
'theinternetrules',
'TheNomad',
'Thinklab',
'ThousandEyes',
'Thumbor',
'Thumbshots',
'ThumbSniper',
Expand All @@ -1296,6 +1336,7 @@ class Crawlers extends AbstractProvider
'truwoGPS',
'TryJsoup',
'TulipChain',
'TuneIn-DirMon',
'Turingos',
'Turnitin',
'tweetedtimes',
Expand Down Expand Up @@ -1323,6 +1364,7 @@ class Crawlers extends AbstractProvider
'updown\.io daemon',
'Upflow',
'Uptimia',
'uptrends',
'URL Verifier',
'Urlcheckr',
'URLitor',
Expand All @@ -1335,6 +1377,7 @@ class Crawlers extends AbstractProvider
'URL\/Emacs',
'Vacuum',
'Vagabondo',
'Valimail\/',
'VB Project',
'vBSEO',
'VCI',
Expand Down Expand Up @@ -1400,6 +1443,7 @@ class Crawlers extends AbstractProvider
'WebLeacher',
'webmastercoffee',
'webmon ',
'WebMon\/',
'WebPix',
'WebReaper',
'WebSauger',
Expand Down Expand Up @@ -1431,6 +1475,7 @@ class Crawlers extends AbstractProvider
'WhatsMyIP',
'WhatWeb',
'WhereGoes\?',
'wheresitup',
'Whibse',
'WhoAPI\/',
'WhoRunsCoinHive',
Expand All @@ -1455,6 +1500,7 @@ class Crawlers extends AbstractProvider
'wpif',
'wprecon\.com survey',
'WPScan',
'WPUmbrella',
'wscheck',
'Wtrace',
'WWW-Collector-E',
Expand Down
51 changes: 51 additions & 0 deletions tests/data/user_agent/crawlers.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3794,3 +3794,54 @@ NYU Internet Census (https://scan.lol; research@scan.lol)
Download Demon/3.5.0.11
webbandit/4.xx.0
Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)
Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx
Apache/2.2.2 (Fedora) (internal dummy connection)
Castro 2, Episode Duration Lookup
Chrome Privacy Preserving Prefetch Proxy
Cloudflare Custom Hostname Verification
Mozilla/5.0 (compatible; Ezgif/1.0; +https://ezgif.com/about)
kiwitcms-gitops/0.1
kiwitcms-gitops/1
azure-logic-apps/1.0 (workflow xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; version xxxxxxxxxxxxxxxxxxxx) microsoft-flow/1.0
Prometheus/2.40.5
Sectigo DCV
Mozilla/5.0 (compatible; SerpReputationManagementAgent/1.0)
TuneIn-DirMon/1.0
Valimail/1.0 (domain scanner)
phpMyAdmin
phpMyAdmin/reporting-script
Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322) 360JK yunjiankong 427691
BBC-Forge-URL-Monitor-Twisted
Blackbox Exporter/0.24.0
Blackbox Exporter/0.20.0-rc.0
Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; Catchpoint) like Gecko
Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36
Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 CloudflareDiagnostics/1.0
Cloudflare-diagnostics
Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/)
Mozilla/5.0 (compatible; Cloudflare-Smart-Transit/1.0; +https://www.cloudflare.com/
collectd/5.5.1
Datadog Agent/5.10.1
Datadog/Synthetics
Grafana/11.1.0
munin/http_loadtime
Oh Dear Sitemap check https://ohdear.app
rocketmonitor 1.0
sixy.ch/1.0
ThousandEyes-Dragonfly-x1
WebMon/2.0.0
Mozilla/5.0 (compatible; AlertSite; +https://www.alertsite.com/)
Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)
BinaryCanary/1.0 (+https://www.binarycanary.com)
Mozilla/5.0 (compatible; EdgeWatch/1.1; https://about.edgewatch.com/)
Hydrozen.io/1.0
Mozilla/5.0 (compatible; MainWP/4.2.7.1; +http://mainwp.com)
ManageWP
NETVIGIE
Netumo/0.2 (+https://www.netumo.app/)
Mozilla/5.0 (compatible; ServerDensity; +https://www.serverdensity.com/)
WPUmbrella
pingping.io/1.0
solarwinds/1.0 ( www.solarwinds.com/solarwinds-observability)
uptrends
wheresitup.com/1.1
1 change: 0 additions & 1 deletion tests/data/user_agent/devices.txt
Original file line number Diff line number Diff line change
Expand Up @@ -99824,7 +99824,6 @@ Mozilla/5.0 (Linux; Android 7.0; SM-A520F Build/NRD90M) AppleWebKit/537.36 (KHTM
Mozilla/5.0 (Linux; Android 7.0) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Focus/1.0 Chrome/59.0.3071.125 Mobile Safari/537.36
Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Mobile/14E304 (4299244784)
Mozilla/5.0 (Linux; Android 7.1.1; Wileyfox Swift Build/N6F26Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36
Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36
Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299214304)
Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; 2014818 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 UCBrowser/11.5.0.1015 Mobile Safari/537.36
Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299207328)
Expand Down