Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions src/Fixtures/Crawlers.php
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,8 @@ class Crawlers extends AbstractProvider
'BlogSearch',
'Blogtrottr',
'BlowFish',
'Bluesky Cardyb',
'bluesky-domain-status-classifier',
'boitho\.com-dc',
'Boost\.Beast',
'BPImageWalker',
Expand Down Expand Up @@ -276,6 +278,7 @@ class Crawlers extends AbstractProvider
'Clarsentia',
'Claude-User',
'claude-web',
'ClickUpLinkUnfurler',
'clips\.ua\.ac\.be',
'Cloud mapping',
'CloudEndure',
Expand Down Expand Up @@ -416,6 +419,7 @@ class Crawlers extends AbstractProvider
'faviconkit',
'FavOrg',
'FediAct Service',
'FediList Agent',
'Feed Wrangler',
'Feedable\/',
'Feedbin',
Expand Down Expand Up @@ -457,6 +461,7 @@ class Crawlers extends AbstractProvider
'free thumbnails',
'Freeuploader',
'FreshRSS',
'Friendica',
'frontman',
'Funnelback',
'Fuzz Faster U Fool',
Expand Down Expand Up @@ -513,22 +518,32 @@ class Crawlers extends AbstractProvider
'Google-Agent',
'Google-AMPHTML',
'Google-Apps-Script',
'Google-Assess',
'Google-BusinessLinkVerification',
'Google-Calendar-Importer',
'Google-Cloud-Scheduler',
'Google-CWS',
'Google-Firebase',
'Google-HotelAdsVerifier',
'Google-HTTP-Java-Client',
'Google-ilp',
'Google-InspectionTool',
'Google-Lens',
'Google-Podcast',
'Google-Publisher-Plugin',
'Google-Read-Aloud',
'Google-Safety',
'Google-SearchByImage',
'Google-Shopping-Quality',
'Google-Site-Verification',
'Google-Sites-Thumbnails',
'Google-SMTP-STS',
'Google-speakr',
'Google-Structured-Data-Testing-Tool',
'Google-Transparency-Report',
'google-xrawler',
'Google-Youtube-Links',
'GoogleAdSenseInfeed',
'GoogleAgent-Mariner',
'GoogleAssociationService',
'GoogleDocs',
Expand Down Expand Up @@ -634,6 +649,7 @@ class Crawlers extends AbstractProvider
'Image Fetch',
'Image Sucker',
'ImageEngine\/',
'ImageProxy\/',
'ImageVisu\/',
'Imagga',
'imagineeasy',
Expand Down Expand Up @@ -703,6 +719,7 @@ class Crawlers extends AbstractProvider
'JS-Kit',
'JungleKeyThumbnail',
'JustView',
'kakaotalk-scrap',
'Kaspersky Lab CFR link resolver',
'Kelny\/',
'Kerrigan\/',
Expand Down Expand Up @@ -804,7 +821,9 @@ class Crawlers extends AbstractProvider
'MeltwaterNews',
'Melvil Rawi',
'MemGator',
'meta-externalads',
'meta-externalagent',
'Meta-ExternalHit\/',
'Metaspinner',
'MetaURI',
'MFC_Tear_Sample',
Expand Down Expand Up @@ -863,6 +882,7 @@ class Crawlers extends AbstractProvider
'NameFo\.com',
'Nameprotect',
'nationalarchives',
'NaverJapan',
'Navroad',
'nbertaupete95',
'NearSite',
Expand Down Expand Up @@ -913,6 +933,7 @@ class Crawlers extends AbstractProvider
'Norton-Safeweb',
'Notifixious',
'notifyninja',
'Notion\/[0-9.]+ \(https:\/\/notion\.so',
'NotionEmbedder',
'NovaAct',
'nuhk',
Expand All @@ -937,6 +958,7 @@ class Crawlers extends AbstractProvider
'Open Source RSS',
'OpenCalaisSemanticProxy',
'Openfind',
'OpenGraph\.io',
'OpenLinkProfiler',
'Openstat\/',
'OpenVAS',
Expand Down Expand Up @@ -1121,6 +1143,7 @@ class Crawlers extends AbstractProvider
'SearchSight',
'SearchWP',
'search\.thunderstone',
'secureurl\.fwdcdn\.com',
'Seeker',
'semanticdiscovery',
'semanticjuice',
Expand Down Expand Up @@ -1150,6 +1173,7 @@ class Crawlers extends AbstractProvider
'sexsearcher',
'Seznam',
'Shap-User',
'Shareaholic',
'Shelob',
'Shodan',
'Shoppimon',
Expand Down Expand Up @@ -1194,6 +1218,7 @@ class Crawlers extends AbstractProvider
'SMRF URL Expander',
'SMUrlExpander',
'Snake',
'Snap ?URL ?Preview',
'Snappy',
'SnapSearch',
'Snarfer\/',
Expand Down Expand Up @@ -1229,6 +1254,8 @@ class Crawlers extends AbstractProvider
'Statastico\/',
'Statically-',
'StatusCake',
'SteamChatImageProxy',
'SteamChatURLLookup',
'Steeler',
'Stratagems Kumo',
'Stripe\/',
Expand Down Expand Up @@ -1340,6 +1367,7 @@ class Crawlers extends AbstractProvider
'VCI',
'Verity',
'via ggpht\.com GoogleImageProxy',
'ViberUrlDownloader',
'Virusdie',
'visionutils',
'Visual Rights Group',
Expand All @@ -1350,6 +1378,7 @@ class Crawlers extends AbstractProvider
'voyager\/',
'VSAgent\/',
'VSB-TUO\/',
'vuhuvRBT',
'Vulnbusters Meter',
'VYU2',
'w3af\.org',
Expand Down Expand Up @@ -1447,6 +1476,7 @@ class Crawlers extends AbstractProvider
'WordPress\/',
'WordupinfoSearch',
'Word\/',
'workona-favicon-service',
'worldping-api',
'wotbox',
'WP Engine Install Performance API',
Expand Down Expand Up @@ -1474,6 +1504,7 @@ class Crawlers extends AbstractProvider
'Yaanb',
'yacy',
'Yahoo Link Preview',
'Yahoo:LinkExpander',
'YahooCacheSystem',
'YahooMailProxy',
'YahooYSMcm',
Expand All @@ -1498,6 +1529,7 @@ class Crawlers extends AbstractProvider
'Zemanta Aggregator',
'Zend\\\\Http\\\\Client',
'Zend_Http_Client',
'Zendesk Webhook',
'Zermelo',
'Zeus ',
'zgrab',
Expand Down
33 changes: 33 additions & 0 deletions tests/data/user_agent/crawlers.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3794,3 +3794,36 @@ NYU Internet Census (https://scan.lol; research@scan.lol)
Download Demon/3.5.0.11
webbandit/4.xx.0
Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)
Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb/1.1; +mailto:support@bsky.app) Chrome/W.X.Y.Z Safari/537.36
bluesky-domain-status-classifier/1.0
ClickUpLinkUnfurler/1.0 (+https://clickup.com)
FediList Agent/3 (https://fediverse.one/)
FediList Agent/3 (https://fedilist.com/)
Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz
kakaotalk-scrap/1.0; +https://devtalk.kakao.com/t/scrap/33984
Mozilla/5.0 (compatible; Meta-ExternalHit/1.1; +http://www.facebook.com/externalhit_uatext.php)
meta-externalads
Notion/1.0 (https://notion.so; team@makenotion.com)
Mozilla/5.0 (compatible; OpenGraph.io/1.1; +https://opengraph.io/;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36
Mozilla/5.0 (compatible; Shareaholic/1.0; +http://www.shareaholic.com/steve)
Mozilla/5.0 (compatible; SnapURLPreview/1.0; +https://www.snapchat.com)
SteamChatImageProxy
Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)
ViberUrlDownloader
workona-favicon-service/1.0.0
Yahoo:LinkExpander:Slingstone
Zendesk Webhook
RamblerMail/6.0 (incompatible; ImageProxy/6.0)
Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (via secureurl.fwdcdn.com - mail.ukr.net proxy)
Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google-BusinessLinkVerification) Chrome/143.0.7499.192 Safari/537.36
Google-Cloud-Scheduler
Mozilla/5.0 (compatible; Google-CWS)
Google-ilp
Google-Lens
Google-Safety
Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Shopping-Quality +http://www.google.com/merchants/tos/extend/DE/tos.html)
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google-Sites-Thumbnails) Chrome/120.0.6099.71 Safari/537.36
Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; GoogleAdSenseInfeed) Chrome/102.0.5005.61 Mobile Safari/537.36
Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/)
Mozilla/5.0 (compatible; vuhuvRBT/2.0; +https://vuhuv.com/rbt.html)
1 change: 1 addition & 0 deletions tests/data/user_agent/devices.txt
Original file line number Diff line number Diff line change
Expand Up @@ -168537,3 +168537,4 @@ Mozilla/5.0 (Linux; Android 13; octopus Build/R152-16765.49.0) AppleWebKit/537.3
Mozilla/5.0 (Linux; x86_64) LibWeb+LibJS/1.0 Ladybird/1.0
Mozilla/5.0 (Linux; x86_64) Ladybird/1.0
Mozilla/5.0 (macOS; AArch64) Ladybird/1.0 Chrome/146.0.0.0 AppleWebKit/537.36 Safari/537.36
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Notion/7.31.3 Chrome/148.0.7778.265 Electron/42.4.1 Safari/537.36 WantsServiceWorker