diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..c48301c 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -220,6 +220,8 @@ class Crawlers extends AbstractProvider 'BlogSearch', 'Blogtrottr', 'BlowFish', + 'Bluesky Cardyb', + 'bluesky-domain-status-classifier', 'boitho\.com-dc', 'Boost\.Beast', 'BPImageWalker', @@ -276,6 +278,7 @@ class Crawlers extends AbstractProvider 'Clarsentia', 'Claude-User', 'claude-web', + 'ClickUpLinkUnfurler', 'clips\.ua\.ac\.be', 'Cloud mapping', 'CloudEndure', @@ -416,6 +419,7 @@ class Crawlers extends AbstractProvider 'faviconkit', 'FavOrg', 'FediAct Service', + 'FediList Agent', 'Feed Wrangler', 'Feedable\/', 'Feedbin', @@ -457,6 +461,7 @@ class Crawlers extends AbstractProvider 'free thumbnails', 'Freeuploader', 'FreshRSS', + 'Friendica', 'frontman', 'Funnelback', 'Fuzz Faster U Fool', @@ -513,22 +518,32 @@ class Crawlers extends AbstractProvider 'Google-Agent', 'Google-AMPHTML', 'Google-Apps-Script', + 'Google-Assess', + 'Google-BusinessLinkVerification', 'Google-Calendar-Importer', + 'Google-Cloud-Scheduler', + 'Google-CWS', 'Google-Firebase', 'Google-HotelAdsVerifier', 'Google-HTTP-Java-Client', + 'Google-ilp', 'Google-InspectionTool', + 'Google-Lens', 'Google-Podcast', 'Google-Publisher-Plugin', 'Google-Read-Aloud', + 'Google-Safety', 'Google-SearchByImage', + 'Google-Shopping-Quality', 'Google-Site-Verification', + 'Google-Sites-Thumbnails', 'Google-SMTP-STS', 'Google-speakr', 'Google-Structured-Data-Testing-Tool', 'Google-Transparency-Report', 'google-xrawler', 'Google-Youtube-Links', + 'GoogleAdSenseInfeed', 'GoogleAgent-Mariner', 'GoogleAssociationService', 'GoogleDocs', @@ -634,6 +649,7 @@ class Crawlers extends AbstractProvider 'Image Fetch', 'Image Sucker', 'ImageEngine\/', + 'ImageProxy\/', 'ImageVisu\/', 'Imagga', 'imagineeasy', @@ -703,6 +719,7 @@ class Crawlers extends AbstractProvider 'JS-Kit', 'JungleKeyThumbnail', 'JustView', + 'kakaotalk-scrap', 'Kaspersky Lab CFR link resolver', 'Kelny\/', 'Kerrigan\/', @@ -804,7 +821,9 @@ class Crawlers extends AbstractProvider 'MeltwaterNews', 'Melvil Rawi', 'MemGator', + 'meta-externalads', 'meta-externalagent', + 'Meta-ExternalHit\/', 'Metaspinner', 'MetaURI', 'MFC_Tear_Sample', @@ -863,6 +882,7 @@ class Crawlers extends AbstractProvider 'NameFo\.com', 'Nameprotect', 'nationalarchives', + 'NaverJapan', 'Navroad', 'nbertaupete95', 'NearSite', @@ -913,6 +933,7 @@ class Crawlers extends AbstractProvider 'Norton-Safeweb', 'Notifixious', 'notifyninja', + 'Notion\/[0-9.]+ \(https:\/\/notion\.so', 'NotionEmbedder', 'NovaAct', 'nuhk', @@ -937,6 +958,7 @@ class Crawlers extends AbstractProvider 'Open Source RSS', 'OpenCalaisSemanticProxy', 'Openfind', + 'OpenGraph\.io', 'OpenLinkProfiler', 'Openstat\/', 'OpenVAS', @@ -1121,6 +1143,7 @@ class Crawlers extends AbstractProvider 'SearchSight', 'SearchWP', 'search\.thunderstone', + 'secureurl\.fwdcdn\.com', 'Seeker', 'semanticdiscovery', 'semanticjuice', @@ -1150,6 +1173,7 @@ class Crawlers extends AbstractProvider 'sexsearcher', 'Seznam', 'Shap-User', + 'Shareaholic', 'Shelob', 'Shodan', 'Shoppimon', @@ -1194,6 +1218,7 @@ class Crawlers extends AbstractProvider 'SMRF URL Expander', 'SMUrlExpander', 'Snake', + 'Snap ?URL ?Preview', 'Snappy', 'SnapSearch', 'Snarfer\/', @@ -1229,6 +1254,8 @@ class Crawlers extends AbstractProvider 'Statastico\/', 'Statically-', 'StatusCake', + 'SteamChatImageProxy', + 'SteamChatURLLookup', 'Steeler', 'Stratagems Kumo', 'Stripe\/', @@ -1340,6 +1367,7 @@ class Crawlers extends AbstractProvider 'VCI', 'Verity', 'via ggpht\.com GoogleImageProxy', + 'ViberUrlDownloader', 'Virusdie', 'visionutils', 'Visual Rights Group', @@ -1350,6 +1378,7 @@ class Crawlers extends AbstractProvider 'voyager\/', 'VSAgent\/', 'VSB-TUO\/', + 'vuhuvRBT', 'Vulnbusters Meter', 'VYU2', 'w3af\.org', @@ -1447,6 +1476,7 @@ class Crawlers extends AbstractProvider 'WordPress\/', 'WordupinfoSearch', 'Word\/', + 'workona-favicon-service', 'worldping-api', 'wotbox', 'WP Engine Install Performance API', @@ -1474,6 +1504,7 @@ class Crawlers extends AbstractProvider 'Yaanb', 'yacy', 'Yahoo Link Preview', + 'Yahoo:LinkExpander', 'YahooCacheSystem', 'YahooMailProxy', 'YahooYSMcm', @@ -1498,6 +1529,7 @@ class Crawlers extends AbstractProvider 'Zemanta Aggregator', 'Zend\\\\Http\\\\Client', 'Zend_Http_Client', + 'Zendesk Webhook', 'Zermelo', 'Zeus ', 'zgrab', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..bc683ae 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,36 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb/1.1; +mailto:support@bsky.app) Chrome/W.X.Y.Z Safari/537.36 +bluesky-domain-status-classifier/1.0 +ClickUpLinkUnfurler/1.0 (+https://clickup.com) +FediList Agent/3 (https://fediverse.one/) +FediList Agent/3 (https://fedilist.com/) +Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz +kakaotalk-scrap/1.0; +https://devtalk.kakao.com/t/scrap/33984 +Mozilla/5.0 (compatible; Meta-ExternalHit/1.1; +http://www.facebook.com/externalhit_uatext.php) +meta-externalads +Notion/1.0 (https://notion.so; team@makenotion.com) +Mozilla/5.0 (compatible; OpenGraph.io/1.1; +https://opengraph.io/;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 +Mozilla/5.0 (compatible; Shareaholic/1.0; +http://www.shareaholic.com/steve) +Mozilla/5.0 (compatible; SnapURLPreview/1.0; +https://www.snapchat.com) +SteamChatImageProxy +Valve/Steam HTTP Client 1.0 (SteamChatURLLookup) +ViberUrlDownloader +workona-favicon-service/1.0.0 +Yahoo:LinkExpander:Slingstone +Zendesk Webhook +RamblerMail/6.0 (incompatible; ImageProxy/6.0) +Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (via secureurl.fwdcdn.com - mail.ukr.net proxy) +Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36 +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google-BusinessLinkVerification) Chrome/143.0.7499.192 Safari/537.36 +Google-Cloud-Scheduler +Mozilla/5.0 (compatible; Google-CWS) +Google-ilp +Google-Lens +Google-Safety +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Shopping-Quality +http://www.google.com/merchants/tos/extend/DE/tos.html) +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google-Sites-Thumbnails) Chrome/120.0.6099.71 Safari/537.36 +Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; GoogleAdSenseInfeed) Chrome/102.0.5005.61 Mobile Safari/537.36 +Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/) +Mozilla/5.0 (compatible; vuhuvRBT/2.0; +https://vuhuv.com/rbt.html) diff --git a/tests/data/user_agent/devices.txt b/tests/data/user_agent/devices.txt index a0d9607..4f7e4d8 100644 --- a/tests/data/user_agent/devices.txt +++ b/tests/data/user_agent/devices.txt @@ -168537,3 +168537,4 @@ Mozilla/5.0 (Linux; Android 13; octopus Build/R152-16765.49.0) AppleWebKit/537.3 Mozilla/5.0 (Linux; x86_64) LibWeb+LibJS/1.0 Ladybird/1.0 Mozilla/5.0 (Linux; x86_64) Ladybird/1.0 Mozilla/5.0 (macOS; AArch64) Ladybird/1.0 Chrome/146.0.0.0 AppleWebKit/537.36 Safari/537.36 +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Notion/7.31.3 Chrome/148.0.7778.265 Electron/42.4.1 Safari/537.36 WantsServiceWorker