Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/Fixtures/Crawlers.php
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,21 @@ class Crawlers extends AbstractProvider
'AllSubmitter',
'alyze\.info',
'amagit',
'amazon-kendra',
'amazon-QBusiness',
'amazon-Quick-on-behalf-of',
'AmazonBuyForMe',
'amphp\/http-client',
'Amzn-User',
'Anarchie',
'AndroidDownloadManager',
'Anemone',
'AngleSharp',
'annotate_google',
'Anomura',
'Anthill',
'anthropic-ai',
'Anthropic-Claude',
'Anturis Agent',
'Ant\.com',
'AnyEvent-HTTP\/',
Expand Down Expand Up @@ -200,6 +207,7 @@ class Crawlers extends AbstractProvider
'BigBozz',
'Bigfoot',
'biglotron',
'bigsur\.ai',
'BingLocalSearch',
'BingPreview',
'binlar',
Expand Down Expand Up @@ -257,6 +265,7 @@ class Crawlers extends AbstractProvider
'cg-eye',
'changedetection',
'ChangesMeter',
'Character-AI',
'Charlotte',
'ChatGPT-User',
'chatterino-api-cache',
Expand All @@ -280,12 +289,15 @@ class Crawlers extends AbstractProvider
'Cloud mapping',
'CloudEndure',
'CloudFlare-AlwaysOnline',
'Cloudflare-AutoRAG',
'Cloudflare-Healthchecks',
'Cloudflare-SSLDetector',
'Cloudinary',
'cmcm\.com',
'coccoc',
'cognitiveseo',
'cohere-ai',
'Cohere-Command',
'ColdFusion',
'colly -',
'CommaFeed',
Expand Down Expand Up @@ -337,6 +349,7 @@ class Crawlers extends AbstractProvider
'DeuSu',
'developers\.google\.com\/\+\/web\/snippet\/',
'Devil',
'Devin\/',
'Digg',
'Digincore',
'DigitalPebble',
Expand Down Expand Up @@ -467,6 +480,7 @@ class Crawlers extends AbstractProvider
'GeedoProductSearch',
'GeedoShopProductFinder',
'geek-tools',
'Gemini-Deep-Research',
'Genieo',
'GentleSource',
'GetCode',
Expand All @@ -481,6 +495,7 @@ class Crawlers extends AbstractProvider
'Ghost Inspector',
'GigablastOpenSource',
'GIS-LABS',
'GitHubCopilotChat',
'GitLab\/',
'github-camo',
'GitHub-Hookshot',
Expand Down Expand Up @@ -518,6 +533,8 @@ class Crawlers extends AbstractProvider
'Google-HotelAdsVerifier',
'Google-HTTP-Java-Client',
'Google-InspectionTool',
'Google-NotebookLM',
'Google-Pinpoint',
'Google-Podcast',
'Google-Publisher-Plugin',
'Google-Read-Aloud',
Expand All @@ -530,6 +547,7 @@ class Crawlers extends AbstractProvider
'google-xrawler',
'Google-Youtube-Links',
'GoogleAgent-Mariner',
'GoogleAgent-URLContext',
'GoogleAssociationService',
'GoogleDocs',
'GoogleHC\/',
Expand Down Expand Up @@ -723,6 +741,8 @@ class Crawlers extends AbstractProvider
'kulturarw3',
'KumKie',
'l9tcpid\/',
'laion-huggingface-processor',
'LAIONDownloader',
'Larbin',
'Lavf\/',
'leakix\.net',
Expand All @@ -737,6 +757,7 @@ class Crawlers extends AbstractProvider
'Licorne',
'Liferea\/',
'Lighthouse',
'Lightpanda',
'Lightspeedsystems',
'Likse',
'limber\.io',
Expand Down Expand Up @@ -788,6 +809,7 @@ class Crawlers extends AbstractProvider
'Majestic12',
'makecontact\/',
'Mandrill',
'Manus-User',
'MapperCmd',
'marketinggrader',
'MarkMonitor',
Expand Down Expand Up @@ -1092,6 +1114,7 @@ class Crawlers extends AbstractProvider
'RuxitSynthetic',
'RyowlEngine',
'safe-agent-scanner',
'SalamandraVLM',
'SalesIntelligent',
'Saleslift',
'SAP NetWeaver Application Server',
Expand Down Expand Up @@ -1289,6 +1312,7 @@ class Crawlers extends AbstractProvider
'Traackr\.com',
'tracemyfile',
'Trackuity',
'Trae-Agent',
'TrapitAgent',
'Trendiction',
'Trendsmap',
Expand All @@ -1304,12 +1328,14 @@ class Crawlers extends AbstractProvider
'twibble',
'Twice',
'Twikle',
'TwinAgent',
'Twingly',
'Twisted PageGetter',
'Typhoeus',
'ubermetrics-technologies',
'uclassify',
'UdmSearch',
'UdV-AI-Research',
'ultimate_sitemap_parser',
'unchaos',
'undici',
Expand All @@ -1333,6 +1359,7 @@ class Crawlers extends AbstractProvider
'URLy Warning',
'URLy\.Warning',
'URL\/Emacs',
'UseAI\/',
'Vacuum',
'Vagabondo',
'VB Project',
Expand Down
32 changes: 32 additions & 0 deletions tests/data/user_agent/crawlers.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3794,3 +3794,35 @@ NYU Internet Census (https://scan.lol; research@scan.lol)
Download Demon/3.5.0.11
webbandit/4.xx.0
Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)
Agent/AmazonBuyForMe
AmazonBuyForMe
Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/119.0.6045.214 Safari/537.36
Mozilla/5.0 (compatible; bigsur.ai/1.0)
bigsur.ai
Mozilla/5.0 (compatible; Character-AI/1.0; +https://character.ai/)
Mozilla/5.0 (compatible; Devin/1.0)
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36; Devin/1.0; +devin.ai
Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36
GitHubCopilotChat/0.8.0
Google-Pinpoint
Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36; Manus-User/1.0
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36; Manus-User/1.0
Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Anthropic-Claude/1.0; +https://www.anthropic.com)
cohere-ai
Mozilla/5.0 (compatible; Cohere-AI)
Mozilla/5.0 (compatible; Cohere-Command/1.0; +https://cohere.com/)
Mozilla/5.0 (compatible; UdV-AI-Research/1.0)
Anomura/1.2 (+https://www.direqt-search.com)
Cloudflare-AutoRAG
amazon-kendra
LAIONDownloader
laion-huggingface-processor
Lightpanda
SalamandraVLM
anthropic-ai
Google-NotebookLM
GoogleAgent-URLContext
Trae-Agent
TwinAgent
Mozilla/5.0 (compatible; UseAI/1.0; +https://agents.urls.use.ai) AppleWebKit/537.36
amazon-QBusiness
1 change: 0 additions & 1 deletion tests/data/user_agent/devices.txt
Original file line number Diff line number Diff line change
Expand Up @@ -168411,7 +168411,6 @@ Mozilla/5.0 (Linux; Android 16; SM-S721B Build/BP4A.251205.006; wv) AppleWebKit/
Mozilla/5.0 (Linux; Android 14; C51 Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/155.0.8059.43 Mobile Safari/537.36
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 OPR/136.0.0.0
Mozilla/5.0 (iPhone; CPU iPhone OS 26_5_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/432.8.954074404 Mobile/15E148 Safari/604.1
Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36; Manus-User/1.0
Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3.1 Safari/605.1.15 Ddg/26.3.1
Mozilla/5.0 (iPhone; CPU iPhone OS 26_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/437.4.973319807 Mobile/15E148 Safari/604.1
Mozilla/5.0 (Linux; Android 16; CPH2663 Build/UKQ1.231108.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/155.0.8059.43 Mobile Safari/537.36
Expand Down