From 401ea58a4e7883931a3247592cf78683ba70687d Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Mon, 16 Nov 2026 22:49:06 +0000 Subject: [PATCH] Detect AI agents, assistants and training crawlers Add 27 signatures for AI tools that weren't detected: - Agents and assistants fetching pages for a user: Amazon (Amzn-User, AmazonBuyForMe, amazon-QBusiness), Gemini Deep Research, GoogleAgent-URLContext, Google NotebookLM and Pinpoint, Devin, Manus, Character.AI, GitHub Copilot Chat, Trae, TwinAgent, UseAI, bigsur.ai. - Training and research crawlers: Anthropic's older tokens (anthropic-ai, Anthropic-Claude), Cohere, LAION, SalamandraVLM, UdV-AI-Research. - Search and retrieval: Cloudflare AutoRAG, Amazon Kendra, Anomura. - Lightpanda, a headless browser built for AI agents and scraping. Devin and UseAI are matched with a trailing slash because Devin is a common name. Gemini is matched only as Gemini-Deep-Research, since phones and the Gemini iOS app also carry the name. The Manus-User UA in devices.txt moves to crawlers.txt: Manus is an agent doing the browsing, not a person viewing the page. --- src/Fixtures/Crawlers.php | 27 +++++++++++++++++++++++++ tests/data/user_agent/crawlers.txt | 32 ++++++++++++++++++++++++++++++ tests/data/user_agent/devices.txt | 1 - 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..4c49d3e 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -128,14 +128,21 @@ class Crawlers extends AbstractProvider 'AllSubmitter', 'alyze\.info', 'amagit', + 'amazon-kendra', + 'amazon-QBusiness', 'amazon-Quick-on-behalf-of', + 'AmazonBuyForMe', 'amphp\/http-client', + 'Amzn-User', 'Anarchie', 'AndroidDownloadManager', 'Anemone', 'AngleSharp', 'annotate_google', + 'Anomura', 'Anthill', + 'anthropic-ai', + 'Anthropic-Claude', 'Anturis Agent', 'Ant\.com', 'AnyEvent-HTTP\/', @@ -200,6 +207,7 @@ class Crawlers extends AbstractProvider 'BigBozz', 'Bigfoot', 'biglotron', + 'bigsur\.ai', 'BingLocalSearch', 'BingPreview', 'binlar', @@ -257,6 +265,7 @@ class Crawlers extends AbstractProvider 'cg-eye', 'changedetection', 'ChangesMeter', + 'Character-AI', 'Charlotte', 'ChatGPT-User', 'chatterino-api-cache', @@ -280,12 +289,15 @@ class Crawlers extends AbstractProvider 'Cloud mapping', 'CloudEndure', 'CloudFlare-AlwaysOnline', + 'Cloudflare-AutoRAG', 'Cloudflare-Healthchecks', 'Cloudflare-SSLDetector', 'Cloudinary', 'cmcm\.com', 'coccoc', 'cognitiveseo', + 'cohere-ai', + 'Cohere-Command', 'ColdFusion', 'colly -', 'CommaFeed', @@ -337,6 +349,7 @@ class Crawlers extends AbstractProvider 'DeuSu', 'developers\.google\.com\/\+\/web\/snippet\/', 'Devil', + 'Devin\/', 'Digg', 'Digincore', 'DigitalPebble', @@ -467,6 +480,7 @@ class Crawlers extends AbstractProvider 'GeedoProductSearch', 'GeedoShopProductFinder', 'geek-tools', + 'Gemini-Deep-Research', 'Genieo', 'GentleSource', 'GetCode', @@ -481,6 +495,7 @@ class Crawlers extends AbstractProvider 'Ghost Inspector', 'GigablastOpenSource', 'GIS-LABS', + 'GitHubCopilotChat', 'GitLab\/', 'github-camo', 'GitHub-Hookshot', @@ -518,6 +533,8 @@ class Crawlers extends AbstractProvider 'Google-HotelAdsVerifier', 'Google-HTTP-Java-Client', 'Google-InspectionTool', + 'Google-NotebookLM', + 'Google-Pinpoint', 'Google-Podcast', 'Google-Publisher-Plugin', 'Google-Read-Aloud', @@ -530,6 +547,7 @@ class Crawlers extends AbstractProvider 'google-xrawler', 'Google-Youtube-Links', 'GoogleAgent-Mariner', + 'GoogleAgent-URLContext', 'GoogleAssociationService', 'GoogleDocs', 'GoogleHC\/', @@ -723,6 +741,8 @@ class Crawlers extends AbstractProvider 'kulturarw3', 'KumKie', 'l9tcpid\/', + 'laion-huggingface-processor', + 'LAIONDownloader', 'Larbin', 'Lavf\/', 'leakix\.net', @@ -737,6 +757,7 @@ class Crawlers extends AbstractProvider 'Licorne', 'Liferea\/', 'Lighthouse', + 'Lightpanda', 'Lightspeedsystems', 'Likse', 'limber\.io', @@ -788,6 +809,7 @@ class Crawlers extends AbstractProvider 'Majestic12', 'makecontact\/', 'Mandrill', + 'Manus-User', 'MapperCmd', 'marketinggrader', 'MarkMonitor', @@ -1092,6 +1114,7 @@ class Crawlers extends AbstractProvider 'RuxitSynthetic', 'RyowlEngine', 'safe-agent-scanner', + 'SalamandraVLM', 'SalesIntelligent', 'Saleslift', 'SAP NetWeaver Application Server', @@ -1289,6 +1312,7 @@ class Crawlers extends AbstractProvider 'Traackr\.com', 'tracemyfile', 'Trackuity', + 'Trae-Agent', 'TrapitAgent', 'Trendiction', 'Trendsmap', @@ -1304,12 +1328,14 @@ class Crawlers extends AbstractProvider 'twibble', 'Twice', 'Twikle', + 'TwinAgent', 'Twingly', 'Twisted PageGetter', 'Typhoeus', 'ubermetrics-technologies', 'uclassify', 'UdmSearch', + 'UdV-AI-Research', 'ultimate_sitemap_parser', 'unchaos', 'undici', @@ -1333,6 +1359,7 @@ class Crawlers extends AbstractProvider 'URLy Warning', 'URLy\.Warning', 'URL\/Emacs', + 'UseAI\/', 'Vacuum', 'Vagabondo', 'VB Project', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..75f6e01 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,35 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +Agent/AmazonBuyForMe +AmazonBuyForMe +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/119.0.6045.214 Safari/537.36 +Mozilla/5.0 (compatible; bigsur.ai/1.0) +bigsur.ai +Mozilla/5.0 (compatible; Character-AI/1.0; +https://character.ai/) +Mozilla/5.0 (compatible; Devin/1.0) +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36; Devin/1.0; +devin.ai +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36 +GitHubCopilotChat/0.8.0 +Google-Pinpoint +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36; Manus-User/1.0 +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36; Manus-User/1.0 +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Anthropic-Claude/1.0; +https://www.anthropic.com) +cohere-ai +Mozilla/5.0 (compatible; Cohere-AI) +Mozilla/5.0 (compatible; Cohere-Command/1.0; +https://cohere.com/) +Mozilla/5.0 (compatible; UdV-AI-Research/1.0) +Anomura/1.2 (+https://www.direqt-search.com) +Cloudflare-AutoRAG +amazon-kendra +LAIONDownloader +laion-huggingface-processor +Lightpanda +SalamandraVLM +anthropic-ai +Google-NotebookLM +GoogleAgent-URLContext +Trae-Agent +TwinAgent +Mozilla/5.0 (compatible; UseAI/1.0; +https://agents.urls.use.ai) AppleWebKit/537.36 +amazon-QBusiness diff --git a/tests/data/user_agent/devices.txt b/tests/data/user_agent/devices.txt index a0d9607..9308062 100644 --- a/tests/data/user_agent/devices.txt +++ b/tests/data/user_agent/devices.txt @@ -168411,7 +168411,6 @@ Mozilla/5.0 (Linux; Android 16; SM-S721B Build/BP4A.251205.006; wv) AppleWebKit/ Mozilla/5.0 (Linux; Android 14; C51 Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/155.0.8059.43 Mobile Safari/537.36 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 OPR/136.0.0.0 Mozilla/5.0 (iPhone; CPU iPhone OS 26_5_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/432.8.954074404 Mobile/15E148 Safari/604.1 -Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36; Manus-User/1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3.1 Safari/605.1.15 Ddg/26.3.1 Mozilla/5.0 (iPhone; CPU iPhone OS 26_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/437.4.973319807 Mobile/15E148 Safari/604.1 Mozilla/5.0 (Linux; Android 16; CPH2663 Build/UKQ1.231108.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/155.0.8059.43 Mobile Safari/537.36