diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..66b59f1 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -29,11 +29,14 @@ class Crawlers extends AbstractProvider '^Calypso v\/', '^COMODO DCV', '^Corax', + '^DA [0-9]', '^DangDang', '^Dart\/[0-9]', '^DavClnt', + '^Deno\/', '^DHSH', '^docker\/[0-9]', + '^ds9 [0-9]', '^Expanse', '^ExposureScanner', '^FDM ', @@ -45,6 +48,7 @@ class Crawlers extends AbstractProvider '^Hello from Palo Alto Networks', '^HTTPClient\/', '^HTTPing', + '^insomnia\/', '^IronFountain-Leads', '^Java\/', '^Jeode\/', @@ -56,12 +60,14 @@ class Crawlers extends AbstractProvider '^Netlab360', '^NG\/[0-9\.]', '^NING\/', + '^node$', '^npm\/', '^Nuclei', '^opencode(\/|$)', '^PHP-AYMAPI\/', '^PHP\/', '^pip\/', + '^Playwright\/', '^pnpm\/', '^RMA\/', '^Ruby|Ruby\/[0-9]', @@ -146,6 +152,7 @@ class Crawlers extends AbstractProvider 'Apache-HttpClient', 'ApacheBench', 'Apexoo', + 'Apidog\/', 'apimon\.de', 'APIs-Google', 'AportWorm\/', @@ -157,6 +164,7 @@ class Crawlers extends AbstractProvider 'arachnode', 'Arachnophilia', 'aria2', + 'ArtelLeadRadar', 'Arukereso', 'asafaweb', 'Asana\/', @@ -168,6 +176,7 @@ class Crawlers extends AbstractProvider 'asynchttp', 'Attach', 'attohttpc', + 'Authory\/', 'autocite', 'AutomaticWPTester', 'Autonomy', @@ -176,6 +185,7 @@ class Crawlers extends AbstractProvider 'axios\/', 'a\.pr-cy\.ru', 'B-l-i-t-z-B-O-T', + 'babashka\.http-client', 'Backlink-Ceck', 'BacklinkHttpStatus', 'BackStreet', @@ -239,6 +249,7 @@ class Crawlers extends AbstractProvider 'Burf Search', 'Butterfly\/', 'BuzzSumo', + 'BW\/[0-9.]+; bit\.ly\/', 'CAAM\/[0-9]', 'caam dot crwlr at gmail dot com', 'CakePHP', @@ -267,6 +278,7 @@ class Crawlers extends AbstractProvider 'Chirp\/', 'chkme\.com', 'Chlooe', + 'Choosito', 'Chromaxa', 'CirrusExplorer', 'CISPA Vulnerability Notification', @@ -288,6 +300,7 @@ class Crawlers extends AbstractProvider 'cognitiveseo', 'ColdFusion', 'colly -', + 'colly\/[0-9]', 'CommaFeed', 'Commons-HttpClient', 'commonscan', @@ -299,11 +312,13 @@ class Crawlers extends AbstractProvider 'copyright sheriff', 'CopyRightCheck', 'Copyscape', + 'CopyvioDetector', 'cortex\/', 'Cosmos4j\.feedback', 'Covario-IDS', 'Cpanel-HTTP-Client', 'cpp-httplib', + 'cpprestsdk\/', 'Craw\/', 'Crescent', 'Criteo', @@ -334,6 +349,7 @@ class Crawlers extends AbstractProvider 'deeris', 'delve\.ai', 'Demon', + 'DepSpid', 'DeuSu', 'developers\.google\.com\/\+\/web\/snippet\/', 'Devil', @@ -348,15 +364,21 @@ class Crawlers extends AbstractProvider 'dlvr', 'DMBrowser', 'DNSPod-reporting', + 'Do Not Track Verifier', 'docoloc', 'Dolphin http client', + 'domain research project', 'DomainAppender', 'DomainLabz', + 'domainreanimator', 'Domains Project\/', 'Donuts Content Explorer', + 'Dormouse\/', 'dotMailer content retrieval', 'dotSemantic', 'downforeveryoneorjustme', + 'Download Master', + 'Download Ninja', 'Download Wonder', 'downnotifier', 'DowntimeDetector', @@ -439,6 +461,7 @@ class Crawlers extends AbstractProvider 'Fiery%20Feeds', 'Filestack', 'Fimap', + 'FinderGo\/', 'findlink', 'findthatfile', 'FlashGet', @@ -451,6 +474,7 @@ class Crawlers extends AbstractProvider 'flynxapp', 'foda-scanner\/[0-9]', 'forensiq', + 'FormFinder\/', 'ForusP', 'FoundSeoTool', 'fragFINN\.de', @@ -458,6 +482,7 @@ class Crawlers extends AbstractProvider 'Freeuploader', 'FreshRSS', 'frontman', + 'Functionize', 'Funnelback', 'Fuzz Faster U Fool', 'G-i-g-a-b-o-t', @@ -577,6 +602,7 @@ class Crawlers extends AbstractProvider 'heritrix', 'Hexometer', 'HF-Research\/', + 'HiDownload', 'historious', 'hkedcity', 'hledejLevne\.cz', @@ -593,6 +619,7 @@ class Crawlers extends AbstractProvider 'HTMLparser', 'htmlyse', 'HTTP Banner Detection', + 'HTTP Retriever', 'http-get', 'HTTP-Header-Abfrage', 'http-kit', @@ -605,6 +632,7 @@ class Crawlers extends AbstractProvider 'HTTPie', 'HTTPMon', 'httpRequest', + 'httprs\/', 'httpscheck', 'httpssites_power', 'httpunit', @@ -629,6 +657,7 @@ class Crawlers extends AbstractProvider 'IDwhois\/', 'Iframely', 'igdeSpyder', + 'iGetter\/', 'iGooglePortal', 'IlTrovatore', 'Image Fetch', @@ -640,6 +669,7 @@ class Crawlers extends AbstractProvider 'img2dataset', 'imgproxy\/[0-9]', 'imgsizer', + 'ImportDomains', 'InAGist', 'inbound\.li parser', 'InDesign%20CC', @@ -651,6 +681,7 @@ class Crawlers extends AbstractProvider 'InfoWizards Reciprocal Link', 'Infrawatch\/[0-9]', 'inpwrd\.com', + 'Inspici', 'instabid', 'Instapaper', 'Integrity', @@ -683,6 +714,7 @@ class Crawlers extends AbstractProvider 'JAHHO', 'janforman', 'Jaunt\/', + 'Java-http-client\/', 'Java.*outbrain', 'javelin\.io', 'Jbrofuzz', @@ -701,10 +733,14 @@ class Crawlers extends AbstractProvider 'Joomla', 'Jorgee', 'JS-Kit', + 'jsdom\/', 'JungleKeyThumbnail', 'JustView', + 'Kapere', 'Kaspersky Lab CFR link resolver', + 'Kaupr\/', 'Kelny\/', + 'KenyaDomainScorer', 'Kerrigan\/', 'KeyCDN', 'Keyword Density', @@ -723,13 +759,16 @@ class Crawlers extends AbstractProvider 'kulturarw3', 'KumKie', 'l9tcpid\/', + 'LAC_IAHarvester', 'Larbin', 'Lavf\/', 'leakix\.net', 'LeechFTP', 'LeechGet', + 'LetrixLabs', 'letsencrypt', 'Lftp', + 'Library Of Congress Web Archiving', 'LibVLC', 'LibWeb', 'Libwhisker', @@ -744,6 +783,7 @@ class Crawlers extends AbstractProvider 'LinkAlarm\/', 'LinkAnalyser', 'link-check', + 'LinkBloom', 'linkCheck', 'linkdex', 'LinkExaminer', @@ -769,6 +809,7 @@ class Crawlers extends AbstractProvider 'ltx71', 'lua-resty-http', 'Lucee \(CFML Engine\)', + 'LumeWebScan', 'Lush Http Client', 'lwp-request', 'lwp-trivial', @@ -777,6 +818,7 @@ class Crawlers extends AbstractProvider 'LYT\.SR', 'L\.webis', 'mabontland', + 'MaCoCu', 'MacOutlook\/', 'MagentaNews\/', 'Mag-Net', @@ -789,6 +831,7 @@ class Crawlers extends AbstractProvider 'makecontact\/', 'Mandrill', 'MapperCmd', + 'MarketGoo', 'marketinggrader', 'MarkMonitor', 'MarkWatch', @@ -808,6 +851,7 @@ class Crawlers extends AbstractProvider 'Metaspinner', 'MetaURI', 'MFC_Tear_Sample', + 'MFHttpScan', 'Microsearch', 'Microsoft Data Access', 'Microsoft Office', @@ -843,6 +887,7 @@ class Crawlers extends AbstractProvider 'Moreover', 'Morfeus Fucking Scanner', 'Morning Paper', + 'Morningscore', 'MovableType', 'mowser', 'Mozlila\/', @@ -853,6 +898,7 @@ class Crawlers extends AbstractProvider 'mShots', 'MuckRack\/', 'muhstik-scan', + 'MuscatFerret', 'MVAClient', 'MxToolbox\/', 'myseosnapshot', @@ -945,6 +991,7 @@ class Crawlers extends AbstractProvider 'Orbiter', 'OrgProbe\/', 'orion-semantics', + 'ORTc\.me', 'Outlook-Express', 'Outlook-iOS', 'Owler', @@ -955,6 +1002,7 @@ class Crawlers extends AbstractProvider 'page scorer', 'Page Valet', 'page2rss', + 'pageburst', 'PageFreezer', 'PageGrabber', 'PagePeeker', @@ -977,12 +1025,15 @@ class Crawlers extends AbstractProvider 'peerindex', 'Peew', 'PeoplePal', + 'perlclient\/', 'Perlu -', 'Perplexity-User', 'PhantomJS Screenshoter', 'PhantomJS\/', + 'PhishBucketRust', 'Photon\/', 'php-requests', + 'PHP-SOAP\/', 'phpservermon', 'Pi-Monster', 'Picscout', @@ -1015,6 +1066,7 @@ class Crawlers extends AbstractProvider 'Port Monitor', 'postano', 'postfix-mta-sts-resolver', + 'PostitLeadDiscovery', 'PostmanRuntime', 'postplanner\.com', 'PostPost', @@ -1030,7 +1082,9 @@ class Crawlers extends AbstractProvider 'probethenet', 'Project ?25499', 'Project-Resonance', + 'project_patchwatch', 'prospectb2b', + 'ProspectorBR', 'Protopage', 'ProWebWalker', 'proximic', @@ -1038,6 +1092,7 @@ class Crawlers extends AbstractProvider 'pshtt, https scanning', 'PTST ', 'PTST\/[0-9]+', + 'puf\/[0-9]', 'pulsetic\.com', 'Pump', 'Python-httplib2', @@ -1059,6 +1114,7 @@ class Crawlers extends AbstractProvider 'Radian6', 'RadioPublicImageResizer', 'Railgun\/', + 'Rainmeter WebParser', 'RankActive', 'RankFlex', 'RankSonicSiteAuditor', @@ -1078,7 +1134,10 @@ class Crawlers extends AbstractProvider 'RepoMonkey', 'request\.js', 'reqwest\/', + 'Research JLU', + 'researchscan\.comsys', 'ResponseCodeTest', + 'REST::Client\/', 'RestSharp', 'Riddler', 'Rival IQ', @@ -1100,6 +1159,8 @@ class Crawlers extends AbstractProvider 'sc-downloader', 'scalaj-http', 'Scamadviser-Frontend', + 'ScamadviserExternalHit', + 'scan@aegis\.network', 'ScanAlert', 'scan\.lol', 'Scoop', @@ -1121,6 +1182,7 @@ class Crawlers extends AbstractProvider 'SearchSight', 'SearchWP', 'search\.thunderstone', + 'Secweb-Sectxt', 'Seeker', 'semanticdiscovery', 'semanticjuice', @@ -1140,6 +1202,7 @@ class Crawlers extends AbstractProvider 'SEOkicks', 'SeoLik\.ru', 'SEOlizer', + 'seolyt', 'Seomoz', 'SEOprofiler', 'seoscanners', @@ -1160,6 +1223,7 @@ class Crawlers extends AbstractProvider 'Sideqik', 'Siege', 'Silktide', + 'Simbiat Software', 'SimplePie', 'SimplyFast', 'Siphon', @@ -1169,6 +1233,7 @@ class Crawlers extends AbstractProvider 'Site24x7', 'SiteBar', 'Sitebeam', + 'sitebulb\.com', 'Sitebulb\/', 'SiteCondor', 'SiteExplorer', @@ -1209,6 +1274,7 @@ class Crawlers extends AbstractProvider 'SpamExperts', 'Spammen', 'Spanner', + 'Sparkler\/', 'Spawning-AI', 'spaziodati', 'SPDYCheck', @@ -1220,6 +1286,7 @@ class Crawlers extends AbstractProvider 'spray-can', 'Sprinklr ', 'spyonweb', + 'sqalix', 'sqlmap', 'Sqlworm', 'Sqworm', @@ -1252,6 +1319,7 @@ class Crawlers extends AbstractProvider 'sysscan', 'Szukacz', 'T0PHackTeam', + 'TagInspector', 'tAkeOut', 'Tarantula\/', 'Taringa UGC', @@ -1295,6 +1363,7 @@ class Crawlers extends AbstractProvider 'trendspottr', 'truwoGPS', 'TryJsoup', + 'TTD-Content', 'TulipChain', 'Turingos', 'Turnitin', @@ -1309,6 +1378,8 @@ class Crawlers extends AbstractProvider 'Typhoeus', 'ubermetrics-technologies', 'uclassify', + 'uclient-fetch', + 'UCSBNetworkMeasurement', 'UdmSearch', 'ultimate_sitemap_parser', 'unchaos', @@ -1333,6 +1404,7 @@ class Crawlers extends AbstractProvider 'URLy Warning', 'URLy\.Warning', 'URL\/Emacs', + 'UtilMind HTTPGet', 'Vacuum', 'Vagabondo', 'VB Project', @@ -1371,6 +1443,7 @@ class Crawlers extends AbstractProvider 'Web Enhancer', 'Web Fetch', 'Web Fuck', + 'Web Image Collector', 'Web Pix', 'Web Sauger', 'Web spyder', @@ -1470,6 +1543,7 @@ class Crawlers extends AbstractProvider 'Xenu Link Sleuth', 'XING-contenttabreceiver', 'xpymep([0-9]?)\.exe', + 'xtate\/', 'Y!J-[A-Z][A-Z][A-Z]', 'Yaanb', 'yacy', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..4232560 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,80 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +Google Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64; CONTACT; scan@aegis.network) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 +Mozilla/5.0 ArtelLeadRadar/4.0 +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BW/1.1; bit.ly/2W6Px8S) Chrome/84.0.4147.105 Safari/537.36 +Choosito/1.0 +http://www.choosito.com/ +Mozilla/5.0 (compatible; CopyvioDetector/0.4.dev0; +wikipedia.earwig[at]gmail.com) +ds9 2.000.ec2(+http://www.deepsearchnine.com/ds9.html) +ds9 2.000.ec2 +Mozilla/4.0 (compatible; DepSpid/5.07; +http://about.depspid.net) +support@domainreanimator.com +domain research project (+https://trentwil.es/domains.html) +Mozilla/5.0 (compatible; Dormouse/1.0) +Electronic Frontier Foundation's Do Not Track Verifier (for questions or concerns email dnt-policy@eff.org) +Mozilla/5.0 (compatible; FinderGo/1.0) +Mozilla/5.0 (compatible; FormFinder/1.0) +Mozilla/5.0 (X11; Windows x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Functionize +Mozilla/5.0 (compatible; ImportDomains/1.0) +Mozilla/6.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Inspici (www.inspici.com) +Mozilla/5.0 (compatible; Kaupr/1.0) +Mozilla/5.0 (compatible; KenyaDomainScorer/2.1) +Mozilla/5.0 (compatible; LAC_IAHarvester/3.3.0; +https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx) +LetrixLabs/1.0 +LinkBloom/2.7.11 (Educational Domain Discovery; +https://hamidsoltani.com/linkbloom) +Mozilla/5.0 (compatible; LumeWebScan/2.0; +https://lumeweaver.com/) +Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/) +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15 MarketGoo/2.1 +Mozilla/5.0 (Morningscore/1.0) +Mozilla/4.0 (compatible; MuscatFerret/1.7; http://www.webtop.com/) +Mozilla/5.0 (compatible; ORT/v1.0.0; visit ORTc.me ) +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; pageburst) Chrome/98.0.4758.102 Safari/537.36 +Mozilla/5.0 (compatible; PhishBucketRust/1.0) +Mozilla/5.0 (compatible; PostitLeadDiscovery/2.0; public business contact research) +Mozilla/5.0 project_patchwatch +Mozilla/5.0 (compatible; ProspectorBR/1.0; +prospeccao-adtech; contato via site) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36 +Mozilla/5.0 researchscan.comsys.rwth-aachen.de +Rainmeter WebParser plugin +Research JLU +ScamadviserExternalHit/1.0 +Secweb-Sectxt/1.0 +Mozilla/5.0 (compatible; seolyt/1.1; +https://seolyt.com) +Mozilla/5.0 (compatible; seoLyt/1.0; +https://seolyt.com) +Simbiat Software +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Mobile Safari/537.36 +https://sitebulb.com +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Sparkler/0.2.0-SNAPSHOT +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) TagInspector/500.1 Chrome/90.0.4430.72 Safari/537.36 Edg/90.0.818.42 +Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content) +Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 UCSBNetworkMeasurement/2023 (contact; stijn; at; ucsb.edu;) +sqalix-geral-worker/1.0 +xtate/2.12.4 +Mozilla/5.0 xtate/2.12.4 +Mozilla/5.0 (compatible; Authory/1.0; +https://authory.com/) +Library Of Congress Web Archiving +Download Master +Download Ninja 3.0 +HiDownload +Kapere (http://www.kapere.com) +MFHttpScan +UtilMind HTTPGet +Web Image Collector +DA 7.0 +colly/2.1.0 +iGetter/2 (Macintosh; U; PPC Mac OS X; en) +puf/0.93.2a (Linux 2.4.18; i686) +Mozilla/5.0 (win32) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/15.2.1 +Apidog/1.0.0 (https://apidog.com) +HTTP Retriever +Java-http-client/11.0.13 +PHP-SOAP/8.5.6 +REST::Client/273 +insomnia/2022.6.0 +babashka.http-client/0.4.14 +cpprestsdk/2.8.0 +httprs/0.1.0 +perlclient/1.0 +uclient-fetch +Deno/1.40 +Playwright/1.40 +node