From f85cdced159dc671a6db852e950c40606a7b0c5d Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Mon, 16 Nov 2026 22:55:21 +0000 Subject: [PATCH] Detect uptime monitors, observability tools and service agents Add 46 signatures for automated checks that weren't detected: uptime and synthetic monitors (Catchpoint, ThousandEyes, AlertSite, Uptrends, Oh Dear, Azure AppInsights and others), observability agents (Prometheus, Grafana, Datadog, collectd, Munin, Blackbox Exporter, SolarWinds), WordPress management services (MainWP, ManageWP, WP Umbrella), Cloudflare's diagnostic and verification fetchers, and other service agents such as Power Automate, Sectigo DCV, Valimail, Chrome's privacy-preserving prefetch proxy and Apache's internal dummy connection. The Catchpoint UA in devices.txt moves to crawlers.txt. --- src/Fixtures/Crawlers.php | 46 +++++++++++++++++++++++++++ tests/data/user_agent/crawlers.txt | 51 ++++++++++++++++++++++++++++++ tests/data/user_agent/devices.txt | 1 - 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php index 97cb4ea..9773c18 100644 --- a/src/Fixtures/Crawlers.php +++ b/src/Fixtures/Crawlers.php @@ -83,6 +83,7 @@ class Crawlers extends AbstractProvider '192\.comAgent', '2GDPR\/', '2ip\.ru', + '360JK', '404enemy', '7Siters', '80legs', @@ -121,6 +122,7 @@ class Crawlers extends AbstractProvider 'akka-http\/', 'akula\/', 'alertra', + 'AlertSite', 'alexa site audit', 'Alibaba\.Security\.Heimdall', 'Alligator', @@ -128,6 +130,7 @@ class Crawlers extends AbstractProvider 'AllSubmitter', 'alyze\.info', 'amagit', + 'Amazon Route 53 Health Check', 'amazon-Quick-on-behalf-of', 'amphp\/http-client', 'Anarchie', @@ -151,6 +154,7 @@ class Crawlers extends AbstractProvider 'AportWorm\/', 'AppBeat\/', 'AppEngine-Google', + 'AppInsights', 'AppleSyndication', 'Aprc\/[0-9]', 'Arachmo', @@ -175,6 +179,7 @@ class Crawlers extends AbstractProvider 'AWS Security Scanner', 'axios\/', 'a\.pr-cy\.ru', + 'azure-logic-apps', 'B-l-i-t-z-B-O-T', 'Backlink-Ceck', 'BacklinkHttpStatus', @@ -192,6 +197,7 @@ class Crawlers extends AbstractProvider 'baypup\/', 'BazQux', 'BBBike', + 'BBC-Forge-URL-Monitor', 'BCKLINKS', 'BDFetch', 'BegunAdvertising', @@ -200,6 +206,7 @@ class Crawlers extends AbstractProvider 'BigBozz', 'Bigfoot', 'biglotron', + 'BinaryCanary', 'BingLocalSearch', 'BingPreview', 'binlar', @@ -211,6 +218,7 @@ class Crawlers extends AbstractProvider 'Black Hole', 'Blackboard Safeassign', 'blackboardally', + 'Blackbox Exporter', 'BlackWidow', 'BlockNote\.Net', 'BlogBridge', @@ -246,6 +254,7 @@ class Crawlers extends AbstractProvider 'Camoufox', 'Canary%20Mail', 'CaretNail', + 'Catchpoint', 'catexplorador', 'CC Metadata Scaper', 'Cegbfeieh', @@ -268,6 +277,7 @@ class Crawlers extends AbstractProvider 'chkme\.com', 'Chlooe', 'Chromaxa', + 'Chrome Privacy Preserving Prefetch Proxy', 'CirrusExplorer', 'CISPA Vulnerability Notification', 'CISPA Web Analyser', @@ -279,14 +289,18 @@ class Crawlers extends AbstractProvider 'clips\.ua\.ac\.be', 'Cloud mapping', 'CloudEndure', + 'Cloudflare Custom Hostname Verification', + 'Cloudflare-?Diagnostics', 'CloudFlare-AlwaysOnline', 'Cloudflare-Healthchecks', + 'Cloudflare-Smart-Transit', 'Cloudflare-SSLDetector', 'Cloudinary', 'cmcm\.com', 'coccoc', 'cognitiveseo', 'ColdFusion', + 'collectd\/', 'colly -', 'CommaFeed', 'Commons-HttpClient', @@ -321,6 +335,8 @@ class Crawlers extends AbstractProvider 'DareBoost', 'DatabaseDriverMysqli', 'DataCha0s', + 'Datadog Agent', + 'Datadog\/Synthetics', 'DatadogSynthetics', 'Datafeedwatch', 'Datanyze', @@ -335,6 +351,7 @@ class Crawlers extends AbstractProvider 'delve\.ai', 'Demon', 'DeuSu', + 'developers\.cloudflare\.com\/security-center', 'developers\.google\.com\/\+\/web\/snippet\/', 'Devil', 'Digg', @@ -377,6 +394,7 @@ class Crawlers extends AbstractProvider 'ECCP', 'eContext\/', 'Ecxi', + 'EdgeWatch', 'EirGrabber', 'ElectricMonk', 'elefent', @@ -387,6 +405,7 @@ class Crawlers extends AbstractProvider 'Embed PHP Library', 'Embedly', 'endo\/', + 'Episode Duration Lookup', 'europarchive\.org', 'evc-batch', 'EventMachine HttpClient', @@ -402,6 +421,7 @@ class Crawlers extends AbstractProvider 'Express WebPictures', 'Extreme Picture Finder', 'EyeNetIE', + 'Ezgif\/', 'ezooms', 'facebookcatalog', 'facebookexternalhit', @@ -545,6 +565,7 @@ class Crawlers extends AbstractProvider 'GoZilla', 'grabify', 'GrabNet', + 'Grafana\/', 'Grafula', 'Grammarly', 'GrapeFX', @@ -619,6 +640,7 @@ class Crawlers extends AbstractProvider 'HubSpot ', 'HubSpot-Link-Resolver', 'Humanlinks', + 'Hydrozen\.io', 'i2kconnect\/', 'IAB-Tech-Lab', 'Iblog', @@ -657,6 +679,7 @@ class Crawlers extends AbstractProvider 'integromedb', 'Intelliseek', 'InterGET', + 'internal dummy connection', 'Internet Ninja', 'InternetMeasurement', 'InternetSeer', @@ -712,6 +735,7 @@ class Crawlers extends AbstractProvider 'khttp\/', 'KickFire', 'KimonoLabs\/', + 'kiwitcms-gitops', 'Kml-Google', 'knows\.is', 'KOCMOHABT', @@ -785,8 +809,10 @@ class Crawlers extends AbstractProvider 'MailChimp', 'Mail\.Ru', 'Mailpit\/', + 'MainWP', 'Majestic12', 'makecontact\/', + 'ManageWP', 'Mandrill', 'MapperCmd', 'marketinggrader', @@ -853,6 +879,7 @@ class Crawlers extends AbstractProvider 'mShots', 'MuckRack\/', 'muhstik-scan', + 'munin\/', 'MVAClient', 'MxToolbox\/', 'myseosnapshot', @@ -881,7 +908,9 @@ class Crawlers extends AbstractProvider 'NetSystemsResearch', 'nettle', 'NetTrack', + 'Netumo', 'Netvibes', + 'NETVIGIE', 'NetworkingExtension', 'NetZIP', 'Neustar WPM', @@ -930,6 +959,7 @@ class Crawlers extends AbstractProvider 'Offline Navigator', 'odin\.com', 'OgScrper', + 'Oh Dear', 'okhttp', 'omgili', 'OMSC', @@ -983,6 +1013,7 @@ class Crawlers extends AbstractProvider 'PhantomJS\/', 'Photon\/', 'php-requests', + 'phpMyAdmin', 'phpservermon', 'Pi-Monster', 'Picscout', @@ -993,6 +1024,7 @@ class Crawlers extends AbstractProvider 'PingAdmin\.Ru', 'Pingdom', 'Pingoscope', + 'pingping\.io', 'PingSpot', 'ping\.blo\.gs', 'pinterest\.com', @@ -1030,6 +1062,7 @@ class Crawlers extends AbstractProvider 'probethenet', 'Project ?25499', 'Project-Resonance', + 'Prometheus\/', 'prospectb2b', 'Protopage', 'ProWebWalker', @@ -1084,6 +1117,7 @@ class Crawlers extends AbstractProvider 'Rival IQ', 'Robosourcer', 'Robozilla', + 'rocketmonitor', 'ROI Hunter', 'RPT-HTTPClient', 'RSiteAuditor', @@ -1121,6 +1155,7 @@ class Crawlers extends AbstractProvider 'SearchSight', 'SearchWP', 'search\.thunderstone', + 'Sectigo DCV', 'Seeker', 'semanticdiscovery', 'semanticjuice', @@ -1146,6 +1181,8 @@ class Crawlers extends AbstractProvider 'SEOsearch', 'seositecheckup', 'SEOstats', + 'SerpReputationManagementAgent', + 'Server ?Density', 'servernfo', 'sexsearcher', 'Seznam', @@ -1184,6 +1221,7 @@ class Crawlers extends AbstractProvider 'SiteTruth', 'Sitevigil', 'sitexy\.com', + 'sixy\.ch', 'SkypeUriPreview', 'Slack\/', 'sli-systems\.com', @@ -1202,6 +1240,7 @@ class Crawlers extends AbstractProvider 'Snoopy', 'SnowHaze Search', 'sogou web', + 'solarwinds', 'SortSite', 'Sottopop', 'sovereign\.ai', @@ -1276,6 +1315,7 @@ class Crawlers extends AbstractProvider 'theinternetrules', 'TheNomad', 'Thinklab', + 'ThousandEyes', 'Thumbor', 'Thumbshots', 'ThumbSniper', @@ -1296,6 +1336,7 @@ class Crawlers extends AbstractProvider 'truwoGPS', 'TryJsoup', 'TulipChain', + 'TuneIn-DirMon', 'Turingos', 'Turnitin', 'tweetedtimes', @@ -1323,6 +1364,7 @@ class Crawlers extends AbstractProvider 'updown\.io daemon', 'Upflow', 'Uptimia', + 'uptrends', 'URL Verifier', 'Urlcheckr', 'URLitor', @@ -1335,6 +1377,7 @@ class Crawlers extends AbstractProvider 'URL\/Emacs', 'Vacuum', 'Vagabondo', + 'Valimail\/', 'VB Project', 'vBSEO', 'VCI', @@ -1400,6 +1443,7 @@ class Crawlers extends AbstractProvider 'WebLeacher', 'webmastercoffee', 'webmon ', + 'WebMon\/', 'WebPix', 'WebReaper', 'WebSauger', @@ -1431,6 +1475,7 @@ class Crawlers extends AbstractProvider 'WhatsMyIP', 'WhatWeb', 'WhereGoes\?', + 'wheresitup', 'Whibse', 'WhoAPI\/', 'WhoRunsCoinHive', @@ -1455,6 +1500,7 @@ class Crawlers extends AbstractProvider 'wpif', 'wprecon\.com survey', 'WPScan', + 'WPUmbrella', 'wscheck', 'Wtrace', 'WWW-Collector-E', diff --git a/tests/data/user_agent/crawlers.txt b/tests/data/user_agent/crawlers.txt index 211fe7f..3557323 100644 --- a/tests/data/user_agent/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -3794,3 +3794,54 @@ NYU Internet Census (https://scan.lol; research@scan.lol) Download Demon/3.5.0.11 webbandit/4.xx.0 Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com) +Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx +Apache/2.2.2 (Fedora) (internal dummy connection) +Castro 2, Episode Duration Lookup +Chrome Privacy Preserving Prefetch Proxy +Cloudflare Custom Hostname Verification +Mozilla/5.0 (compatible; Ezgif/1.0; +https://ezgif.com/about) +kiwitcms-gitops/0.1 +kiwitcms-gitops/1 +azure-logic-apps/1.0 (workflow xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; version xxxxxxxxxxxxxxxxxxxx) microsoft-flow/1.0 +Prometheus/2.40.5 +Sectigo DCV +Mozilla/5.0 (compatible; SerpReputationManagementAgent/1.0) +TuneIn-DirMon/1.0 +Valimail/1.0 (domain scanner) +phpMyAdmin +phpMyAdmin/reporting-script +Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322) 360JK yunjiankong 427691 +BBC-Forge-URL-Monitor-Twisted +Blackbox Exporter/0.24.0 +Blackbox Exporter/0.20.0-rc.0 +Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; Catchpoint) like Gecko +Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 CloudflareDiagnostics/1.0 +Cloudflare-diagnostics +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/) +Mozilla/5.0 (compatible; Cloudflare-Smart-Transit/1.0; +https://www.cloudflare.com/ +collectd/5.5.1 +Datadog Agent/5.10.1 +Datadog/Synthetics +Grafana/11.1.0 +munin/http_loadtime +Oh Dear Sitemap check https://ohdear.app +rocketmonitor 1.0 +sixy.ch/1.0 +ThousandEyes-Dragonfly-x1 +WebMon/2.0.0 +Mozilla/5.0 (compatible; AlertSite; +https://www.alertsite.com/) +Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights) +BinaryCanary/1.0 (+https://www.binarycanary.com) +Mozilla/5.0 (compatible; EdgeWatch/1.1; https://about.edgewatch.com/) +Hydrozen.io/1.0 +Mozilla/5.0 (compatible; MainWP/4.2.7.1; +http://mainwp.com) +ManageWP +NETVIGIE +Netumo/0.2 (+https://www.netumo.app/) +Mozilla/5.0 (compatible; ServerDensity; +https://www.serverdensity.com/) +WPUmbrella +pingping.io/1.0 +solarwinds/1.0 ( www.solarwinds.com/solarwinds-observability) +uptrends +wheresitup.com/1.1 diff --git a/tests/data/user_agent/devices.txt b/tests/data/user_agent/devices.txt index a0d9607..e49992c 100644 --- a/tests/data/user_agent/devices.txt +++ b/tests/data/user_agent/devices.txt @@ -99824,7 +99824,6 @@ Mozilla/5.0 (Linux; Android 7.0; SM-A520F Build/NRD90M) AppleWebKit/537.36 (KHTM Mozilla/5.0 (Linux; Android 7.0) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Focus/1.0 Chrome/59.0.3071.125 Mobile Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Mobile/14E304 (4299244784) Mozilla/5.0 (Linux; Android 7.1.1; Wileyfox Swift Build/N6F26Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36 -Mozilla/5.0 (compatible; Windows NT 6.1; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299214304) Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; 2014818 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 UCBrowser/11.5.0.1015 Mobile Safari/537.36 Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Mobile/14F89 (4299207328)