docs: add Common Recipes section for well-known files

README.md

@@ -285,6 +285,81 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
 ### Using `httpx` as a library
 `httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder
 
+## Common Recipes
+
+Below are practical one-liners for common use cases leveraging httpx's composable primitives. These recipes are validated in `runner/wellknown_recipes_test.go`.
+
+Use `-mdc` with DSL helpers such as `contains(content_type, ...)` and `contains(body, ...)` to match response metadata and body content. (`-mr`/`-ms` match the full raw response; use `-mdc` for structured field matching.)
+
+### Well-known files
+
+**security.txt**  
+Probe for a valid [RFC 9116](https://www.rfc-editor.org/rfc/rfc9116.html) security.txt file at the standard paths (`/.well-known/security.txt`, `/security.txt`):
+
+```bash
+echo target.com | httpx -path '/.well-known/security.txt,/security.txt' -mc 200 -mdc 'contains(content_type, "text/plain") && contains(body, "Contact:") && contains_any(body, "mailto:", "https://")'
+```
+
+- `-path` tests custom path(s)
+- `-mc 200` matches HTTP 200
+- `-mdc` matches using DSL expressions on response fields such as `content_type` and `body`
+
+**robots.txt**
+
+```bash
+echo target.com | httpx -path '/robots.txt' -mc 200 -mdc 'contains(content_type, "text/plain")'
+```
+
+**sitemap.xml**
+
+```bash
+echo target.com | httpx -path '/sitemap.xml' -mc 200 -mdc 'contains_any(content_type, "application/xml", "text/xml") && contains(body, "<urlset")'
+```
+
+**humans.txt**
+
+```bash
+echo target.com | httpx -path '/humans.txt' -mc 200 -mdc 'contains(content_type, "text/plain")'
+```
+
+**ads.txt**
+
+```bash
+echo target.com | httpx -path '/ads.txt' -mc 200 -mdc 'contains(content_type, "text/plain") && contains(body, "google.com")'
+```
+
+### Well-known URIs
+
+**OpenID Connect discovery** ([spec](https://openid.net/specs/openid-connect-discovery-1_0.html))
+
+```bash
+echo target.com | httpx -path '/.well-known/openid-configuration' -mc 200 -mdc 'contains(content_type, "application/json") && contains(body, "\"issuer\"")'
+```
+
+**Apple Universal Links**
+
+```bash
+echo target.com | httpx -path '/.well-known/apple-app-site-association,/.well-known/apple-app-site-association.json' -mc 200 -mdc 'contains(content_type, "application/json") && contains(body, "\"applinks\"")'
+```
+
+**Android App Links**
+
+```bash
+echo target.com | httpx -path '/.well-known/assetlinks.json' -mc 200 -mdc 'contains(content_type, "application/json") && contains(body, "\"android_app\"")'
+```
+
+**crossdomain.xml** (legacy Flash policy file)
+
+```bash
+echo target.com | httpx -path '/crossdomain.xml' -mc 200 -mdc 'contains_any(content_type, "application/xml", "text/xml") && contains(body, "cross-domain-policy")'
+```
+
+**Other well-known URIs** ([IANA registry](https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml)):
+
+```bash
+echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-password,/.well-known/openid-configuration' -mc 200
+```
+
 # Notes
 
 - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.

runner/wellknown_recipes.go

@@ -0,0 +1,152 @@
+package runner
+
+// WellKnownRecipe documents a composable httpx one-liner for probing well-known resources.
+type WellKnownRecipe struct {
+	Name            string
+	Paths           string
+	MatchStatusCode string
+	MatchCondition  string
+}
+
+// WellKnownRecipes returns README-documented recipes validated by TestWellKnownRecipes.
+func WellKnownRecipes() []WellKnownRecipe {
+	return []WellKnownRecipe{
+		{
+			Name:            "security.txt",
+			Paths:           "/.well-known/security.txt,/security.txt",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "text/plain") && contains(body, "Contact:") && contains_any(body, "mailto:", "https://")`,
+		},
+		{
+			Name:            "robots.txt",
+			Paths:           "/robots.txt",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "text/plain")`,
+		},
+		{
+			Name:            "sitemap.xml",
+			Paths:           "/sitemap.xml",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains_any(content_type, "application/xml", "text/xml") && contains(body, "<urlset")`,
+		},
+		{
+			Name:            "humans.txt",
+			Paths:           "/humans.txt",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "text/plain")`,
+		},
+		{
+			Name:            "ads.txt",
+			Paths:           "/ads.txt",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "text/plain") && contains(body, "google.com")`,
+		},
+		{
+			Name:            "openid-configuration",
+			Paths:           "/.well-known/openid-configuration",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "application/json") && contains(body, "\"issuer\"")`,
+		},
+		{
+			Name:            "apple-app-site-association",
+			Paths:           "/.well-known/apple-app-site-association,/.well-known/apple-app-site-association.json",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "application/json") && contains(body, "\"applinks\"")`,
+		},
+		{
+			Name:            "assetlinks.json",
+			Paths:           "/.well-known/assetlinks.json",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains(content_type, "application/json") && contains(body, "\"android_app\"")`,
+		},
+		{
+			Name:            "crossdomain.xml",
+			Paths:           "/crossdomain.xml",
+			MatchStatusCode: "200",
+			MatchCondition:  `contains_any(content_type, "application/xml", "text/xml") && contains(body, "cross-domain-policy")`,
+		},
+		{
+			Name:            "well-known-uri-batch",
+			Paths:           "/.well-known/security.txt,/.well-known/change-password,/.well-known/openid-configuration",
+			MatchStatusCode: "200",
+		},
+	}
+}
+
+// wellKnownFixture describes an HTTP response used by recipe tests.
+type wellKnownFixture struct {
+	statusCode  int
+	contentType string
+	body        string
+}
+
+// wellKnownFixtures maps request paths to HTTP responses used by recipe tests.
+var wellKnownFixtures = map[string]wellKnownFixture{
+	"/.well-known/security.txt": {
+		statusCode:  200,
+		contentType: "text/plain; charset=utf-8",
+		body:        "Contact: mailto:security@example.com\nPreferred-Languages: en\n",
+	},
+	"/security.txt": {
+		statusCode:  200,
+		contentType: "text/plain; charset=utf-8",
+		body:        "Contact: https://example.com/security\nPreferred-Languages: en\n",
+	},
+	"/robots.txt": {
+		statusCode:  200,
+		contentType: "text/plain",
+		body:        "User-agent: *\nDisallow: /admin\n",
+	},
+	"/sitemap.xml": {
+		statusCode:  200,
+		contentType: "application/xml",
+		body:        `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"></urlset>`,
+	},
+	"/humans.txt": {
+		statusCode:  200,
+		contentType: "text/plain",
+		body:        "/* TEAM */\nDeveloper: Example Dev\n",
+	},
+	"/ads.txt": {
+		statusCode:  200,
+		contentType: "text/plain",
+		body:        "google.com, pub-0000000000000000, DIRECT, f08c47fec0942fa0\n",
+	},
+	"/.well-known/openid-configuration": {
+		statusCode:  200,
+		contentType: "application/json",
+		body:        `{"issuer":"https://example.com","authorization_endpoint":"https://example.com/auth"}`,
+	},
+	"/.well-known/apple-app-site-association": {
+		statusCode:  200,
+		contentType: "application/json",
+		body:        `{"applinks":{"apps":[],"details":[]}}`,
+	},
+	"/.well-known/apple-app-site-association.json": {
+		statusCode:  200,
+		contentType: "application/json",
+		body:        `{"applinks":{"apps":[],"details":[]}}`,
+	},
+	"/.well-known/assetlinks.json": {
+		statusCode:  200,
+		contentType: "application/json",
+		body:        `[{"relation":["delegate_permission/common.handle_all_urls"],"target":{"namespace":"android_app","package_name":"com.example.app"}}]`,
+	},
+	"/crossdomain.xml": {
+		statusCode:  200,
+		contentType: "text/xml",
+		body:        `<?xml version="1.0"?><!DOCTYPE cross-domain-policy SYSTEM "http://www.adobe.com/xml/dtds/cross-domain-policy.dtd"><cross-domain-policy></cross-domain-policy>`,
+	},
+	"/.well-known/change-password": {
+		statusCode:  200,
+		contentType: "text/html",
+		body:        "<html><body>Change password</body></html>",
+	},
+}
+
+// soft404Fixture is an HTML error page that should not match strict well-known recipes.
+var soft404Fixture = wellKnownFixture{
+	statusCode:  200,
+	contentType: "text/html; charset=utf-8",
+	body:        "<html><head><title>Not Found</title></head><body><h1>404</h1></body></html>",
+}

runner/wellknown_recipes_test.go

@@ -0,0 +1,123 @@
+package runner
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/httpx/common/stringz"
+	sliceutil "github.com/projectdiscovery/utils/slice"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWellKnownRecipes(t *testing.T) {
+	for _, recipe := range WellKnownRecipes() {
+		t.Run(recipe.Name, func(t *testing.T) {
+			matched := false
+			for path, fixture := range wellKnownFixtures {
+				if !recipeMatchesPath(recipe, path) {
+					continue
+				}
+				result := resultFromWellKnownFixture(path, fixture)
+				if recipeMatches(recipe, result) {
+					matched = true
+					break
+				}
+			}
+			require.True(t, matched, "recipe %q should match at least one fixture", recipe.Name)
+		})
+	}
+}
+
+func TestWellKnownRecipeSecurityTxtRejectsSoft404(t *testing.T) {
+	recipe := WellKnownRecipes()[0]
+	result := resultFromWellKnownFixture("/.well-known/security.txt", soft404Fixture)
+	require.False(t, recipeMatches(recipe, result), "security.txt recipe should not match HTML soft-404 pages")
+}
+
+func TestWellKnownRecipeSecurityTxtRejectsMissingContact(t *testing.T) {
+	recipe := WellKnownRecipes()[0]
+	result := resultFromWellKnownFixture("/.well-known/security.txt", wellKnownFixture{
+		statusCode:  http.StatusOK,
+		contentType: "text/plain",
+		body:        "Preferred-Languages: en\n",
+	})
+	require.False(t, recipeMatches(recipe, result), "security.txt recipe should require a Contact field")
+}
+
+func TestWellKnownRecipeAdsTxtRejectsInvalidBody(t *testing.T) {
+	recipe := WellKnownRecipes()[4]
+	result := resultFromWellKnownFixture("/ads.txt", wellKnownFixture{
+		statusCode:  http.StatusOK,
+		contentType: "text/plain",
+		body:        "example.com, DIRECT\n",
+	})
+	require.False(t, recipeMatches(recipe, result), "ads.txt recipe should require an authorized digital seller entry")
+}
+
+func TestWellKnownRecipesHTTPProbe(t *testing.T) {
+	ts := newWellKnownTestServer(t)
+	defer ts.Close()
+
+	recipe := WellKnownRecipes()[len(WellKnownRecipes())-1]
+	for _, path := range strings.Split(recipe.Paths, ",") {
+		t.Run(path, func(t *testing.T) {
+			resp, err := http.Get(ts.URL + path)
+			require.NoError(t, err)
+			defer func() { _ = resp.Body.Close() }()
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func recipeMatchesPath(recipe WellKnownRecipe, path string) bool {
+	for _, recipePath := range strings.Split(recipe.Paths, ",") {
+		if recipePath == path {
+			return true
+		}
+	}
+	return false
+}
+
+func recipeMatches(recipe WellKnownRecipe, result Result) bool {
+	if recipe.MatchStatusCode != "" {
+		codes, err := stringz.StringToSliceInt(recipe.MatchStatusCode)
+		if err != nil || !sliceutil.Contains(codes, result.StatusCode) {
+			return false
+		}
+	}
+	if recipe.MatchCondition != "" && !evalDslExpr(result, recipe.MatchCondition) {
+		return false
+	}
+	return true
+}
+
+func resultFromWellKnownFixture(path string, fixture wellKnownFixture) Result {
+	url := "http://example.com" + path
+	contentType := fixture.contentType
+	if idx := strings.Index(contentType, ";"); idx >= 0 {
+		contentType = strings.TrimSpace(contentType[:idx])
+	}
+	return Result{
+		StatusCode:   fixture.statusCode,
+		ContentType:  contentType,
+		ResponseBody: fixture.body,
+		URL:          url,
+		str:          url,
+	}
+}
+
+func newWellKnownTestServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fixture, ok := wellKnownFixtures[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", fixture.contentType)
+		w.WriteHeader(fixture.statusCode)
+		_, _ = w.Write([]byte(fixture.body))
+	}))
+}