Skip to content

feat(js): add provider-prefixed and structural secret rules #361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vmfunc merged 1 commit into from
Aug 22, 2026
Merged

feat(js): add provider-prefixed and structural secret rules #361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 126 additions & 1 deletion internal/scan/js/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
package js

import (
"encoding/base64"
"math"
"regexp"
"strings"
Expand All @@ -39,12 +40,14 @@ const (

// secretRules is the credential regex bank. the matching group (or the whole
// match when there's no group) is what gets reported; minEntropy gates the
// generic high-entropy rules so we don't flag every short literal.
// generic high-entropy rules so we don't flag every short literal. validate
// runs after the gates for rules where shape alone isn't proof.
var secretRules = []struct {
name string
re *regexp.Regexp
minEntropy float64
requireDigit bool
validate func(string) bool
}{
{
// aws access key ids are fixed-shape and unmistakable.
Expand Down Expand Up @@ -142,6 +145,76 @@ var secretRules = []struct {
re: regexp.MustCompile(`\b(hooks\.slack\.com/services/T[0-9A-Za-z_]+/B[0-9A-Za-z_]+/[0-9A-Za-z]{24})\b`),
minEntropy: noEntropyGate,
},
{
// pypi tokens all share the pypi-AgEIcHlwaS5vcmc prefix, the base64
// encoding of a fixed macaroon header, so it's effectively unforgeable.
name: "pypi api token",
re: regexp.MustCompile(`\b(pypi-AgEIcHlwaS5vcmc[0-9A-Za-z_-]{50,})\b`),
minEntropy: noEntropyGate,
},
{
// legacy openai secret keys embed a fixed T3BlbkFJ marker (base64 for
// "OpenAI") between two random halves.
name: "openai api key",
re: regexp.MustCompile(`\b(sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})\b`),
minEntropy: noEntropyGate,
},
{
// current-generation project and service-account keys.
name: "openai project api key",
re: regexp.MustCompile(`\b(sk-(?:proj|svcacct)-[A-Za-z0-9_-]{20,})\b`),
minEntropy: noEntropyGate,
},
{
name: "square access token",
re: regexp.MustCompile(`\b(sq0atp-[0-9A-Za-z_-]{22}|sq0csp-[0-9A-Za-z_-]{43})\b`),
minEntropy: noEntropyGate,
},
{
// mailgun api keys, key- then a 32-char hex blob.
name: "mailgun api key",
re: regexp.MustCompile(`\b(key-[0-9a-f]{32})\b`),
minEntropy: noEntropyGate,
},
{
// discord bot tokens: base64 user id, a timestamp segment, then an HMAC.
name: "discord bot token",
re: regexp.MustCompile(`\b([MNOP][A-Za-z0-9_-]{23}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,38})\b`),
minEntropy: noEntropyGate,
},
{
// discord incoming-webhook urls embed the secret in the path.
name: "discord webhook url",
re: regexp.MustCompile(`\b(discord(?:app)?\.com/api/webhooks/[0-9]{17,20}/[A-Za-z0-9_-]{60,68})`),
minEntropy: noEntropyGate,
},
{
name: "new relic license key",
re: regexp.MustCompile(`\b(NRAK-[A-Z0-9]{27})\b`),
minEntropy: noEntropyGate,
},
{
// cloudinary connection urls carry the api key and secret in the userinfo.
name: "cloudinary url",
re: regexp.MustCompile(`\b(cloudinary://[0-9]{10,20}:[A-Za-z0-9_-]{20,}@[A-Za-z0-9_-]+)`),
minEntropy: noEntropyGate,
},
{
// jwts have no fixed prefix; validate decodes the header to rule out
// arbitrary dotted base64url blobs.
name: "jwt",
re: regexp.MustCompile(`\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b`),
minEntropy: noEntropyGate,
validate: isStructuredJWT,
},
{
// validate drops the countless doc/template examples that use a
// placeholder password.
name: "database connection string",
re: regexp.MustCompile(`\b((?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|rediss|amqp|amqps)://[^:\s"'` + "`" + `/@]+:[^@\s"'` + "`" + `/]+@[^\s"'` + "`" + `]+)`),
minEntropy: noEntropyGate,
validate: hasRealConnStringPassword,
},
{
// generic apikey/secret/token = "<value>" assignments; the value is in
// group 2 and only reported if it looks random (entropy gate) and carries
Expand Down Expand Up @@ -185,6 +258,11 @@ func ScanSecrets(content, srcURL string) []SecretMatch {
continue
}

// structural validation for rules whose shape alone isn't proof.
if rule.validate != nil && !rule.validate(value) {
continue
}

// dedupe per source so a key referenced twice is one finding.
key := rule.name + "\x00" + value
if _, ok := seen[key]; ok {
Expand Down Expand Up @@ -217,6 +295,53 @@ func hasDigit(s string) bool {
return false
}

// alg is mandatory per RFC 7519; requiring both fields keeps arbitrary
// dot-separated base64url blobs from being mistaken for a jwt.
const (
jwtAlgField = `"alg"`
jwtTypField = `"typ"`
)

// connStringPasswordRe pulls the userinfo password out of a scheme://user:pass@host
// connection string, for filtering placeholder credentials post-match.
var connStringPasswordRe = regexp.MustCompile(`://[^:@/\s]*:([^@/\s]+)@`)

// placeholderPasswords are stand-ins that show up constantly in docs, sample
// configs and .env.example files; matching one means the string isn't a real
// leaked credential.
var placeholderPasswords = map[string]struct{}{
"password": {}, "pass": {}, "passwd": {}, "xxxx": {}, "xxxxx": {},
"changeme": {}, "yourpassword": {}, "example": {}, "test": {},
"123456": {}, "secret": {}, "admin": {}, "root": {}, "pwd": {},
}

// isStructuredJWT confirms the header segment decodes to jwt-shaped json.
func isStructuredJWT(token string) bool {
parts := strings.Split(token, ".")
if len(parts) != 3 {
return false
}

header, err := base64.RawURLEncoding.DecodeString(parts[0])
if err != nil {
return false
}

h := string(header)
return strings.Contains(h, jwtAlgField) && strings.Contains(h, jwtTypField)
}

// hasRealConnStringPassword rejects known placeholder passwords.
func hasRealConnStringPassword(value string) bool {
m := connStringPasswordRe.FindStringSubmatch(value)
if len(m) < 2 {
return true
}

_, placeholder := placeholderPasswords[strings.ToLower(m[1])]
return !placeholder
}

// shannonEntropy is the per-character shannon entropy (bits) of s, used to tell
// random-looking secrets apart from plain words. empty input is zero entropy.
func shannonEntropy(s string) float64 {
Expand Down
171 changes: 171 additions & 0 deletions internal/scan/js/secrets_provider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
/*
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
: :
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
: ▄█ █ █▀ · BSD 3-Clause License :
: :
: (c) 2022-2026 vmfunc, xyzeva, :
: lunchcat alumni & contributors :
: :
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
*/

package js

import (
"fmt"
"strings"
"testing"
)

// split into fragments so the file itself never carries a contiguous token a
// secret scanner would flag.
var (
provPyPI = "pypi-AgEIcHlwaS5vcmc" + strings.Repeat("1jKlMn0p", 7)
provOpenAILeg = "sk-" + "aB3dEfGh1jKlMn0pQrSt" + "T3BlbkFJ" + "uVwXyZ012345abcdefgh"
provOpenAIProj = "sk-proj-" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd"
provSquare = "sq0atp-" + "aB3dEfGh1jKlMn0pQrSt-U"
provMailgun = "key-" + "0123456789abcdef0123456789abcdef"
provDiscordBot = "M" + "TIzNDU2Nzg5MDEyMzQ1Njc4" + "." + "GaBcDe" + "." + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcdef"
provDiscordHook = "discord.com/api/webhooks/" + "123456789012345678" + "/" + strings.Repeat("aB3dEfGh1j", 6) + "abcdefgh"
provNewRelic = "NRAK-" + "AB3DEFGH1JKLMN0PQRSTUVWXYZZ"
provCloudinary = "cloudinary://" + "123456789012345" + ":" + "aB3dEfGh1jKlMn0pQrStUvWxYz" + "@my-cloud"
provMongoURI = "mongodb+srv://" + "dbadmin" + ":" + "tR7q!zK2vLp9xC" + "@cluster0.example.mongodb.net/prod"
provMongoPlace = "mongodb://" + "user" + ":" + "password" + "@localhost:27017/app"

// a real jwt (rfc 7519 example header/payload), signature is dummy.
provJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" +
".eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIn0" +
".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
)

// each provider rule added on top of the existing bank, plus the two cases the
// shape alone cannot decide: a dotted base64url blob that is not a jwt, and a
// connection string whose password is a documentation placeholder.
func TestScanSecretsProviderRules(t *testing.T) {
tests := []struct {
name string
content string
wantRule string // "" means the content must produce no match
}{
{
name: "pypi api token",
content: fmt.Sprintf(`password = %q`, provPyPI),
wantRule: "pypi api token",
},
{
name: "openai legacy api key",
content: fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAILeg),
wantRule: "openai api key",
},
{
name: "openai project api key",
content: fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAIProj),
wantRule: "openai project api key",
},
{
name: "square access token",
content: fmt.Sprintf(`squareToken = %q`, provSquare),
wantRule: "square access token",
},
{
name: "mailgun api key",
content: fmt.Sprintf(`MAILGUN_KEY=%q`, provMailgun),
wantRule: "mailgun api key",
},
{
name: "discord bot token",
content: fmt.Sprintf(`client.login(%q)`, provDiscordBot),
wantRule: "discord bot token",
},
{
name: "discord webhook url",
content: fmt.Sprintf(`fetch("https://%s")`, provDiscordHook),
wantRule: "discord webhook url",
},
{
name: "new relic license key",
content: fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, provNewRelic),
wantRule: "new relic license key",
},
{
name: "cloudinary url",
content: fmt.Sprintf(`CLOUDINARY_URL=%q`, provCloudinary),
wantRule: "cloudinary url",
},
{
name: "jwt with a valid header",
content: fmt.Sprintf(`const token = %q;`, provJWT),
wantRule: "jwt",
},
{
name: "three dotted base64url blobs without a jwt header",
content: `const s = "eyJhYmNkZWZnaGlq.aGVsbG93b3JsZDEy.c2lnbmF0dXJlYmxvYmhlcmU";`,
},
{
name: "connection string with real credentials",
content: fmt.Sprintf(`const uri = %q;`, provMongoURI),
wantRule: "database connection string",
},
{
name: "connection string with a placeholder password",
content: fmt.Sprintf(`// example: %s`, provMongoPlace),
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ScanSecrets(tt.content, "https://example.test/app.js")

if tt.wantRule == "" {
if len(got) != 0 {
t.Fatalf("got %d matches (%s), want none", len(got), got[0].Rule)
}
return
}

var rules []string
for i := range got {
rules = append(rules, got[i].Rule)
if got[i].Rule == tt.wantRule {
return
}
}
t.Fatalf("rule %q did not fire, got %v", tt.wantRule, rules)
})
}
}

// the rules added here must not overlap each other or the provider-prefixed
// rules already in the bank: one credential in one script is one finding, not
// two. the generic assignment rule is excluded because it claims any quoted
// high-entropy value behind a token/password/secret keyword, so it already
// doubles up with every prefixed rule on main; that is pre-existing and not
// something these rules introduce.
func TestProviderRulesDoNotDuplicateExistingCoverage(t *testing.T) {
content := strings.Join([]string{
fmt.Sprintf(`password = %q`, provPyPI),
fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAILeg),
fmt.Sprintf(`squareToken = %q`, provSquare),
fmt.Sprintf(`MAILGUN_KEY=%q`, provMailgun),
fmt.Sprintf(`client.login(%q)`, provDiscordBot),
fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, provNewRelic),
fmt.Sprintf(`CLOUDINARY_URL=%q`, provCloudinary),
fmt.Sprintf(`const token = %q;`, provJWT),
fmt.Sprintf(`const uri = %q;`, provMongoURI),
}, "\n")

seen := make(map[string]int)
for _, m := range ScanSecrets(content, "https://example.test/app.js") {
if m.Rule == "generic secret assignment" {
continue
}
seen[m.Match]++
}

for value, n := range seen {
if n > 1 {
t.Errorf("value %q reported %d times, want 1", value, n)
}
}
}
Loading