diff --git a/internal/redaction/control_split.go b/internal/redaction/control_split.go new file mode 100644 index 000000000..dacf88bdd --- /dev/null +++ b/internal/redaction/control_split.go @@ -0,0 +1,547 @@ +package redaction + +import ( + "fmt" + "regexp" + "regexp/syntax" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// A CREDENTIAL IS STILL A CREDENTIAL WITH AN INVISIBLE BYTE IN THE MIDDLE. +// +// The shape matchers (openaiKeyPattern, textSecretPatterns) all describe a +// CONTIGUOUS run of body characters, and none of their character classes admits +// a control or format character. So a single NUL, ESC or zero-width space +// dropped into the body of a key ends the match, and RedactString emits the two +// fragments untouched. Whoever reads that output next, a terminal that eats the +// escape, a log viewer, a JSON consumer that strips control bytes, or a person +// copying the text, sees the original key back, because nothing about those +// characters was ever displayed. Reported in #969 for NUL and ESC; every other +// Cc and Cf character behaves the same way, which is why this is written against +// the classes rather than against the two bytes named there. +// +// HOW: each shape is rewritten so that a run of separators may sit between any +// two characters it consumes, and matched against the ORIGINAL text. Nothing is +// removed before matching, so there are no offsets to map back, and the leading +// \b of every shape is judged by the regexp engine against the real neighbour: +// a key written after a NUL starts at a boundary, exactly as the unsplit key +// does, and a key glued to the end of a word does not. +// +// An earlier version removed the separators first and matched the compacted +// copy. That lost the difference between a separator inside a credential and one +// next to it, and the fixes for that (check the boundary against the source +// afterwards, map offsets back) left a hole: in "xAKIAAKIAIOSFODNN7..." +// the leftmost compacted match began at the decoy AKIA, was then rejected for +// the "x" in front of it, and had already consumed the start of the real key, so +// the real key was never considered. Resuming the scan one byte after each +// rejection would close that, but it is quadratic for the greedy shapes on text +// like "ask-ask-ask-..." with a single NUL in it. Keeping the boundary inside +// the pattern lets RE2 do it in one linear pass. +// +// This pass runs AFTER the contiguous matchers, over their output, and never +// across a replacement. A credential that is whole is claimed by its own strict +// match, so two whole credentials with a separator between them stay two +// replacements with the separator still between them. Two split credentials do +// too: see spansFor. +// +// A match never starts or ends on a separator, so the separators on either side +// of a credential stay where they were: "AKIA..." becomes +// "[REDACTED]", as it does for the unsplit key. +// +// ONE KNOWN LIMIT, from running the contiguous matchers first. When the part of +// a split key BEFORE its first separator is already a complete credential on +// its own, the contiguous matcher claims that part and stops at the separator, +// and what follows is judged by itself. That is the rest of the key, which is no +// credential alone, so it stays visible: a fragment of the key, never the whole +// of it, because the part in front is replaced. For the same reason a credential +// glued straight onto that rest is left, where the unsplit key would have run on +// and swallowed it. Carrying the contiguous match across the separator instead +// would bring back what running first exists to prevent, since nothing tells the +// rest of a key from prose written after a whole one: that prose would be eaten, +// and a following JWT would lose its header. +// +// WHAT THIS DELIBERATELY DOES NOT TREAT AS A SEPARATOR: tab, newline and +// carriage return. Those three are real text structure rather than invisible +// filler, so a credential broken across a line boundary is still two visible +// fragments here and is not covered. Treating them as gaps would also let a +// match span a line break and collapse unrelated lines into one replacement. A +// credential split that way is visible to whoever reads it; the ones handled +// here are not. Invalid UTF-8 is out for the same reason: the engine reads an +// invalid byte as U+FFFD, which is drawn. + +// splitSecretSeparator reports a character that can sit inside a credential +// without being seen. Cc covers the C0 and C1 controls (NUL, ESC, DEL, NEL, CSI) +// and Cf the format characters (zero-width space and joiner, word joiner, BOM, +// soft hyphen). Tab, newline and carriage return are excluded: see above. +func splitSecretSeparator(r rune) bool { + switch r { + case '\t', '\n', '\r': + return false + } + return unicode.Is(unicode.Cc, r) || unicode.Is(unicode.Cf, r) +} + +// containsSplitSeparator is the cheap gate in front of the gap-tolerant pass, so +// text with nothing invisible in it, which is nearly all text, costs one scan and +// no allocation. ASCII is settled from the byte alone; only a byte that could +// begin a multi-byte rune needs decoding, and the C1 and Cf blocks all start +// with a lead byte at or above 0xC2. +func containsSplitSeparator(value string) bool { + for i := 0; i < len(value); i++ { + b := value[i] + if b < utf8.RuneSelf { + if b < 0x20 || b == 0x7f { + if b == '\t' || b == '\n' || b == '\r' { + continue + } + return true + } + continue + } + r, width := utf8.DecodeRuneInString(value[i:]) + if splitSecretSeparator(r) { + return true + } + i += width - 1 + } + return false +} + +// stripSplitSeparators returns value with every splitSecretSeparator removed. +// Used only on a matched credential, to judge it by its visible characters. +func stripSplitSeparators(value string) string { + var builder strings.Builder + builder.Grow(len(value)) + for offset := 0; offset < len(value); { + r, width := utf8.DecodeRuneInString(value[offset:]) + if !splitSecretSeparator(r) { + builder.WriteString(value[offset : offset+width]) + } + offset += width + } + return builder.String() +} + +// splitSeparatorClass is splitSecretSeparator as a regexp character class, +// generated from the same Unicode tables and the same three exclusions so the +// class and the predicate cannot disagree. +var splitSeparatorClass = func() string { + var class strings.Builder + class.WriteByte('[') + emit := func(lo, hi rune) { + fmt.Fprintf(&class, `\x{%x}-\x{%x}`, lo, hi) + } + for _, table := range []*unicode.RangeTable{unicode.Cc, unicode.Cf} { + visit := func(lo, hi, stride uint32) { + for r := lo; r <= hi; r += stride { + start := rune(r) + if !splitSecretSeparator(start) { + continue + } + end := start + if stride == 1 { + for next := rune(r) + 1; uint32(next) <= hi && splitSecretSeparator(next); next++ { + end = next + } + r = uint32(end) + } + emit(start, end) + } + } + for _, r := range table.R16 { + visit(uint32(r.Lo), uint32(r.Hi), uint32(r.Stride)) + } + for _, r := range table.R32 { + visit(r.Lo, r.Hi, r.Stride) + } + } + class.WriteByte(']') + return class.String() +}() + +// splitOpenAIPattern and splitTextPatterns are the shape matchers with a run of +// separators allowed between every two characters they consume, derived from +// the originals rather than written out a second time so the two lists cannot +// drift apart. +var ( + splitOpenAIPattern = gapTolerant(openaiKeyPattern) + splitTextPatterns = gapTolerantAll(textSecretPatterns) +) + +// splitShape is one credential shape as the split pass uses it. +type splitShape struct { + // free finds the shape anywhere, leftmost first. anchored matches it only + // where the text begins, and whole only when it spans the entire text. + free, anchored, whole *regexp.Regexp + // prefix is the literal every match of the shape begins with. + prefix string + // accept is the test a match has to pass to be a credential, or nil when + // every match is one. + accept func(match string) bool +} + +var splitShapes = func() []splitShape { + shapes := []splitShape{newSplitShape(openaiKeyPattern, splitOpenAIPattern, func(match string) bool { + return openAIShapeIsSecret(stripSplitSeparators(match)) + })} + for i, pattern := range textSecretPatterns { + shapes = append(shapes, newSplitShape(pattern, splitTextPatterns[i], nil)) + } + return shapes +}() + +func newSplitShape(original, free *regexp.Regexp, accept func(string) bool) splitShape { + return splitShape{ + free: free, + anchored: regexp.MustCompile(`\A(?:` + free.String() + `)`), + whole: regexp.MustCompile(`\A(?:` + free.String() + `)\z`), + prefix: leadingLiteral(original), + accept: accept, + } +} + +// leadingLiteral is the literal every match of pattern begins with, read past a +// leading \b, or "" when the pattern does not open with one. +func leadingLiteral(pattern *regexp.Regexp) string { + tree, err := syntax.Parse(pattern.String(), syntax.Perl) + if err != nil { + panic(fmt.Sprintf("redaction: parse %q: %v", pattern.String(), err)) + } + nodes := []*syntax.Regexp{tree} + if tree.Op == syntax.OpConcat { + nodes = tree.Sub + } + for _, node := range nodes { + if node.Op == syntax.OpWordBoundary { + continue + } + if node.Op == syntax.OpLiteral { + return string(node.Rune) + } + return "" + } + return "" +} + +// maxRunOnCandidates bounds the anchored matches tried inside one match, so a +// match full of lookalike starts still costs a fixed number of them. +const maxRunOnCandidates = 8 + +// spansFor turns one match of the shape into the stretches to replace. +// +// A MATCH CAN RUN ON INTO THE NEXT CREDENTIAL OF ITS OWN SHAPE. Gaps are allowed +// inside a match, and a separator between two credentials looks exactly like one +// inside a credential, so a greedy body runs through it into whatever follows. A +// credential of another shape is still found, because every shape is matched on +// its own and the union is replaced. One of the SAME shape is not: matches of one +// pattern never overlap, so the second credential cannot start inside the first +// one's match. Most bodies swallow it whole, so nothing leaks, but a JWT stops at +// the next token's first dot, and two split JWTs side by side left the second +// one's payload and signature in the clear. And a match the filter rejects hides +// whatever it ran into, so kebab-case prose in front of a split key took the key +// down with it. Reported by @jatmn. +// +// So when a match runs through a run of separators into a position where the +// same shape matches again and reaches at least as far, that credential is taken +// on its own, and the first is cut at the separators if it is a complete match by +// itself. The separators between the two then stay outside both replacements, as +// they do between two whole credentials, and each piece is filtered on its own. +// When the first is not complete alone, it keeps the whole of its match and the +// union joins the two into one replacement. +// +// ONE STEP PER MATCH, NOT THE WHOLE CHAIN. The second credential's own match can +// run on into a third, and following that from here would walk the rest of a +// chain once for every match inside it, which is quadratic: a megabyte of split +// JWTs ran for minutes. The scan's own next match starts inside the second +// credential and takes the step after it, so the chain is still covered, and in +// a chain of three or more the markers after the first pair can join into one. +func (shape splitShape) spansFor(text string, start, end int) [][]int { + next, gap, nextEnd := shape.runOn(text, start, end) + if next < 0 { + return shape.keep(nil, text, start, end) + } + var spans [][]int + if shape.whole.MatchString(text[start:gap]) { + spans = shape.keep(spans, text, start, gap) + } else { + spans = shape.keep(spans, text, start, end) + } + return shape.keep(spans, text, next, nextEnd) +} + +func (shape splitShape) keep(spans [][]int, text string, start, end int) [][]int { + if shape.accept != nil && !shape.accept(text[start:end]) { + return spans + } + return append(spans, []int{start, end}) +} + +// runOn looks inside the match [start, end) for where it ran through a run of +// separators into another credential of the same shape. It returns where that +// credential begins, where the separator run in front of it begins, and where the +// credential's own match ends, or next = -1 when there is none. +// +// Latest first, because a match reaches into the next credential only as far as +// its own pattern allows. Only where the text after the separators opens with the +// shape's literal prefix, which is where a match of the shape can begin; the +// separator in front supplies the word boundary, so matching from there judges it +// the same as the whole text would. +func (shape splitShape) runOn(text string, start, end int) (next, gap, nextEnd int) { + tried := 0 + for at := end; at > start && tried < maxRunOnCandidates; { + r, size := utf8.DecodeLastRuneInString(text[start:at]) + if !splitSecretSeparator(r) { + at -= size + continue + } + candidate := at + for at > start { + r, size = utf8.DecodeLastRuneInString(text[start:at]) + if !splitSecretSeparator(r) { + break + } + at -= size + } + if !hasGappedPrefix(text[candidate:], shape.prefix) { + continue + } + tried++ + if match := shape.anchored.FindStringIndex(text[candidate:]); match != nil && candidate+match[1] >= end { + return candidate, at, candidate + match[1] + } + } + return -1, 0, 0 +} + +// hasGappedPrefix reports whether text opens with prefix, allowing a run of +// separators between its characters the way the shapes themselves do. A plain +// prefix test missed a token split inside its first few characters, and the run +// that swallowed it then went unnoticed. +func hasGappedPrefix(text, prefix string) bool { + at := 0 + for index, want := range prefix { + if index > 0 { + for at < len(text) { + r, size := utf8.DecodeRuneInString(text[at:]) + if !splitSecretSeparator(r) { + break + } + at += size + } + } + if at >= len(text) { + return false + } + r, size := utf8.DecodeRuneInString(text[at:]) + if r != want { + return false + } + at += size + } + return true +} + +func gapTolerantAll(patterns []*regexp.Regexp) []*regexp.Regexp { + tolerant := make([]*regexp.Regexp, 0, len(patterns)) + for _, pattern := range patterns { + tolerant = append(tolerant, gapTolerant(pattern)) + } + return tolerant +} + +// gapTolerant rewrites pattern so that a run of separators may appear before +// every character it consumes except the first. Zero-width assertions, the +// leading \b in particular, are left exactly where they were. +func gapTolerant(pattern *regexp.Regexp) *regexp.Regexp { + tree, err := syntax.Parse(pattern.String(), syntax.Perl) + if err != nil { + panic(fmt.Sprintf("redaction: parse %q: %v", pattern.String(), err)) + } + separator, err := syntax.Parse(splitSeparatorClass, syntax.Perl) + if err != nil { + panic(fmt.Sprintf("redaction: parse separator class: %v", err)) + } + consumed := false + return regexp.MustCompile(withGaps(tree, separator, &consumed).String()) +} + +// withGaps inserts the separator run before each consuming atom. consumed +// tracks whether anything has been consumed yet on this path, because the first +// character of a match must be a real one: a leading gap would let a match start +// on a separator and swallow it. +func withGaps(node, separator *syntax.Regexp, consumed *bool) *syntax.Regexp { + switch node.Op { + case syntax.OpLiteral: + parts := make([]*syntax.Regexp, 0, len(node.Rune)) + for _, r := range node.Rune { + atom := &syntax.Regexp{Op: syntax.OpLiteral, Rune: []rune{r}, Flags: node.Flags} + parts = append(parts, gapBefore(atom, separator, consumed)) + } + if len(parts) == 1 { + return parts[0] + } + return &syntax.Regexp{Op: syntax.OpConcat, Sub: parts} + case syntax.OpCharClass, syntax.OpAnyChar, syntax.OpAnyCharNotNL: + return gapBefore(node, separator, consumed) + case syntax.OpConcat, syntax.OpCapture: + for i := range node.Sub { + node.Sub[i] = withGaps(node.Sub[i], separator, consumed) + } + return node + case syntax.OpAlternate: + start, after := *consumed, *consumed + for i := range node.Sub { + branch := start + node.Sub[i] = withGaps(node.Sub[i], separator, &branch) + after = after || branch + } + *consumed = after + return node + case syntax.OpStar, syntax.OpPlus, syntax.OpQuest, syntax.OpRepeat: + // Every shape opens with a literal prefix, so a repetition is never the + // first thing a match consumes. If one ever is, its first iteration would + // need no gap and the rest would; fail loudly rather than guess. + if !*consumed { + panic("redaction: gap-tolerant rewrite reached a repetition before any literal") + } + node.Sub[0] = withGaps(node.Sub[0], separator, consumed) + return node + case syntax.OpWordBoundary, syntax.OpNoWordBoundary, syntax.OpBeginLine, syntax.OpEndLine, + syntax.OpBeginText, syntax.OpEndText, syntax.OpEmptyMatch: + return node + default: + panic(fmt.Sprintf("redaction: gap-tolerant rewrite does not handle %v", node.Op)) + } +} + +func gapBefore(atom, separator *syntax.Regexp, consumed *bool) *syntax.Regexp { + if !*consumed { + *consumed = true + return atom + } + gap := &syntax.Regexp{Op: syntax.OpStar, Sub: []*syntax.Regexp{separator}} + return &syntax.Regexp{Op: syntax.OpConcat, Sub: []*syntax.Regexp{gap, atom}} +} + +// openAIShapeIsSecret is the digit/kebab-case filter RedactString applies to an +// openaiKeyPattern match, named so both callers state the same rule once. +func openAIShapeIsSecret(match string) bool { + if knownOpenAIKeyPrefix(match) || secretMatchHasDigit(match) { + return true + } + return !strings.Contains(strings.TrimPrefix(match, "sk-"), "-") +} + +// redactControlSplitSecrets replaces credentials whose body is interrupted by +// invisible characters. It is a no-op for text containing none of them, which is +// nearly all text, so the contiguous path keeps its behavior exactly. +// +// It runs over the contiguous matchers' output and stops at every replacement +// they left behind: those mark text already accounted for, and a match reaching +// across one would be reaching out of one credential and into the next. +// +// It runs at most twice. The second pass exists for a credential glued straight +// onto the end of another split one: it has no word boundary of its own until +// the first is replaced, and the replacement then supplies one, the same way the +// contiguous shapes cascade into each other. Two passes and no more keeps the +// cost linear; a longer chain of glued split credentials is left as the +// contiguous matchers leave a chain of glued whole ones. +func redactControlSplitSecrets(value, replacement string) string { + if !containsSplitSeparator(value) { + return value + } + once := redactSplitRegions(value, replacement, true) + if once == value { + return value + } + // Ungated: the credential the second pass exists for is often contiguous + // itself (a whole JWT glued onto a split AWS key), so its region can hold no + // separator at all once the first pass has cut the text around it. + return redactSplitRegions(once, replacement, false) +} + +func redactSplitRegions(value, replacement string, gated bool) string { + if replacement == "" { + return redactSplitRegion(value, replacement, 0, gated) + } + regions := strings.Split(value, replacement) + if len(regions) == 1 { + return redactSplitRegion(value, replacement, 0, gated) + } + previous := byte(0) + for i, region := range regions { + regions[i] = redactSplitRegion(region, replacement, previous, gated) + previous = replacement[len(replacement)-1] + } + return strings.Join(regions, replacement) +} + +// redactSplitRegion runs every gap-tolerant shape over one stretch of text with +// no replacement inside it and replaces the UNION of what they found. +// +// Independently, not one after another. An unbounded body with gaps allowed will +// run on across a separator into whatever follows, and when that is the start of +// another credential it takes that credential's prefix with it: in +// "sk-ant-…buildeyJ….eyJ….sig" the Anthropic shape stops only at the +// JWT's first dot. Applied in sequence, the JWT shape then finds its header gone +// and the payload and signature are left in the clear. Matched against the same +// text, the JWT shape still finds the whole token, and the union covers both. +// +// previous is the byte immediately before the region in the original, or zero at +// the start of the string. It is lent to the engine in front of the region so a +// leading \b is judged against the real neighbour, and a match is never allowed +// to begin on it. +func redactSplitRegion(value, replacement string, previous byte, gated bool) string { + if gated && !containsSplitSeparator(value) { + return value + } + text, lead := value, 0 + if previous != 0 { + text, lead = string([]byte{previous})+value, 1 + } + var spans [][]int + for _, shape := range splitShapes { + for _, match := range shape.free.FindAllStringIndex(text, -1) { + if match[0] >= lead { + spans = append(spans, shape.spansFor(text, match[0], match[1])...) + } + } + } + return spliceSpans(text, spans, replacement)[lead:] +} + +// spliceSpans replaces every named range of value with replacement, merging +// ranges that overlap so one stretch of text is never replaced twice. +func spliceSpans(value string, spans [][]int, replacement string) string { + if len(spans) == 0 { + return value + } + sort.Slice(spans, func(i, j int) bool { + if spans[i][0] != spans[j][0] { + return spans[i][0] < spans[j][0] + } + return spans[i][1] > spans[j][1] + }) + var builder strings.Builder + builder.Grow(len(value)) + written := 0 + for _, span := range spans { + if span[0] < written { + // Overlaps the stretch just replaced: extend it rather than emit a + // second marker, which would claim two credentials where there was one + // stretch of text. + if span[1] > written { + written = span[1] + } + continue + } + builder.WriteString(value[written:span[0]]) + builder.WriteString(replacement) + written = span[1] + } + builder.WriteString(value[written:]) + return builder.String() +} diff --git a/internal/redaction/control_split_test.go b/internal/redaction/control_split_test.go new file mode 100644 index 000000000..851aa0f4b --- /dev/null +++ b/internal/redaction/control_split_test.go @@ -0,0 +1,622 @@ +package redaction + +import ( + "fmt" + "regexp" + "runtime" + "strings" + "testing" + "unicode" +) + +const ( + awsKey = "AKIAIOSFODNN7EXAMPLE" + anthropicKey = "sk-ant-api03-aaaaaaaaaaaaaaaaaaaaaaaa0123456789ABCD" + githubKey = "ghp_cccccccccccccccccccccccccccccccccccc" + openaiKey = "sk-proj-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +) + +// rejoin drops every character a reader would never see, which is what a +// terminal, a log viewer or a copy-paste does to this output. If the secret is +// back after that, the redaction did not happen. +func rejoin(value string) string { + return strings.Map(func(r rune) rune { + if splitSecretSeparator(r) { + return -1 + } + return r + }, value) +} + +// AN INVISIBLE BYTE INSIDE A KEY MUST NOT END THE MATCH. +// +// Every shape matcher describes a contiguous body, so one NUL or ESC in the +// middle of a key ended the match and both fragments were emitted. Dropping the +// byte — which is what the next reader does, because it was never displayed — +// put the original key back. Reported in #969 for NUL and ESC; this covers the +// classes those two belong to, because every member behaves the same way. +func TestRedactStringRedactsSecretsSplitByInvisibleCharacters(t *testing.T) { + separators := map[string]string{ + "NUL": "\x00", + "ESC": "\x1b", + "BEL": "\x07", + "backspace": "\x08", + "vertical tab": "\x0b", + "form feed": "\x0c", + "DEL": "\x7f", + "C1 NEL": "\u0085", + "C1 CSI": "\u009b", + "zero width space": "\u200b", + "zero width joiner": "\u200d", + "word joiner": "\u2060", + "byte order mark": "\ufeff", + "soft hyphen": "\u00ad", + } + // Every entry has to BE a separator, or its leg passes vacuously: inserting an + // ordinary character into a key also stops the secret matching, and the rejoin + // below would then find nothing. An escape mangled on the way into this file is + // exactly how that happens. + for name, separator := range separators { + if runes := []rune(separator); len(runes) != 1 || !splitSecretSeparator(runes[0]) { + t.Fatalf("SETUP INVALID: %s is %q, which is not a single invisible separator", name, separator) + } + } + secrets := map[string]string{"aws": awsKey, "anthropic": anthropicKey, "github": githubKey, "openai": openaiKey} + for secretName, secret := range secrets { + // The premise: the unsplit form already redacts, so a failure below is + // about the separator and not about the shape being unknown. + if out := RedactString(secret, Options{}); strings.Contains(out, secret) { + t.Fatalf("SETUP INVALID: the unsplit %s key is not redacted at all: %q", secretName, out) + } + for sepName, separator := range separators { + for _, at := range []int{1, len(secret) / 2, len(secret) - 1} { + split := secret[:at] + separator + secret[at:] + out := RedactString(split, Options{}) + if got := rejoin(out); strings.Contains(got, secret) { + t.Errorf("%s key split by %s at byte %d survives: %q rejoins to %q", secretName, sepName, at, out, got) + } + } + } + } +} + +// The replacement has to cover the ORIGINAL bytes, separators included. Leaving +// the separator behind would emit `[REDACTED]\x00[REDACTED]` and tell a reader +// there were two secrets, and leaving either fragment behind is the leak itself. +func TestRedactStringCoversTheSeparatorsInsideASplitSecret(t *testing.T) { + for _, testCase := range []struct { + name string + value string + want string + }{ + {"one separator", "key=" + awsKey[:8] + "\x00" + awsKey[8:], "key=[REDACTED]"}, + {"several separators", "key=" + strings.Join(strings.Split(awsKey, ""), "\x1b"), "key=[REDACTED]"}, + {"adjacent separators", "key=" + awsKey[:8] + "\x00\x1b\u200b" + awsKey[8:], "key=[REDACTED]"}, + {"two split secrets", awsKey[:8] + "\x00" + awsKey[8:] + " and " + githubKey[:8] + "\x00" + githubKey[8:], "[REDACTED] and [REDACTED]"}, + {"separator before the secret", "\x00" + awsKey, "\x00[REDACTED]"}, + {"separator after the secret", awsKey + "\x00", "[REDACTED]\x00"}, + {"text around it survives", "before " + awsKey[:4] + "\x00" + awsKey[4:] + " after", "before [REDACTED] after"}, + } { + t.Run(testCase.name, func(t *testing.T) { + if got := RedactString(testCase.value, Options{}); got != testCase.want { + t.Errorf("RedactString(...) = %q, want %q", got, testCase.want) + } + }) + } +} + +// NOTHING ELSE MOVES. The compaction exists to find a match; it must never +// reach text that has no secret in it, and text with no invisible character at +// all must take exactly the path it took before. +func TestRedactStringLeavesOrdinaryTextWithControlCharactersAlone(t *testing.T) { + for _, value := range []string{ + "a plain sentence", + "a sentence\x00with a NUL in it", + "\x1b[31mcolored output\x1b[0m", + "tab\tseparated\tcolumns", + "line one\nline two\r\nline three", + "zero\u200bwidth\u200bspaces", + "", "\x00", "\x00\x00\x00", + "not-a-secret-just-a-long-kebab-case-identifier-here", + "sk-some-kebab-case-value-without-any-digits-at-all", + } { + if got := RedactString(value, Options{}); got != value { + t.Errorf("RedactString(%q) = %q, want it unchanged", value, got) + } + } +} + +// The openai filter that keeps kebab-case identifiers out of the redactor has +// to apply on the split path too, or working around the false positive would +// become a matter of inserting a control byte. +func TestSplitPathKeepsTheOpenAIKebabCaseFilter(t *testing.T) { + kebab := "sk-some-kebab-case-value-without-digits" + split := kebab[:6] + "\x00" + kebab[6:] + if got := RedactString(split, Options{}); got != split { + t.Errorf("a kebab-case identifier split by a NUL was redacted: %q", got) + } + // ... while a real key of the same family, split the same way, is not spared. + realKey := openaiKey[:6] + "\x00" + openaiKey[6:] + if got := RedactString(realKey, Options{}); strings.Contains(rejoin(got), openaiKey) { + t.Errorf("a real openai key split by a NUL survived: %q", got) + } +} + +// TAB, NEWLINE AND CARRIAGE RETURN ARE OUT OF SCOPE ON PURPOSE, and this says +// so in a place that fails if someone changes it without meaning to. They are +// real text structure: stripping them would let one match span a line break and +// replace unrelated lines, and a credential broken across a line is visible to +// whoever reads it rather than hidden from them. +func TestLineStructureIsNotTreatedAsAnInvisibleSeparator(t *testing.T) { + for name, separator := range map[string]string{"tab": "\t", "newline": "\n", "carriage return": "\r"} { + if splitSecretSeparator([]rune(separator)[0]) { + t.Errorf("%s is treated as an invisible separator; if that is now wanted, this test and the note in control_split.go both need to change", name) + } + split := awsKey[:8] + separator + awsKey[8:] + if got := RedactString(split, Options{}); got != split { + t.Errorf("%s split was redacted: %q. That may be an improvement, but it is a scope change this test is here to make deliberate", name, got) + } + } +} + +// Malformed UTF-8 must not move the span mapping, which works in the source's +// own byte offsets. A decoder that measured an invalid byte as the three bytes +// of U+FFFD would shift every later span. +// The exact output is asserted, not just the absence of the key. An invalid +// byte measured as three would still cover the secret while moving the +// replacement's edges onto the bytes around it, which reads as a pass if the +// test only asks whether the secret is gone. +func TestSplitRedactionHandlesMalformedUTF8(t *testing.T) { + for _, testCase := range []struct { + name string + value string + want string + }{ + {"invalid byte before the secret", "\xff" + awsKey[:8] + "\x00" + awsKey[8:], "\xff[REDACTED]"}, + // An invalid byte is NOT one of the separators: it decodes to U+FFFD and + // is drawn, so a key broken by one is visible to whoever reads it, the + // same argument that keeps tab and newline out. It is kept in the + // compacted text and so it still ends the match. + {"invalid byte inside the split", awsKey[:8] + "\x00\xff" + awsKey[8:], awsKey[:8] + "\x00\xff" + awsKey[8:]}, + {"invalid byte after the secret", awsKey[:8] + "\x00" + awsKey[8:] + "\xfe", "[REDACTED]\xfe"}, + {"invalid bytes on both sides", "\xff\xfe" + awsKey[:8] + "\x00" + awsKey[8:] + "\xfd\xfc", "\xff\xfe[REDACTED]\xfd\xfc"}, + {"invalid byte then text then the secret", "\xffpre " + awsKey[:8] + "\x00" + awsKey[8:] + " post", "\xffpre [REDACTED] post"}, + {"only invalid bytes", "\xff\xfe\x00\xff", "\xff\xfe\x00\xff"}, + } { + t.Run(testCase.name, func(t *testing.T) { + got := RedactString(testCase.value, Options{}) + if got != testCase.want { + t.Errorf("RedactString(%q) = %q, want %q", testCase.value, got, testCase.want) + } + if strings.Contains(rejoin(got), awsKey) { + t.Errorf("the key survived: %q", got) + } + }) + } +} + +// The compaction is only a lens for finding the match. A string with no +// separator at all must come back byte-identical, including its invalid bytes. +func TestCompactionDoesNotRewriteTextItDoesNotRedact(t *testing.T) { + for _, value := range []string{"plain", "with\xffinvalid", "tab\there", "\xef\xbb\xbfbom at the front"} { + if got := redactControlSplitSecrets(value, RedactedSecret); got != value { + t.Errorf("redactControlSplitSecrets(%q) = %q, want it unchanged", value, got) + } + } +} + +// The separators the tests below need by name, built from their code points so +// no escape in this file can be mangled into the character it stands for. +var ( + nulSeparator = string(rune(0x00)) + escSeparator = string(rune(0x1b)) + zwspSeparator = string(rune(0x200b)) +) + +const ( + slackKey = "xoxb-EXAMPLE-NOT-A-REAL-TOKEN-AAAAAAAAAA" + googleKey = "AIzaSyD-0123456789abcdefghijklmnopqrstuv" + jwtToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r0W1gFWFOEjXkPY" + otherJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiI5ODc2NTQzMjEwIn0.QWxpY2VCb2JDYXJvbERhdmVFdmVGcmFua0dyYWNlSA" +) + +// splitInTheMiddle puts one separator inside the body of a secret. +func splitInTheMiddle(secret, separator string) string { + half := len(secret) / 2 + return secret[:half] + separator + secret[half:] +} + +// A SEPARATOR IN FRONT OF A KEY IS A DELIMITER, ONE INSIDE IT IS FILLER. +// +// #969 asks for one thing: the split form is redacted the same as the unsplit +// form. That is the whole assertion here, run over every kind of byte a key can +// follow. It catches the two ways of getting it wrong from opposite sides. Drop +// the separators and then ask the pattern for a word boundary, and a key written +// after a NUL stops redacting the moment a second separator lands inside it, +// even though the unsplit key after that same NUL redacts today. Ignore the +// boundary instead, and a key glued to the end of a word starts redacting when +// it is split, although the unsplit form never did. +func TestSplitRedactionMatchesTheUnsplitVerdictAtEveryLeadingBoundary(t *testing.T) { + leaders := map[string]string{ + "start of string": "", + "space": "before ", + "quote": `{"k":"`, + "equals": "key=", + "NUL": "prefix" + nulSeparator, + "ESC": "prefix" + escSeparator, + "zero width": "prefix" + zwspSeparator, + "letter": "prefix", + "digit": "7", + "underscore": "field_", + } + secrets := map[string]string{"aws": awsKey, "github": githubKey, "openai": openaiKey, "slack": slackKey, "google": googleKey} + for secretName, secret := range secrets { + for leaderName, leader := range leaders { + unsplit := leader + secret + split := leader + splitInTheMiddle(secret, escSeparator) + wantRedacted := !strings.Contains(RedactString(unsplit, Options{}), secret) + out := RedactString(split, Options{}) + gotRedacted := !strings.Contains(rejoin(out), secret) + if gotRedacted == wantRedacted { + continue + } + if wantRedacted { + t.Errorf("%s key after a %s leader: the unsplit form redacts and the split form does not: %q rejoins to %q", + secretName, leaderName, out, rejoin(out)) + continue + } + t.Errorf("%s key after a %s leader: the unsplit form is left alone and the split form is redacted: %q", + secretName, leaderName, out) + } + } +} + +// A MATCH MUST NOT REACH OUT OF ONE CREDENTIAL AND INTO THE NEXT. +// +// The JWT shapes end on a run of body characters with no trailing boundary, so +// a matcher reading a copy with the separators removed can start in the key in +// front and run through the header of the JWT behind it, replacing both with one +// marker and eating the delimiter between them. The contiguous pass claims whole +// credentials before this one runs, which is what keeps them apart. +func TestSplitRedactionKeepsNeighbouringCredentialsApart(t *testing.T) { + separators := map[string]string{"NUL": nulSeparator, "ESC": escSeparator, "zero width": zwspSeparator} + leaders := map[string]string{ + "jwt": jwtToken, + "github": githubKey, + "anthropic": anthropicKey, + "openai": openaiKey, + "slack": slackKey, + "google": googleKey, + "aws": awsKey, + } + for leaderName, leader := range leaders { + for sepName, separator := range separators { + value := leader + separator + otherJWT + want := RedactedSecret + separator + RedactedSecret + if got := RedactString(value, Options{}); got != want { + t.Errorf("a %s key %s a JWT: RedactString(...) = %q, want %q", leaderName, sepName, got, want) + } + } + } +} + +// splitThroughout puts a separator a quarter of the way into every dot-separated +// segment of a token. Early, so the part in front is too short to be a +// credential by itself and the strict pass cannot claim it; a split whose front +// part is complete alone is the documented limit, not what these tests are about. +func splitThroughout(secret, separator string) string { + parts := strings.Split(secret, ".") + for i, part := range parts { + quarter := len(part) / 4 + parts[i] = part[:quarter] + separator + part[quarter:] + } + return strings.Join(parts, ".") +} + +// TWO SPLIT CREDENTIALS OF ONE SHAPE ARE TWO CREDENTIALS. +// +// Matches of one pattern never overlap, so when the first one's body runs on +// through the separator between them, the second cannot start a match of its +// own. Whole neighbours are safe because the strict pass claims them first; +// these are both split. A JWT body stops at the next token's first dot and a +// classic GitHub body at the next token's underscore, which left the rest of the +// second credential in the clear, and the bodies that swallow the next key whole +// joined both keys and the separator between them into one replacement. +func TestSplitRedactionKeepsTwoSplitCredentialsOfOneShapeApart(t *testing.T) { + separators := map[string]string{"NUL": nulSeparator, "ESC": escSeparator, "zero width": zwspSeparator} + pairs := map[string][2]string{ + "jwt": {jwtToken, otherJWT}, + "github": {githubKey, "ghp_ffffffffffffffffffffffffffffffffffff"}, + "openai": {openaiKey, "sk-proj-dddddddddddddddddddddddddddddddd"}, + "anthropic": {anthropicKey, "sk-ant-api03-eeeeeeeeeeeeeeeeeeeeeeee9876543210WXYZ"}, + "slack": {slackKey, "xoxb-ANOTHER-NOT-REAL-TOKEN-BBBBBBBBBBBB"}, + } + for name, pair := range pairs { + for sepName, separator := range separators { + value := splitThroughout(pair[0], escSeparator) + separator + splitThroughout(pair[1], escSeparator) + want := RedactedSecret + separator + RedactedSecret + if got := RedactString(value, Options{}); got != want { + t.Errorf("two split %s credentials with %s between them: RedactString(...) = %q, want %q", name, sepName, got, want) + } + } + } +} + +// The search for where the next credential begins is bounded, so it has to be +// spent on positions where one can begin. Here the second token's header is split +// at every character inside the part the first match swallowed, which puts dozens +// of separator runs between the end of that match and the real boundary. +func TestSplitRedactionFindsTheNextCredentialPastItsOwnSplits(t *testing.T) { + parts := strings.Split(otherJWT, ".") + header := strings.Join(strings.Split(parts[0], ""), escSeparator) + second := header + "." + strings.Join(parts[1:], ".") + value := splitThroughout(jwtToken, escSeparator) + nulSeparator + second + want := RedactedSecret + nulSeparator + RedactedSecret + if got := RedactString(value, Options{}); got != want { + t.Errorf("RedactString(...) = %q, want %q", got, want) + } +} + +// And a chain of them leaks nothing, however the markers fall. +func TestSplitRedactionLeavesNothingOfAChainOfSplitJWTs(t *testing.T) { + chain := splitThroughout(jwtToken, escSeparator) + nulSeparator + + splitThroughout(otherJWT, escSeparator) + nulSeparator + + splitThroughout(jwtToken, zwspSeparator) + out := rejoin(RedactString(chain, Options{})) + for _, segment := range append(strings.Split(jwtToken, "."), strings.Split(otherJWT, ".")...) { + if strings.Contains(out, segment) { + t.Errorf("a JWT segment survived a chain of three split tokens: %q in %q", segment, out) + } + } +} + +// A CHAIN OF SPLIT CREDENTIALS COSTS WHAT ITS LENGTH COSTS. +// +// The run-on search once followed a chain to its end from every match inside it, +// which is quadratic in the chain: a megabyte of split JWTs back to back +// allocated seventeen gigabytes and ran for ten minutes. Measured as allocation, +// which that walk multiplies with the chain and a linear pass does not. +func TestSplitRedactionStaysLinearOnAChainOfSplitJWTs(t *testing.T) { + unit := splitThroughout(jwtToken, escSeparator) + nulSeparator + chain := strings.Repeat(unit, (64<<10)/len(unit)) + signature := strings.Split(jwtToken, ".")[2] + if strings.Contains(rejoin(RedactString(chain, Options{})), signature) { + t.Fatal("SETUP INVALID: the chain is not redacted, so the run-on search never ran") + } + limit := int64(200 * len(chain)) + if used := redactAllocBytes(chain); used > limit { + t.Errorf("redacting a %d byte chain of split JWTs allocated %d bytes, over the %d byte limit", len(chain), used, limit) + } +} + +// A MATCH THE FILTER REJECTS MUST NOT HIDE THE KEY IT RAN INTO. +// +// The OpenAI shape is filtered, because kebab-case with no digit in it is prose +// and not a key. A rejected match still spans the text it matched, and its body +// runs on through a separator into whatever follows, so a split key after a +// kebab-case name was rejected along with the name and printed in full. +func TestSplitRedactionDoesNotLetARejectedNeighbourHideASplitKey(t *testing.T) { + const prose = "sk-some-kebab-case-name-here-ok" + const key = "sk-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh" + if !strings.Contains(RedactString(prose, Options{}), prose) { + t.Fatal("SETUP INVALID: the prose is redacted on its own, so it is not a rejected match") + } + if strings.Contains(RedactString(key, Options{}), key) { + t.Fatal("SETUP INVALID: the key is not redacted even whole, so it is not a credential") + } + value := prose + nulSeparator + splitInTheMiddle(key, escSeparator) + want := prose + nulSeparator + RedactedSecret + if got := RedactString(value, Options{}); got != want { + t.Errorf("RedactString(...) = %q, want %q", got, want) + } +} + +// ORDINARY TEXT AFTER A KEY IS NOT PART OF THE KEY. Removing the separator +// between them joins the word behind it onto the end of the key, and an +// unbounded shape then carries the replacement over text that was never secret. +func TestSplitRedactionLeavesTextBehindAKeyAlone(t *testing.T) { + for _, testCase := range []struct { + name string + value string + want string + }{ + {"word", openaiKey + nulSeparator + "ordinary prose here", RedactedSecret + nulSeparator + "ordinary prose here"}, + {"escape then word", openaiKey + escSeparator + "ordinary", RedactedSecret + escSeparator + "ordinary"}, + {"zero width then word", githubKey + zwspSeparator + "ordinary", RedactedSecret + zwspSeparator + "ordinary"}, + {"digits", awsKey + nulSeparator + "0123456789", RedactedSecret + nulSeparator + "0123456789"}, + } { + t.Run(testCase.name, func(t *testing.T) { + if got := RedactString(testCase.value, Options{}); got != testCase.want { + t.Errorf("RedactString(...) = %q, want %q", got, testCase.want) + } + }) + } +} + +// THE OFFSET TABLE IS SIZED BY THE MATCHES, NOT BY THE TEXT IT SEARCHED. +// +// RedactString is handed whole command output, verification stdout and +// notification bodies, none of which is bounded before it runs, and an index +// carrying a start and an end for every source byte costs sixteen bytes per byte +// of input on a 64-bit build. Measured against the same text with no separator +// in it, so the regex work and the output copy cancel out and what is left is +// what the split path added. +func TestSplitRedactionDoesNotIndexEverySourceByte(t *testing.T) { + const line = "plain log output line with no secret in it " + filler := strings.Repeat(line, (1<<20)/len(line)) + split := filler + " " + splitInTheMiddle(awsKey, escSeparator) + plain := filler + " " + awsKey + if got := RedactString(split, Options{}); !strings.HasSuffix(got, RedactedSecret) { + t.Fatalf("SETUP INVALID: the split key in the large input is not redacted, so nothing indexes it: %q", got[len(got)-64:]) + } + // Four times the input still fails an index of two ints per source byte, + // which needs sixteen, and clears the copy this path actually makes. + limit := int64(4 * len(split)) + overhead := redactAllocBytes(split) - redactAllocBytes(plain) + if overhead > limit { + t.Errorf("redacting %d bytes with a split key allocated %d bytes more than the same text without one, over the %d byte limit", + len(split), overhead, limit) + } +} + +// redactAllocBytes is the smallest number of bytes any of three RedactString +// calls allocated. Smallest rather than mean because a garbage collection or an +// unrelated goroutine can only ever add to the figure. +func redactAllocBytes(value string) int64 { + smallest := int64(-1) + for i := 0; i < 3; i++ { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + out := RedactString(value, Options{}) + runtime.ReadMemStats(&after) + if len(out) == 0 { + panic("RedactString returned nothing") + } + used := int64(after.TotalAlloc - before.TotalAlloc) + if smallest < 0 || used < smallest { + smallest = used + } + } + return smallest +} + +// A REPLACEMENT ALREADY ON THE PAGE IS A BARRIER. The default marker is spelled +// with brackets, which no credential body admits, so it stops a match by +// itself. A caller is free to supply one made of ordinary word characters, and +// then the text left by the contiguous pass would read as the middle of a key +// and carry the replacement out over both delimiters and the text behind them. +func TestSplitRedactionStopsAtACallerSuppliedReplacement(t *testing.T) { + options := Options{Replacement: "REDACTED"} + value := "sk-aaaaa" + nulSeparator + awsKey + nulSeparator + "bbbbbbbbbbbbbbb" + want := "sk-aaaaa" + nulSeparator + "REDACTED" + nulSeparator + "bbbbbbbbbbbbbbb" + if got := RedactString(value, options); got != want { + t.Errorf("RedactString(...) = %q, want %q", got, want) + } +} + +// A REJECTED MATCH MUST NOT SHADOW THE REAL KEY BEHIND IT. Raised by CodeRabbit +// on #1067. With the separators removed before matching, the leftmost AWS match +// in "xAKIAAKIAIOSFODNN7EXAMPLE" began at the decoy AKIA, was rejected +// for the "x" in front of it, and had already consumed the start of the real key, +// so the real key was never tried. "ask-" in front of a split OpenAI key does the +// same through "sk-", which is the ordinary-prose version of it. Each input is +// checked against its unsplit counterpart, which main already redacts. +func TestSplitRedactionDoesNotLetARejectedMatchShadowTheRealKey(t *testing.T) { + soh := string(rune(0x01)) + for _, testCase := range []struct { + name string + split string + unsplit string + }{ + {"decoy AKIA before a split key", "xAKIA" + nulSeparator + "AKIAIOSFODNN7E" + soh + "XAMPLE", "xAKIA" + nulSeparator + awsKey}, + {"ask- before a split OpenAI key", "ask-" + nulSeparator + splitInTheMiddle(openaiKey, soh), "ask-" + nulSeparator + openaiKey}, + {"task- before a split OpenAI key", "task-" + escSeparator + splitInTheMiddle(openaiKey, zwspSeparator), "task-" + escSeparator + openaiKey}, + } { + t.Run(testCase.name, func(t *testing.T) { + want := RedactString(testCase.unsplit, Options{}) + if strings.Contains(want, awsKey) || strings.Contains(want, openaiKey) { + t.Fatalf("SETUP INVALID: main does not redact the unsplit form either: %q", want) + } + if got := RedactString(testCase.split, Options{}); got != want { + t.Errorf("RedactString(split) = %q, want the unsplit form's %q", got, want) + } + }) + } +} + +// AN UNBOUNDED BODY MUST NOT TAKE A NEIGHBOUR'S PREFIX WITH IT. With gaps allowed +// between body characters, a split key's body runs on across a separator into +// whatever follows. When that is a JWT, it stops only at the JWT's first dot, and +// if the shapes were applied one after another the JWT shape would then find its +// header gone and leave the payload and signature in the clear. The shapes are +// matched against the same text and their union replaced, so it cannot. +func TestSplitRedactionKeepsANeighbourWhosePrefixASplitBodyWouldSwallow(t *testing.T) { + cut := len(jwtToken) / 2 + splitJWT := jwtToken[:cut] + nulSeparator + jwtToken[cut:] + for _, testCase := range []struct { + name string + value string + }{ + {"split Anthropic key, a word, then a split JWT", splitInTheMiddle(anthropicKey, zwspSeparator) + escSeparator + "build" + escSeparator + splitJWT}, + {"split GitHub key, then a split JWT", splitInTheMiddle(githubKey, nulSeparator) + escSeparator + splitJWT}, + } { + t.Run(testCase.name, func(t *testing.T) { + got := rejoin(RedactString(testCase.value, Options{})) + for _, piece := range strings.Split(jwtToken, ".") { + if strings.Contains(got, piece) { + t.Errorf("part of the JWT survives: %q in %q", piece, got) + } + } + }) + } +} + +// A CREDENTIAL GLUED ONTO A SPLIT ONE GETS ITS BOUNDARY FROM THE REPLACEMENT. +// Contiguous shapes cascade: once an AWS key is replaced, a JWT written straight +// after it has the "]" in front of it and matches. A split key is replaced by +// the second pass, so the JWT behind it needs a pass of its own after that one, +// and the JWT is often whole, which means its region may hold no separator at +// all by then. The second case adds a whole key elsewhere so that the strict +// pass has already cut the text into regions. +func TestSplitRedactionCascadesIntoACredentialGluedOntoASplitOne(t *testing.T) { + csi := string(rune(0x9b)) + for _, testCase := range []struct { + name string + value string + }{ + {"whole JWT glued onto a split AWS key", splitInTheMiddle(awsKey, escSeparator) + jwtToken}, + {"same, with a whole key elsewhere in the text", "user?q=" + "ASIAIOSFODNN7EXA" + csi + "MPLE" + jwtToken + "'" + awsKey + string(rune(0x7f)) + "ok"}, + } { + t.Run(testCase.name, func(t *testing.T) { + got := rejoin(RedactString(testCase.value, Options{})) + for _, piece := range strings.Split(jwtToken, ".") { + if strings.Contains(got, piece) { + t.Errorf("part of the JWT survives: %q in %q", piece, got) + } + } + }) + } +} + +// THE CLASS AND THE PREDICATE ARE ONE FACT. splitSeparatorClass is generated +// from the same tables as splitSecretSeparator; this walks every code point and +// fails on the first one they disagree about. +func TestSplitSeparatorClassAgreesWithThePredicate(t *testing.T) { + class := regexp.MustCompile("^" + splitSeparatorClass + "$") + for r := rune(0); r <= unicode.MaxRune; r++ { + if r >= 0xd800 && r <= 0xdfff { + continue // surrogates are not valid in a Go string + } + if got, want := class.MatchString(string(r)), splitSecretSeparator(r); got != want { + t.Fatalf("U+%04X: class matches = %v, predicate says %v", r, got, want) + } + } +} + +// ON TEXT WITH NO SEPARATOR, EVERY GAP-TOLERANT SHAPE IS ITS ORIGINAL. The +// rewrite only adds optional gaps, so a contiguous key has to produce exactly the +// same match; anything else means the rewrite changed what a shape is. +func TestGapTolerantShapesMatchContiguousKeysLikeTheOriginals(t *testing.T) { + inputs := []string{ + "key=" + awsKey + " and " + githubKey, + "x" + awsKey, + anthropicKey + "." + openaiKey, + jwtToken + " then " + slackKey + ", " + googleKey, + "sk-this-is-kebab-case-prose-not-a-key", + "github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyzABCDEFGH/glpat-abcdefghij0123456789", + } + originals := append([]*regexp.Regexp{openaiKeyPattern}, textSecretPatterns...) + tolerant := append([]*regexp.Regexp{splitOpenAIPattern}, splitTextPatterns...) + if len(originals) != len(tolerant) { + t.Fatalf("SETUP INVALID: %d original shapes and %d gap-tolerant ones", len(originals), len(tolerant)) + } + for i := range originals { + for _, input := range inputs { + want := fmt.Sprint(originals[i].FindAllStringIndex(input, -1)) + if got := fmt.Sprint(tolerant[i].FindAllStringIndex(input, -1)); got != want { + t.Errorf("shape %s on %q: gap-tolerant matched %s, original %s", originals[i], input, got, want) + } + } + } +} diff --git a/internal/redaction/redaction.go b/internal/redaction/redaction.go index e65312821..211239eac 100644 --- a/internal/redaction/redaction.go +++ b/internal/redaction/redaction.go @@ -227,8 +227,7 @@ func RedactString(value string, options Options) string { // openai keys first so the filter can drop kebab-case false positives // before any other pattern rewrites nearby text. redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string { - if !knownOpenAIKeyPrefix(match) && !secretMatchHasDigit(match) && - strings.Contains(strings.TrimPrefix(match, "sk-"), "-") { + if !openAIShapeIsSecret(match) { return match } return replacement @@ -236,6 +235,11 @@ func RedactString(value string, options Options) string { for _, pattern := range textSecretPatterns { redacted = pattern.ReplaceAllString(redacted, replacement) } + // Credentials broken up by invisible characters last, over what the + // contiguous matchers left: a whole credential is already claimed by its own + // strict match, so this pass cannot reach out of one and into the next. It is + // a no-op for text with no such character, which is nearly all text. + redacted = redactControlSplitSecrets(redacted, replacement) return redacted }