fix(ci): redact well-known secret forms and use attest config digest (D-1020, D-1270)

free/ci/cmd/v15.go

@@ -12,8 +12,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
@@ -25,6 +23,7 @@ import (
 	"syscall"
 
 	"github.com/nself-org/plugins/free/ci/internal/adhoc"
+	"github.com/nself-org/plugins/free/ci/internal/attest"
 	"github.com/nself-org/plugins/free/ci/internal/config"
 	"github.com/nself-org/plugins/free/ci/internal/drill"
 	"github.com/nself-org/plugins/free/ci/internal/model"
@@ -188,27 +187,30 @@ func parseBare(args []string) (RunOptions, error) {
 	return o, nil
 }
 
+// configDigest is a seam so a test can fail the digest.
+var configDigest = func(cfg model.PipelineConfig) (string, error) { return attest.ConfigDigest(cfg) }
+
 // preparePlan runs E621, config load, detection and the planner.
 func preparePlan(ctx context.Context, o RunOptions) (*plan.Plan, string, func(plan.PlannedJob) bool, error) {
 	if err := config.CheckLeftover(o.Root); err != nil {
 		return nil, "", nil, err
 	}
-	cfg, digest := model.PipelineConfig{Version: 1}, "builtin-defaults"
+	cfg := model.PipelineConfig{Version: 1}
 	path := firstOf(bare.cfg, filepath.Join(o.Root, config.FileName))
 	loaded, err := config.Load(path)
 	switch {
 	case err == nil:
 		cfg = loaded.Config
-		if raw, e := os.ReadFile(path); e == nil {
-			sum := sha256.Sum256(raw)
-			digest = hex.EncodeToString(sum[:])
-		}
 	case !errors.Is(err, os.ErrNotExist) || bare.cfg != "":
 		return nil, "", nil, err
 	}
 	if cfg, err = drill.Expand(o.Root, cfg, o.Pipeline, o.Trigger); err != nil { // drills_dir subdirectories become kind drill jobs (P7-CI-02)
 		return nil, "", nil, err
 	}
+	digest, err := configDigest(cfg) // the effective config, built-in defaults included (D-1270)
+	if err != nil {
+		return nil, "", nil, err
+	}
 	facts, checks, err := planChecks(o.Root, cfg)
 	if err != nil {
 		return nil, "", nil, err

free/ci/cmd/v15_digest_test.go

@@ -0,0 +1,99 @@
+package main
+
+// Purpose: the config digest preparePlan hands the coordinator (D-1270): the
+// attest digest of the effective config on every path, never a placeholder,
+// and a digest failure is an error rather than a silent fallback.
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/nself-org/plugins/free/ci/internal/attest"
+	"github.com/nself-org/plugins/free/ci/internal/evidence"
+	"github.com/nself-org/plugins/free/ci/internal/model"
+)
+
+var hexDigest = regexp.MustCompile(`^(sha256:)?[0-9a-f]{64}$`)
+
+func digestRepo(t *testing.T, yaml string) RunOptions {
+	t.Helper()
+	dir := t.TempDir()
+	if yaml != "" {
+		p := filepath.Join(dir, ".nself", "ci.yaml")
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(yaml), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	t.Setenv("NSELF_CI_BASE_SHA", "")
+	bare.cfg, bare.base = "", ""
+	return RunOptions{Root: dir}
+}
+
+func TestPreparePlanDigestWithConfig(t *testing.T) {
+	_, a, _, err := preparePlan(context.Background(), digestRepo(t, ciYAML))
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, b, _, err := preparePlan(context.Background(), digestRepo(t, strings.Replace(ciYAML, "unit", "other", 1)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !hexDigest.MatchString(a) || a == b {
+		t.Fatalf("config digests %q and %q must be distinct attest digests", a, b)
+	}
+}
+
+func TestPreparePlanDigestWithoutConfig(t *testing.T) {
+	_, got, _, err := preparePlan(context.Background(), digestRepo(t, ""))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want, err := attest.ConfigDigest(model.PipelineConfig{Version: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got == "builtin-defaults" || got != want {
+		t.Fatalf("no-config digest = %q, want the attest digest %q", got, want)
+	}
+}
+
+func TestPreparePlanDigestErrorIsReturned(t *testing.T) {
+	boom := errors.New("digest boom")
+	old := configDigest
+	configDigest = func(model.PipelineConfig) (string, error) { return "", boom }
+	defer func() { configDigest = old }()
+	for _, yaml := range []string{ciYAML, ""} {
+		if _, _, _, err := preparePlan(context.Background(), digestRepo(t, yaml)); !errors.Is(err, boom) {
+			t.Fatalf("preparePlan err = %v, want the digest error", err)
+		}
+	}
+}
+
+// The digest preparePlan hands the coordinator is the one evidence.InputsOf
+// records for the same tree, with and without a config.
+func TestPreparePlanDigestMatchesInputsOf(t *testing.T) {
+	for name, yaml := range map[string]string{"with ci.yaml": ciYAML, "without ci.yaml": ""} {
+		t.Run(name, func(t *testing.T) {
+			o := digestRepo(t, yaml)
+			_, got, _, err := preparePlan(context.Background(), o)
+			if err != nil {
+				t.Fatal(err)
+			}
+			in, err := evidence.InputsOf(os.DirFS(o.Root))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != in.Config {
+				t.Fatalf("preparePlan digest = %q, InputsOf config = %q", got, in.Config)
+			}
+		})
+	}
+}

free/ci/internal/adhoc/argv.go

@@ -4,11 +4,12 @@ package adhoc
 // (it is exec'd as given); a redacted copy is the only form stored or shown.
 // Inputs:  the argv and the process-wide redaction registry.
 // Outputs: Recorded (a redacted, bounded one-line form) and Register (values
-// that look like secrets join the registry so job output is masked too).
+//          that look like secrets join the registry so job output is masked too).
 // Constraints: token shapes (ghp_, sk_live_, AKIA, JWT...) are masked by the
-// P7-TRUST-02 redactor with no help; a value that follows a secret-looking name
-// (`--token=V`, `PASSWORD=V`, `--api-key V`) is registered, then masked. Nothing
-// is guessed beyond that: an argument that is only a word stays visible.
+//          P7-TRUST-02 redactor with no help; a value that follows a secret-looking name
+//          (`--token=V`, `PASSWORD=V`, `--api-key V`) is registered, then masked. Well-known
+//          positional forms (`-pVALUE`, `-u user:pass`, `-H X-Key: val`) are also caught.
+//          Nothing else is guessed: an argument that is only a word stays visible.
 
 import (
 	"regexp"
@@ -23,7 +24,56 @@ const maxRecorded = 240
 
 var secretName = regexp.MustCompile(`(?i)(token|secret|passw(or)?d|passwd|credential|api[-_]?key|private[-_]?key|access[-_]?key|auth)`)
 
-// secretValues returns the argv values that follow a secret-looking name.
+// mentionsMySQLClient reports whether argv names a MySQL-family client
+// (mysql, mysqldump, mariadb, mariadb-dump...), directly or behind a wrapper
+// such as `docker exec db mysql`.
+func mentionsMySQLClient(argv []string) bool {
+	for _, a := range argv {
+		base := a
+		if i := strings.LastIndexAny(base, `/\`); i >= 0 {
+			base = base[i+1:]
+		}
+		if strings.HasPrefix(base, "mysql") || strings.HasPrefix(base, "mariadb") {
+			return true
+		}
+	}
+	return false
+}
+
+// isMySQLPassword reports whether a is the attached-password form `-pVALUE`.
+func isMySQLPassword(a string) bool { return strings.HasPrefix(a, "-p") && len(a) > 2 }
+
+// wellKnownSecrets returns values carried by well-known positional forms that
+// do not use a secret-looking flag name: mysql-style `-pVALUE`, curl-style
+// `-u user:pass` and `-H "X-Key: value"` headers whose name looks secret-like.
+func wellKnownSecrets(argv []string) [][]byte {
+	var out [][]byte
+	mysql := mentionsMySQLClient(argv)
+	for i, a := range argv {
+		if len(a) < 2 || a[0] != '-' {
+			continue
+		}
+		switch {
+		case mysql && isMySQLPassword(a):
+			// mysql/mariadb: -pS3cret (no space). Only when a MySQL-family client is in
+			// the command, so `go test -parallel 4`, `-pkgdir`, `-print` stay visible.
+			out = append(out, []byte(a[2:]))
+		case a == "-u" && i+1 < len(argv) && strings.Contains(argv[i+1], ":"):
+			// curl -u user:pass
+			out = append(out, []byte(argv[i+1]))
+		case a == "-H" && i+1 < len(argv):
+			// -H "X-Api-Key: value" — mask the value when the header name is secret-looking.
+			hdr := argv[i+1]
+			if name, value, ok := strings.Cut(hdr, ":"); ok && secretName.MatchString(name) {
+				out = append(out, []byte(strings.TrimSpace(value)))
+			}
+		}
+	}
+	return out
+}
+
+// secretValues returns the argv values that follow a secret-looking name plus
+// any well-known positional forms.
 func secretValues(argv []string) [][]byte {
 	var out [][]byte
 	for i, a := range argv {
@@ -39,6 +89,7 @@ func secretValues(argv []string) [][]byte {
 			out = append(out, []byte(argv[i+1]))
 		}
 	}
+	out = append(out, wellKnownSecrets(argv)...)
 	return out
 }
 
@@ -61,8 +112,13 @@ func Recorded(r *redact.Redactor, argv []string) string {
 	for _, v := range secretValues(argv) {
 		hidden[string(v)] = true
 	}
+	mysql := mentionsMySQLClient(argv)
 	parts := make([]string, len(argv))
 	for i, a := range argv {
+		if mysql && isMySQLPassword(a) { // masked by position: a value too short for the registry is hidden too
+			parts[i] = "-p[redacted]"
+			continue
+		}
 		if name, value, ok := strings.Cut(a, "="); ok && hidden[value] {
 			a = name + "=[redacted]"
 		} else if hidden[a] {

free/ci/internal/adhoc/argv_test.go

@@ -72,3 +72,79 @@ func TestDashPrefixedSecretValuesAreRedacted(t *testing.T) {
 		})
 	}
 }
+
+func TestWellKnownSecretFormsAreRedacted(t *testing.T) {
+	cases := []struct {
+		name   string
+		argv   []string
+		secret string
+	}{
+		{"mysql -pVALUE", []string{"mysql", "-pS3cretPass", "db"}, "S3cretPass"},
+		{"mariadb path -pVALUE", []string{"/usr/bin/mariadb", "-pS3cretPass", "db"}, "S3cretPass"},
+		{"mysql behind docker exec", []string{"docker", "exec", "db", "mysql", "-pS3cretPass"}, "S3cretPass"},
+		{"curl -u user:pass", []string{"curl", "-u", "admin:s3cret", "https://x"}, "admin:s3cret"},
+		{"header X-Api-Key", []string{"curl", "-H", "X-Api-Key: tok123456789", "https://x"}, "tok123456789"},
+		{"header Authorization", []string{"curl", "-H", "Authorization: Bearer abcdefghij", "https://x"}, "Bearer abcdefghij"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := redact.New()
+			Register(r, tc.argv)
+			got := Recorded(r, tc.argv)
+			if strings.Contains(got, tc.secret) {
+				t.Errorf("recorded argv holds %q: %s", tc.secret, got)
+			}
+			f := r.NewFilter()
+			out := string(append(f.Redact([]byte("line "+tc.secret+" end\n")), f.Flush()...))
+			if strings.Contains(out, tc.secret) {
+				t.Errorf("job output still holds %q: %s", tc.secret, out)
+			}
+		})
+	}
+}
+
+func TestNonSecretDashFlagsNotRedacted(t *testing.T) {
+	// -port, -pidfile, -print, -path and plain positional args must stay visible,
+	// in the recorded line and in job output.
+	argv := []string{"myapp", "-port=8080", "-pidfile=/tmp/pid", "-print", "-path=/srv/app", "plainarg"}
+	r := redact.New()
+	Register(r, argv)
+	got := Recorded(r, argv)
+	if want := strings.Join(argv, " "); got != want {
+		t.Errorf("recorded argv = %q, want %q", got, want)
+	}
+	f := r.NewFilter()
+	out := string(append(f.Redact([]byte("pidfile=/tmp/pid print path=/srv/app port=8080 plainarg\n")), f.Flush()...))
+	if strings.Contains(out, redact.Placeholder) {
+		t.Errorf("job output was masked: %q", out)
+	}
+}
+
+// A -pVALUE too short for the registry still has to be hidden in Recorded.
+func TestShortMySQLPasswordMaskedInRecorded(t *testing.T) {
+	argv := []string{"mysql", "-pab", "db"}
+	r := redact.New()
+	if tooShort := Register(r, argv); tooShort != 1 {
+		t.Fatalf("short secret count = %d, want 1", tooShort)
+	}
+	if got, want := Recorded(r, argv), "mysql -p[redacted] db"; got != want {
+		t.Errorf("recorded argv = %q, want %q", got, want)
+	}
+}
+
+// Flags that merely start with -p belong to other tools (go test, make...) and
+// must stay visible: redacting them would corrupt the job's output.
+func TestDashPFlagsOfOtherToolsAreNotRedacted(t *testing.T) {
+	argv := []string{"go", "test", "-parallel", "4", "-pkgdir=/tmp/pkg", "-print-errors", "-p4", "./..."}
+	r := redact.New()
+	Register(r, argv)
+	got := Recorded(r, argv)
+	if want := strings.Join(argv, " "); got != want {
+		t.Errorf("recorded argv = %q, want %q", got, want)
+	}
+	f := r.NewFilter()
+	out := string(append(f.Redact([]byte("ok parallel pkgdir print-errors\n")), f.Flush()...))
+	if strings.Contains(out, redact.Placeholder) {
+		t.Errorf("job output was masked: %q", out)
+	}
+}