diff --git a/free/ci/cmd/v15.go b/free/ci/cmd/v15.go index a2376996c..0b47616d6 100644 --- a/free/ci/cmd/v15.go +++ b/free/ci/cmd/v15.go @@ -12,8 +12,6 @@ package main import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "flag" "fmt" @@ -25,6 +23,7 @@ import ( "syscall" "github.com/nself-org/plugins/free/ci/internal/adhoc" + "github.com/nself-org/plugins/free/ci/internal/attest" "github.com/nself-org/plugins/free/ci/internal/config" "github.com/nself-org/plugins/free/ci/internal/drill" "github.com/nself-org/plugins/free/ci/internal/model" @@ -188,27 +187,30 @@ func parseBare(args []string) (RunOptions, error) { return o, nil } +// configDigest is a seam so a test can fail the digest. +var configDigest = func(cfg model.PipelineConfig) (string, error) { return attest.ConfigDigest(cfg) } + // preparePlan runs E621, config load, detection and the planner. func preparePlan(ctx context.Context, o RunOptions) (*plan.Plan, string, func(plan.PlannedJob) bool, error) { if err := config.CheckLeftover(o.Root); err != nil { return nil, "", nil, err } - cfg, digest := model.PipelineConfig{Version: 1}, "builtin-defaults" + cfg := model.PipelineConfig{Version: 1} path := firstOf(bare.cfg, filepath.Join(o.Root, config.FileName)) loaded, err := config.Load(path) switch { case err == nil: cfg = loaded.Config - if raw, e := os.ReadFile(path); e == nil { - sum := sha256.Sum256(raw) - digest = hex.EncodeToString(sum[:]) - } case !errors.Is(err, os.ErrNotExist) || bare.cfg != "": return nil, "", nil, err } if cfg, err = drill.Expand(o.Root, cfg, o.Pipeline, o.Trigger); err != nil { // drills_dir subdirectories become kind drill jobs (P7-CI-02) return nil, "", nil, err } + digest, err := configDigest(cfg) // the effective config, built-in defaults included (D-1270) + if err != nil { + return nil, "", nil, err + } facts, checks, err := planChecks(o.Root, cfg) if err != nil { return nil, "", nil, err diff --git a/free/ci/cmd/v15_digest_test.go b/free/ci/cmd/v15_digest_test.go new file mode 100644 index 000000000..e58d0d0dc --- /dev/null +++ b/free/ci/cmd/v15_digest_test.go @@ -0,0 +1,99 @@ +package main + +// Purpose: the config digest preparePlan hands the coordinator (D-1270): the +// attest digest of the effective config on every path, never a placeholder, +// and a digest failure is an error rather than a silent fallback. + +import ( + "context" + "errors" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/nself-org/plugins/free/ci/internal/attest" + "github.com/nself-org/plugins/free/ci/internal/evidence" + "github.com/nself-org/plugins/free/ci/internal/model" +) + +var hexDigest = regexp.MustCompile(`^(sha256:)?[0-9a-f]{64}$`) + +func digestRepo(t *testing.T, yaml string) RunOptions { + t.Helper() + dir := t.TempDir() + if yaml != "" { + p := filepath.Join(dir, ".nself", "ci.yaml") + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + } + t.Setenv("NSELF_CI_BASE_SHA", "") + bare.cfg, bare.base = "", "" + return RunOptions{Root: dir} +} + +func TestPreparePlanDigestWithConfig(t *testing.T) { + _, a, _, err := preparePlan(context.Background(), digestRepo(t, ciYAML)) + if err != nil { + t.Fatal(err) + } + _, b, _, err := preparePlan(context.Background(), digestRepo(t, strings.Replace(ciYAML, "unit", "other", 1))) + if err != nil { + t.Fatal(err) + } + if !hexDigest.MatchString(a) || a == b { + t.Fatalf("config digests %q and %q must be distinct attest digests", a, b) + } +} + +func TestPreparePlanDigestWithoutConfig(t *testing.T) { + _, got, _, err := preparePlan(context.Background(), digestRepo(t, "")) + if err != nil { + t.Fatal(err) + } + want, err := attest.ConfigDigest(model.PipelineConfig{Version: 1}) + if err != nil { + t.Fatal(err) + } + if got == "builtin-defaults" || got != want { + t.Fatalf("no-config digest = %q, want the attest digest %q", got, want) + } +} + +func TestPreparePlanDigestErrorIsReturned(t *testing.T) { + boom := errors.New("digest boom") + old := configDigest + configDigest = func(model.PipelineConfig) (string, error) { return "", boom } + defer func() { configDigest = old }() + for _, yaml := range []string{ciYAML, ""} { + if _, _, _, err := preparePlan(context.Background(), digestRepo(t, yaml)); !errors.Is(err, boom) { + t.Fatalf("preparePlan err = %v, want the digest error", err) + } + } +} + +// The digest preparePlan hands the coordinator is the one evidence.InputsOf +// records for the same tree, with and without a config. +func TestPreparePlanDigestMatchesInputsOf(t *testing.T) { + for name, yaml := range map[string]string{"with ci.yaml": ciYAML, "without ci.yaml": ""} { + t.Run(name, func(t *testing.T) { + o := digestRepo(t, yaml) + _, got, _, err := preparePlan(context.Background(), o) + if err != nil { + t.Fatal(err) + } + in, err := evidence.InputsOf(os.DirFS(o.Root)) + if err != nil { + t.Fatal(err) + } + if got != in.Config { + t.Fatalf("preparePlan digest = %q, InputsOf config = %q", got, in.Config) + } + }) + } +} diff --git a/free/ci/internal/adhoc/argv.go b/free/ci/internal/adhoc/argv.go index e82d25288..2e51780a1 100644 --- a/free/ci/internal/adhoc/argv.go +++ b/free/ci/internal/adhoc/argv.go @@ -4,11 +4,12 @@ package adhoc // (it is exec'd as given); a redacted copy is the only form stored or shown. // Inputs: the argv and the process-wide redaction registry. // Outputs: Recorded (a redacted, bounded one-line form) and Register (values -// that look like secrets join the registry so job output is masked too). +// that look like secrets join the registry so job output is masked too). // Constraints: token shapes (ghp_, sk_live_, AKIA, JWT...) are masked by the -// P7-TRUST-02 redactor with no help; a value that follows a secret-looking name -// (`--token=V`, `PASSWORD=V`, `--api-key V`) is registered, then masked. Nothing -// is guessed beyond that: an argument that is only a word stays visible. +// P7-TRUST-02 redactor with no help; a value that follows a secret-looking name +// (`--token=V`, `PASSWORD=V`, `--api-key V`) is registered, then masked. Well-known +// positional forms (`-pVALUE`, `-u user:pass`, `-H X-Key: val`) are also caught. +// Nothing else is guessed: an argument that is only a word stays visible. import ( "regexp" @@ -23,7 +24,56 @@ const maxRecorded = 240 var secretName = regexp.MustCompile(`(?i)(token|secret|passw(or)?d|passwd|credential|api[-_]?key|private[-_]?key|access[-_]?key|auth)`) -// secretValues returns the argv values that follow a secret-looking name. +// mentionsMySQLClient reports whether argv names a MySQL-family client +// (mysql, mysqldump, mariadb, mariadb-dump...), directly or behind a wrapper +// such as `docker exec db mysql`. +func mentionsMySQLClient(argv []string) bool { + for _, a := range argv { + base := a + if i := strings.LastIndexAny(base, `/\`); i >= 0 { + base = base[i+1:] + } + if strings.HasPrefix(base, "mysql") || strings.HasPrefix(base, "mariadb") { + return true + } + } + return false +} + +// isMySQLPassword reports whether a is the attached-password form `-pVALUE`. +func isMySQLPassword(a string) bool { return strings.HasPrefix(a, "-p") && len(a) > 2 } + +// wellKnownSecrets returns values carried by well-known positional forms that +// do not use a secret-looking flag name: mysql-style `-pVALUE`, curl-style +// `-u user:pass` and `-H "X-Key: value"` headers whose name looks secret-like. +func wellKnownSecrets(argv []string) [][]byte { + var out [][]byte + mysql := mentionsMySQLClient(argv) + for i, a := range argv { + if len(a) < 2 || a[0] != '-' { + continue + } + switch { + case mysql && isMySQLPassword(a): + // mysql/mariadb: -pS3cret (no space). Only when a MySQL-family client is in + // the command, so `go test -parallel 4`, `-pkgdir`, `-print` stay visible. + out = append(out, []byte(a[2:])) + case a == "-u" && i+1 < len(argv) && strings.Contains(argv[i+1], ":"): + // curl -u user:pass + out = append(out, []byte(argv[i+1])) + case a == "-H" && i+1 < len(argv): + // -H "X-Api-Key: value" — mask the value when the header name is secret-looking. + hdr := argv[i+1] + if name, value, ok := strings.Cut(hdr, ":"); ok && secretName.MatchString(name) { + out = append(out, []byte(strings.TrimSpace(value))) + } + } + } + return out +} + +// secretValues returns the argv values that follow a secret-looking name plus +// any well-known positional forms. func secretValues(argv []string) [][]byte { var out [][]byte for i, a := range argv { @@ -39,6 +89,7 @@ func secretValues(argv []string) [][]byte { out = append(out, []byte(argv[i+1])) } } + out = append(out, wellKnownSecrets(argv)...) return out } @@ -61,8 +112,13 @@ func Recorded(r *redact.Redactor, argv []string) string { for _, v := range secretValues(argv) { hidden[string(v)] = true } + mysql := mentionsMySQLClient(argv) parts := make([]string, len(argv)) for i, a := range argv { + if mysql && isMySQLPassword(a) { // masked by position: a value too short for the registry is hidden too + parts[i] = "-p[redacted]" + continue + } if name, value, ok := strings.Cut(a, "="); ok && hidden[value] { a = name + "=[redacted]" } else if hidden[a] { diff --git a/free/ci/internal/adhoc/argv_test.go b/free/ci/internal/adhoc/argv_test.go index a6681f08a..0b9825953 100644 --- a/free/ci/internal/adhoc/argv_test.go +++ b/free/ci/internal/adhoc/argv_test.go @@ -72,3 +72,79 @@ func TestDashPrefixedSecretValuesAreRedacted(t *testing.T) { }) } } + +func TestWellKnownSecretFormsAreRedacted(t *testing.T) { + cases := []struct { + name string + argv []string + secret string + }{ + {"mysql -pVALUE", []string{"mysql", "-pS3cretPass", "db"}, "S3cretPass"}, + {"mariadb path -pVALUE", []string{"/usr/bin/mariadb", "-pS3cretPass", "db"}, "S3cretPass"}, + {"mysql behind docker exec", []string{"docker", "exec", "db", "mysql", "-pS3cretPass"}, "S3cretPass"}, + {"curl -u user:pass", []string{"curl", "-u", "admin:s3cret", "https://x"}, "admin:s3cret"}, + {"header X-Api-Key", []string{"curl", "-H", "X-Api-Key: tok123456789", "https://x"}, "tok123456789"}, + {"header Authorization", []string{"curl", "-H", "Authorization: Bearer abcdefghij", "https://x"}, "Bearer abcdefghij"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := redact.New() + Register(r, tc.argv) + got := Recorded(r, tc.argv) + if strings.Contains(got, tc.secret) { + t.Errorf("recorded argv holds %q: %s", tc.secret, got) + } + f := r.NewFilter() + out := string(append(f.Redact([]byte("line "+tc.secret+" end\n")), f.Flush()...)) + if strings.Contains(out, tc.secret) { + t.Errorf("job output still holds %q: %s", tc.secret, out) + } + }) + } +} + +func TestNonSecretDashFlagsNotRedacted(t *testing.T) { + // -port, -pidfile, -print, -path and plain positional args must stay visible, + // in the recorded line and in job output. + argv := []string{"myapp", "-port=8080", "-pidfile=/tmp/pid", "-print", "-path=/srv/app", "plainarg"} + r := redact.New() + Register(r, argv) + got := Recorded(r, argv) + if want := strings.Join(argv, " "); got != want { + t.Errorf("recorded argv = %q, want %q", got, want) + } + f := r.NewFilter() + out := string(append(f.Redact([]byte("pidfile=/tmp/pid print path=/srv/app port=8080 plainarg\n")), f.Flush()...)) + if strings.Contains(out, redact.Placeholder) { + t.Errorf("job output was masked: %q", out) + } +} + +// A -pVALUE too short for the registry still has to be hidden in Recorded. +func TestShortMySQLPasswordMaskedInRecorded(t *testing.T) { + argv := []string{"mysql", "-pab", "db"} + r := redact.New() + if tooShort := Register(r, argv); tooShort != 1 { + t.Fatalf("short secret count = %d, want 1", tooShort) + } + if got, want := Recorded(r, argv), "mysql -p[redacted] db"; got != want { + t.Errorf("recorded argv = %q, want %q", got, want) + } +} + +// Flags that merely start with -p belong to other tools (go test, make...) and +// must stay visible: redacting them would corrupt the job's output. +func TestDashPFlagsOfOtherToolsAreNotRedacted(t *testing.T) { + argv := []string{"go", "test", "-parallel", "4", "-pkgdir=/tmp/pkg", "-print-errors", "-p4", "./..."} + r := redact.New() + Register(r, argv) + got := Recorded(r, argv) + if want := strings.Join(argv, " "); got != want { + t.Errorf("recorded argv = %q, want %q", got, want) + } + f := r.NewFilter() + out := string(append(f.Redact([]byte("ok parallel pkgdir print-errors\n")), f.Flush()...)) + if strings.Contains(out, redact.Placeholder) { + t.Errorf("job output was masked: %q", out) + } +}