diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000000..f44158d39728 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,66 @@ +name: codspeed + +# Default to 'contents: read', which grants actions to read commits. +# +# If any permission is set, any permission not included in the list is +# implicitly set to "none". +# +# see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + # "workflow_dispatch" also allows CodSpeed to trigger backtest performance + # analysis to generate initial data. + workflow_dispatch: + push: + branches: + - 'master' + - '[0-9]+.[0-9]+' + - '[0-9]+.x' + tags: + - 'v*' + pull_request: + +jobs: + benchmark: + runs-on: ubuntu-24.04 + permissions: + contents: read # required for actions/checkout + id-token: write # required for OIDC authentication with CodSpeed + steps: + - + name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - + name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: "1.26.4" + cache: false + - + name: Prepare + run: | + # run in go modules mode to prevent traversing to nested modules + ln -s vendor.mod go.mod + ln -s vendor.sum go.sum + - + name: Run benchmarks + uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 + with: + mode: walltime + run: | + go test -bench=. \ + ./cli/command/commands/ \ + ./cli/command/container/ \ + ./cli/command/formatter/ \ + ./cli/command/system/ \ + ./cli/config/ \ + ./opts/ \ + ./templates/ diff --git a/README.md b/README.md index 1183834102b4..c1b7f0c92c3c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/docker/cli)](https://goreportcard.com/report/github.com/docker/cli) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/docker/cli/badge)](https://scorecard.dev/viewer/?uri=github.com/docker/cli) [![Codecov](https://img.shields.io/codecov/c/github/docker/cli?logo=codecov)](https://codecov.io/gh/docker/cli) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/maksimtech/cli?utm_source=badge) ## About diff --git a/TESTING.md b/TESTING.md index ab1a09e32db7..9918c695f14c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -32,6 +32,36 @@ Fakes, and testing utilities can be found in [internal/test](https://godoc.org/github.com/docker/cli/internal/test) and [gotest.tools](https://godoc.org/gotest.tools). +## Benchmarks + +Performance sensitive code should be covered by benchmarks. Benchmarks use the +standard Go [testing](https://pkg.go.dev/testing#hdr-Benchmarks) conventions and +live next to the unit tests in `_test.go` files, named using the convention: + +``` +Benchmark[] +``` + +Prefer [`b.Loop()`](https://pkg.go.dev/testing#B.Loop) over `for i := 0; i < b.N; i++`, +and call `b.ReportAllocs()` to keep track of allocations. + +Benchmarks can be run locally with: + +```shell +go test -bench=. ./templates/ +``` + +Benchmarks are also run continuously in CI through +[CodSpeed](https://app.codspeed.io/maksimtech/cli), which reports the +performance impact of a pull request. The packages that are benchmarked in CI +are listed in [.github/workflows/codspeed.yml](.github/workflows/codspeed.yml). +To run them the same way CodSpeed does, install the +[CodSpeed CLI](https://codspeed.io/docs/cli) and run: + +```shell +codspeed run --skip-upload --mode walltime -- go test -bench=. ./templates/ +``` + ## End-to-End Test Suite The end-to-end test suite tests a cli binary against a real API backend. diff --git a/cli/command/commands/commands_test.go b/cli/command/commands/commands_test.go new file mode 100644 index 000000000000..bb11b56bd885 --- /dev/null +++ b/cli/command/commands/commands_test.go @@ -0,0 +1,20 @@ +package commands + +import ( + "testing" + + "github.com/docker/cli/internal/test" + "github.com/spf13/cobra" +) + +// BenchmarkAddCommands measures the cost of constructing the command-tree. +// The command-tree is constructed on every invocation of the CLI, and as +// part of generating shell-completion scripts and documentation. +func BenchmarkAddCommands(b *testing.B) { + dockerCLI := test.NewFakeCli(nil) + + b.ReportAllocs() + for b.Loop() { + AddCommands(&cobra.Command{Use: "docker"}, dockerCLI) + } +} diff --git a/cli/command/container/opts_test.go b/cli/command/container/opts_test.go index 0781e99687c0..5cacb06277ca 100644 --- a/cli/command/container/opts_test.go +++ b/cli/command/container/opts_test.go @@ -1157,3 +1157,58 @@ func TestConvertToStandardNotation(t *testing.T) { } } } + +// BenchmarkParseRun measures parsing the flags of a "docker run" invocation, +// which includes constructing the flag-set, and converting the options to +// the container-, host-, and networking-config. +func BenchmarkParseRun(b *testing.B) { + for _, tc := range []struct { + doc string + args []string + }{ + { + doc: "minimal", + args: []string{"ubuntu", "bash"}, + }, + { + doc: "many flags", + args: []string{ + "--hostname", "my-hostname", + "--user", "1000:1000", + "--workdir", "/some/workdir", + "--env", "FOO=bar", + "--env", "SOME_OTHER_VAR=some-other-value", + "--label", "com.example.label=some-value", + "--publish", "8080:80/tcp", + "--publish", "127.0.0.1:8443:443", + "--expose", "9000-9010", + "--volume", "/tmp/source:/mnt/source:ro", + "--mount", "type=volume,source=my-volume,target=/data,readonly", + "--tmpfs", "/run:size=1m", + "--network", "my-network", + "--dns", "1.1.1.1", + "--add-host", "example.com:127.0.0.1", + "--cap-add", "NET_ADMIN", + "--cap-drop", "MKNOD", + "--memory", "512m", + "--cpus", "1.5", + "--restart", "on-failure:5", + "--ulimit", "nofile=1024:2048", + "--health-cmd", "curl -f http://localhost/ || exit 1", + "--health-interval", "30s", + "--log-driver", "json-file", + "--log-opt", "max-size=10m", + "ubuntu", "bash", + }, + }, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, _, _, err := parseRun(tc.args); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/command/formatter/container_test.go b/cli/command/formatter/container_test.go index 1c68aeaea059..c6610fd6ec58 100644 --- a/cli/command/formatter/container_test.go +++ b/cli/command/formatter/container_test.go @@ -953,3 +953,66 @@ func TestDisplayablePorts(t *testing.T) { assert.Check(t, is.Equal(port.expected, actual)) } } + +// genContainers generates a list of containers to be used as a realistic +// input for benchmarks; the containers have all fields set that are used +// by the various formats. +func genContainers(count int) []container.Summary { + created := time.Now().Add(-72 * time.Hour).Unix() + containers := make([]container.Summary, 0, count) + for i := range count { + containers = append(containers, container.Summary{ + ID: fmt.Sprintf("%064x", i), + Names: []string{fmt.Sprintf("/container_%d", i), fmt.Sprintf("/other_name_%d", i)}, + Image: "docker.io/library/ubuntu:24.04", + ImageID: fmt.Sprintf("sha256:%064x", i), + Command: `/bin/sh -c "while true; do echo hello world; sleep 1; done"`, + Created: created, + Ports: []container.PortSummary{ + {IP: netip.MustParseAddr("0.0.0.0"), PrivatePort: 80, PublicPort: uint16(30000 + i), Type: "tcp"}, + {IP: netip.MustParseAddr("::"), PrivatePort: 443, PublicPort: uint16(40000 + i), Type: "tcp"}, + {PrivatePort: 8080, Type: "tcp"}, + }, + SizeRw: 123456789, + SizeRootFs: 987654321, + Labels: map[string]string{ + "com.docker.compose.project": "some-project", + "com.docker.compose.service": fmt.Sprintf("service-%d", i), + "org.opencontainers.image.source": "https://github.com/docker/cli", + }, + State: "running", + Status: "Up 3 days (healthy)", + Mounts: []container.MountPoint{ + {Type: "volume", Name: fmt.Sprintf("volume-%d", i), Destination: "/data"}, + {Type: "bind", Source: "/tmp/source", Destination: "/mnt/source"}, + }, + }) + } + return containers +} + +func BenchmarkContainerWrite(b *testing.B) { + containers := genContainers(100) + for _, tc := range []struct { + doc string + format Format + }{ + {doc: "table", format: NewContainerFormat("table", false, false)}, + {doc: "table-with-size", format: NewContainerFormat("table", false, true)}, + {doc: "quiet", format: NewContainerFormat("table", true, false)}, + {doc: "raw", format: NewContainerFormat("raw", false, false)}, + {doc: "json", format: NewContainerFormat("json", false, false)}, + {doc: "custom", format: NewContainerFormat(`{{.ID}}: {{.Names}} {{.Ports}} {{.Labels}} {{.Mounts}}`, false, false)}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + out := bytes.NewBuffer(nil) + for b.Loop() { + out.Reset() + if err := ContainerWrite(Context{Format: tc.format, Output: out, Trunc: true}, containers); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/command/formatter/image_test.go b/cli/command/formatter/image_test.go index c3906687c4fa..c212eeb3e74c 100644 --- a/cli/command/formatter/image_test.go +++ b/cli/command/formatter/image_test.go @@ -367,3 +367,60 @@ func TestImageContextWriteWithNoImage(t *testing.T) { }) } } + +// genImages generates a list of images to be used as a realistic input for +// benchmarks; the images have all fields set that are used by the various +// formats. +func genImages(count int) []image.Summary { + created := time.Now().AddDate(0, 0, -1).Unix() + images := make([]image.Summary, 0, count) + for i := range count { + images = append(images, image.Summary{ + ID: fmt.Sprintf("sha256:%064x", i), + Created: created, + RepoTags: []string{ + fmt.Sprintf("docker.io/library/image-%d:latest", i), + fmt.Sprintf("example.com/some/longer/name/image-%d:v1.2.3", i), + }, + RepoDigests: []string{ + fmt.Sprintf("docker.io/library/image-%d@sha256:%064x", i, i), + }, + Size: 123456789, + SharedSize: 12345678, + Containers: 3, + Labels: map[string]string{"org.opencontainers.image.source": "https://github.com/docker/cli"}, + }) + } + return images +} + +func BenchmarkImageWrite(b *testing.B) { + images := genImages(100) + for _, tc := range []struct { + doc string + format Format + digest bool + }{ + {doc: "table", format: NewImageFormat("table", false, false)}, + {doc: "table-with-digest", format: NewImageFormat("table", false, true), digest: true}, + {doc: "quiet", format: NewImageFormat("table", true, false)}, + {doc: "raw", format: NewImageFormat("raw", false, false)}, + {doc: "json", format: NewImageFormat("json", false, false)}, + {doc: "custom", format: NewImageFormat(`{{.Repository}}:{{.Tag}} {{.ID}} {{.Size}}`, false, false)}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + out := bytes.NewBuffer(nil) + for b.Loop() { + out.Reset() + ctx := ImageContext{ + Context: Context{Format: tc.format, Output: out, Trunc: true}, + Digest: tc.digest, + } + if err := ImageWrite(ctx, images); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/config/config_test.go b/cli/config/config_test.go index 922641a726ad..9d266bb7c456 100644 --- a/cli/config/config_test.go +++ b/cli/config/config_test.go @@ -482,3 +482,39 @@ func TestSetDir(t *testing.T) { SetDir(expected) assert.Check(t, is.Equal(Dir(), expected)) } + +// benchConfig is a realistic configuration file, as loaded on every +// invocation of the CLI. +const benchConfig = `{ + "auths": { + "https://index.docker.io/v1/": {"auth": "am9lam9lOmhlbGxv", "email": "user@example.com"}, + "registry.example.com": {"auth": "am9lam9lOmhlbGxv"}, + "registry.example.com:5000": {"auth": "am9lam9lOmhlbGxv"}, + "some-other-registry.example.com": {"auth": "am9lam9lOmhlbGxv", "identitytoken": "super-secret-token"} + }, + "credsStore": "desktop", + "credHelpers": { + "registry.example.com": "secretservice", + "other.example.com": "pass" + }, + "psFormat": "table {{.ID}}\\t{{.Image}}\\t{{.Command}}\\t{{.Status}}", + "imagesFormat": "table {{.Repository}}\\t{{.Tag}}\\t{{.ID}}\\t{{.Size}}", + "detachKeys": "ctrl-e,e", + "currentContext": "default", + "plugins": { + "buildx": {"defaultBuilder": "default"}, + "compose": {"someOption": "someValue"} + }, + "aliases": {"builder": "buildx"}, + "features": {"containerd-snapshotter": "true"}, + "experimental": "enabled" +}` + +func BenchmarkLoadFromReader(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := LoadFromReader(strings.NewReader(benchConfig)); err != nil { + b.Fatal(err) + } + } +} diff --git a/opts/mount_test.go b/opts/mount_test.go index 2014c98d55c4..545caa1a98a9 100644 --- a/opts/mount_test.go +++ b/opts/mount_test.go @@ -586,3 +586,25 @@ func TestMountOptSetBindRecursive(t *testing.T) { }, m.Value())) }) } + +func BenchmarkMountOptSet(b *testing.B) { + for _, tc := range []struct { + doc string + value string + }{ + {doc: "volume", value: "type=volume,source=my-volume,target=/data"}, + {doc: "volume-with-opts", value: "type=volume,source=my-volume,target=/data,readonly,volume-nocopy,volume-driver=local,volume-label=foo=bar,volume-opt=type=nfs,volume-opt=device=:/some/path"}, + {doc: "bind", value: "type=bind,source=/home/path,target=/target,readonly,bind-propagation=rprivate"}, + {doc: "tmpfs", value: "type=tmpfs,target=/target,tmpfs-size=1m,tmpfs-mode=0700"}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + var m MountOpt + if err := m.Set(tc.value); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/templates/templates_test.go b/templates/templates_test.go index ed1ee5b95d13..b69d1ad3cb67 100644 --- a/templates/templates_test.go +++ b/templates/templates_test.go @@ -229,3 +229,44 @@ func TestJoinElements(t *testing.T) { }) } } + +func BenchmarkParse(b *testing.B) { + for _, tc := range []struct { + doc string + format string + }{ + {doc: "simple", format: `{{.ID}}`}, + {doc: "json", format: `{{json .}}`}, + {doc: "table", format: "{{.ID}}\t{{.Image}}\t{{.Command}}\t{{.RunningFor}}\t{{.Status}}\t{{.Ports}}\t{{.Names}}"}, + {doc: "functions", format: `{{pad (upper (truncate .Name 12)) 1 1}}{{join (split .Names ",") " "}}`}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := Parse(tc.format); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkExecute(b *testing.B) { + tmpl, err := Parse(`{{pad (upper (truncate .Name 12)) 1 1}}{{json .Labels}}{{join (split .Names ",") " "}}`) + assert.NilError(b, err) + + data := map[string]any{ + "Name": "some-container-name", + "Names": "first,second,third", + "Labels": map[string]string{"label1": "value1", "label2": "value2"}, + } + + b.ReportAllocs() + var buf bytes.Buffer + for b.Loop() { + buf.Reset() + if err := tmpl.Execute(&buf, data); err != nil { + b.Fatal(err) + } + } +}