Add continuous benchmarking with CodSpeed

.github/workflows/codspeed.yml

@@ -0,0 +1,66 @@
+name: codspeed
+
+# Default to 'contents: read', which grants actions to read commits.
+#
+# If any permission is set, any permission not included in the list is
+# implicitly set to "none".
+#
+# see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  # "workflow_dispatch" also allows CodSpeed to trigger backtest performance
+  # analysis to generate initial data.
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+    tags:
+      - 'v*'
+  pull_request:
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read # required for actions/checkout
+      id-token: write # required for OIDC authentication with CodSpeed
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      -
+        name: Set up Go
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: "1.26.4"
+          cache: false
+      -
+        name: Prepare
+        run: |
+          # run in go modules mode to prevent traversing to nested modules
+          ln -s vendor.mod go.mod
+          ln -s vendor.sum go.sum
+      -
+        name: Run benchmarks
+        uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3
+        with:
+          mode: walltime
+          run: |
+            go test -bench=. \
+              ./cli/command/commands/ \
+              ./cli/command/container/ \
+              ./cli/command/formatter/ \
+              ./cli/command/system/ \
+              ./cli/config/ \
+              ./opts/ \
+              ./templates/

README.md

@@ -6,6 +6,7 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/docker/cli)](https://goreportcard.com/report/github.com/docker/cli)
 [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/docker/cli/badge)](https://scorecard.dev/viewer/?uri=github.com/docker/cli)
 [![Codecov](https://img.shields.io/codecov/c/github/docker/cli?logo=codecov)](https://codecov.io/gh/docker/cli)
+[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/maksimtech/cli?utm_source=badge)
 
 ## About
 

TESTING.md

@@ -32,6 +32,36 @@ Fakes, and testing utilities can be found in
 [internal/test](https://godoc.org/github.com/docker/cli/internal/test) and
 [gotest.tools](https://godoc.org/gotest.tools).
 
+## Benchmarks
+
+Performance sensitive code should be covered by benchmarks. Benchmarks use the
+standard Go [testing](https://pkg.go.dev/testing#hdr-Benchmarks) conventions and
+live next to the unit tests in `_test.go` files, named using the convention:
+
+```
+Benchmark<Function Name>[<Test Case Name>]
+```
+
+Prefer [`b.Loop()`](https://pkg.go.dev/testing#B.Loop) over `for i := 0; i < b.N; i++`,
+and call `b.ReportAllocs()` to keep track of allocations.
+
+Benchmarks can be run locally with:
+
+```shell
+go test -bench=. ./templates/
+```
+
+Benchmarks are also run continuously in CI through
+[CodSpeed](https://app.codspeed.io/maksimtech/cli), which reports the
+performance impact of a pull request. The packages that are benchmarked in CI
+are listed in [.github/workflows/codspeed.yml](.github/workflows/codspeed.yml).
+To run them the same way CodSpeed does, install the
+[CodSpeed CLI](https://codspeed.io/docs/cli) and run:
+
+```shell
+codspeed run --skip-upload --mode walltime -- go test -bench=. ./templates/
+```
+
 ## End-to-End Test Suite
 
 The end-to-end test suite tests a cli binary against a real API backend.

cli/command/commands/commands_test.go

@@ -0,0 +1,20 @@
+package commands
+
+import (
+	"testing"
+
+	"github.com/docker/cli/internal/test"
+	"github.com/spf13/cobra"
+)
+
+// BenchmarkAddCommands measures the cost of constructing the command-tree.
+// The command-tree is constructed on every invocation of the CLI, and as
+// part of generating shell-completion scripts and documentation.
+func BenchmarkAddCommands(b *testing.B) {
+	dockerCLI := test.NewFakeCli(nil)
+
+	b.ReportAllocs()
+	for b.Loop() {
+		AddCommands(&cobra.Command{Use: "docker"}, dockerCLI)
+	}
+}

cli/command/container/opts_test.go

@@ -1157,3 +1157,58 @@ func TestConvertToStandardNotation(t *testing.T) {
 		}
 	}
 }
+
+// BenchmarkParseRun measures parsing the flags of a "docker run" invocation,
+// which includes constructing the flag-set, and converting the options to
+// the container-, host-, and networking-config.
+func BenchmarkParseRun(b *testing.B) {
+	for _, tc := range []struct {
+		doc  string
+		args []string
+	}{
+		{
+			doc:  "minimal",
+			args: []string{"ubuntu", "bash"},
+		},
+		{
+			doc: "many flags",
+			args: []string{
+				"--hostname", "my-hostname",
+				"--user", "1000:1000",
+				"--workdir", "/some/workdir",
+				"--env", "FOO=bar",
+				"--env", "SOME_OTHER_VAR=some-other-value",
+				"--label", "com.example.label=some-value",
+				"--publish", "8080:80/tcp",
+				"--publish", "127.0.0.1:8443:443",
+				"--expose", "9000-9010",
+				"--volume", "/tmp/source:/mnt/source:ro",
+				"--mount", "type=volume,source=my-volume,target=/data,readonly",
+				"--tmpfs", "/run:size=1m",
+				"--network", "my-network",
+				"--dns", "1.1.1.1",
+				"--add-host", "example.com:127.0.0.1",
+				"--cap-add", "NET_ADMIN",
+				"--cap-drop", "MKNOD",
+				"--memory", "512m",
+				"--cpus", "1.5",
+				"--restart", "on-failure:5",
+				"--ulimit", "nofile=1024:2048",
+				"--health-cmd", "curl -f http://localhost/ || exit 1",
+				"--health-interval", "30s",
+				"--log-driver", "json-file",
+				"--log-opt", "max-size=10m",
+				"ubuntu", "bash",
+			},
+		},
+	} {
+		b.Run(tc.doc, func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				if _, _, _, err := parseRun(tc.args); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cli/command/formatter/container_test.go

@@ -953,3 +953,66 @@ func TestDisplayablePorts(t *testing.T) {
 		assert.Check(t, is.Equal(port.expected, actual))
 	}
 }
+
+// genContainers generates a list of containers to be used as a realistic
+// input for benchmarks; the containers have all fields set that are used
+// by the various formats.
+func genContainers(count int) []container.Summary {
+	created := time.Now().Add(-72 * time.Hour).Unix()
+	containers := make([]container.Summary, 0, count)
+	for i := range count {
+		containers = append(containers, container.Summary{
+			ID:      fmt.Sprintf("%064x", i),
+			Names:   []string{fmt.Sprintf("/container_%d", i), fmt.Sprintf("/other_name_%d", i)},
+			Image:   "docker.io/library/ubuntu:24.04",
+			ImageID: fmt.Sprintf("sha256:%064x", i),
+			Command: `/bin/sh -c "while true; do echo hello world; sleep 1; done"`,
+			Created: created,
+			Ports: []container.PortSummary{
+				{IP: netip.MustParseAddr("0.0.0.0"), PrivatePort: 80, PublicPort: uint16(30000 + i), Type: "tcp"},
+				{IP: netip.MustParseAddr("::"), PrivatePort: 443, PublicPort: uint16(40000 + i), Type: "tcp"},
+				{PrivatePort: 8080, Type: "tcp"},
+			},
+			SizeRw:     123456789,
+			SizeRootFs: 987654321,
+			Labels: map[string]string{
+				"com.docker.compose.project":      "some-project",
+				"com.docker.compose.service":      fmt.Sprintf("service-%d", i),
+				"org.opencontainers.image.source": "https://github.com/docker/cli",
+			},
+			State:  "running",
+			Status: "Up 3 days (healthy)",
+			Mounts: []container.MountPoint{
+				{Type: "volume", Name: fmt.Sprintf("volume-%d", i), Destination: "/data"},
+				{Type: "bind", Source: "/tmp/source", Destination: "/mnt/source"},
+			},
+		})
+	}
+	return containers
+}
+
+func BenchmarkContainerWrite(b *testing.B) {
+	containers := genContainers(100)
+	for _, tc := range []struct {
+		doc    string
+		format Format
+	}{
+		{doc: "table", format: NewContainerFormat("table", false, false)},
+		{doc: "table-with-size", format: NewContainerFormat("table", false, true)},
+		{doc: "quiet", format: NewContainerFormat("table", true, false)},
+		{doc: "raw", format: NewContainerFormat("raw", false, false)},
+		{doc: "json", format: NewContainerFormat("json", false, false)},
+		{doc: "custom", format: NewContainerFormat(`{{.ID}}: {{.Names}} {{.Ports}} {{.Labels}} {{.Mounts}}`, false, false)},
+	} {
+		b.Run(tc.doc, func(b *testing.B) {
+			b.ReportAllocs()
+			out := bytes.NewBuffer(nil)
+			for b.Loop() {
+				out.Reset()
+				if err := ContainerWrite(Context{Format: tc.format, Output: out, Trunc: true}, containers); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cli/command/formatter/image_test.go

@@ -367,3 +367,60 @@ func TestImageContextWriteWithNoImage(t *testing.T) {
 		})
 	}
 }
+
+// genImages generates a list of images to be used as a realistic input for
+// benchmarks; the images have all fields set that are used by the various
+// formats.
+func genImages(count int) []image.Summary {
+	created := time.Now().AddDate(0, 0, -1).Unix()
+	images := make([]image.Summary, 0, count)
+	for i := range count {
+		images = append(images, image.Summary{
+			ID:      fmt.Sprintf("sha256:%064x", i),
+			Created: created,
+			RepoTags: []string{
+				fmt.Sprintf("docker.io/library/image-%d:latest", i),
+				fmt.Sprintf("example.com/some/longer/name/image-%d:v1.2.3", i),
+			},
+			RepoDigests: []string{
+				fmt.Sprintf("docker.io/library/image-%d@sha256:%064x", i, i),
+			},
+			Size:       123456789,
+			SharedSize: 12345678,
+			Containers: 3,
+			Labels:     map[string]string{"org.opencontainers.image.source": "https://github.com/docker/cli"},
+		})
+	}
+	return images
+}
+
+func BenchmarkImageWrite(b *testing.B) {
+	images := genImages(100)
+	for _, tc := range []struct {
+		doc    string
+		format Format
+		digest bool
+	}{
+		{doc: "table", format: NewImageFormat("table", false, false)},
+		{doc: "table-with-digest", format: NewImageFormat("table", false, true), digest: true},
+		{doc: "quiet", format: NewImageFormat("table", true, false)},
+		{doc: "raw", format: NewImageFormat("raw", false, false)},
+		{doc: "json", format: NewImageFormat("json", false, false)},
+		{doc: "custom", format: NewImageFormat(`{{.Repository}}:{{.Tag}} {{.ID}} {{.Size}}`, false, false)},
+	} {
+		b.Run(tc.doc, func(b *testing.B) {
+			b.ReportAllocs()
+			out := bytes.NewBuffer(nil)
+			for b.Loop() {
+				out.Reset()
+				ctx := ImageContext{
+					Context: Context{Format: tc.format, Output: out, Trunc: true},
+					Digest:  tc.digest,
+				}
+				if err := ImageWrite(ctx, images); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cli/config/config_test.go

@@ -482,3 +482,39 @@ func TestSetDir(t *testing.T) {
 	SetDir(expected)
 	assert.Check(t, is.Equal(Dir(), expected))
 }
+
+// benchConfig is a realistic configuration file, as loaded on every
+// invocation of the CLI.
+const benchConfig = `{
+	"auths": {
+		"https://index.docker.io/v1/": {"auth": "am9lam9lOmhlbGxv", "email": "user@example.com"},
+		"registry.example.com": {"auth": "am9lam9lOmhlbGxv"},
+		"registry.example.com:5000": {"auth": "am9lam9lOmhlbGxv"},
+		"some-other-registry.example.com": {"auth": "am9lam9lOmhlbGxv", "identitytoken": "super-secret-token"}
+	},
+	"credsStore": "desktop",
+	"credHelpers": {
+		"registry.example.com": "secretservice",
+		"other.example.com": "pass"
+	},
+	"psFormat": "table {{.ID}}\\t{{.Image}}\\t{{.Command}}\\t{{.Status}}",
+	"imagesFormat": "table {{.Repository}}\\t{{.Tag}}\\t{{.ID}}\\t{{.Size}}",
+	"detachKeys": "ctrl-e,e",
+	"currentContext": "default",
+	"plugins": {
+		"buildx": {"defaultBuilder": "default"},
+		"compose": {"someOption": "someValue"}
+	},
+	"aliases": {"builder": "buildx"},
+	"features": {"containerd-snapshotter": "true"},
+	"experimental": "enabled"
+}`
+
+func BenchmarkLoadFromReader(b *testing.B) {
+	b.ReportAllocs()
+	for b.Loop() {
+		if _, err := LoadFromReader(strings.NewReader(benchConfig)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

opts/mount_test.go

@@ -586,3 +586,25 @@ func TestMountOptSetBindRecursive(t *testing.T) {
 		}, m.Value()))
 	})
 }
+
+func BenchmarkMountOptSet(b *testing.B) {
+	for _, tc := range []struct {
+		doc   string
+		value string
+	}{
+		{doc: "volume", value: "type=volume,source=my-volume,target=/data"},
+		{doc: "volume-with-opts", value: "type=volume,source=my-volume,target=/data,readonly,volume-nocopy,volume-driver=local,volume-label=foo=bar,volume-opt=type=nfs,volume-opt=device=:/some/path"},
+		{doc: "bind", value: "type=bind,source=/home/path,target=/target,readonly,bind-propagation=rprivate"},
+		{doc: "tmpfs", value: "type=tmpfs,target=/target,tmpfs-size=1m,tmpfs-mode=0700"},
+	} {
+		b.Run(tc.doc, func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				var m MountOpt
+				if err := m.Set(tc.value); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}