metrics: expose fractional cgroup CPU quota as process_available_cpu_cores

cpu_quota.go

@@ -0,0 +1,147 @@
+package metrics
+
+import (
+	"os"
+	"path"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+func availableCPUCores() float64 {
+	fallback := float64(runtime.NumCPU())
+	if runtime.GOOS != "linux" {
+		return fallback
+	}
+	membership, err := os.ReadFile("/proc/self/cgroup")
+	if err != nil {
+		return fallback
+	}
+	return availableCPUCoresFromCgroup(string(membership), cpuMountInfo(), os.ReadFile, fallback)
+}
+
+// Cache mountinfo, whose generation can block on an unresponsive mount. Only
+// one read is attempted, so a stuck mount cannot leak a goroutine on each scrape.
+var cpuMountInfo = sync.OnceValue(func() string {
+	ch := make(chan string, 1)
+	go func() {
+		data, _ := os.ReadFile("/proc/self/mountinfo")
+		ch <- string(data)
+	}()
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case data := <-ch:
+		return data
+	case <-timer.C:
+		return ""
+	}
+})
+
+// Use the smallest positive quota across the process's visible cgroup ancestors.
+// Quotas are shared upper bounds, not a reservation or a GOMAXPROCS setting.
+func availableCPUCoresFromCgroup(membership, mountinfo string, readFile func(string) ([]byte, error), fallback float64) float64 {
+	var quota float64
+	for line := range strings.SplitSeq(mountinfo, "\n") {
+		before, after, ok := strings.Cut(line, " - ")
+		if !ok {
+			continue
+		}
+		mount := strings.Fields(before)
+		fs := strings.Fields(after)
+		if len(mount) < 5 || len(fs) < 3 {
+			continue
+		}
+		v2 := fs[0] == "cgroup2"
+		if !v2 && (fs[0] != "cgroup" || !hasCPUController(fs[2])) {
+			continue
+		}
+		root := unescapeCPUMountPath(mount[3])
+		mountpoint := unescapeCPUMountPath(mount[4])
+		if !path.IsAbs(root) || !path.IsAbs(mountpoint) {
+			continue
+		}
+		root, mountpoint = path.Clean(root), path.Clean(mountpoint)
+		group := cpuCgroupPath(membership, v2)
+		if group == "" {
+			continue
+		}
+		var relative string
+		if group != root {
+			var underRoot bool
+			relative, underRoot = strings.CutPrefix(group, strings.TrimSuffix(root, "/")+"/")
+			if !underRoot {
+				continue
+			}
+		}
+		for dir := path.Join(mountpoint, relative); ; dir = path.Dir(dir) {
+			q := readCPUQuota(dir, v2, readFile)
+			if q > 0 && (quota == 0 || q < quota) {
+				quota = q
+			}
+			if dir == mountpoint {
+				break
+			}
+		}
+	}
+	if quota > 0 {
+		return quota
+	}
+	return fallback
+}
+
+func cpuCgroupPath(membership string, v2 bool) string {
+	for line := range strings.SplitSeq(membership, "\n") {
+		fields := strings.SplitN(line, ":", 3)
+		if len(fields) != 3 || !path.IsAbs(fields[2]) {
+			continue
+		}
+		if (v2 && fields[0] == "0" && fields[1] == "") || (!v2 && hasCPUController(fields[1])) {
+			return path.Clean(fields[2])
+		}
+	}
+	return ""
+}
+
+func hasCPUController(controllers string) bool {
+	return strings.Contains(","+controllers+",", ",cpu,")
+}
+
+func unescapeCPUMountPath(s string) string {
+	return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(s)
+}
+
+func readCPUQuota(dir string, v2 bool, readFile func(string) ([]byte, error)) float64 {
+	var fields []string
+	if v2 {
+		data, err := readFile(path.Join(dir, "cpu.max"))
+		if err != nil {
+			return 0
+		}
+		fields = strings.Fields(string(data))
+	} else {
+		quota, err := readFile(path.Join(dir, "cpu.cfs_quota_us"))
+		if err != nil {
+			return 0
+		}
+		period, err := readFile(path.Join(dir, "cpu.cfs_period_us"))
+		if err != nil {
+			return 0
+		}
+		fields = []string{strings.TrimSpace(string(quota)), strings.TrimSpace(string(period))}
+	}
+	if len(fields) != 2 {
+		return 0
+	}
+	quota, err := strconv.ParseUint(fields[0], 10, 64)
+	if err != nil || quota == 0 {
+		return 0
+	}
+	period, err := strconv.ParseUint(fields[1], 10, 64)
+	if err != nil || period == 0 {
+		return 0
+	}
+	return float64(quota) / float64(period)
+}

cpu_quota_test.go

@@ -0,0 +1,129 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestAvailableCPUCoresQuota(t *testing.T) {
+	v2Mount := "30 23 0:26 / /cg rw - cgroup2 cgroup rw\n"
+	v1Mount := "31 23 0:27 / /cpu rw - cgroup cgroup rw,cpu,cpuacct\n"
+	for _, tc := range []struct {
+		name, membership, mounts string
+		files                    map[string]string
+		want                     float64
+	}{
+		{"v2 fractional", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "150000 100000\n"}, 1.5},
+		{"v1 fractional", "2:cpu,cpuacct:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "50000\n", "/cpu/app/cpu.cfs_period_us": "100000\n"}, 0.5},
+		{"v2 unlimited", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "max 100000"}, 8},
+		{"v1 unlimited", "2:cpu:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "-1", "/cpu/app/cpu.cfs_period_us": "100000"}, 8},
+		{"parent quota", "0::/group/app", v2Mount, map[string]string{"/cg/group/app/cpu.max": "max 100000", "/cg/group/cpu.max": "150000 100000"}, 1.5},
+		{"minimum quota", "0::/group/app", v2Mount, map[string]string{"/cg/group/app/cpu.max": "300000 100000", "/cg/group/cpu.max": "150000 100000"}, 1.5},
+		{"v1 parent", "2:cpu:/app", v1Mount, map[string]string{"/cpu/cpu.cfs_quota_us": "250000", "/cpu/cpu.cfs_period_us": "100000"}, 2.5},
+		{"mount root", "0::/group/app", "30 23 0:26 /group /cg rw - cgroup2 cgroup rw", map[string]string{"/cg/app/cpu.max": "250000 100000", "/cpu.max": "50000 100000"}, 2.5},
+		{"namespace root", "0::/", v2Mount, map[string]string{"/cg/cpu.max": "50000 100000"}, 0.5},
+		{"unrelated mount root", "0::/other/app", "30 23 0:26 /group /cg rw - cgroup2 cgroup rw", map[string]string{"/cg/cpu.max": "50000 100000"}, 8},
+		{"hybrid", "0::/app\n2:cpu,cpuacct:/app", v2Mount + v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "150000", "/cpu/app/cpu.cfs_period_us": "100000"}, 1.5},
+		{"escaped mount", "0::/app", "30 23 0:26 / /my\\040cg rw - cgroup2 cgroup rw", map[string]string{"/my cg/app/cpu.max": "150000 100000"}, 1.5},
+		{"quota above host count", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "1600000 100000"}, 16},
+		{"missing quota", "0::/app", v2Mount, nil, 8},
+		{"missing membership", "", v2Mount, nil, 8},
+		{"missing mounts", "0::/app", "", nil, 8},
+		{"malformed mounts", "0::/app", "x - cgroup2\n", nil, 8},
+		{"zero period", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "100000 0"}, 8},
+		{"malformed quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "NaN 100000"}, 8},
+		{"negative quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "-2 100000"}, 8},
+		{"missing period", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "100000"}, 8},
+		{"zero quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "0 100000"}, 8},
+		{"v1 bad period", "2:cpu:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "150000", "/cpu/app/cpu.cfs_period_us": "0"}, 8},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			readFile := func(name string) ([]byte, error) {
+				data, ok := tc.files[name]
+				if !ok {
+					return nil, os.ErrNotExist
+				}
+				return []byte(data), nil
+			}
+			if got := availableCPUCoresFromCgroup(tc.membership, tc.mounts, readFile, 8); got != tc.want {
+				t.Fatalf("got %g cores; want %g", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestAvailableCPUCoresQuotaChanges(t *testing.T) {
+	quota := "150000 100000"
+	readFile := func(name string) ([]byte, error) {
+		if name == "/cg/cpu.max" {
+			return []byte(quota), nil
+		}
+		return nil, os.ErrNotExist
+	}
+	for _, want := range []float64{1.5, 0.5} {
+		quota = fmt.Sprintf("%.0f 100000", want*100000)
+		got := availableCPUCoresFromCgroup("0::/", "30 23 0:26 / /cg rw - cgroup2 cgroup rw", readFile, 8)
+		if got != want {
+			t.Fatalf("got %g cores after quota change; want %g", got, want)
+		}
+	}
+}
+
+func TestProcessAvailableCPUCoresExport(t *testing.T) {
+	previous := runtime.GOMAXPROCS(1)
+	t.Cleanup(func() { runtime.GOMAXPROCS(previous) })
+	for _, tc := range []struct {
+		name  string
+		write func(io.Writer)
+		want  bool
+	}{
+		{"process", WriteProcessMetrics, true},
+		{"proc", WriteProcMetrics, true},
+		{"prometheus enabled", func(w io.Writer) { WritePrometheus(w, true) }, true},
+		{"prometheus disabled", func(w io.Writer) { WritePrometheus(w, false) }, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			tc.write(&b)
+			count := 0
+			for line := range strings.SplitSeq(b.String(), "\n") {
+				if value, ok := strings.CutPrefix(line, "process_available_cpu_cores "); ok {
+					count++
+					v, err := strconv.ParseFloat(value, 64)
+					if err != nil || !(v > 0) {
+						t.Fatalf("invalid available cores %q", value)
+					}
+					if runtime.GOOS != "linux" && v != float64(runtime.NumCPU()) {
+						t.Fatalf("got %g cores; want NumCPU=%d", v, runtime.NumCPU())
+					}
+				}
+			}
+			wantCount := 0
+			if tc.want {
+				wantCount = 1
+			}
+			if count != wantCount {
+				t.Fatalf("got %d samples; want %d", count, wantCount)
+			}
+			if runtime.GOMAXPROCS(0) != 1 {
+				t.Fatal("collection modified GOMAXPROCS")
+			}
+		})
+	}
+}
+
+func TestProcessAvailableCPUCoresMetadata(t *testing.T) {
+	ExposeMetadata(true)
+	t.Cleanup(func() { ExposeMetadata(false) })
+	var b bytes.Buffer
+	WriteProcMetrics(&b)
+	if !strings.Contains(b.String(), "# TYPE process_available_cpu_cores gauge\n") {
+		t.Fatal("missing gauge metadata")
+	}
+}

metrics.go

@@ -125,6 +125,10 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) {
 //
 //   - process_cpu_seconds_total - CPU time spent by the process
 //
+//   - process_available_cpu_cores - the smallest visible cgroup CPU quota in cores,
+//     including fractional cores, or runtime.NumCPU() if no quota can be read.
+//     This is independent of GOMAXPROCS. Ancestor quotas may be shared with other processes.
+//
 //   - process_major_pagefaults_total - page faults resulted in disk IO
 //
 //   - process_minor_pagefaults_total - page faults resolved without disk IO
@@ -248,7 +252,7 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) {
 // See also WriteFDMetrics.
 func WriteProcessMetrics(w io.Writer) {
 	writeGoMetrics(w)
-	writeProcessMetrics(w)
+	WriteProcMetrics(w)
 	writePushMetrics(w)
 }
 
@@ -261,6 +265,7 @@ func WriteGoMetrics(w io.Writer) {
 // WriteProcMetrics writes OS-level process metrics to w by reading
 // the /proc filesystem (CPU, memory, file descriptors, PSI, etc.).
 func WriteProcMetrics(w io.Writer) {
+	WriteGaugeFloat64(w, "process_available_cpu_cores", availableCPUCores())
 	writeProcessMetrics(w)
 }