From 8ca2ea2faba0d244c0aa921c1f6872de699c7035 Mon Sep 17 00:00:00 2001 From: xiao Date: Sun, 11 Oct 2026 01:05:34 +0800 Subject: [PATCH] metrics: expose fractional cgroup CPU quota as process_available_cpu_cores --- cpu_quota.go | 147 ++++++++++++++++++++++++++++++++++++++++++++++ cpu_quota_test.go | 129 ++++++++++++++++++++++++++++++++++++++++ metrics.go | 7 ++- 3 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 cpu_quota.go create mode 100644 cpu_quota_test.go diff --git a/cpu_quota.go b/cpu_quota.go new file mode 100644 index 0000000..8dc773b --- /dev/null +++ b/cpu_quota.go @@ -0,0 +1,147 @@ +package metrics + +import ( + "os" + "path" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +func availableCPUCores() float64 { + fallback := float64(runtime.NumCPU()) + if runtime.GOOS != "linux" { + return fallback + } + membership, err := os.ReadFile("/proc/self/cgroup") + if err != nil { + return fallback + } + return availableCPUCoresFromCgroup(string(membership), cpuMountInfo(), os.ReadFile, fallback) +} + +// Cache mountinfo, whose generation can block on an unresponsive mount. Only +// one read is attempted, so a stuck mount cannot leak a goroutine on each scrape. +var cpuMountInfo = sync.OnceValue(func() string { + ch := make(chan string, 1) + go func() { + data, _ := os.ReadFile("/proc/self/mountinfo") + ch <- string(data) + }() + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case data := <-ch: + return data + case <-timer.C: + return "" + } +}) + +// Use the smallest positive quota across the process's visible cgroup ancestors. +// Quotas are shared upper bounds, not a reservation or a GOMAXPROCS setting. +func availableCPUCoresFromCgroup(membership, mountinfo string, readFile func(string) ([]byte, error), fallback float64) float64 { + var quota float64 + for line := range strings.SplitSeq(mountinfo, "\n") { + before, after, ok := strings.Cut(line, " - ") + if !ok { + continue + } + mount := strings.Fields(before) + fs := strings.Fields(after) + if len(mount) < 5 || len(fs) < 3 { + continue + } + v2 := fs[0] == "cgroup2" + if !v2 && (fs[0] != "cgroup" || !hasCPUController(fs[2])) { + continue + } + root := unescapeCPUMountPath(mount[3]) + mountpoint := unescapeCPUMountPath(mount[4]) + if !path.IsAbs(root) || !path.IsAbs(mountpoint) { + continue + } + root, mountpoint = path.Clean(root), path.Clean(mountpoint) + group := cpuCgroupPath(membership, v2) + if group == "" { + continue + } + var relative string + if group != root { + var underRoot bool + relative, underRoot = strings.CutPrefix(group, strings.TrimSuffix(root, "/")+"/") + if !underRoot { + continue + } + } + for dir := path.Join(mountpoint, relative); ; dir = path.Dir(dir) { + q := readCPUQuota(dir, v2, readFile) + if q > 0 && (quota == 0 || q < quota) { + quota = q + } + if dir == mountpoint { + break + } + } + } + if quota > 0 { + return quota + } + return fallback +} + +func cpuCgroupPath(membership string, v2 bool) string { + for line := range strings.SplitSeq(membership, "\n") { + fields := strings.SplitN(line, ":", 3) + if len(fields) != 3 || !path.IsAbs(fields[2]) { + continue + } + if (v2 && fields[0] == "0" && fields[1] == "") || (!v2 && hasCPUController(fields[1])) { + return path.Clean(fields[2]) + } + } + return "" +} + +func hasCPUController(controllers string) bool { + return strings.Contains(","+controllers+",", ",cpu,") +} + +func unescapeCPUMountPath(s string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(s) +} + +func readCPUQuota(dir string, v2 bool, readFile func(string) ([]byte, error)) float64 { + var fields []string + if v2 { + data, err := readFile(path.Join(dir, "cpu.max")) + if err != nil { + return 0 + } + fields = strings.Fields(string(data)) + } else { + quota, err := readFile(path.Join(dir, "cpu.cfs_quota_us")) + if err != nil { + return 0 + } + period, err := readFile(path.Join(dir, "cpu.cfs_period_us")) + if err != nil { + return 0 + } + fields = []string{strings.TrimSpace(string(quota)), strings.TrimSpace(string(period))} + } + if len(fields) != 2 { + return 0 + } + quota, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil || quota == 0 { + return 0 + } + period, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil || period == 0 { + return 0 + } + return float64(quota) / float64(period) +} diff --git a/cpu_quota_test.go b/cpu_quota_test.go new file mode 100644 index 0000000..1cc72ad --- /dev/null +++ b/cpu_quota_test.go @@ -0,0 +1,129 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "os" + "runtime" + "strconv" + "strings" + "testing" +) + +func TestAvailableCPUCoresQuota(t *testing.T) { + v2Mount := "30 23 0:26 / /cg rw - cgroup2 cgroup rw\n" + v1Mount := "31 23 0:27 / /cpu rw - cgroup cgroup rw,cpu,cpuacct\n" + for _, tc := range []struct { + name, membership, mounts string + files map[string]string + want float64 + }{ + {"v2 fractional", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "150000 100000\n"}, 1.5}, + {"v1 fractional", "2:cpu,cpuacct:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "50000\n", "/cpu/app/cpu.cfs_period_us": "100000\n"}, 0.5}, + {"v2 unlimited", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "max 100000"}, 8}, + {"v1 unlimited", "2:cpu:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "-1", "/cpu/app/cpu.cfs_period_us": "100000"}, 8}, + {"parent quota", "0::/group/app", v2Mount, map[string]string{"/cg/group/app/cpu.max": "max 100000", "/cg/group/cpu.max": "150000 100000"}, 1.5}, + {"minimum quota", "0::/group/app", v2Mount, map[string]string{"/cg/group/app/cpu.max": "300000 100000", "/cg/group/cpu.max": "150000 100000"}, 1.5}, + {"v1 parent", "2:cpu:/app", v1Mount, map[string]string{"/cpu/cpu.cfs_quota_us": "250000", "/cpu/cpu.cfs_period_us": "100000"}, 2.5}, + {"mount root", "0::/group/app", "30 23 0:26 /group /cg rw - cgroup2 cgroup rw", map[string]string{"/cg/app/cpu.max": "250000 100000", "/cpu.max": "50000 100000"}, 2.5}, + {"namespace root", "0::/", v2Mount, map[string]string{"/cg/cpu.max": "50000 100000"}, 0.5}, + {"unrelated mount root", "0::/other/app", "30 23 0:26 /group /cg rw - cgroup2 cgroup rw", map[string]string{"/cg/cpu.max": "50000 100000"}, 8}, + {"hybrid", "0::/app\n2:cpu,cpuacct:/app", v2Mount + v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "150000", "/cpu/app/cpu.cfs_period_us": "100000"}, 1.5}, + {"escaped mount", "0::/app", "30 23 0:26 / /my\\040cg rw - cgroup2 cgroup rw", map[string]string{"/my cg/app/cpu.max": "150000 100000"}, 1.5}, + {"quota above host count", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "1600000 100000"}, 16}, + {"missing quota", "0::/app", v2Mount, nil, 8}, + {"missing membership", "", v2Mount, nil, 8}, + {"missing mounts", "0::/app", "", nil, 8}, + {"malformed mounts", "0::/app", "x - cgroup2\n", nil, 8}, + {"zero period", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "100000 0"}, 8}, + {"malformed quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "NaN 100000"}, 8}, + {"negative quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "-2 100000"}, 8}, + {"missing period", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "100000"}, 8}, + {"zero quota", "0::/app", v2Mount, map[string]string{"/cg/app/cpu.max": "0 100000"}, 8}, + {"v1 bad period", "2:cpu:/app", v1Mount, map[string]string{"/cpu/app/cpu.cfs_quota_us": "150000", "/cpu/app/cpu.cfs_period_us": "0"}, 8}, + } { + t.Run(tc.name, func(t *testing.T) { + readFile := func(name string) ([]byte, error) { + data, ok := tc.files[name] + if !ok { + return nil, os.ErrNotExist + } + return []byte(data), nil + } + if got := availableCPUCoresFromCgroup(tc.membership, tc.mounts, readFile, 8); got != tc.want { + t.Fatalf("got %g cores; want %g", got, tc.want) + } + }) + } +} + +func TestAvailableCPUCoresQuotaChanges(t *testing.T) { + quota := "150000 100000" + readFile := func(name string) ([]byte, error) { + if name == "/cg/cpu.max" { + return []byte(quota), nil + } + return nil, os.ErrNotExist + } + for _, want := range []float64{1.5, 0.5} { + quota = fmt.Sprintf("%.0f 100000", want*100000) + got := availableCPUCoresFromCgroup("0::/", "30 23 0:26 / /cg rw - cgroup2 cgroup rw", readFile, 8) + if got != want { + t.Fatalf("got %g cores after quota change; want %g", got, want) + } + } +} + +func TestProcessAvailableCPUCoresExport(t *testing.T) { + previous := runtime.GOMAXPROCS(1) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + for _, tc := range []struct { + name string + write func(io.Writer) + want bool + }{ + {"process", WriteProcessMetrics, true}, + {"proc", WriteProcMetrics, true}, + {"prometheus enabled", func(w io.Writer) { WritePrometheus(w, true) }, true}, + {"prometheus disabled", func(w io.Writer) { WritePrometheus(w, false) }, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var b bytes.Buffer + tc.write(&b) + count := 0 + for line := range strings.SplitSeq(b.String(), "\n") { + if value, ok := strings.CutPrefix(line, "process_available_cpu_cores "); ok { + count++ + v, err := strconv.ParseFloat(value, 64) + if err != nil || !(v > 0) { + t.Fatalf("invalid available cores %q", value) + } + if runtime.GOOS != "linux" && v != float64(runtime.NumCPU()) { + t.Fatalf("got %g cores; want NumCPU=%d", v, runtime.NumCPU()) + } + } + } + wantCount := 0 + if tc.want { + wantCount = 1 + } + if count != wantCount { + t.Fatalf("got %d samples; want %d", count, wantCount) + } + if runtime.GOMAXPROCS(0) != 1 { + t.Fatal("collection modified GOMAXPROCS") + } + }) + } +} + +func TestProcessAvailableCPUCoresMetadata(t *testing.T) { + ExposeMetadata(true) + t.Cleanup(func() { ExposeMetadata(false) }) + var b bytes.Buffer + WriteProcMetrics(&b) + if !strings.Contains(b.String(), "# TYPE process_available_cpu_cores gauge\n") { + t.Fatal("missing gauge metadata") + } +} diff --git a/metrics.go b/metrics.go index c91b1ca..c374b41 100644 --- a/metrics.go +++ b/metrics.go @@ -125,6 +125,10 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // // - process_cpu_seconds_total - CPU time spent by the process // +// - process_available_cpu_cores - the smallest visible cgroup CPU quota in cores, +// including fractional cores, or runtime.NumCPU() if no quota can be read. +// This is independent of GOMAXPROCS. Ancestor quotas may be shared with other processes. +// // - process_major_pagefaults_total - page faults resulted in disk IO // // - process_minor_pagefaults_total - page faults resolved without disk IO @@ -248,7 +252,7 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // See also WriteFDMetrics. func WriteProcessMetrics(w io.Writer) { writeGoMetrics(w) - writeProcessMetrics(w) + WriteProcMetrics(w) writePushMetrics(w) } @@ -261,6 +265,7 @@ func WriteGoMetrics(w io.Writer) { // WriteProcMetrics writes OS-level process metrics to w by reading // the /proc filesystem (CPU, memory, file descriptors, PSI, etc.). func WriteProcMetrics(w io.Writer) { + WriteGaugeFloat64(w, "process_available_cpu_cores", availableCPUCores()) writeProcessMetrics(w) }