diff --git a/apps/api/internal/infrastructure/registrycache/cleanup.go b/apps/api/internal/infrastructure/registrycache/cleanup.go index feda7c4e..49cc7919 100644 --- a/apps/api/internal/infrastructure/registrycache/cleanup.go +++ b/apps/api/internal/infrastructure/registrycache/cleanup.go @@ -33,6 +33,7 @@ const ( registryConfigPath = registryConfigDir + "/" + registryConfigKey registryRepoDir = registryDataDir + "/docker/registry/v2/repositories" tagLinkMarker = "/_manifests/tags/" + revisionLinkMarker = "/_manifests/revisions/sha256/" buildCacheSuffix = "-buildcache" staleUploadMinutes = 1440 @@ -45,6 +46,7 @@ const ( freshTagGrace = time.Hour ) +// A manifest in a repository. Tag is empty for one that no tag points at anymore, like a build cache the next build overwrote. type TagInfo struct { Repo string Tag string @@ -56,9 +58,16 @@ type TagInfo struct { } func (self TagInfo) Key() string { + if self.Untagged() { + return self.ManifestKey() + } return self.Repo + ":" + self.Tag } +func (self TagInfo) Untagged() bool { + return self.Tag == "" +} + func (self TagInfo) ManifestKey() string { return self.Repo + "@" + self.Digest } @@ -149,13 +158,17 @@ func (self *Cleaner) Run(ctx context.Context, thresholdBytes int64, pruneAll boo log.Warnf("registry cleanup: over threshold with nothing prunable, grow the registry volume") } - deleted := 0 + deleted, deletedUntagged := 0, 0 for _, tag := range plan { if err := self.registry.DeleteManifest(ctx, tag.Repo, tag.Digest); err != nil { log.Warnf("registry cleanup: failed to delete %s: %v", tag.Key(), err) continue } - deleted++ + if tag.Untagged() { + deletedUntagged++ + } else { + deleted++ + } log.Infof("registry cleanup: deleted %s", tag.Key()) } for _, child := range orphanedChildren(tags, plan) { @@ -172,14 +185,15 @@ func (self *Cleaner) Run(ctx context.Context, thresholdBytes int64, pruneAll boo if err != nil { return nil, err } - return cleanupResult(used, usedAfter, thresholdBytes, deleted), nil + return cleanupResult(used, usedAfter, thresholdBytes, deleted, deletedUntagged), nil } -func cleanupResult(usedBefore, usedAfter, thresholdBytes int64, deleted int) *models.RegistryCleanupResult { +func cleanupResult(usedBefore, usedAfter, thresholdBytes int64, deleted, deletedUntagged int) *models.RegistryCleanupResult { result := &models.RegistryCleanupResult{ - Outcome: models.RegistryCleanupCleaned, - FreedBytes: max(usedBefore-usedAfter, 0), - DeletedImages: deleted, + Outcome: models.RegistryCleanupCleaned, + FreedBytes: max(usedBefore-usedAfter, 0), + DeletedImages: deleted, + DeletedUntagged: deletedUntagged, } if usedAfter >= thresholdBytes { result.Outcome = models.RegistryCleanupOverThreshold @@ -211,8 +225,11 @@ func planDeletions(tags []TagInfo, inUse map[string]bool, target int64, freshSin candidates = append(candidates, tag) } - // Oldest first: a cache in use is rewritten by every build, so a stale one sorts ahead of it + // Untagged manifests are worthless and go first, the rest oldest first: a cache in use is rewritten by every build, so a stale one sorts ahead of it sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].Untagged() != candidates[j].Untagged() { + return candidates[i].Untagged() + } return candidates[i].ModTime < candidates[j].ModTime }) @@ -220,7 +237,7 @@ func planDeletions(tags []TagInfo, inUse map[string]bool, target int64, freshSin var freed int64 planned := map[string]bool{} for _, tag := range candidates { - if freed >= target { + if freed >= target && !tag.Untagged() { break } // Deletion is by digest, so a manifest a protected tag also points at is off limits @@ -289,6 +306,10 @@ func protectedTags(tags []TagInfo, inUse map[string]bool, freshSince int64) map[ if inUse[tag.Key()] || inUse[tag.Repo+":"+tag.Digest] || tag.ModTime >= freshSince { protected[tag.Key()] = true } + // A push writes the manifest before the tag, so a fresh untagged one may still be getting its tag + if tag.Untagged() { + continue + } key := newestKey{repo: tag.Repo, buildCache: tag.IsBuildCache()} if current, ok := newest[key]; !ok || tag.ModTime > current.ModTime { newest[key] = tag @@ -409,33 +430,81 @@ func (self *Cleaner) inventory(ctx context.Context, pod string) ([]TagInfo, erro continue } for _, tag := range tags { - manifest, err := self.registry.Manifest(ctx, repo, tag) + info, err := self.describe(ctx, repo, tag, tag, modTimes[repo+":"+tag]) if err != nil { - log.Warnf("registry cleanup: failed to read manifest %s:%s: %v", repo, tag, err) - continue - } - blobs := map[string]int64{} - if err := collectBlobs(ctx, self.registry, repo, tag, blobs, 0); err != nil { - log.Warnf("registry cleanup: failed to size %s:%s: %v", repo, tag, err) + log.Warnf("registry cleanup: %v", err) continue } - children := make([]string, 0, len(manifest.Manifests)) - for _, child := range manifest.Manifests { - children = append(children, child.Digest) - } - inventory = append(inventory, TagInfo{ - Repo: repo, - Tag: tag, - Digest: manifest.Digest, - Blobs: blobs, - ModTime: modTimes[repo+":"+tag], - Children: children, - }) + inventory = append(inventory, info) + } + } + + // Overwriting a tag leaves its old manifest in the repository, where only a deletion by digest reaches it + revisions, err := self.revisionModTimes(ctx, pod) + if err != nil { + log.Warnf("registry cleanup: failed to list stored manifests, leaving untagged ones alone: %v", err) + return inventory, nil + } + for _, revision := range untaggedRevisions(revisions, inventory) { + info, err := self.describe(ctx, revision.repo, revision.digest, "", revision.modTime) + if err != nil { + log.Warnf("registry cleanup: %v", err) + continue } + inventory = append(inventory, info) } return inventory, nil } +func (self *Cleaner) describe(ctx context.Context, repo, ref, tag string, modTime int64) (TagInfo, error) { + manifest, err := self.registry.Manifest(ctx, repo, ref) + if err != nil { + return TagInfo{}, fmt.Errorf("failed to read manifest %s/%s: %w", repo, ref, err) + } + blobs := map[string]int64{} + if err := collectBlobs(ctx, self.registry, repo, ref, blobs, 0); err != nil { + return TagInfo{}, fmt.Errorf("failed to size %s/%s: %w", repo, ref, err) + } + children := make([]string, 0, len(manifest.Manifests)) + for _, child := range manifest.Manifests { + children = append(children, child.Digest) + } + return TagInfo{ + Repo: repo, + Tag: tag, + Digest: manifest.Digest, + Blobs: blobs, + ModTime: modTime, + Children: children, + }, nil +} + +type manifestRevision struct { + repo string + digest string + modTime int64 +} + +// The children of a tagged index are untagged too, but deleting them breaks the index +func untaggedRevisions(revisions []manifestRevision, tagged []TagInfo) []manifestRevision { + referenced := map[string]bool{} + for _, tag := range tagged { + referenced[tag.ManifestKey()] = true + for _, child := range tag.Children { + referenced[tag.Repo+"@"+child] = true + } + } + + var untagged []manifestRevision + for _, revision := range revisions { + if referenced[revision.repo+"@"+revision.digest] { + continue + } + untagged = append(untagged, revision) + } + return untagged +} + func (self *Cleaner) inUseRefs(ctx context.Context) (map[string]bool, error) { pods, err := self.clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) if err != nil { @@ -571,6 +640,52 @@ func tagKeyFromLinkPath(path string) (string, bool) { return repo + ":" + tag, true } +func (self *Cleaner) revisionModTimes(ctx context.Context, pod string) ([]manifestRevision, error) { + command := fmt.Sprintf(`[ -d %[1]s ] || exit 0; find %[1]s -path '*%[2]s*/link' -exec stat -c '%%Y %%n' {} +`, registryRepoDir, revisionLinkMarker) + out, err := self.exec(ctx, pod, []string{"sh", "-c", command}) + if err != nil { + return nil, err + } + return parseRevisionModTimes(out), nil +} + +func parseRevisionModTimes(output string) []manifestRevision { + var revisions []manifestRevision + for line := range strings.SplitSeq(output, "\n") { + fields := strings.SplitN(strings.TrimSpace(line), " ", 2) + if len(fields) != 2 { + continue + } + epoch, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + continue + } + revision, ok := revisionFromLinkPath(fields[1]) + if !ok { + continue + } + revision.modTime = epoch + revisions = append(revisions, revision) + } + return revisions +} + +func revisionFromLinkPath(path string) (manifestRevision, bool) { + trimmed := strings.TrimPrefix(path, registryRepoDir+"/") + if trimmed == path { + return manifestRevision{}, false + } + repo, rest, found := strings.Cut(trimmed, revisionLinkMarker) + if !found { + return manifestRevision{}, false + } + hex, rest, found := strings.Cut(rest, "/") + if !found || rest != "link" || repo == "" || hex == "" { + return manifestRevision{}, false + } + return manifestRevision{repo: repo, digest: "sha256:" + hex}, true +} + // Distribution's own purger only drops uploads older than 7 days, too slow for a full volume func (self *Cleaner) pruneStaleUploads(ctx context.Context, pod string) error { command := fmt.Sprintf( diff --git a/apps/api/internal/infrastructure/registrycache/cleanup_test.go b/apps/api/internal/infrastructure/registrycache/cleanup_test.go index 7aa98eeb..5bf4b93f 100644 --- a/apps/api/internal/infrastructure/registrycache/cleanup_test.go +++ b/apps/api/internal/infrastructure/registrycache/cleanup_test.go @@ -287,19 +287,112 @@ func TestPlanDeletionsPruneAllKeepsProtectedTags(t *testing.T) { } func TestCleanupResultOutcome(t *testing.T) { - cleaned := cleanupResult(20*gib, 10*gib, 16*gib, 4) + cleaned := cleanupResult(20*gib, 10*gib, 16*gib, 4, 7) assert.Equal(t, models.RegistryCleanupCleaned, cleaned.Outcome) assert.Equal(t, 10*gib, cleaned.FreedBytes) assert.Equal(t, 4, cleaned.DeletedImages) + assert.Equal(t, 7, cleaned.DeletedUntagged) - stuck := cleanupResult(20*gib, 20*gib, 16*gib, 0) + stuck := cleanupResult(20*gib, 20*gib, 16*gib, 0, 0) assert.Equal(t, models.RegistryCleanupOverThreshold, stuck.Outcome) assert.Zero(t, stuck.FreedBytes) - grew := cleanupResult(10*gib, 11*gib, 16*gib, 0) + grew := cleanupResult(10*gib, 11*gib, 16*gib, 0, 0) assert.Zero(t, grew.FreedBytes) } +func untagged(repo, digest string, modTime int64, blobs map[string]int64) TagInfo { + return TagInfo{Repo: repo, Digest: "sha256:" + digest, Blobs: blobs, ModTime: modTime} +} + +// An overwritten build cache keeps its blobs on the volume although no tag reaches it anymore +func TestPlanDeletionsDeletesEveryUntaggedManifestFirst(t *testing.T) { + tags := []TagInfo{ + tag("app", "newest", 300, map[string]int64{"n": 100}), + tag("app", "old", 100, map[string]int64{"o": 100}), + tag("app", "current-buildcache", 300, map[string]int64{"c": 100}), + untagged("app", "cache-1", 200, map[string]int64{"c1": 100}), + untagged("app", "cache-2", 250, map[string]int64{"c2": 100}), + } + + plan := planDeletions(tags, nil, 50, noFreshTags) + + assert.Equal(t, []string{"app@sha256:cache-1", "app@sha256:cache-2"}, keys(plan)) +} + +func TestPlanDeletionsUntaggedBeforeTaggedImages(t *testing.T) { + tags := []TagInfo{ + tag("app", "newest", 300, map[string]int64{"n": 100}), + tag("app", "old", 100, map[string]int64{"o": 100}), + untagged("app", "cache-1", 200, map[string]int64{"c1": 100}), + } + + plan := planDeletions(tags, nil, 150, noFreshTags) + + assert.Equal(t, []string{"app@sha256:cache-1", "app:old"}, keys(plan)) +} + +func TestPlanDeletionsKeepsFreshUntaggedManifests(t *testing.T) { + tags := []TagInfo{ + tag("app", "newest", 300, map[string]int64{"n": 100}), + untagged("app", "being-tagged", 290, map[string]int64{"b": 100}), + untagged("app", "stale", 100, map[string]int64{"s": 100}), + } + + plan := planDeletions(tags, nil, math.MaxInt64, 280) + + assert.Equal(t, []string{"app@sha256:stale"}, keys(plan)) +} + +func TestPlanDeletionsUntaggedNeverCountsAsNewest(t *testing.T) { + tags := []TagInfo{ + tag("app", "image", 100, map[string]int64{"i": 100}), + untagged("app", "newer-untagged", 200, map[string]int64{"u": 100}), + } + + plan := planDeletions(tags, nil, math.MaxInt64, noFreshTags) + + assert.Equal(t, []string{"app@sha256:newer-untagged"}, keys(plan)) +} + +func TestUntaggedRevisions(t *testing.T) { + tagged := []TagInfo{ + {Repo: "app", Tag: "v1", Digest: "sha256:index", Children: []string{"sha256:amd64", "sha256:attest"}}, + {Repo: "app", Tag: "v1-buildcache", Digest: "sha256:cache"}, + } + revisions := []manifestRevision{ + {repo: "app", digest: "sha256:index", modTime: 1}, + {repo: "app", digest: "sha256:amd64", modTime: 1}, + {repo: "app", digest: "sha256:attest", modTime: 1}, + {repo: "app", digest: "sha256:cache", modTime: 1}, + {repo: "app", digest: "sha256:old-cache", modTime: 2}, + {repo: "other", digest: "sha256:cache", modTime: 3}, + } + + got := untaggedRevisions(revisions, tagged) + + assert.Equal(t, []manifestRevision{ + {repo: "app", digest: "sha256:old-cache", modTime: 2}, + {repo: "other", digest: "sha256:cache", modTime: 3}, + }, got) +} + +func TestParseRevisionModTimes(t *testing.T) { + output := `1788280993 /var/lib/registry/docker/registry/v2/repositories/group/sub/_manifests/revisions/sha256/aaaa/link +1788280994 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/revisions/sha256/bbbb/link +1788280995 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/revisions/sha256/cccc/signatures/sha256/dddd/link +1788280996 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/tags/latest/current/link +garbage line +` + + got := parseRevisionModTimes(output) + + assert.Equal(t, []manifestRevision{ + {repo: "group/sub", digest: "sha256:aaaa", modTime: 1788280993}, + {repo: "tezara", digest: "sha256:bbbb", modTime: 1788280994}, + }, got) +} + func cleanupJob(name string, created int64, finished bool) batchv1.Job { job := batchv1.Job{ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/apps/api/internal/models/registry_cache.go b/apps/api/internal/models/registry_cache.go index defcef11..e4bd6821 100644 --- a/apps/api/internal/models/registry_cache.go +++ b/apps/api/internal/models/registry_cache.go @@ -13,10 +13,11 @@ const ( // RegistryCleanupResult is what a cleanup run reports when it exits. type RegistryCleanupResult struct { - Outcome RegistryCleanupOutcome `json:"outcome,omitempty" required:"false" enum:"under_threshold,cleaned,over_threshold,skipped" doc:"under_threshold: nothing to do. cleaned: usage is under the threshold. over_threshold: usage is still over the threshold because everything left is protected. skipped: builds or another cleanup kept running"` - FreedBytes int64 `json:"freed_bytes"` - DeletedImages int `json:"deleted_images"` - Error string `json:"error,omitempty" required:"false"` + Outcome RegistryCleanupOutcome `json:"outcome,omitempty" required:"false" enum:"under_threshold,cleaned,over_threshold,skipped" doc:"under_threshold: nothing to do. cleaned: usage is under the threshold. over_threshold: usage is still over the threshold because everything left is protected. skipped: builds or another cleanup kept running"` + FreedBytes int64 `json:"freed_bytes"` + DeletedImages int `json:"deleted_images"` + DeletedUntagged int `json:"deleted_untagged" doc:"Manifests no tag pointed at anymore, mostly build caches the next build overwrote"` + Error string `json:"error,omitempty" required:"false"` } // RegistryCacheCleanupRun summarizes the most recent cleanup job execution. diff --git a/apps/docs/content/operations/system-settings.mdx b/apps/docs/content/operations/system-settings.mdx index 14854112..7e7e01bd 100644 --- a/apps/docs/content/operations/system-settings.mdx +++ b/apps/docs/content/operations/system-settings.mdx @@ -39,7 +39,7 @@ The bar shows how full the volume is, with a mark at the cleanup threshold. Usag ### Cleanup -Cleanup runs every hour. When usage is over the threshold, it deletes the oldest images until usage is under it again. It never deletes: +Cleanup runs every hour. When usage is over the threshold, it first deletes what nothing points at anymore: every build overwrites the build cache of the one before it, and the old cache stays on the volume until cleanup removes it. If usage is still over the threshold, it deletes the oldest images until usage is under it again. It never deletes: - Images that are deployed - Images pushed in the last hour @@ -47,7 +47,7 @@ Cleanup runs every hour. When usage is over the threshold, it deletes the oldest Cleanup waits until no build is running before it deletes anything. Builds never wait for cleanup. If a build starts while cleanup is deleting and part of its image gets removed, the build notices and pushes the image again. -The page shows the last run: how much space it freed and how many images it deleted. **Still over the threshold** means everything left is protected, so grow the volume. +The page shows the last run: how much space it freed and how many images and overwritten caches it deleted. **Still over the threshold** means everything left is protected, so grow the volume. **Clean Up Now** runs cleanup right away and deletes every image it is allowed to delete, even when usage is under the threshold. Redeploying a deployment whose image was deleted builds it again. diff --git a/apps/web/src/components/system/settings/registry-tab-content.tsx b/apps/web/src/components/system/settings/registry-tab-content.tsx index 5cb0d02a..7245cea7 100644 --- a/apps/web/src/components/system/settings/registry-tab-content.tsx +++ b/apps/web/src/components/system/settings/registry-tab-content.tsx @@ -39,6 +39,7 @@ import type { RegistryCacheCleanupRun, RegistryCacheConfig, RegistryCacheStats, + RegistryCleanupResult, } from "@/lib/server/client.gen"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { @@ -292,7 +293,7 @@ function describeCleanup(run: RegistryCacheCleanupRun): { const result = run.result; const freed = result ? `Freed ${formatBytes(result.freed_bytes)}` : ""; - const deleted = result ? pluralize(result.deleted_images, "image") : ""; + const deleted = result ? describeDeleted(result) : ""; switch (result?.outcome) { case "under_threshold": @@ -319,6 +320,14 @@ function describeCleanup(run: RegistryCacheCleanupRun): { return { Icon: CircleCheckIcon, tone: "success", text: "Finished" }; } +function describeDeleted(result: RegistryCleanupResult) { + const images = pluralize(result.deleted_images, "image"); + if (!result.deleted_untagged) return images; + const caches = pluralize(result.deleted_untagged, "overwritten cache"); + if (!result.deleted_images) return caches; + return `${images} and ${caches}`; +} + function LastCleanupLine({ Icon, tone, diff --git a/apps/web/src/lib/server/client.gen.ts b/apps/web/src/lib/server/client.gen.ts index 949616bd..0472bd1d 100644 --- a/apps/web/src/lib/server/client.gen.ts +++ b/apps/web/src/lib/server/client.gen.ts @@ -2662,6 +2662,7 @@ export const ReferenceableVariablesResponseBodySchema = z export const RegistryCleanupResultSchema = z .object({ deleted_images: z.number(), + deleted_untagged: z.number(), // Manifests no tag pointed at anymore, mostly build caches the next build overwrote error: z.string().optional(), freed_bytes: z.number(), outcome: z.enum(['under_threshold', 'cleaned', 'over_threshold', 'skipped']).optional(), // under_threshold: nothing to do. cleaned: usage is under the threshold. over_threshold: usage is still over the threshold because everything left is protected. skipped: builds or another cleanup kept running