api | web | docs: delete overwritten build caches in registry cleanup

apps/api/internal/infrastructure/registrycache/cleanup.go

@@ -33,6 +33,7 @@ const (
 	registryConfigPath = registryConfigDir + "/" + registryConfigKey
 	registryRepoDir    = registryDataDir + "/docker/registry/v2/repositories"
 	tagLinkMarker      = "/_manifests/tags/"
+	revisionLinkMarker = "/_manifests/revisions/sha256/"
 
 	buildCacheSuffix   = "-buildcache"
 	staleUploadMinutes = 1440
@@ -45,6 +46,7 @@ const (
 	freshTagGrace = time.Hour
 )
 
+// A manifest in a repository. Tag is empty for one that no tag points at anymore, like a build cache the next build overwrote.
 type TagInfo struct {
 	Repo    string
 	Tag     string
@@ -56,9 +58,16 @@ type TagInfo struct {
 }
 
 func (self TagInfo) Key() string {
+	if self.Untagged() {
+		return self.ManifestKey()
+	}
 	return self.Repo + ":" + self.Tag
 }
 
+func (self TagInfo) Untagged() bool {
+	return self.Tag == ""
+}
+
 func (self TagInfo) ManifestKey() string {
 	return self.Repo + "@" + self.Digest
 }
@@ -149,13 +158,17 @@ func (self *Cleaner) Run(ctx context.Context, thresholdBytes int64, pruneAll boo
 		log.Warnf("registry cleanup: over threshold with nothing prunable, grow the registry volume")
 	}
 
-	deleted := 0
+	deleted, deletedUntagged := 0, 0
 	for _, tag := range plan {
 		if err := self.registry.DeleteManifest(ctx, tag.Repo, tag.Digest); err != nil {
 			log.Warnf("registry cleanup: failed to delete %s: %v", tag.Key(), err)
 			continue
 		}
-		deleted++
+		if tag.Untagged() {
+			deletedUntagged++
+		} else {
+			deleted++
+		}
 		log.Infof("registry cleanup: deleted %s", tag.Key())
 	}
 	for _, child := range orphanedChildren(tags, plan) {
@@ -172,14 +185,15 @@ func (self *Cleaner) Run(ctx context.Context, thresholdBytes int64, pruneAll boo
 	if err != nil {
 		return nil, err
 	}
-	return cleanupResult(used, usedAfter, thresholdBytes, deleted), nil
+	return cleanupResult(used, usedAfter, thresholdBytes, deleted, deletedUntagged), nil
 }
 
-func cleanupResult(usedBefore, usedAfter, thresholdBytes int64, deleted int) *models.RegistryCleanupResult {
+func cleanupResult(usedBefore, usedAfter, thresholdBytes int64, deleted, deletedUntagged int) *models.RegistryCleanupResult {
 	result := &models.RegistryCleanupResult{
-		Outcome:       models.RegistryCleanupCleaned,
-		FreedBytes:    max(usedBefore-usedAfter, 0),
-		DeletedImages: deleted,
+		Outcome:         models.RegistryCleanupCleaned,
+		FreedBytes:      max(usedBefore-usedAfter, 0),
+		DeletedImages:   deleted,
+		DeletedUntagged: deletedUntagged,
 	}
 	if usedAfter >= thresholdBytes {
 		result.Outcome = models.RegistryCleanupOverThreshold
@@ -211,16 +225,19 @@ func planDeletions(tags []TagInfo, inUse map[string]bool, target int64, freshSin
 		candidates = append(candidates, tag)
 	}
 
-	// Oldest first: a cache in use is rewritten by every build, so a stale one sorts ahead of it
+	// Untagged manifests are worthless and go first, the rest oldest first: a cache in use is rewritten by every build, so a stale one sorts ahead of it
 	sort.SliceStable(candidates, func(i, j int) bool {
+		if candidates[i].Untagged() != candidates[j].Untagged() {
+			return candidates[i].Untagged()
+		}
 		return candidates[i].ModTime < candidates[j].ModTime
 	})
 
 	var plan []TagInfo
 	var freed int64
 	planned := map[string]bool{}
 	for _, tag := range candidates {
-		if freed >= target {
+		if freed >= target && !tag.Untagged() {
 			break
 		}
 		// Deletion is by digest, so a manifest a protected tag also points at is off limits
@@ -289,6 +306,10 @@ func protectedTags(tags []TagInfo, inUse map[string]bool, freshSince int64) map[
 		if inUse[tag.Key()] || inUse[tag.Repo+":"+tag.Digest] || tag.ModTime >= freshSince {
 			protected[tag.Key()] = true
 		}
+		// A push writes the manifest before the tag, so a fresh untagged one may still be getting its tag
+		if tag.Untagged() {
+			continue
+		}
 		key := newestKey{repo: tag.Repo, buildCache: tag.IsBuildCache()}
 		if current, ok := newest[key]; !ok || tag.ModTime > current.ModTime {
 			newest[key] = tag
@@ -409,33 +430,81 @@ func (self *Cleaner) inventory(ctx context.Context, pod string) ([]TagInfo, erro
 			continue
 		}
 		for _, tag := range tags {
-			manifest, err := self.registry.Manifest(ctx, repo, tag)
+			info, err := self.describe(ctx, repo, tag, tag, modTimes[repo+":"+tag])
 			if err != nil {
-				log.Warnf("registry cleanup: failed to read manifest %s:%s: %v", repo, tag, err)
-				continue
-			}
-			blobs := map[string]int64{}
-			if err := collectBlobs(ctx, self.registry, repo, tag, blobs, 0); err != nil {
-				log.Warnf("registry cleanup: failed to size %s:%s: %v", repo, tag, err)
+				log.Warnf("registry cleanup: %v", err)
 				continue
 			}
-			children := make([]string, 0, len(manifest.Manifests))
-			for _, child := range manifest.Manifests {
-				children = append(children, child.Digest)
-			}
-			inventory = append(inventory, TagInfo{
-				Repo:     repo,
-				Tag:      tag,
-				Digest:   manifest.Digest,
-				Blobs:    blobs,
-				ModTime:  modTimes[repo+":"+tag],
-				Children: children,
-			})
+			inventory = append(inventory, info)
+		}
+	}
+
+	// Overwriting a tag leaves its old manifest in the repository, where only a deletion by digest reaches it
+	revisions, err := self.revisionModTimes(ctx, pod)
+	if err != nil {
+		log.Warnf("registry cleanup: failed to list stored manifests, leaving untagged ones alone: %v", err)
+		return inventory, nil
+	}
+	for _, revision := range untaggedRevisions(revisions, inventory) {
+		info, err := self.describe(ctx, revision.repo, revision.digest, "", revision.modTime)
+		if err != nil {
+			log.Warnf("registry cleanup: %v", err)
+			continue
 		}
+		inventory = append(inventory, info)
 	}
 	return inventory, nil
 }
 
+func (self *Cleaner) describe(ctx context.Context, repo, ref, tag string, modTime int64) (TagInfo, error) {
+	manifest, err := self.registry.Manifest(ctx, repo, ref)
+	if err != nil {
+		return TagInfo{}, fmt.Errorf("failed to read manifest %s/%s: %w", repo, ref, err)
+	}
+	blobs := map[string]int64{}
+	if err := collectBlobs(ctx, self.registry, repo, ref, blobs, 0); err != nil {
+		return TagInfo{}, fmt.Errorf("failed to size %s/%s: %w", repo, ref, err)
+	}
+	children := make([]string, 0, len(manifest.Manifests))
+	for _, child := range manifest.Manifests {
+		children = append(children, child.Digest)
+	}
+	return TagInfo{
+		Repo:     repo,
+		Tag:      tag,
+		Digest:   manifest.Digest,
+		Blobs:    blobs,
+		ModTime:  modTime,
+		Children: children,
+	}, nil
+}
+
+type manifestRevision struct {
+	repo    string
+	digest  string
+	modTime int64
+}
+
+// The children of a tagged index are untagged too, but deleting them breaks the index
+func untaggedRevisions(revisions []manifestRevision, tagged []TagInfo) []manifestRevision {
+	referenced := map[string]bool{}
+	for _, tag := range tagged {
+		referenced[tag.ManifestKey()] = true
+		for _, child := range tag.Children {
+			referenced[tag.Repo+"@"+child] = true
+		}
+	}
+
+	var untagged []manifestRevision
+	for _, revision := range revisions {
+		if referenced[revision.repo+"@"+revision.digest] {
+			continue
+		}
+		untagged = append(untagged, revision)
+	}
+	return untagged
+}
+
 func (self *Cleaner) inUseRefs(ctx context.Context) (map[string]bool, error) {
 	pods, err := self.clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{})
 	if err != nil {
@@ -571,6 +640,52 @@ func tagKeyFromLinkPath(path string) (string, bool) {
 	return repo + ":" + tag, true
 }
 
+func (self *Cleaner) revisionModTimes(ctx context.Context, pod string) ([]manifestRevision, error) {
+	command := fmt.Sprintf(`[ -d %[1]s ] || exit 0; find %[1]s -path '*%[2]s*/link' -exec stat -c '%%Y %%n' {} +`, registryRepoDir, revisionLinkMarker)
+	out, err := self.exec(ctx, pod, []string{"sh", "-c", command})
+	if err != nil {
+		return nil, err
+	}
+	return parseRevisionModTimes(out), nil
+}
+
+func parseRevisionModTimes(output string) []manifestRevision {
+	var revisions []manifestRevision
+	for line := range strings.SplitSeq(output, "\n") {
+		fields := strings.SplitN(strings.TrimSpace(line), " ", 2)
+		if len(fields) != 2 {
+			continue
+		}
+		epoch, err := strconv.ParseInt(fields[0], 10, 64)
+		if err != nil {
+			continue
+		}
+		revision, ok := revisionFromLinkPath(fields[1])
+		if !ok {
+			continue
+		}
+		revision.modTime = epoch
+		revisions = append(revisions, revision)
+	}
+	return revisions
+}
+
+func revisionFromLinkPath(path string) (manifestRevision, bool) {
+	trimmed := strings.TrimPrefix(path, registryRepoDir+"/")
+	if trimmed == path {
+		return manifestRevision{}, false
+	}
+	repo, rest, found := strings.Cut(trimmed, revisionLinkMarker)
+	if !found {
+		return manifestRevision{}, false
+	}
+	hex, rest, found := strings.Cut(rest, "/")
+	if !found || rest != "link" || repo == "" || hex == "" {
+		return manifestRevision{}, false
+	}
+	return manifestRevision{repo: repo, digest: "sha256:" + hex}, true
+}
+
 // Distribution's own purger only drops uploads older than 7 days, too slow for a full volume
 func (self *Cleaner) pruneStaleUploads(ctx context.Context, pod string) error {
 	command := fmt.Sprintf(

apps/api/internal/infrastructure/registrycache/cleanup_test.go

@@ -287,19 +287,112 @@ func TestPlanDeletionsPruneAllKeepsProtectedTags(t *testing.T) {
 }
 
 func TestCleanupResultOutcome(t *testing.T) {
-	cleaned := cleanupResult(20*gib, 10*gib, 16*gib, 4)
+	cleaned := cleanupResult(20*gib, 10*gib, 16*gib, 4, 7)
 	assert.Equal(t, models.RegistryCleanupCleaned, cleaned.Outcome)
 	assert.Equal(t, 10*gib, cleaned.FreedBytes)
 	assert.Equal(t, 4, cleaned.DeletedImages)
+	assert.Equal(t, 7, cleaned.DeletedUntagged)
 
-	stuck := cleanupResult(20*gib, 20*gib, 16*gib, 0)
+	stuck := cleanupResult(20*gib, 20*gib, 16*gib, 0, 0)
 	assert.Equal(t, models.RegistryCleanupOverThreshold, stuck.Outcome)
 	assert.Zero(t, stuck.FreedBytes)
 
-	grew := cleanupResult(10*gib, 11*gib, 16*gib, 0)
+	grew := cleanupResult(10*gib, 11*gib, 16*gib, 0, 0)
 	assert.Zero(t, grew.FreedBytes)
 }
 
+func untagged(repo, digest string, modTime int64, blobs map[string]int64) TagInfo {
+	return TagInfo{Repo: repo, Digest: "sha256:" + digest, Blobs: blobs, ModTime: modTime}
+}
+
+// An overwritten build cache keeps its blobs on the volume although no tag reaches it anymore
+func TestPlanDeletionsDeletesEveryUntaggedManifestFirst(t *testing.T) {
+	tags := []TagInfo{
+		tag("app", "newest", 300, map[string]int64{"n": 100}),
+		tag("app", "old", 100, map[string]int64{"o": 100}),
+		tag("app", "current-buildcache", 300, map[string]int64{"c": 100}),
+		untagged("app", "cache-1", 200, map[string]int64{"c1": 100}),
+		untagged("app", "cache-2", 250, map[string]int64{"c2": 100}),
+	}
+
+	plan := planDeletions(tags, nil, 50, noFreshTags)
+
+	assert.Equal(t, []string{"app@sha256:cache-1", "app@sha256:cache-2"}, keys(plan))
+}
+
+func TestPlanDeletionsUntaggedBeforeTaggedImages(t *testing.T) {
+	tags := []TagInfo{
+		tag("app", "newest", 300, map[string]int64{"n": 100}),
+		tag("app", "old", 100, map[string]int64{"o": 100}),
+		untagged("app", "cache-1", 200, map[string]int64{"c1": 100}),
+	}
+
+	plan := planDeletions(tags, nil, 150, noFreshTags)
+
+	assert.Equal(t, []string{"app@sha256:cache-1", "app:old"}, keys(plan))
+}
+
+func TestPlanDeletionsKeepsFreshUntaggedManifests(t *testing.T) {
+	tags := []TagInfo{
+		tag("app", "newest", 300, map[string]int64{"n": 100}),
+		untagged("app", "being-tagged", 290, map[string]int64{"b": 100}),
+		untagged("app", "stale", 100, map[string]int64{"s": 100}),
+	}
+
+	plan := planDeletions(tags, nil, math.MaxInt64, 280)
+
+	assert.Equal(t, []string{"app@sha256:stale"}, keys(plan))
+}
+
+func TestPlanDeletionsUntaggedNeverCountsAsNewest(t *testing.T) {
+	tags := []TagInfo{
+		tag("app", "image", 100, map[string]int64{"i": 100}),
+		untagged("app", "newer-untagged", 200, map[string]int64{"u": 100}),
+	}
+
+	plan := planDeletions(tags, nil, math.MaxInt64, noFreshTags)
+
+	assert.Equal(t, []string{"app@sha256:newer-untagged"}, keys(plan))
+}
+
+func TestUntaggedRevisions(t *testing.T) {
+	tagged := []TagInfo{
+		{Repo: "app", Tag: "v1", Digest: "sha256:index", Children: []string{"sha256:amd64", "sha256:attest"}},
+		{Repo: "app", Tag: "v1-buildcache", Digest: "sha256:cache"},
+	}
+	revisions := []manifestRevision{
+		{repo: "app", digest: "sha256:index", modTime: 1},
+		{repo: "app", digest: "sha256:amd64", modTime: 1},
+		{repo: "app", digest: "sha256:attest", modTime: 1},
+		{repo: "app", digest: "sha256:cache", modTime: 1},
+		{repo: "app", digest: "sha256:old-cache", modTime: 2},
+		{repo: "other", digest: "sha256:cache", modTime: 3},
+	}
+
+	got := untaggedRevisions(revisions, tagged)
+
+	assert.Equal(t, []manifestRevision{
+		{repo: "app", digest: "sha256:old-cache", modTime: 2},
+		{repo: "other", digest: "sha256:cache", modTime: 3},
+	}, got)
+}
+
+func TestParseRevisionModTimes(t *testing.T) {
+	output := `1788280993 /var/lib/registry/docker/registry/v2/repositories/group/sub/_manifests/revisions/sha256/aaaa/link
+1788280994 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/revisions/sha256/bbbb/link
+1788280995 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/revisions/sha256/cccc/signatures/sha256/dddd/link
+1788280996 /var/lib/registry/docker/registry/v2/repositories/tezara/_manifests/tags/latest/current/link
+garbage line
+`
+
+	got := parseRevisionModTimes(output)
+
+	assert.Equal(t, []manifestRevision{
+		{repo: "group/sub", digest: "sha256:aaaa", modTime: 1788280993},
+		{repo: "tezara", digest: "sha256:bbbb", modTime: 1788280994},
+	}, got)
+}
+
 func cleanupJob(name string, created int64, finished bool) batchv1.Job {
 	job := batchv1.Job{ObjectMeta: metav1.ObjectMeta{
 		Name:              name,

apps/api/internal/models/registry_cache.go

@@ -13,10 +13,11 @@ const (
 
 // RegistryCleanupResult is what a cleanup run reports when it exits.
 type RegistryCleanupResult struct {
-	Outcome       RegistryCleanupOutcome `json:"outcome,omitempty" required:"false" enum:"under_threshold,cleaned,over_threshold,skipped" doc:"under_threshold: nothing to do. cleaned: usage is under the threshold. over_threshold: usage is still over the threshold because everything left is protected. skipped: builds or another cleanup kept running"`
-	FreedBytes    int64                  `json:"freed_bytes"`
-	DeletedImages int                    `json:"deleted_images"`
-	Error         string                 `json:"error,omitempty" required:"false"`
+	Outcome         RegistryCleanupOutcome `json:"outcome,omitempty" required:"false" enum:"under_threshold,cleaned,over_threshold,skipped" doc:"under_threshold: nothing to do. cleaned: usage is under the threshold. over_threshold: usage is still over the threshold because everything left is protected. skipped: builds or another cleanup kept running"`
+	FreedBytes      int64                  `json:"freed_bytes"`
+	DeletedImages   int                    `json:"deleted_images"`
+	DeletedUntagged int                    `json:"deleted_untagged" doc:"Manifests no tag pointed at anymore, mostly build caches the next build overwrote"`
+	Error           string                 `json:"error,omitempty" required:"false"`
 }
 
 // RegistryCacheCleanupRun summarizes the most recent cleanup job execution.