From 52649da1ddf0d7a6aabe5f1bd4e04e4d2208a149 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Mon, 3 Aug 2026 23:26:19 +0100 Subject: [PATCH 1/3] chore(omni): add hand-owned versions.yaml, drop duplicate unraid machine class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Desired Talos/Kubernetes versions get one hand-owned home. talos-image.yaml stays the machine-written build ledger — it is rewritten wholesale by the talos-images workflow and records what was built, not what should run. unraid-control-plane.yaml was a hand-written duplicate of what omni-on-unraid's renderer generates for unraid-cp, and had already drifted: memory 4096 against the renderer's 6144, and no installImage at all. Refs: openspec/changes/automate-talos-upgrade-pipeline (tasks 1.4, 2.1) --- .../machine-classes/unraid-control-plane.yaml | 19 --------------- omni/versions.yaml | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) delete mode 100644 omni/machine-classes/unraid-control-plane.yaml create mode 100644 omni/versions.yaml diff --git a/omni/machine-classes/unraid-control-plane.yaml b/omni/machine-classes/unraid-control-plane.yaml deleted file mode 100644 index e3b4b44..0000000 --- a/omni/machine-classes/unraid-control-plane.yaml +++ /dev/null @@ -1,19 +0,0 @@ -metadata: - namespace: default - type: MachineClasses.omni.sidero.dev - id: unraid-cp - -spec: - autoprovision: - providerid: libvirt - kernelargs: [] - metavalues: [] - providerdata: | - cores: 4 - memory: 4096 - disk_size: 40 - storage_pool: "omni-domains" - network_interfaces: - - driver: "virtio" - network_name: "default" - grpctunnel: 0 diff --git a/omni/versions.yaml b/omni/versions.yaml new file mode 100644 index 0000000..edd9959 --- /dev/null +++ b/omni/versions.yaml @@ -0,0 +1,24 @@ +--- +# Desired Talos / Kubernetes versions. HAND-OWNED — edit this file to upgrade. +# +# Build outputs (schematic_id, oci_image_ocid, installer and extension image +# refs) live in talos-image.yaml, which is rewritten wholesale by the +# talos-images build workflow. Never put desired versions there: that file +# records what has been built, not what is intended to run. +# +# Clusters inherit the global pin. A cluster entry overrides it. +# +# Kubernetes is never inherited across a diverging Talos target: a cluster that +# overrides `talos` without `kubernetes` derives its Kubernetes version from +# the supported versions of its own Talos target, not from the global pin. + +talos: v1.13.7 +kubernetes: v1.36.3 + +clusters: + # Sacrificial canary. Always takes a prerelease first; recovery is destroy + # and re-provision at the previous pin, never a Talos downgrade. + unraid-lab: {} + + # No etcd restore point yet — promotion is gated on a verified backup. + oci-lab: {} From c243d1d2f4face717d7a08b4a103e13ee3994354 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Mon, 3 Aug 2026 23:32:17 +0100 Subject: [PATCH 2/3] fix(ci): check deprecations against desired k8s, drop orphaned update workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pluto job read kubernetes_version from omni/talos-image.yaml, which is the build ledger — it records what was last built, not what clusters run. With the ledger at v1.37.0-beta.0 and clusters on v1.36.3, the deprecation gate was checking the wrong target entirely. It now reads the desired pin from omni/versions.yaml and fails if that pin is missing. talos-image-update.yml was a second, competing PR-opening path: a repository_dispatch receiver that sed-bumps talos_version in the ledger and opens its own PR on a different branch from the one talos-images/build.yml opens. No sender exists in any repo (grepped all of ~/syscode/git), so it is orphaned — but if it ever fired it would bump talos_version without schematic_id or oci_image_ocid, producing a ledger pointing at a version whose image was never built. That is root cause #4 of the 2026-08-03 incident. Refs: openspec/changes/automate-talos-upgrade-pipeline --- .github/workflows/ci.yml | 9 +++++- .github/workflows/talos-image-update.yml | 41 ------------------------ 2 files changed, 8 insertions(+), 42 deletions(-) delete mode 100644 .github/workflows/talos-image-update.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e50f58d..b2937f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,14 @@ jobs: - name: Read target Kubernetes version id: k8s run: | - K8S_VERSION=$(grep '^kubernetes_version:' omni/talos-image.yaml | awk '{print $2}') + # Desired version, from versions.yaml — NOT talos-image.yaml, which + # is the build ledger and records what was last built. Reading the + # ledger here checked deprecations against a version no cluster runs. + K8S_VERSION=$(grep '^kubernetes:' omni/versions.yaml | awk '{print $2}') + if [[ -z "$K8S_VERSION" ]]; then + echo "ERROR: no kubernetes pin in omni/versions.yaml" >&2 + exit 1 + fi echo "version=${K8S_VERSION}" >> "$GITHUB_OUTPUT" echo "Checking against Kubernetes ${K8S_VERSION}" diff --git a/.github/workflows/talos-image-update.yml b/.github/workflows/talos-image-update.yml deleted file mode 100644 index 2a9ddb6..0000000 --- a/.github/workflows/talos-image-update.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Update Talos image - -on: - repository_dispatch: - types: [talos-image-update] - -jobs: - update: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@v6 - - - name: Update omni/talos-image.yaml - env: - TALOS_VERSION: ${{ github.event.client_payload.talos_version }} - run: | - sed -i "s|^talos_version:.*|talos_version: ${TALOS_VERSION}|" omni/talos-image.yaml - sed -i "s|^updated_at:.*|updated_at: \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"|" omni/talos-image.yaml - - - name: Open pull request - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TALOS_VERSION: ${{ github.event.client_payload.talos_version }} - run: | - BRANCH="talos-image-update/${TALOS_VERSION}" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "${BRANCH}" - git add omni/talos-image.yaml - git diff --cached --quiet && echo "No changes" && exit 0 - git commit -m "chore: update Talos installer image to ${TALOS_VERSION}" - git push origin "${BRANCH}" - gh pr create \ - --title "chore: update Talos installer image to ${TALOS_VERSION}" \ - --body "Auto-generated by talos-images build workflow." \ - --base main \ - --head "${BRANCH}" From f72d85eb5f208159dc9f3079c025baeef8fd0d34 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Mon, 3 Aug 2026 23:42:21 +0100 Subject: [PATCH 3/3] feat(ci): fail the build on version drift against versions.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing reconciled the derived files when versions.yaml changed without needing a new image — the build workflow only opens a PR when it has something to build, so flipping a cluster back to an already-built version left machine classes and templates stale with no failure anywhere. The checker caught live drift on its first run: both OCI machine classes still installed v1.14.0-beta.1 while oci-lab targets v1.13.7, so any node provisioned into oci-lab would have installed the beta. Repaired to v1.13.7. Kubernetes is never inherited across a diverging Talos target: a cluster that overrides talos without kubernetes is an error, not an inherit, because the global kubernetes pin belongs to a different Talos minor. Refs: openspec/changes/automate-talos-upgrade-pipeline --- .github/workflows/ci.yml | 5 + omni/machine-classes/control-plane.yaml | 2 +- omni/machine-classes/worker.yaml | 2 +- scripts/check-version-drift.py | 150 ++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 scripts/check-version-drift.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2937f0..41abe90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,11 @@ jobs: pip install pyyaml --quiet python3 scripts/validate.py + - name: Check for version drift against versions.yaml + run: | + python3 scripts/check-version-drift.py --selftest + python3 scripts/check-version-drift.py + pluto: name: API deprecation check runs-on: ubuntu-latest diff --git a/omni/machine-classes/control-plane.yaml b/omni/machine-classes/control-plane.yaml index 0b9275e..01065cc 100644 --- a/omni/machine-classes/control-plane.yaml +++ b/omni/machine-classes/control-plane.yaml @@ -4,7 +4,7 @@ metadata: type: MachineClasses.omni.sidero.dev id: oci-cp spec: - installImage: ghcr.io/syscode-labs/talos-images/installer:v1.14.0-beta.1 + installImage: ghcr.io/syscode-labs/talos-images/installer:v1.13.7 matchLabels: # Synced into Omni; UI can select this class after machines join. # Labels may be assigned by automation or, for lab recovery, in Omni UI. diff --git a/omni/machine-classes/worker.yaml b/omni/machine-classes/worker.yaml index 4889bb6..2626aad 100644 --- a/omni/machine-classes/worker.yaml +++ b/omni/machine-classes/worker.yaml @@ -4,7 +4,7 @@ metadata: type: MachineClasses.omni.sidero.dev id: oci-worker spec: - installImage: ghcr.io/syscode-labs/talos-images/installer:v1.14.0-beta.1 + installImage: ghcr.io/syscode-labs/talos-images/installer:v1.13.7 matchLabels: # Synced into Omni; UI can select this class after machines join. # Labels may be assigned by automation or, for lab recovery, in Omni UI. diff --git a/scripts/check-version-drift.py b/scripts/check-version-drift.py new file mode 100644 index 0000000..7380fb6 --- /dev/null +++ b/scripts/check-version-drift.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Fail if any version-bearing file disagrees with omni/versions.yaml. + +versions.yaml is the hand-owned declaration of what should run. Every other +version reference in this repo is derived from it and must agree. Without this +check, editing versions.yaml in a way that needs no new image (flipping a +cluster back to a version already built, say) leaves the derived files stale, +because nothing rebuilds them — the build workflow only opens a PR when it has +an image to build. + +That is the failure this repo already lived through: the ledger and machine +classes moved to v1.14.0-beta.1 while oci-lab.yaml stayed on v1.13.7, and +nothing failed until a cluster refused its config. + +Runs identically locally and in CI — `python3 scripts/check-version-drift.py`. +""" + +import os +import re +import sys + +import yaml + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Files owned by a cluster: which cluster, and which versions to check. +# unraid-lab's machine classes and template are generated in omni-on-unraid +# from this same file, so they are deliberately absent here. +OCI_LAB_FILES = [ + "omni/cluster-templates/oci-lab.yaml", + "omni/machine-classes/control-plane.yaml", + "omni/machine-classes/worker.yaml", +] + + +def resolve(versions, cluster): + """Resolve (talos, kubernetes) for a cluster. + + Kubernetes is never inherited across a diverging Talos target: a cluster + that overrides talos without kubernetes cannot keep the global Kubernetes + pin, which belongs to a different Talos minor. + """ + entry = (versions.get("clusters") or {}).get(cluster) or {} + talos = entry.get("talos") or versions.get("talos") + k8s = entry.get("kubernetes") + if k8s is None: + if entry.get("talos") and entry["talos"] != versions.get("talos"): + return talos, None # caller reports: needs an explicit pin + k8s = versions.get("kubernetes") + return talos, k8s + + +def main(): + path = os.path.join(ROOT, "omni/versions.yaml") + with open(path) as fh: + versions = yaml.safe_load(fh) or {} + + if not versions.get("talos") or not versions.get("kubernetes"): + print("ERROR: omni/versions.yaml must set both talos and kubernetes") + return 1 + + talos, k8s = resolve(versions, "oci-lab") + errors = [] + + if k8s is None: + errors.append( + "oci-lab overrides talos to %s but sets no kubernetes version. " + "Derive it from that Talos release and pin it explicitly — the " + "global kubernetes pin belongs to a different Talos minor." % talos + ) + k8s = "" + + for rel in OCI_LAB_FILES: + full = os.path.join(ROOT, rel) + if not os.path.exists(full): + errors.append("%s: missing" % rel) + continue + text = open(full).read() + + if "machine-classes" in rel: + found = re.findall(r"^\s*installImage:.*:(\S+)$", text, re.M) + if not found: + errors.append("%s: no installImage found" % rel) + for tag in found: + # Tags may carry a variant suffix, e.g. v1.13.7-libvirt. + if not tag.startswith(talos): + errors.append( + "%s: installImage tag %s does not match oci-lab's " + "Talos pin %s" % (rel, tag, talos) + ) + else: + doc = yaml.safe_load(text) or {} + got_talos = (doc.get("talos") or {}).get("version") + got_k8s = (doc.get("kubernetes") or {}).get("version") + if got_talos != talos: + errors.append( + "%s: talos.version is %s, versions.yaml says %s" + % (rel, got_talos, talos) + ) + if got_k8s != k8s: + errors.append( + "%s: kubernetes.version is %s, versions.yaml says %s" + % (rel, got_k8s, k8s) + ) + + if errors: + print("Version drift against omni/versions.yaml:\n") + for err in errors: + print(" - %s" % err) + print( + "\nversions.yaml is the source of truth. Update the derived files " + "to match it, or correct the pin." + ) + return 1 + + print("No version drift. oci-lab: Talos %s, Kubernetes %s" % (talos, k8s)) + return 0 + + +def selftest(): + g = {"talos": "v1.13.7", "kubernetes": "v1.36.3"} + + v = dict(g, clusters={"a": {}}) + assert resolve(v, "a") == ("v1.13.7", "v1.36.3"), "inherit both" + + v = dict(g, clusters={"a": None}) + assert resolve(v, "a") == ("v1.13.7", "v1.36.3"), "null entry inherits" + + v = dict(g, clusters={}) + assert resolve(v, "missing") == ("v1.13.7", "v1.36.3"), "absent cluster inherits" + + # Diverging Talos must not silently keep the global Kubernetes pin. + v = dict(g, clusters={"a": {"talos": "v1.14.0"}}) + assert resolve(v, "a") == ("v1.14.0", None), "diverged talos needs explicit k8s" + + v = dict(g, clusters={"a": {"talos": "v1.14.0", "kubernetes": "v1.37.0"}}) + assert resolve(v, "a") == ("v1.14.0", "v1.37.0"), "both overridden" + + # Restating the global pin is not divergence. + v = dict(g, clusters={"a": {"talos": "v1.13.7"}}) + assert resolve(v, "a") == ("v1.13.7", "v1.36.3"), "redundant pin still inherits" + + print("selftest ok") + return 0 + + +if __name__ == "__main__": + if "--selftest" in sys.argv: + sys.exit(selftest()) + sys.exit(main())