diff --git a/.github/workflows/archinfra-k8s-docker-r1.yml b/.github/workflows/archinfra-k8s-docker-r1.yml new file mode 100644 index 0000000..2fdd037 --- /dev/null +++ b/.github/workflows/archinfra-k8s-docker-r1.yml @@ -0,0 +1,162 @@ +name: Archinfra Kubernetes Docker Runtime r1 + +on: + workflow_dispatch: + pull_request: + paths: + - "archinfra/**" + - "docker/**" + - "registry/**" + - "k8s/**" + - ".github/workflows/archinfra-k8s-docker-r1.yml" + push: + branches: + - main + paths: + - "archinfra/releases/v1.36.4-r1.env" + - "archinfra/scripts/**" + - "docker/**" + - "registry/**" + - "k8s/**" + - ".github/workflows/archinfra-k8s-docker-r1.yml" + +permissions: + contents: read + packages: write + +concurrency: + group: archinfra-kubernetes-runtime-v1.36.4-r1 + cancel-in-progress: false + +env: + RELEASE_FILE: ${{ github.workspace }}/archinfra/releases/v1.36.4-r1.env + +jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate deterministic runtime release + shell: bash + run: | + set -euo pipefail + bash -n archinfra/scripts/build-k8s-docker.sh + source archinfra/releases/v1.36.4-r1.env + test "$RELEASE_VERSION" = "v1.36.4-r1" + test "$ARCH" = "amd64" + test "$RUNTIME_PROFILE" = "docker" + test "$KUBERNETES_VERSION" = "1.36.4" + test "$SEALOS_VERSION" = "5.1.1" + test "$DOCKER_VERSION" = "28.5.2" + test "$SEALOS_CACHE_DIGEST" != "" + test "$DOCKER_CACHE_DIGEST" != "" + test "$CRICTL_CACHE_DIGEST" != "" + test "$KUBERNETES_CACHE_DIGEST" != "" + echo "release lock validation OK" + + build: + if: ${{ github.event_name != 'pull_request' }} + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install deterministic build host tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + buildah \ + skopeo \ + jq \ + file \ + uidmap \ + fuse-overlayfs + buildah --version + skopeo --version + jq --version + + - name: Build and publish Cluster Image to GHCR + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash archinfra/scripts/build-k8s-docker.sh + + - name: Mirror verified Runtime image to Alibaba Cloud ACR + if: ${{ vars.PUBLISH_ALIYUN == 'true' }} + shell: bash + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ALIYUN_REGISTRY: ${{ vars.ALIYUN_REGISTRY }} + ALIYUN_NAMESPACE: ${{ vars.ALIYUN_NAMESPACE }} + ALIYUN_USERNAME: ${{ secrets.ALIYUN_USERNAME }} + ALIYUN_PASSWORD: ${{ secrets.ALIYUN_PASSWORD }} + run: | + set -euo pipefail + + source archinfra/releases/v1.36.4-r1.env + PROVENANCE="out/runtime-v1.36.4-r1.provenance.env" + test -s "$PROVENANCE" + + : "${ALIYUN_REGISTRY:?organization variable ALIYUN_REGISTRY is required}" + : "${ALIYUN_NAMESPACE:?organization variable ALIYUN_NAMESPACE is required}" + : "${ALIYUN_USERNAME:?organization secret ALIYUN_USERNAME is required}" + : "${ALIYUN_PASSWORD:?organization secret ALIYUN_PASSWORD is required}" + + ALIYUN_IMAGE="${ALIYUN_REGISTRY}/${ALIYUN_NAMESPACE}/kubernetes-runtime:${RELEASE_VERSION}-${ARCH}" + + SRC_DIGEST="$(skopeo inspect \ + --creds "$GHCR_USER:$GHCR_TOKEN" \ + "docker://$FINAL_IMAGE" | jq -r '.Digest')" + test "$SRC_DIGEST" = "$(grep '^FINAL_DIGEST=' "$PROVENANCE" | cut -d= -f2-)" + + echo "Mirroring:" + echo " source: $FINAL_IMAGE@$SRC_DIGEST" + echo " target: $ALIYUN_IMAGE" + + skopeo copy \ + --all \ + --preserve-digests \ + --src-creds "$GHCR_USER:$GHCR_TOKEN" \ + --dest-creds "$ALIYUN_USERNAME:$ALIYUN_PASSWORD" \ + "docker://$FINAL_IMAGE" \ + "docker://$ALIYUN_IMAGE" + + DST_DIGEST="$(skopeo inspect \ + --creds "$ALIYUN_USERNAME:$ALIYUN_PASSWORD" \ + "docker://$ALIYUN_IMAGE" | jq -r '.Digest')" + + if [[ "$DST_DIGEST" != "$SRC_DIGEST" ]]; then + echo "ERROR: Aliyun ACR digest mismatch: source=$SRC_DIGEST target=$DST_DIGEST" >&2 + exit 1 + fi + + cat >> "$PROVENANCE" <> "$GITHUB_STEP_SUMMARY" + + echo "Aliyun ACR mirror VERIFIED: $ALIYUN_IMAGE@$DST_DIGEST" + + - name: Upload runtime provenance + if: always() + uses: actions/upload-artifact@v4 + with: + name: kubernetes-runtime-v1.36.4-r1-provenance + path: out/ + if-no-files-found: warn diff --git a/archinfra/releases/v1.36.4-r1.env b/archinfra/releases/v1.36.4-r1.env new file mode 100644 index 0000000..435ad56 --- /dev/null +++ b/archinfra/releases/v1.36.4-r1.env @@ -0,0 +1,48 @@ +# Archinfra Kubernetes runtime release lock +# This file consumes the VERIFIED cache build produced by archinfra/kubernetes-cache. +# Runtime release builds MUST NOT resolve component versions dynamically. + +RELEASE_VERSION=v1.36.4-r1 +ARCH=amd64 +RUNTIME_PROFILE=docker + +KUBERNETES_VERSION=1.36.4 +SEALOS_VERSION=5.1.1 +DOCKER_VERSION=28.5.2 +CRI_DOCKERD_VERSION=0.4.4 +CRICTL_VERSION=1.36.0 +REGISTRY_VERSION=3.1.1 +DOCKER_BUNDLED_CONTAINERD_VERSION=1.7.28 +DOCKER_BUNDLED_RUNC_VERSION=1.3.3 + +# Provenance of the verified cache release consumed by this runtime. +CACHE_BUILD_RUN_ID=34090624151 +CACHE_BUILD_GIT_SHA=cc49c691384e488c57f7d2871e41491c384856bb + +SEALOS_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:sealos-v5.1.1-amd64 +SEALOS_CACHE_DIGEST=sha256:7cf5261c3b3f7db31e3f29e8452b82bb0b05b016aeb237b88d69b0c6d2a10523 + +DOCKER_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:cri-docker-v28.5.2-cridockerd-v0.4.4-registry-v3.1.1-amd64 +DOCKER_CACHE_DIGEST=sha256:6dc872804628b98223e56e8e0ef7464ae14310f62c4f790601d9572305bd0a1c + +CRICTL_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:cri-v1.36-amd64 +CRICTL_CACHE_DIGEST=sha256:3f672ab56c5a0687292fadb6fc8f6fd351ae82e36a7d5b98ff5440d93000e2b6 + +KUBERNETES_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:kubernetes-v1.36.4-amd64 +KUBERNETES_CACHE_DIGEST=sha256:2e7ac7992dcfe6df24970ad9531d552b4995cbd17dea712e3d15769d0c3172f3 + +# Source-artifact checks retained as defense in depth after OCI digest pinning. +DOCKER_SOURCE_SHA256=ea90cfd12e1eeb12aa1c971741adb8bd4ed88e2a574eaac13f5029a1dbc6300d +CRI_DOCKERD_SOURCE_SHA256=109b7540053a507dd85ad7b9fee9cee9caae55feedcc28365d3e7ab4fb2172d5 +CRICTL_SOURCE_SHA256=83855e114566a8a8c44c548d515670f51de3a5e1da8b2effb59870e2f10c25a3 +KUBERNETES_IMAGE_LIST_SHA256=3ba2f001a417a25085fd9d44407730d49d539d5122fac44530f77658df8bced1 + +KUBEADM_ETCD_VERSION=3.6.8-0 +KUBEADM_COREDNS_VERSION=v1.14.2 +KUBEADM_PAUSE_VERSION=3.10.2 + +# Sealos HA helper. r1 records the resolved registry digest during the runtime build; +# it will be digest-pinned in the runtime build lock after the first VERIFIED build. +LVSCARE_IMAGE=ghcr.io/labring/lvscare:v5.1.1 + +FINAL_IMAGE=ghcr.io/archinfra/kubernetes-runtime/kubernetes-docker:v1.36.4-r1-amd64 diff --git a/archinfra/scripts/build-k8s-docker.sh b/archinfra/scripts/build-k8s-docker.sh new file mode 100644 index 0000000..5914070 --- /dev/null +++ b/archinfra/scripts/build-k8s-docker.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RELEASE_FILE="${RELEASE_FILE:-$REPO_ROOT/archinfra/releases/v1.36.4-r1.env}" +OUT_DIR="${OUT_DIR:-$REPO_ROOT/out}" +WORK_DIR="${WORK_DIR:-${RUNNER_TEMP:-/tmp}/archinfra-kubernetes-runtime}" + +[[ -f "$RELEASE_FILE" ]] || { echo "ERROR: release file not found: $RELEASE_FILE" >&2; exit 1; } +# shellcheck disable=SC1090 +source "$RELEASE_FILE" + +GHCR_USER="${GHCR_USER:?GHCR_USER is required}" +GHCR_TOKEN="${GHCR_TOKEN:?GHCR_TOKEN is required}" + +mkdir -p "$OUT_DIR" "$WORK_DIR" +ROOT="$WORK_DIR/rootfs" +rm -rf "$ROOT" +mkdir -p "$ROOT" + +log() { printf '[archinfra-runtime] %s\n' "$*"; } +fail() { echo "ERROR: $*" >&2; exit 1; } +require_cmd() { + local cmd + for cmd in "$@"; do + command -v "$cmd" >/dev/null 2>&1 || fail "required command not found: $cmd" + done +} +sha256_of() { sudo sha256sum "$1" | awk '{print $1}'; } +assert_sha256() { + local expected="$1" file="$2" actual + actual="$(sha256_of "$file")" + [[ "$actual" == "$expected" ]] || fail "SHA256 mismatch: $file expected=$expected actual=$actual" + log "sha256 OK: $(basename "$file") = $actual" +} +verify_registry_digest() { + local image="$1" expected="$2" actual + actual="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$image" | jq -r '.Digest')" + log "registry digest: $image -> $actual" + [[ "$actual" == "$expected" ]] || fail "registry digest mismatch: image=$image expected=$expected actual=$actual" +} + +require_cmd buildah skopeo jq sha256sum tar file sed grep install +[[ "$ARCH" == amd64 ]] || fail "r1 supports amd64 only (got: $ARCH)" +[[ "$RUNTIME_PROFILE" == docker ]] || fail "r1 supports Docker profile only (got: $RUNTIME_PROFILE)" + +# Canonical provenance references. Buildah 1.33 on Ubuntu 24.04 cannot reliably +# consume BuildKit OCI indexes with SBOM/provenance by repo@digest, so the release +# builder verifies each tag's live registry digest first and then pulls that verified tag. +# The cache tags are Archinfra-controlled release tags, not floating aliases. +SEALOS_CACHE_REF="${SEALOS_CACHE_IMAGE}@${SEALOS_CACHE_DIGEST}" +DOCKER_CACHE_REF="${DOCKER_CACHE_IMAGE}@${DOCKER_CACHE_DIGEST}" +CRICTL_CACHE_REF="${CRICTL_CACHE_IMAGE}@${CRICTL_CACHE_DIGEST}" +KUBERNETES_CACHE_REF="${KUBERNETES_CACHE_IMAGE}@${KUBERNETES_CACHE_DIGEST}" + +cleanup() { + sudo buildah umount --all >/dev/null 2>&1 || true + sudo buildah rm --all >/dev/null 2>&1 || true +} +trap cleanup EXIT + +log "release=$RELEASE_VERSION arch=$ARCH profile=$RUNTIME_PROFILE" +log "cache build git sha=$CACHE_BUILD_GIT_SHA run=$CACHE_BUILD_RUN_ID" + +# Authenticate rootful Buildah because Sealos also uses the root container storage. +printf '%s' "$GHCR_TOKEN" | sudo buildah login --username "$GHCR_USER" --password-stdin ghcr.io >/dev/null + +# Fail closed before any cache content is consumed. +verify_registry_digest "$SEALOS_CACHE_IMAGE" "$SEALOS_CACHE_DIGEST" +verify_registry_digest "$DOCKER_CACHE_IMAGE" "$DOCKER_CACHE_DIGEST" +verify_registry_digest "$CRICTL_CACHE_IMAGE" "$CRICTL_CACHE_DIGEST" +verify_registry_digest "$KUBERNETES_CACHE_IMAGE" "$KUBERNETES_CACHE_DIGEST" + +mount_cache() { + local image="$1" cid + cid="$(sudo buildah from --platform "linux/$ARCH" "$image")" + sudo buildah mount "$cid" +} + +log "mount digest-verified cache artifacts" +MOUNT_SEALOS="$(mount_cache "$SEALOS_CACHE_IMAGE")" +MOUNT_DOCKER="$(mount_cache "$DOCKER_CACHE_IMAGE")" +MOUNT_CRICTL="$(mount_cache "$CRICTL_CACHE_IMAGE")" +MOUNT_KUBE="$(mount_cache "$KUBERNETES_CACHE_IMAGE")" + +# Rootful Buildah mounts live below /var/lib/containers and must be consumed as root. +# Keep the mount read-only from the builder's point of view: all writes go to $ROOT. +# Defense-in-depth: verify the key bytes inside the digest-verified cache images. +assert_sha256 "$DOCKER_SOURCE_SHA256" "$MOUNT_DOCKER/cri/docker.tgz" +assert_sha256 "$CRI_DOCKERD_SOURCE_SHA256" "$MOUNT_DOCKER/cri/cri-dockerd.tgz" +assert_sha256 "$CRICTL_SOURCE_SHA256" "$MOUNT_CRICTL/cri/crictl.tar.gz" +assert_sha256 "$KUBERNETES_IMAGE_LIST_SHA256" "$MOUNT_KUBE/images/shim/DefaultImageList" + +sudo "$MOUNT_KUBE/bin/kubeadm" version -o short | grep -Fx "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubeadm in cache is not v$KUBERNETES_VERSION" +sudo "$MOUNT_KUBE/bin/kubelet" --version | grep -F "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubelet in cache is not v$KUBERNETES_VERSION" +sudo "$MOUNT_KUBE/bin/kubectl" version --client=true 2>/dev/null | grep -F "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubectl in cache is not v$KUBERNETES_VERSION" + +# Resolve lvscare once for this build and record the immutable digest in provenance. +# Sealos' image save path cannot consume tag@digest here, so we verify the versioned +# tag first and pass that verified tag to Sealos while retaining the immutable ref. +LVSCARE_DIGEST="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$LVSCARE_IMAGE" | jq -r '.Digest')" +[[ "$LVSCARE_DIGEST" == sha256:* ]] || fail "unable to resolve lvscare digest for $LVSCARE_IMAGE" +LVSCARE_REF="${LVSCARE_IMAGE}@${LVSCARE_DIGEST}" +LVSCARE_RUNTIME_IMAGE="$LVSCARE_IMAGE" +log "lvscare verified: $LVSCARE_RUNTIME_IMAGE -> $LVSCARE_DIGEST" + +# Assemble the Docker rootfs overlay while retaining the upstream Sealos lifecycle scripts/configuration. +cp -a "$REPO_ROOT/docker/." "$ROOT/" +cp -a "$REPO_ROOT/registry/." "$ROOT/" +cp -a "$REPO_ROOT/k8s/." "$ROOT/" +mkdir -p "$ROOT/bin" "$ROOT/cri" "$ROOT/opt" "$ROOT/images/shim" "$ROOT/etc/archinfra" + +# Use the exact Sealos binary from the verified cache as the builder and ship its runtime helpers. +sudo install -m 0755 "$MOUNT_SEALOS/sealos/sealos" /usr/local/bin/sealos +sudo install -m 0755 "$MOUNT_SEALOS/sealos/image-cri-shim" "$ROOT/cri/image-cri-shim" +sudo install -m 0755 "$MOUNT_SEALOS/sealos/sealctl" "$ROOT/opt/sealctl" +sealos version | grep -F "$SEALOS_VERSION" >/dev/null || fail "Sealos builder is not $SEALOS_VERSION" + +# Docker profile payload. +sudo cp -a "$MOUNT_DOCKER/cri/docker.tgz" "$ROOT/cri/docker.tgz" +sudo cp -a "$MOUNT_DOCKER/cri/cri-dockerd.tgz" "$ROOT/cri/cri-dockerd.tgz" +sudo install -m 0755 "$MOUNT_DOCKER/cri/registry" "$ROOT/cri/registry" +sudo install -m 0755 "$MOUNT_DOCKER/cri/conntrack" "$ROOT/bin/conntrack" +sudo install -m 0755 "$MOUNT_DOCKER/cri/lsof" "$ROOT/opt/lsof" + +# CRI client aligned with Kubernetes 1.36. +sudo tar -xzf "$MOUNT_CRICTL/cri/crictl.tar.gz" -C "$ROOT/bin" crictl +sudo chmod 0755 "$ROOT/bin/crictl" + +# Return ownership of the assembled overlay to the runner before generating metadata. +sudo chown -R "$(id -u):$(id -g)" "$ROOT" + +# Keep a human-readable immutable release record inside the final Cluster Image. +cat > "$ROOT/etc/archinfra/release.env" < "$ROOT/etc/archinfra/cache-lock.env" < "$ROOT/Kubefile.tmp" +mv "$ROOT/Kubefile.tmp" "$ROOT/Kubefile" + +grep -F "FROM $KUBERNETES_CACHE_IMAGE" "$ROOT/Kubefile" >/dev/null \ + || fail "Kubefile base image was not set to verified Kubernetes cache" + +pauseImage="$(sudo grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" +[[ -n "$pauseImage" ]] || fail "pause image not found in Kubernetes image list" +echo "$LVSCARE_RUNTIME_IMAGE" > "$ROOT/images/shim/LvscareImageList" + +# Normalize executable bits for rootfs scripts and binaries. +find "$ROOT" -type f -exec file {} \; \ + | grep -E '(executable,|/ld-)' \ + | awk -F: '{print $1}' \ + | grep -vE '\.so' \ + | xargs -r chmod a+x + +log "build final Sealos Cluster Image: $FINAL_IMAGE" +sudo sealos build \ + --platform "linux/$ARCH" \ + --label "sealos.io.type=rootfs" \ + --label "sealos.io.version=v1beta1" \ + --label "version=v$KUBERNETES_VERSION" \ + --label "image=$LVSCARE_RUNTIME_IMAGE" \ + --label "io.archinfra.release=$RELEASE_VERSION" \ + --label "io.archinfra.runtime=$RUNTIME_PROFILE" \ + --label "io.archinfra.cache.build-sha=$CACHE_BUILD_GIT_SHA" \ + --label "io.archinfra.cache.kubernetes=$KUBERNETES_CACHE_DIGEST" \ + --label "io.archinfra.cache.docker=$DOCKER_CACHE_DIGEST" \ + --label "io.archinfra.cache.crictl=$CRICTL_CACHE_DIGEST" \ + --label "io.archinfra.cache.sealos=$SEALOS_CACHE_DIGEST" \ + --env "defaultVIP=10.103.97.2" \ + --env "sandboxImage=${pauseImage#*/}" \ + -t "$FINAL_IMAGE" \ + "$ROOT" + +# Verify target architecture before publishing. +arch_inspect="$(sudo buildah inspect "$FINAL_IMAGE" | jq -r '.OCIv1.architecture // .Docker.architecture // empty')" +[[ "$arch_inspect" == "$ARCH" ]] || fail "final image architecture mismatch: expected=$ARCH actual=$arch_inspect" + +log "push final Cluster Image" +sudo sealos login -u "$GHCR_USER" -p "$GHCR_TOKEN" ghcr.io >/dev/null +sudo sealos push "$FINAL_IMAGE" +sudo sealos logout ghcr.io >/dev/null || true + +FINAL_DIGEST="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$FINAL_IMAGE" | jq -r '.Digest')" +[[ "$FINAL_DIGEST" == sha256:* ]] || fail "unable to resolve final image digest" + +PROVENANCE="$OUT_DIR/runtime-v1.36.4-r1.provenance.env" +cat > "$PROVENANCE" </dev/null 2>&1 || exit source common.sh + +# docker.service expects docker group and containerd runtime binaries. +# k8s 1.36 + cri-dockerd requires a complete docker runtime stack. +getent group docker >/dev/null 2>&1 || groupadd -r docker + if ! command_exists docker; then lsb_dist=$(get_distribution) lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" @@ -30,10 +35,16 @@ if ! command_exists docker; then tar --strip-components=1 -zxvf ../cri/docker.tgz -C /usr/bin # shellcheck disable=SC2046 chmod a+x $(tar -tf ../cri/docker.tgz | while read -r binary; do echo "/usr/bin/${binary##*/}"; done | xargs) + + # verify required runtime binaries exist before starting docker + command_exists containerd || { echo "containerd binary missing"; exit 1; } + command_exists containerd-shim-runc-v2 || { echo "containerd-shim-runc-v2 binary missing"; exit 1; } + systemctl enable docker.service systemctl restart docker.service cp ../etc/daemon.json /etc/docker fi + systemctl daemon-reload systemctl restart docker.service check_status docker