From 045d9e531c88ce7461c7352af87aa7971a682c0c Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 18 Aug 2026 19:08:40 +0100 Subject: [PATCH 01/14] tidy and consolidate CI setup distinct CI jobs that can be reused. - separate build_image related actions into two distinct CI jobs. Create dev image and Create release image Create dev image runs when a PR is created - to build images and upload them to harbor in the stfc-cloud-staging project. Create release image happens when merged into master we also allow building locally using scripts in .github/scripts/build_dev_image.sh and .github/scripts/build_release_image.sh for local testing See updated readme on how to use the new CI jobs Motivation: - build jobs and CI are complex - there's a lot of repeated code between CI jobs - ultimately performing similar jobs: building dev and release images and uploading to harbor. # Changelog simplified the CI such that: - have 2 scripts to handle building and pushing images - used by github action and can be invoked locally for testing. Easy to test the scripts outside of github actions - created a shared action that builds and pushes a dev image when a PR is opened that changes the contents of a image subdirectory. - created an action that build a new release image when a PR is merged to master - modified existing CI that would regularly publish a PR bumping the patch version for rabbit-consumer docker image. Made this apply to all docker images so that we can force dependencies to updated and the docker build step - simplified individual image CI jobs to just testing and linting - not also building the images. This reduces duplicated code. --- .github/scripts/build_dev_image.sh | 49 +++++ .github/scripts/build_image.sh | 49 +++++ .github/scripts/build_release_image.sh | 59 ++++++ .github/scripts/bump_patch_version.sh | 29 +++ .github/scripts/utils.sh | 50 +++++ .github/workflows/bin/version_increment.sh | 20 -- .github/workflows/build_images.yaml | 111 ---------- .github/workflows/cloud_chatops.yaml | 160 --------------- .github/workflows/cloud_monitoring.yaml | 191 ------------------ .github/workflows/codeql.yaml | 77 +++---- .github/workflows/create_dev_image.yaml | 168 +++++++++++++++ .github/workflows/create_release_image.yaml | 109 ++++++++++ .github/workflows/patch_bump_pr.yaml | 86 ++++++++ .github/workflows/rabbit_consumer.yaml | 127 ------------ .../rabbit_consumer_chart_schedule.yaml | 93 --------- .../test_and_lint_cloud_chatops.yaml | 63 ++++++ .../test_and_lint_cloud_monitoring.yaml | 57 ++++++ .../test_and_lint_rabbit_consumer.yaml | 60 ++++++ .../test_and_lint_username_service.yaml | 47 +++++ .github/workflows/username_service.yaml | 117 ----------- README.md | 45 ++++- cloud-monitoring/version.txt | 1 + .../requirements-test.txt | 3 +- stfc-username-service/version.txt | 2 +- 24 files changed, 905 insertions(+), 868 deletions(-) create mode 100644 .github/scripts/build_dev_image.sh create mode 100644 .github/scripts/build_image.sh create mode 100644 .github/scripts/build_release_image.sh create mode 100755 .github/scripts/bump_patch_version.sh create mode 100644 .github/scripts/utils.sh delete mode 100755 .github/workflows/bin/version_increment.sh delete mode 100644 .github/workflows/build_images.yaml delete mode 100644 .github/workflows/cloud_chatops.yaml delete mode 100644 .github/workflows/cloud_monitoring.yaml create mode 100644 .github/workflows/create_dev_image.yaml create mode 100644 .github/workflows/create_release_image.yaml create mode 100644 .github/workflows/patch_bump_pr.yaml delete mode 100644 .github/workflows/rabbit_consumer.yaml delete mode 100644 .github/workflows/rabbit_consumer_chart_schedule.yaml create mode 100644 .github/workflows/test_and_lint_cloud_chatops.yaml create mode 100644 .github/workflows/test_and_lint_cloud_monitoring.yaml create mode 100644 .github/workflows/test_and_lint_rabbit_consumer.yaml create mode 100644 .github/workflows/test_and_lint_username_service.yaml delete mode 100644 .github/workflows/username_service.yaml create mode 100644 cloud-monitoring/version.txt diff --git a/.github/scripts/build_dev_image.sh b/.github/scripts/build_dev_image.sh new file mode 100644 index 00000000..db0d3967 --- /dev/null +++ b/.github/scripts/build_dev_image.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Build a dev image. +# +# Tag format: dev-, e.g. 1.0.0-dev-ase241f +# +# Usage: build_dev_image.sh [sha] +# sha defaults to HEAD. +# +# Env: +# REGISTRY, IMAGE_NAMESPACE required +# PUSH=true push; otherwise the image is loaded locally +# BUILD_CACHE=1 use the registry build cache +# +# Outputs: +# image_path full image reference +# dockerfile path to the image_name's Dockerfile +# +# Local run: +# REGISTRY=harbor.stfc.ac.uk PUSH=false IMAGE_NAMESPACE=stfc-cloud-staging \ +# .github/scripts/build-release-image.sh service-a + +set -euo pipefail + +# shellcheck source=.github/scripts/_common.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" + +# shellcheck source=.github/scripts/build_image.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/build_image.sh" + + +image_path=${1:?usage: build_dev_image.sh [sha]} +sha=${2:-$(git rev-parse HEAD)} +require REGISTRY IMAGE_NAMESPACE + +version=$(read_version "$image_path") +name=$(get_image_name "$image_path") +dockerfile=$(find_dockerfile "$image_path") +short=${sha:0:7} + +image="$REGISTRY/$IMAGE_NAMESPACE/$name:${version}dev.${short}" + +output image "$image" +output dockerfile "$dockerfile" + +build_image "$image_path" "$dockerfile" "$image" + +info "built $image" +summary "dev image: \`$image\`" diff --git a/.github/scripts/build_image.sh b/.github/scripts/build_image.sh new file mode 100644 index 00000000..8eafbb7f --- /dev/null +++ b/.github/scripts/build_image.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Helper function to build an image. +# +# build_image.sh [tag...] +# +# Env: +# +# PUSH=true push to remote registry; +# +# BUILD_CACHE=1 read (and, when pushing, write) a :buildcache tag alongside +# the image. Needs a docker-container buildx driver. +# docker/setup-buildx-action CI job builds this. +# + +build_image() { + local context=$1 dockerfile=$2 + + # tags are positional args - provide at least one + shift 2 + [ $# -gt 0 ] || fail "build_image: at least one tag is required" + + local args=(buildx build "$context" --file "$dockerfile") + + local tag + for tag in "$@"; do + args+=(--tag "$tag") + done + + if [ "${BUILD_CACHE:-0}" = "1" ]; then + local cache_ref="${1%:*}:buildcache" + args+=(--cache-from "type=registry,ref=$cache_ref") + if [ "${PUSH:-false}" = "true" ]; then + args+=(--cache-to "type=registry,ref=$cache_ref,mode=max") + fi + fi + + if [ "${PUSH:-false}" = "true" ]; then + args+=(--push) + else + args+=(--load) + info "PUSH is not 'true' — building locally, not pushing" + fi + + info "+ docker ${args[*]}" + docker "${args[@]}" +} diff --git a/.github/scripts/build_release_image.sh b/.github/scripts/build_release_image.sh new file mode 100644 index 00000000..a15bb2bd --- /dev/null +++ b/.github/scripts/build_release_image.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Build a release image, tagged with latest version. +# +# If the git tag already exists then does nothing. +# +# Usage: build_release_image.sh +# +# Env: +# REGISTRY, IMAGE_NAMESPACE required +# PUSH=true push; otherwise images are loaded locally +# BUILD_CACHE=1 use the registry build cache +# +# Outputs: +# skip "true" or "false" - if git-tag exists matching version, returns true +# tag git tag to set (for CI), - +# image image reference +# latest :latest image reference +# dockerfile path to the image_path's Dockerfile +# +# Local run: +# REGISTRY=harbor.stfc.ac.uk PUSH=false IMAGE_NAMESPACE=stfc-cloud \ +# .github/scripts/build-release-image.sh service-a + +# shellcheck source=.github/scripts/utils +source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" + +# shellcheck source=.github/scripts/build_image.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/build_image.sh" + +image_path=${1:?usage: build-release-image.sh } +require REGISTRY IMAGE_NAMESPACE + +version=$(read_version "$image_path") +name=$(image_name "$image_path") +tag="$name-$version" + +# if the git-tag already exists - do nothing as image likely already exists in registry +if git rev-parse -q --verify "refs/tags/$tag" > /dev/null; then + info "tag $tag already exists — nothing to release - did you remember to bump the version in version.txt?" + output skip true + exit 0 +fi + +dockerfile=$(find_dockerfile "$image_path") +image="$REGISTRY/$IMAGE_NAMESPACE/$name:$version" +latest="$REGISTRY/$IMAGE_NAMESPACE/$name:latest" + +output skip false +output tag "$tag" +output dockerfile "$dockerfile" +output image "$image" +output latest "$latest" + +build_image "$image_path" "$dockerfile" "$image" "$latest" + +info "built $image" +summary "release image: \`$image\`" \ No newline at end of file diff --git a/.github/scripts/bump_patch_version.sh b/.github/scripts/bump_patch_version.sh new file mode 100755 index 00000000..a33ce22b --- /dev/null +++ b/.github/scripts/bump_patch_version.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Bump the patch version of a given version.txt +# Usage: bump_patch_version.sh + +version_file="${1:-}" + +if [[ -z "$version_file" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if [[ ! -f "$version_file" ]]; then + echo "Error: file does not exist: $version_file" >&2 + exit 1 +fi + +version=$(<"$version_file") + + +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: $version_file must contain exactly one version in MAJOR.MINOR.PATCH format" >&2 + exit 1 +fi + +IFS='.' read -r major minor patch <<< "$version" + +printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))" > "$version_file" diff --git a/.github/scripts/utils.sh b/.github/scripts/utils.sh new file mode 100644 index 00000000..eb93ae38 --- /dev/null +++ b/.github/scripts/utils.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Shared helpers. Sourced by the other scripts, not run directly. + +# allows these scripts to be used by github runners +# if run locally will go back to stdout and summaries discarded +: "${GITHUB_OUTPUT:=/dev/stdout}" +: "${GITHUB_STEP_SUMMARY:=/dev/null}" + +# Emit a step output (key=value). +output() { printf '%s=%s\n' "$1" "$2" >> "$GITHUB_OUTPUT"; } + +# Append a line to the job summary. +summary() { printf '%s\n' "$*" >> "$GITHUB_STEP_SUMMARY"; } + +# Logging. All to stderr. +info() { printf '%s\n' "$*" >&2; } +warn() { printf '::warning::%s\n' "$*" >&2; } +fail() { printf '::error::%s\n' "$*" >&2; exit 1; } + +# The image name for a image_path is just its directory name. +get_image_name() { basename "$1"; } + +# Assert that the named variables are set and non-empty. +require() { + local var + for var in "$@"; do + [ -n "${!var:-}" ] || fail "$var is not set" + done +} + +# Read and validate a image_path's version.txt. Prints the version. +read_version() { + local image_path=$1 file version + file="$image_path/version.txt" + [ -f "$file" ] || fail "$file not found" + version=$(tr -d '[:space:]' < "$file") + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || fail "$file is '$version', expected MAJOR.MINOR.PATCH" + printf '%s' "$version" +} + +# Locate a image_path's Dockerfile, at any depth. Prints the path. +find_dockerfile() { + local image_path=$1 dockerfile + dockerfile=$(find "$image_path" -name Dockerfile | head -n1) + [ -n "$dockerfile" ] || fail "no Dockerfile found under $image_path" + printf '%s' "$dockerfile" +} \ No newline at end of file diff --git a/.github/workflows/bin/version_increment.sh b/.github/workflows/bin/version_increment.sh deleted file mode 100755 index 57bbc186..00000000 --- a/.github/workflows/bin/version_increment.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/bash - -set -euo pipefail - -#reads the version.txt file and sets the version to the current version -version=`cat ~0/openstack-rabbit-consumer/version.txt` - -# cuts the $version variable into major, minor and patch numbers, removing the fullstop -major=$(echo $version | cut -f1 -d.) -minor=$(echo $version | cut -f2 -d.) -patch=$(echo $version | cut -f3 -d.) - -#increments the patch by 1 -patch=$((patch + 1)) - -#concatenate the version -newversion="$major.$minor.$patch" - -#overwrites the version.txt file with new new version -printf "$newversion" > ~0/openstack-rabbit-consumer/version.txt diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml deleted file mode 100644 index 5d268a26..00000000 --- a/.github/workflows/build_images.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: docker_images -on: - push: - paths: - - "exporters/**" - - ".github/workflows/codeql.yaml" - - ".github/workflows/build_images.yaml" - pull_request: - paths: - - "exporters/**" - - ".github/workflows/codeql.yaml" - - ".github/workflows/build_images.yaml" - schedule: [ { cron: "14 14 * * TUE" } ] # Every Tuesday at 14:14 -jobs: - setup: - if: github.event_name != 'pull_request' || - !contains('OWNER,MEMBER,COLLABORATOR', - github.event.pull_request.author_association) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} # fix SHA - - name: List all images - id: metadata - run: | - echo "images=$( - find -name Dockerfile | sed -r 's#\./(.+)/Dockerfile#\1#g' | jq -Rsc 'split("\n")[:-1]' - )" >> $GITHUB_OUTPUT - echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - outputs: - images: ${{ steps.metadata.outputs.images }} - sha: ${{ steps.metadata.outputs.sha_short }} - build: - needs: setup - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: ${{ fromJson(needs.setup.outputs.images) }} - steps: - - name: Maximize build space # free up space for building images - uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - build-mount-path: "/var/lib/docker/" - - name: Restart docker # restart the docker service - run: sudo service docker restart - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - name: Login to DockerHub # increase pull rate limit - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to Harbor Staging - if: ${{ github.ref != 'refs/heads/master' }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - name: Login to Harbor - if: ${{ github.ref == 'refs/heads/master' }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - name: Build & push to staging - if: ${{ github.ref != 'refs/heads/master' }} - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:${{ matrix.image }}" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/${{ matrix.image }}:${{ - needs.setup.outputs.sha }}" - - name: Inform of tagged name - if: ${{ github.ref != 'refs/heads/master' }} - run: echo "::notice title=published::harbor.stfc.ac.uk/stfc-cloud-staging/${{ - matrix.image }}:${{ needs.setup.outputs.sha }}" - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat ${{ matrix.image }}/version.txt)" >> $GITHUB_OUTPUT - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - '${{ matrix.image }}/version.txt' - - name: Build & push to prod - if: ${{ github.ref == 'refs/heads/master' }} - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - push: true - context: "{{defaultContext}}:${{ matrix.image }}" - tags: "harbor.stfc.ac.uk/stfc-cloud/${{ matrix.image }}:v${{ - steps.release_tag.outputs.version }}" - finished: - # convenient single job name to apply branch protection to - needs: build - runs-on: ubuntu-latest - steps: [ { run: true } ] diff --git a/.github/workflows/cloud_chatops.yaml b/.github/workflows/cloud_chatops.yaml deleted file mode 100644 index 0c3308d5..00000000 --- a/.github/workflows/cloud_chatops.yaml +++ /dev/null @@ -1,160 +0,0 @@ -name: Cloud Chatops - -permissions: - contents: read - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/cloud_chatops.yaml" - - "cloud-chatops/**" - -jobs: - test_and_lint: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ['ubuntu-22.04','ubuntu-latest'] - python-version: [ "3.12", "3.x" ] - steps: - # https://github.com/actions/checkout/releases - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - # https://github.com/actions/setup-python/releases - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - cd cloud-chatops - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Analyse with pylint - run: | - cd cloud-chatops - pylint . --recursive=true --rcfile=.pylintrc - - - name: Run tests - run: | - cd cloud-chatops - python3 -m pytest tests - - - name: Run tests and collect coverage - run: | - cd cloud-chatops - python3 -m pytest tests --cov-report xml:coverage.xml --cov - - - name: Upload coverage to Codecov - # https://github.com/codecov/codecov-action/releases - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - token: ${{secrets.CODECOV_TOKEN}} - files: cloud-chatops/coverage.xml - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - # https://github.com/docker/setup-buildx-action/releases - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - # https://github.com/docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2/releases - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - # https://github.com/docker/build-push-action/releases - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-chatops" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/cloud-chatops:${{ steps.commit_sha.outputs.sha_short }}" - - check_version_update: - runs-on: ubuntu-latest - steps: - - name: Checkout master - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: 'master' - path: 'main' - - - name: Checkout current working branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - path: 'branch' - - - name: Compare versions - if: ${{ github.ref != 'refs/heads/master' }} - uses: stfc/check-version-action@main - with: - app_version_path: "cloud-chatops/version.txt" - docker_compose_path: "cloud-chatops/docker-compose.yaml" - labels: ${{ toJSON(github.event.pull_request.labels.*.name) }} - - - name: Log App Success - if: ${{ env.app_updated == 'true' }} - run: | - echo "App version has been updated correctly!" - - name: Log Compose Success - if: ${{ env.compose_updated == 'true' }} - run: | - echo "Compose version has been updated correctly!" - - push_prod_image_harbor: - runs-on: ubuntu-latest - needs: [test_and_lint, check_version_update] - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat cloud-chatops/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'cloud-chatops/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-chatops" - tags: "harbor.stfc.ac.uk/stfc-cloud/cloud-chatops:${{ steps.release_tag.outputs.version }}" diff --git a/.github/workflows/cloud_monitoring.yaml b/.github/workflows/cloud_monitoring.yaml deleted file mode 100644 index a92bae33..00000000 --- a/.github/workflows/cloud_monitoring.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: Cloud Monitoring - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/cloud_monitoring.yaml" - - "cloud-monitoring/**" - -jobs: - test_and_lint: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-22.04", "ubuntu-latest"] - python-version: ["3.12", "3.x"] - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - cd cloud-monitoring - python -m pip install --upgrade pip - # Install the package with all optional dependencies for testing - pip install -e ".[test,dev]" - - - name: Check code formatting with Black - run: | - cd cloud-monitoring - black --check --diff cloudmon/ tests/ - - - name: Analyse with pylint - run: | - cd cloud-monitoring - pylint . --recursive=true - - - name: Run tests with Pytest - run: | - cd cloud-monitoring - python -m pytest tests --cov-report xml:coverage.xml --cov - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - token: ${{secrets.CODECOV_TOKEN}} - files: cloud-chatops/coverage.xml - - build_and_push_dev_image: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Checkout the code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-monitoring" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/cloud-monitoring:${{ steps.commit_sha.outputs.sha_short }}" - - build_and_push_prod_image: - runs-on: ubuntu-latest - needs: test_and_lint - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 # Needed for tags - - - name: Check if relevant files changed - id: changes - run: | - # check if any python package files have been changed - CHANGED=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -E '^cloud-monitoring/(cloudmon/.*|tests/.*|pyproject\.toml)$' || true) - if [ -z "$CHANGED" ]; then - echo "changed=false" >> $GITHUB_OUTPUT - else - echo "changed=true" >> $GITHUB_OUTPUT - fi - - - name: Exit if no relevant changes - if: steps.changes.outputs.changed == 'false' - run: echo "No relevant changes, no version bump needed." - - - name: Get latest cloud-monitoring tag - if: steps.changes.outputs.changed == 'true' - id: get_tag - run: | - tag=$(git tag --list 'cloud-monitoring-*' | sort -V | tail -n 1) - if [ -z "$tag" ]; then - tag="cloud-monitoring-0.0.0" - fi - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - - name: Get version from pyproject.toml - id: get_version - run: | - version=$(grep -m1 '^version' cloud-monitoring/pyproject.toml | sed -E 's/version *= *"([^"]+)"/\1/') - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Check if bump is needed - id: check_bump - if: steps.changes.outputs.changed == 'true' - run: | - tag_version="${{ steps.get_tag.outputs.tag#cloud-monitoring- }}" - current_version="${{ steps.get_version.outputs.version }}" - - if [ "$tag_version" = "$current_version" ]; then - echo "same=true" >> "$GITHUB_OUTPUT" - else - echo "same=false" >> "$GITHUB_OUTPUT" - fi - - - name: Bump version (patch) - if: steps.check_bump.outputs.same == 'true' - uses: callowayproject/bump-my-version@e6ecdc3e573698766cd6c2112faeda50bcc2e56a - id: bump - with: - args: patch - github-token: ${{ secrets.GITHUB_TOKEN }} - workdir: cloud-monitoring - - - name: Refresh version after bump - id: final_version - run: | - version=$(grep -m1 '^version' cloud-monitoring/pyproject.toml | sed -E 's/version *= *"([^"]+)"/\1/') - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Create tag for current version - run: | - version="${{ steps.final_version.outputs.version }}" - tag_name="cloud-monitoring-${version}" - - git config user.name "github-actions" - git config user.email "github-actions@github.com" - - # Only tag if it doesn't already exist - if git rev-parse "$tag_name" >/dev/null 2>&1; then - echo "Tag $tag_name already exists. Skipping." - else - git tag "$tag_name" - git push origin "$tag_name" - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-monitoring" - tags: "harbor.stfc.ac.uk/stfc-cloud/cloud-monitoring:${{ steps.final_version.outputs.version }}" diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index f063f3c0..7e4769ff 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -13,11 +13,11 @@ name: "CodeQL Advanced" on: push: - branches: [ "master" ] + branches: ["master"] pull_request: - branches: [ "master" ] + branches: ["master"] schedule: - - cron: '0 12 * * 1' + - cron: "0 12 * * 1" jobs: analyze: @@ -43,8 +43,8 @@ jobs: fail-fast: false matrix: include: - - language: python - build-mode: none + - language: python + build-mode: none # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -54,39 +54,42 @@ jobs: # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Checkout repository + # https://github.com/actions/checkout/releases + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + # https://github.com/github/codeql-action/releases + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa #v4.36.0 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + # https://github.com/github/codeql-action/releases + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa #v4.36.0 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml new file mode 100644 index 00000000..3cf19e50 --- /dev/null +++ b/.github/workflows/create_dev_image.yaml @@ -0,0 +1,168 @@ +name: Create a Dev Image + +on: + pull_request: + branches: + - master + workflow_dispatch: + +env: + REGISTRY: harbor.stfc.ac.uk + IMAGE_NAMESPACE: stfc-cloud-staging + +permissions: + contents: read + packages: write + +jobs: + detect: + runs-on: ubuntu-latest + outputs: + image_paths: ${{ steps.detect.outputs.image_paths }} + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - id: detect + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + + # find changed files + changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + + # for each changed file, find the root directory of the docker image + changed_image_paths=$( + find . \ + -name Dockerfile \ + -not -path './.git/*' \ + -printf '%h\n' | + sed 's|^\./||' | + while read -r dir; do + grep -q "^$dir/" <<< "$changed" && echo "$dir"; + done | + sort -u | + jq -Rc -s 'split("\n") | map(select(length > 0))' + ) + + # each unique path to image directory changed needs a dev images built + echo "image_paths=$changed_image_paths" >> "$GITHUB_OUTPUT" + + printf '%s\n' "$changed_image_paths" + + # version.txt must be bumped, must go up, and must not already be tagged. + version: + needs: detect + if: needs.detect.outputs.image_paths != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: "Check Version Update" + env: + IMAGE_PATH: ${{ matrix.image_path }} + BASE: ${{ github.event.pull_request.base.sha }} + run: | + + new=$(tr -d '[:space:]' < "$IMAGE_PATH/version.txt") + tag="$(basename "$IMAGE_PATH")-$new" + + if [[ ! "$new" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::$IMAGE_PATH/version.txt is '$new', want MAJOR.MINOR.PATCH" + exit 1 + fi + + if git rev-parse -q --verify "refs/tags/$tag" > /dev/null; then + echo "::error::$tag already exists: bump $IMAGE_PATH/version.txt first" + exit 1 + fi + + # A image directory that didn't exist on the base branch is new, so there's + # nothing to compare against. + if ! git cat-file -e "$BASE:$IMAGE_PATH/version.txt" 2> /dev/null; then + echo "$IMAGE_PATH is new at $new" + exit 0 + fi + + old=$(git show "$BASE:$IMAGE_PATH/version.txt" | tr -d '[:space:]') + + if [[ "$old" == "$new" ]]; then + echo "::error::$IMAGE_PATH changed but version.txt is still $old" + exit 1 + fi + + lowest=$(printf '%s\n%s\n' "$old" "$new" | sort -V | head -1) + + if [[ "$lowest" != "$old" ]]; then + echo "::error::$IMAGE_PATH version went backwards: $old -> $new" + exit 1 + fi + + echo "$IMAGE_PATH: $old -> $new" >> "$GITHUB_STEP_SUMMARY" + + # Build + push a dev image — trusted authors only. + # Tag format: dev-, e.g. 1.0.0dev-ase241f + dev-image: + needs: [detect, version] + # The `env` context isn't available in a job-level `if`, so the trusted-bot + # list has to be inline. + if: >- + ${{ needs.detect.outputs.image_paths != '[]' && + (contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) || + contains(fromJSON('["dependabot[bot]"]'), github.event.pull_request.user.login)) }} + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} + steps: + - name: Maximize build space # free up space for building images + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - name: Restart docker # restart the docker service + run: sudo service docker restart + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Login to DockerHub # increase pull rate limit + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Harbor Staging + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.STAGING_HARBOR_USERNAME }} + password: ${{ secrets.STAGING_HARBOR_TOKEN }} + + - env: + IMAGE_PATH: ${{ matrix.image_path }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH: "true" + BUILD_CACHE: "1" + run: bash .github/scripts/build_dev_image.sh "$IMAGE_PATH" "$HEAD_SHA" diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml new file mode 100644 index 00000000..69fe4abd --- /dev/null +++ b/.github/workflows/create_release_image.yaml @@ -0,0 +1,109 @@ +name: Creata a Release Image + +on: + push: + branches: + - master + +# ───────────────────────────────────────────── +# Keep these in sync with pr.yml +# ───────────────────────────────────────────── +env: + REGISTRY: harbor.stfc.ac.uk + IMAGE_NAMESPACE: stfc-cloud + +permissions: + contents: write # needed to push tags + packages: write # needed to push images + +jobs: + detect: + runs-on: ubuntu-latest + outputs: + image_paths: ${{ steps.detect.outputs.image_paths }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: "Find All Charts" + id: detect + env: + BEFORE: ${{ github.event.before }} + run: | + + image_paths=$( + find . \ + -name Dockerfile \ + -not -path './.git/*' \ + -printf '%h\n' | + sed 's|^\./||' | + sort -u | + jq -Rc -s 'split("\n") | map(select(length > 0))' + ) + + echo "image_paths=$image_paths" >> "$GITHUB_OUTPUT" + + echo "$image_paths" + + release: + needs: detect + if: >- + ${{ needs.detect.outputs.image_paths != '[]'}} + + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 1 # tag pushes serialise more cleanly + matrix: + image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} + + steps: + - name: Maximize build space # free up space for building images + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - name: Restart docker # restart the docker service + run: sudo service docker restart + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Login to DockerHub # increase pull rate limit + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Harbor Staging + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.STAGING_HARBOR_USERNAME }} + password: ${{ secrets.STAGING_HARBOR_TOKEN }} + + # No-ops if this version is already tagged. + - id: release + env: + IMAGE_PATH: ${{ matrix.image_path }} + PUSH: "true" + BUILD_CACHE: "1" + run: bash .github/scripts/build_release_image.sh "$IMAGE_PATH" + + # Tag last, so a failed build doesn't leave a tag behind. + - if: steps.release.outputs.skip == 'false' + env: + TAG: ${{ steps.release.outputs.tag }} + run: | + git tag "$TAG" "$GITHUB_SHA" + git push origin "$TAG" diff --git a/.github/workflows/patch_bump_pr.yaml b/.github/workflows/patch_bump_pr.yaml new file mode 100644 index 00000000..32e07cad --- /dev/null +++ b/.github/workflows/patch_bump_pr.yaml @@ -0,0 +1,86 @@ +name: Schedule Image Rebuild + +# This action simply increments the patch version for +# all docker images in this repo and opens a new PR + +# This is so that we update our image dependencies periodically +# currently set for every month + +on: + schedule: + - cron: "0 10 1 * *" # First day of each month at 10.00am + workflow_dispatch: + +permissions: + contents: write # needed to bump versions + packages: write # needed to make branches/PRs + +env: + author: "${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>" + committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + +jobs: + detect: + runs-on: ubuntu-latest + outputs: + images: ${{ steps.detect.outputs.images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - id: detect + name: "Increment patch version for every image in repo" + run: | + set -euo pipefail + image_paths=$(find . -name Dockerfile -printf '%h\n' | sed 's|^\./||' | sort -u ) + echo "images=$(jq -Rc -s 'split("\n") | map(select(length > 0))' <<< "$image_paths")" \ + >> "$GITHUB_OUTPUT" + + bump-chart: + runs-on: ubuntu-latest + needs: detect + strategy: + matrix: + image: ${{ fromJSON(needs.detect.outputs.images) }} + steps: + - name: "Setup Github Token" + # https://github.com/actions/create-github-app-token/releases + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 #v3.2.0 + id: app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # increments ONLY the patch version using the version_incrememnt.sh script and commits the version.txt file to master + - name: Increment Version + env: + VERSION_FILEPATH: ./${{ matrix.image }}/version.txt + run: bash ${{ github.workspace }}/.github/workflows/scripts/bump_patch_version.sh "$VERSION_FILEPATH" + + - name: "Create Pull Request for a patch bump" + id: patch-bump-pr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + env: + pr-title: "Patch bump for ${{ matrix.image }} image" + branch: "${{ matrix.image }}-patch-bump" + with: + token: ${{ steps.app-token.outputs.token }} + commit-message: ${{ env.pr-title }} + author: ${{ env.author }} + committer: ${{ env.committer }} + branch: ${{ env.branch }} + delete-branch: true + title: ${{ env.pr-title }} + labels: | + automated + dependency-update + patch + body: | + Automated PR to bump the patch version for ${{ metrix.image }} + Forces a new build of the image to update dependencies once merged + + **Please test using the dev chart before merging** + --- diff --git a/.github/workflows/rabbit_consumer.yaml b/.github/workflows/rabbit_consumer.yaml deleted file mode 100644 index 59ba0a29..00000000 --- a/.github/workflows/rabbit_consumer.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: Rabbit Consumer - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/rabbit_consumer.yaml" - - "openstack-rabbit-consumer/**" - -jobs: - test_and_lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Required for requests-kerberos - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev --fix-missing - pip install -r openstack-rabbit-consumer/requirements.txt - pip install -r openstack-rabbit-consumer/requirements-test.txt - - - name: Run tests - # Using Python3 to launch the module sets up the Python path for us - run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . - - - name: Analyse with pylint - run: | - cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') - - - name: Prepare coverage - run: | - cd openstack-rabbit-consumer && python -m coverage xml - - - name: Upload coverage to codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - files: openstack-rabbit-consumer/coverage.xml - fail_ci_if_error: true - flags: rabbit_consumer - token: ${{secrets.CODECOV_TOKEN}} - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/openstack-rabbit-consumer:${{ steps.commit_sha.outputs.sha_short }}" - - - name: Inform of tagged name - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud-staging/openstack-rabbit-consumer:${{ steps.commit_sha.outputs.sha_short }}" - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat openstack-rabbit-consumer/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'openstack-rabbit-consumer/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer:v${{ steps.release_tag.outputs.version }}" diff --git a/.github/workflows/rabbit_consumer_chart_schedule.yaml b/.github/workflows/rabbit_consumer_chart_schedule.yaml deleted file mode 100644 index 9382eec0..00000000 --- a/.github/workflows/rabbit_consumer_chart_schedule.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: Rabbit Consumer Schedule - -# Incrementing the patch version on a schedule to pull latest containers dependencies. - -on: - schedule: - - cron: "0 10 1 * *" # First day of each month at 10.00am - -jobs: - test_and_lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - sudo apt-get update --fix-missing - python -m pip install --upgrade pip - # Required for requests-kerberos - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev - pip install -r openstack-rabbit-consumer/requirements.txt - pip install -r openstack-rabbit-consumer/requirements-test.txt - - - name: Run tests - # Using Python3 to launch the module sets up the Python path for us - run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . - - - name: Analyse with pylint - run: | - cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') - - - name: Prepare coverage - run: | - cd openstack-rabbit-consumer && python -m coverage xml - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - #increments ONLY the patch version using the version_incrememnt.sh script and commits the version.txt file to master - - name: Increment Version - run: bash ${{ github.workspace }}/.github/workflows/bin/version_increment.sh - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat openstack-rabbit-consumer/version.txt)" >> $GITHUB_OUTPUT - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer-schedule:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer-schedule:v${{ steps.release_tag.outputs.version }}" - - - name: Commit Changes - uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 - with: - add: ${{ github.workspace }}/openstack-rabbit-consumer/version.txt - default_author: github_actions - committer_name: GitHub Actions - committer_email: actions@github.com - new_branch: rabbitConsumerDependUpdate - - - name: create pull request - run: gh pr create -B master -H rabbitConsumerDependUpdate --title 'Merge rabbitConsumerDependUpdate into master' --body 'Updating the dependencies for Rabbit Consumer Image. Created by Github action' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_and_lint_cloud_chatops.yaml b/.github/workflows/test_and_lint_cloud_chatops.yaml new file mode 100644 index 00000000..e2afefd5 --- /dev/null +++ b/.github/workflows/test_and_lint_cloud_chatops.yaml @@ -0,0 +1,63 @@ +name: Cloud ChatOps Lint & Test + +permissions: + contents: read + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/cloud_chatops.yaml" + - "cloud-chatops/**" + +jobs: + test_and_lint: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-22.04", "ubuntu-latest"] + python-version: ["3.12", "3.x"] + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + cd cloud-chatops + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Check code formatting with Black + run: | + cd cloud-chatops/ + black --check --diff chatops/ tests/ + + - name: Analyse with pylint + run: | + cd cloud-chatops + pylint . --recursive=true --rcfile=.pylintrc + + - name: Run tests + run: | + cd cloud-chatops + python3 -m pytest tests + + - name: Run tests and collect coverage + run: | + cd cloud-chatops + python3 -m pytest tests --cov-report xml:coverage.xml --cov + + - name: Upload coverage to Codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_cloud_monitoring.yaml b/.github/workflows/test_and_lint_cloud_monitoring.yaml new file mode 100644 index 00000000..a4c6b355 --- /dev/null +++ b/.github/workflows/test_and_lint_cloud_monitoring.yaml @@ -0,0 +1,57 @@ +name: Cloud Monitoring Lint & Test + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/cloud_monitoring.yaml" + - "cloud-monitoring/**" + +jobs: + test_and_lint: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-22.04", "ubuntu-latest"] + python-version: ["3.12", "3.x"] + + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + cd cloud-monitoring + python -m pip install --upgrade pip + # Install the package with all optional dependencies for testing + pip install -e ".[test,dev]" + + - name: Check code formatting with Black + run: | + cd cloud-monitoring + black --check --diff cloudmon/ tests/ + + - name: Analyse with pylint + run: | + cd cloud-monitoring + pylint . --recursive=true + + - name: Run tests with Pytest + run: | + cd cloud-monitoring + python -m pytest tests --cov-report xml:coverage.xml --cov + + - name: Upload coverage to Codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_rabbit_consumer.yaml b/.github/workflows/test_and_lint_rabbit_consumer.yaml new file mode 100644 index 00000000..b3323dfb --- /dev/null +++ b/.github/workflows/test_and_lint_rabbit_consumer.yaml @@ -0,0 +1,60 @@ +name: Rabbit Consumer Lint & Test + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/rabbit_consumer.yaml" + - "openstack-rabbit-consumer/**" + +jobs: + test_and_lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Required for requests-kerberos + sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update + sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev --fix-missing + pip install -r openstack-rabbit-consumer/requirements.txt + pip install -r openstack-rabbit-consumer/requirements-test.txt + + - name: Check code formatting with Black + run: | + cd openstack-rabbit-consumer + black --check --diff rabbit_consumer/ tests/ + + - name: Run tests + # Using Python3 to launch the module sets up the Python path for us + run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . + + - name: Analyse with pylint + run: | + cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') + + - name: Prepare coverage + run: | + cd openstack-rabbit-consumer && python -m coverage xml + + - name: Upload coverage to codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + files: openstack-rabbit-consumer/coverage.xml + fail_ci_if_error: true + flags: rabbit_consumer + token: ${{secrets.CODECOV_TOKEN}} diff --git a/.github/workflows/test_and_lint_username_service.yaml b/.github/workflows/test_and_lint_username_service.yaml new file mode 100644 index 00000000..b03cc7ee --- /dev/null +++ b/.github/workflows/test_and_lint_username_service.yaml @@ -0,0 +1,47 @@ +name: Username Service Test and Lint + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/username_service.yaml" + - "stfc-username-service/**" + +jobs: + golangci-lint: + name: runner / golangci-lint + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + # https://github.com/actions/checkout/releases + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: golangci-lint + # https://github.com/golangci/golangci-lint-action/releases + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + working-directory: stfc-username-service/ + + run_tests: + runs-on: ubuntu-latest + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Docker Buildx + # https://github.com/docker/setup-buildx-action/releases + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Run tests in container + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + no-cache: true + cache-to: type=gha,mode=max + target: test-stage + push: false + context: "{{defaultContext}}:stfc-username-service" diff --git a/.github/workflows/username_service.yaml b/.github/workflows/username_service.yaml deleted file mode 100644 index 204acfc9..00000000 --- a/.github/workflows/username_service.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Username Service - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/username_service.yaml" - - "stfc-username-service/**" - -jobs: - golangci-lint: - name: runner / golangci-lint - runs-on: ubuntu-latest - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - - - name: golangci-lint - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a - with: - working-directory: stfc-username-service/ - - run_tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Run tests in container - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - no-cache: true - cache-to: type=gha,mode=max - target: test-stage - push: false - context: "{{defaultContext}}:stfc-username-service" - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: [run_tests, golangci-lint] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:stfc-username-service" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/stfc-username-service:${{ steps.commit_sha.outputs.sha_short }}" - - - name: Inform of tagged name - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud-staging/stfc-username-service:${{ steps.commit_sha.outputs.sha_short }}" - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: [run_tests, golangci-lint] - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat stfc-username-service/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'stfc-username-service/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:stfc-username-service" - tags: "harbor.stfc.ac.uk/stfc-cloud/stfc-username-service:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/stfc-username-service:v${{ steps.release_tag.outputs.version }}" diff --git a/README.md b/README.md index 879686e5..7e57fffd 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,41 @@ -## Building Images Locally +# Cloud Docker Images -`build_images.sh` will build all docker images locally. +This repo stores various Docker images and build scripts that are used by the STFC Cloud -## CI and Automated Builds +# Adding a new docker image -[![CI](https://img.shields.io/github/actions/workflow/status/stfc/cloud-docker-images/build_images.yaml?logo=docker)](https://github.com/stfc/cloud-docker-images/actions/workflows/build_images.yaml?query=branch%3Amaster) -[![codecov](https://codecov.io/gh/stfc/cloud-docker-images/graph/badge.svg?token=BZEBAE0TQD)](https://codecov.io/gh/stfc/cloud-docker-images) +Create a new unique directory and create a `Dockerfile` and `version.txt`. -Images are built: +The `version.txt` will track the latest version of your image for build tagging and version bumps. -- On Pull Requests -- On merge to `master` -- Weekly +If adding source-code - please a new CI github action for performing tests and linting checks -In addition, the successful builds on `master` will be uploaded to [Harbor](https://harbor.stfc.ac.uk) too. + +# Image build scripts and CI + +Docker images are built and pushed to https://harbor.stfc.ac.uk by CI jobs + + +`.github/workflows/create_dev_image.yml` runs when a PR is created and will build and push a dev image following the format: `-dev--` - e.g. `cloud-monitoring-dev-1.0.0-ase34e1` + +`.github/workflows/create_release_image.yml` runs when the PR is merged. A new version - specified in `version.txt` is required to be bumped before merging. Once merged - a new image with that version will be built and pushed to harbor + +both scripts can be invoked locally by utilising `.github/scripts/build_dev_image.sh` and `.github/scripts/build_release_iamge.sh` repectively - see script file for details on how to use it. + +In general, they can be invoked like so: + +```sh +# dev image: dev-, sha defaults to HEAD +REGISTRY=ghcr.io IMAGE_NAMESPACE=my-org ./.github/scripts/build-dev-image.sh cloud-monitoring + +# release image: : and :latest +REGISTRY=ghcr.io IMAGE_NAMESPACE=my-org ./.github/scripts/build-release-image.sh cloud-monitoring +``` + +## Linting and Testing + +Currently linting and testing CI actions are prefixed with `test_and_lint*`. Since each image is built slightly differently, we have separate CI actions for each. These CI jobs do not build images and are run when a PR is opened + +## Regular version bumps + +Each month, we bump the minor version of all our docker images and open a pull request - this forces a new docker image to be built with updated dependencies. diff --git a/cloud-monitoring/version.txt b/cloud-monitoring/version.txt new file mode 100644 index 00000000..3eefcb9d --- /dev/null +++ b/cloud-monitoring/version.txt @@ -0,0 +1 @@ +1.0.0 diff --git a/openstack-rabbit-consumer/requirements-test.txt b/openstack-rabbit-consumer/requirements-test.txt index c74b26f9..f1b5873f 100644 --- a/openstack-rabbit-consumer/requirements-test.txt +++ b/openstack-rabbit-consumer/requirements-test.txt @@ -1,3 +1,4 @@ coverage pylint -pytest \ No newline at end of file +pytest +black \ No newline at end of file diff --git a/stfc-username-service/version.txt b/stfc-username-service/version.txt index 7f207341..7dea76ed 100644 --- a/stfc-username-service/version.txt +++ b/stfc-username-service/version.txt @@ -1 +1 @@ -1.0.1 \ No newline at end of file +1.0.1 From 36be4856cdec5299956e6aba8e00237135935cbd Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 18 Aug 2026 19:49:29 +0100 Subject: [PATCH 02/14] remove nested subdirectory for ipmi-exporter --- exporters/ipmi_exporter/version.txt | 1 - {exporters/ipmi_exporter => ipmi-exporter}/Dockerfile | 0 ipmi-exporter/version.txt | 1 + 3 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 exporters/ipmi_exporter/version.txt rename {exporters/ipmi_exporter => ipmi-exporter}/Dockerfile (100%) create mode 100644 ipmi-exporter/version.txt diff --git a/exporters/ipmi_exporter/version.txt b/exporters/ipmi_exporter/version.txt deleted file mode 100644 index e33692ab..00000000 --- a/exporters/ipmi_exporter/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.10.1 \ No newline at end of file diff --git a/exporters/ipmi_exporter/Dockerfile b/ipmi-exporter/Dockerfile similarity index 100% rename from exporters/ipmi_exporter/Dockerfile rename to ipmi-exporter/Dockerfile diff --git a/ipmi-exporter/version.txt b/ipmi-exporter/version.txt new file mode 100644 index 00000000..4dae2985 --- /dev/null +++ b/ipmi-exporter/version.txt @@ -0,0 +1 @@ +1.10.1 From e0beab35718f41bf40b628d73b40889897b06312 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Mon, 7 Sep 2026 16:22:52 +0100 Subject: [PATCH 03/14] patch bump openstack-rabbit-consumer and stfc-username-service --- openstack-rabbit-consumer/version.txt | 2 +- stfc-username-service/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack-rabbit-consumer/version.txt b/openstack-rabbit-consumer/version.txt index 0fa4ae48..712bd5a6 100644 --- a/openstack-rabbit-consumer/version.txt +++ b/openstack-rabbit-consumer/version.txt @@ -1 +1 @@ -3.3.0 \ No newline at end of file +3.3.1 \ No newline at end of file diff --git a/stfc-username-service/version.txt b/stfc-username-service/version.txt index 7dea76ed..6d7de6e6 100644 --- a/stfc-username-service/version.txt +++ b/stfc-username-service/version.txt @@ -1 +1 @@ -1.0.1 +1.0.2 From be6584138dd306dccfdfada9aea00ccb076e0a5c Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 8 Sep 2026 10:15:49 +0100 Subject: [PATCH 04/14] fix typo login to harbor not staging harbor --- .github/workflows/create_dev_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index 3cf19e50..8c2c8f51 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -153,7 +153,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to Harbor Staging + - name: Login to Harbor uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ${{ env.REGISTRY }} From 8d6d6110f63244b23a8d27747351eeeee89b14ef Mon Sep 17 00:00:00 2001 From: Anish Mudaraddi <81758831+anish-mudaraddi@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:13:05 +0100 Subject: [PATCH 05/14] fix typos Co-authored-by: George Matthews <112955466+gmatthews20@users.noreply.github.com> --- .github/workflows/create_release_image.yaml | 2 +- .github/workflows/patch_bump_pr.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 69fe4abd..2a84afe7 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -26,7 +26,7 @@ jobs: with: fetch-depth: 0 - - name: "Find All Charts" + - name: "Find All Dockerfiles" id: detect env: BEFORE: ${{ github.event.before }} diff --git a/.github/workflows/patch_bump_pr.yaml b/.github/workflows/patch_bump_pr.yaml index 32e07cad..e13ee8a4 100644 --- a/.github/workflows/patch_bump_pr.yaml +++ b/.github/workflows/patch_bump_pr.yaml @@ -37,7 +37,7 @@ jobs: echo "images=$(jq -Rc -s 'split("\n") | map(select(length > 0))' <<< "$image_paths")" \ >> "$GITHUB_OUTPUT" - bump-chart: + bump-version: runs-on: ubuntu-latest needs: detect strategy: @@ -82,5 +82,5 @@ jobs: Automated PR to bump the patch version for ${{ metrix.image }} Forces a new build of the image to update dependencies once merged - **Please test using the dev chart before merging** + **Please test using the dev image before merging** --- From 4d1554819172a17b4b6cb794c93611ed1d7f011c Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 8 Sep 2026 11:49:57 +0100 Subject: [PATCH 06/14] split out long bash from CI into standalone scripts split out detect_changed_images.sh and validate_version.sh from build_dev_image.yaml --- .github/scripts/build_dev_image.sh | 2 +- .github/scripts/build_release_image.sh | 2 +- .github/scripts/detect_changed_images.sh | 22 +++++++++ .github/scripts/validate_version.sh | 40 ++++++++++++++++ .github/workflows/create_dev_image.yaml | 61 ++---------------------- build_images.sh | 5 -- 6 files changed, 68 insertions(+), 64 deletions(-) create mode 100755 .github/scripts/detect_changed_images.sh create mode 100755 .github/scripts/validate_version.sh delete mode 100755 build_images.sh diff --git a/.github/scripts/build_dev_image.sh b/.github/scripts/build_dev_image.sh index db0d3967..207353b8 100644 --- a/.github/scripts/build_dev_image.sh +++ b/.github/scripts/build_dev_image.sh @@ -22,7 +22,7 @@ set -euo pipefail -# shellcheck source=.github/scripts/_common.sh +# shellcheck source=.github/scripts/utils.sh source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" # shellcheck source=.github/scripts/build_image.sh diff --git a/.github/scripts/build_release_image.sh b/.github/scripts/build_release_image.sh index a15bb2bd..0e53d8bf 100644 --- a/.github/scripts/build_release_image.sh +++ b/.github/scripts/build_release_image.sh @@ -23,7 +23,7 @@ set -euo pipefail # REGISTRY=harbor.stfc.ac.uk PUSH=false IMAGE_NAMESPACE=stfc-cloud \ # .github/scripts/build-release-image.sh service-a -# shellcheck source=.github/scripts/utils +# shellcheck source=.github/scripts/utils.sh source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" # shellcheck source=.github/scripts/build_image.sh diff --git a/.github/scripts/detect_changed_images.sh b/.github/scripts/detect_changed_images.sh new file mode 100755 index 00000000..6b55a499 --- /dev/null +++ b/.github/scripts/detect_changed_images.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +base="$1" +head="$2" + +changed=$(git diff --name-only "$base...$head") + +find . -name Dockerfile -not -path './.git/*' -printf '%h\n' | +while read -r dir; do + dir=${dir#./} + + # Only consider directories containing both Dockerfile and version.txt + # as these directories will be the name of the image that has changed + [[ -f "$dir/Dockerfile" && -f "$dir/version.txt" ]] || continue + + # Check whether anything in the image directory changed + if grep -q "^$dir/" <<< "$changed"; then + echo "$dir" + fi +done | +sort -u diff --git a/.github/scripts/validate_version.sh b/.github/scripts/validate_version.sh new file mode 100755 index 00000000..0f540d45 --- /dev/null +++ b/.github/scripts/validate_version.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +image_path="$1" +base="$2" + +version_file="$image_path/version.txt" + +new=$(tr -d '[:space:]' < "$version_file") + +if [[ ! "$new" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: $version_file contains '$new', expected MAJOR.MINOR.PATCH" + exit 1 +fi + +tag="$(basename "$image_path")-$new" + +if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + echo "ERROR: tag $tag already exists" + exit 1 +fi + +if ! git cat-file -e "$base:$version_file" 2>/dev/null; then + echo "$image_path is new at $new" + exit 0 +fi + +old=$(git show "$base:$version_file" | tr -d '[:space:]') + +if [[ "$old" == "$new" ]]; then + echo "ERROR: $version_file was not bumped (still $old)" + exit 1 +fi + +if [[ "$(printf '%s\n%s\n' "$old" "$new" | sort -V | head -1)" != "$old" ]]; then + echo "ERROR: version went backwards: $old -> $new" + exit 1 +fi + +echo "$image_path correctly updated: $old -> $new" \ No newline at end of file diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index 8c2c8f51..255bedc3 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -30,29 +30,11 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - - # find changed files - changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") - - # for each changed file, find the root directory of the docker image - changed_image_paths=$( - find . \ - -name Dockerfile \ - -not -path './.git/*' \ - -printf '%h\n' | - sed 's|^\./||' | - while read -r dir; do - grep -q "^$dir/" <<< "$changed" && echo "$dir"; - done | - sort -u | - jq -Rc -s 'split("\n") | map(select(length > 0))' - ) - - # each unique path to image directory changed needs a dev images built + # convert image paths into json + changed_image_paths=$(.github/scripts/detect_changed_images.sh "$BASE_SHA" "$HEAD_SHA" | jq -Rsc 'split("\n") | map(select(length > 0))') + printf 'changed: %s\n' "$changed_image_paths" echo "image_paths=$changed_image_paths" >> "$GITHUB_OUTPUT" - printf '%s\n' "$changed_image_paths" - # version.txt must be bumped, must go up, and must not already be tagged. version: needs: detect @@ -73,42 +55,7 @@ jobs: IMAGE_PATH: ${{ matrix.image_path }} BASE: ${{ github.event.pull_request.base.sha }} run: | - - new=$(tr -d '[:space:]' < "$IMAGE_PATH/version.txt") - tag="$(basename "$IMAGE_PATH")-$new" - - if [[ ! "$new" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::$IMAGE_PATH/version.txt is '$new', want MAJOR.MINOR.PATCH" - exit 1 - fi - - if git rev-parse -q --verify "refs/tags/$tag" > /dev/null; then - echo "::error::$tag already exists: bump $IMAGE_PATH/version.txt first" - exit 1 - fi - - # A image directory that didn't exist on the base branch is new, so there's - # nothing to compare against. - if ! git cat-file -e "$BASE:$IMAGE_PATH/version.txt" 2> /dev/null; then - echo "$IMAGE_PATH is new at $new" - exit 0 - fi - - old=$(git show "$BASE:$IMAGE_PATH/version.txt" | tr -d '[:space:]') - - if [[ "$old" == "$new" ]]; then - echo "::error::$IMAGE_PATH changed but version.txt is still $old" - exit 1 - fi - - lowest=$(printf '%s\n%s\n' "$old" "$new" | sort -V | head -1) - - if [[ "$lowest" != "$old" ]]; then - echo "::error::$IMAGE_PATH version went backwards: $old -> $new" - exit 1 - fi - - echo "$IMAGE_PATH: $old -> $new" >> "$GITHUB_STEP_SUMMARY" + .github/scripts/validate_version.sh "$IMAGE_PATH" "$BASE" # Build + push a dev image — trusted authors only. # Tag format: dev-, e.g. 1.0.0dev-ase241f diff --git a/build_images.sh b/build_images.sh deleted file mode 100755 index 3c7bf9dd..00000000 --- a/build_images.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -# Builds all docker images in all subirectories -find -name Dockerfile -execdir docker build . \; -echo "All images built successfully" From 264ea6daa9be10e78bbf749713696c51b404ace2 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 15 Sep 2026 13:45:37 +0100 Subject: [PATCH 07/14] remove patch bump PR github action this github action periodically opened a PR that would just bump the patch version of each chart, forcing a new dev image to be created. this was done to update the underlying apt package dependencies periodically for the chart. I removed this because there was an argument made that the reviewer had nothing to review - hence we need to re-think this PR - potentially by only opening when a dependency changed and doocumenting what dependencies changed in the PR --- .github/workflows/patch_bump_pr.yaml | 86 ---------------------------- 1 file changed, 86 deletions(-) delete mode 100644 .github/workflows/patch_bump_pr.yaml diff --git a/.github/workflows/patch_bump_pr.yaml b/.github/workflows/patch_bump_pr.yaml deleted file mode 100644 index e13ee8a4..00000000 --- a/.github/workflows/patch_bump_pr.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: Schedule Image Rebuild - -# This action simply increments the patch version for -# all docker images in this repo and opens a new PR - -# This is so that we update our image dependencies periodically -# currently set for every month - -on: - schedule: - - cron: "0 10 1 * *" # First day of each month at 10.00am - workflow_dispatch: - -permissions: - contents: write # needed to bump versions - packages: write # needed to make branches/PRs - -env: - author: "${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>" - committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" - -jobs: - detect: - runs-on: ubuntu-latest - outputs: - images: ${{ steps.detect.outputs.images }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - id: detect - name: "Increment patch version for every image in repo" - run: | - set -euo pipefail - image_paths=$(find . -name Dockerfile -printf '%h\n' | sed 's|^\./||' | sort -u ) - echo "images=$(jq -Rc -s 'split("\n") | map(select(length > 0))' <<< "$image_paths")" \ - >> "$GITHUB_OUTPUT" - - bump-version: - runs-on: ubuntu-latest - needs: detect - strategy: - matrix: - image: ${{ fromJSON(needs.detect.outputs.images) }} - steps: - - name: "Setup Github Token" - # https://github.com/actions/create-github-app-token/releases - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 #v3.2.0 - id: app-token - with: - app-id: ${{ vars.APP_ID }} - private-key: ${{ secrets.PRIVATE_KEY }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # increments ONLY the patch version using the version_incrememnt.sh script and commits the version.txt file to master - - name: Increment Version - env: - VERSION_FILEPATH: ./${{ matrix.image }}/version.txt - run: bash ${{ github.workspace }}/.github/workflows/scripts/bump_patch_version.sh "$VERSION_FILEPATH" - - - name: "Create Pull Request for a patch bump" - id: patch-bump-pr - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - env: - pr-title: "Patch bump for ${{ matrix.image }} image" - branch: "${{ matrix.image }}-patch-bump" - with: - token: ${{ steps.app-token.outputs.token }} - commit-message: ${{ env.pr-title }} - author: ${{ env.author }} - committer: ${{ env.committer }} - branch: ${{ env.branch }} - delete-branch: true - title: ${{ env.pr-title }} - labels: | - automated - dependency-update - patch - body: | - Automated PR to bump the patch version for ${{ metrix.image }} - Forces a new build of the image to update dependencies once merged - - **Please test using the dev image before merging** - --- From acc75cb683c6869f9d30bae0ab2873111f3a477f Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Fri, 2 Oct 2026 14:31:33 +0100 Subject: [PATCH 08/14] simplify image build CI jobs scripts were too complex, simplify to just use the docker build-push-action instead of standalone scripts the manual method is simple just a docker build --push - So just document that in the README remove validating the version.txt change as not all versions will be in the format major.minor.patch. Instead we simply check that it has been changed in detect_changed_images.sh --- .github/scripts/build_dev_image.sh | 49 ----------------- .github/scripts/build_image.sh | 49 ----------------- .github/scripts/build_release_image.sh | 59 --------------------- .github/scripts/detect_changed_images.sh | 5 ++ .github/scripts/validate_version.sh | 40 -------------- .github/workflows/create_dev_image.yaml | 58 +++++++++----------- .github/workflows/create_release_image.yaml | 51 +++++++++++++++--- README.md | 19 +++---- 8 files changed, 81 insertions(+), 249 deletions(-) delete mode 100644 .github/scripts/build_dev_image.sh delete mode 100644 .github/scripts/build_image.sh delete mode 100644 .github/scripts/build_release_image.sh delete mode 100755 .github/scripts/validate_version.sh diff --git a/.github/scripts/build_dev_image.sh b/.github/scripts/build_dev_image.sh deleted file mode 100644 index 207353b8..00000000 --- a/.github/scripts/build_dev_image.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Build a dev image. -# -# Tag format: dev-, e.g. 1.0.0-dev-ase241f -# -# Usage: build_dev_image.sh [sha] -# sha defaults to HEAD. -# -# Env: -# REGISTRY, IMAGE_NAMESPACE required -# PUSH=true push; otherwise the image is loaded locally -# BUILD_CACHE=1 use the registry build cache -# -# Outputs: -# image_path full image reference -# dockerfile path to the image_name's Dockerfile -# -# Local run: -# REGISTRY=harbor.stfc.ac.uk PUSH=false IMAGE_NAMESPACE=stfc-cloud-staging \ -# .github/scripts/build-release-image.sh service-a - -set -euo pipefail - -# shellcheck source=.github/scripts/utils.sh -source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" - -# shellcheck source=.github/scripts/build_image.sh -source "$(dirname -- "${BASH_SOURCE[0]}")/build_image.sh" - - -image_path=${1:?usage: build_dev_image.sh [sha]} -sha=${2:-$(git rev-parse HEAD)} -require REGISTRY IMAGE_NAMESPACE - -version=$(read_version "$image_path") -name=$(get_image_name "$image_path") -dockerfile=$(find_dockerfile "$image_path") -short=${sha:0:7} - -image="$REGISTRY/$IMAGE_NAMESPACE/$name:${version}dev.${short}" - -output image "$image" -output dockerfile "$dockerfile" - -build_image "$image_path" "$dockerfile" "$image" - -info "built $image" -summary "dev image: \`$image\`" diff --git a/.github/scripts/build_image.sh b/.github/scripts/build_image.sh deleted file mode 100644 index 8eafbb7f..00000000 --- a/.github/scripts/build_image.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Helper function to build an image. -# -# build_image.sh [tag...] -# -# Env: -# -# PUSH=true push to remote registry; -# -# BUILD_CACHE=1 read (and, when pushing, write) a :buildcache tag alongside -# the image. Needs a docker-container buildx driver. -# docker/setup-buildx-action CI job builds this. -# - -build_image() { - local context=$1 dockerfile=$2 - - # tags are positional args - provide at least one - shift 2 - [ $# -gt 0 ] || fail "build_image: at least one tag is required" - - local args=(buildx build "$context" --file "$dockerfile") - - local tag - for tag in "$@"; do - args+=(--tag "$tag") - done - - if [ "${BUILD_CACHE:-0}" = "1" ]; then - local cache_ref="${1%:*}:buildcache" - args+=(--cache-from "type=registry,ref=$cache_ref") - if [ "${PUSH:-false}" = "true" ]; then - args+=(--cache-to "type=registry,ref=$cache_ref,mode=max") - fi - fi - - if [ "${PUSH:-false}" = "true" ]; then - args+=(--push) - else - args+=(--load) - info "PUSH is not 'true' — building locally, not pushing" - fi - - info "+ docker ${args[*]}" - docker "${args[@]}" -} diff --git a/.github/scripts/build_release_image.sh b/.github/scripts/build_release_image.sh deleted file mode 100644 index 0e53d8bf..00000000 --- a/.github/scripts/build_release_image.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Build a release image, tagged with latest version. -# -# If the git tag already exists then does nothing. -# -# Usage: build_release_image.sh -# -# Env: -# REGISTRY, IMAGE_NAMESPACE required -# PUSH=true push; otherwise images are loaded locally -# BUILD_CACHE=1 use the registry build cache -# -# Outputs: -# skip "true" or "false" - if git-tag exists matching version, returns true -# tag git tag to set (for CI), - -# image image reference -# latest :latest image reference -# dockerfile path to the image_path's Dockerfile -# -# Local run: -# REGISTRY=harbor.stfc.ac.uk PUSH=false IMAGE_NAMESPACE=stfc-cloud \ -# .github/scripts/build-release-image.sh service-a - -# shellcheck source=.github/scripts/utils.sh -source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" - -# shellcheck source=.github/scripts/build_image.sh -source "$(dirname -- "${BASH_SOURCE[0]}")/build_image.sh" - -image_path=${1:?usage: build-release-image.sh } -require REGISTRY IMAGE_NAMESPACE - -version=$(read_version "$image_path") -name=$(image_name "$image_path") -tag="$name-$version" - -# if the git-tag already exists - do nothing as image likely already exists in registry -if git rev-parse -q --verify "refs/tags/$tag" > /dev/null; then - info "tag $tag already exists — nothing to release - did you remember to bump the version in version.txt?" - output skip true - exit 0 -fi - -dockerfile=$(find_dockerfile "$image_path") -image="$REGISTRY/$IMAGE_NAMESPACE/$name:$version" -latest="$REGISTRY/$IMAGE_NAMESPACE/$name:latest" - -output skip false -output tag "$tag" -output dockerfile "$dockerfile" -output image "$image" -output latest "$latest" - -build_image "$image_path" "$dockerfile" "$image" "$latest" - -info "built $image" -summary "release image: \`$image\`" \ No newline at end of file diff --git a/.github/scripts/detect_changed_images.sh b/.github/scripts/detect_changed_images.sh index 6b55a499..8da7b6bb 100755 --- a/.github/scripts/detect_changed_images.sh +++ b/.github/scripts/detect_changed_images.sh @@ -16,6 +16,11 @@ while read -r dir; do # Check whether anything in the image directory changed if grep -q "^$dir/" <<< "$changed"; then + # version.txt must also have changed + if ! grep -q "^$dir/version.txt$" <<< "$changed"; then + echo "ERROR: $dir has changed, but version.txt was not changed - bump the version!" >&2 + exit 1 + fi echo "$dir" fi done | diff --git a/.github/scripts/validate_version.sh b/.github/scripts/validate_version.sh deleted file mode 100755 index 0f540d45..00000000 --- a/.github/scripts/validate_version.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -image_path="$1" -base="$2" - -version_file="$image_path/version.txt" - -new=$(tr -d '[:space:]' < "$version_file") - -if [[ ! "$new" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "ERROR: $version_file contains '$new', expected MAJOR.MINOR.PATCH" - exit 1 -fi - -tag="$(basename "$image_path")-$new" - -if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then - echo "ERROR: tag $tag already exists" - exit 1 -fi - -if ! git cat-file -e "$base:$version_file" 2>/dev/null; then - echo "$image_path is new at $new" - exit 0 -fi - -old=$(git show "$base:$version_file" | tr -d '[:space:]') - -if [[ "$old" == "$new" ]]; then - echo "ERROR: $version_file was not bumped (still $old)" - exit 1 -fi - -if [[ "$(printf '%s\n%s\n' "$old" "$new" | sort -V | head -1)" != "$old" ]]; then - echo "ERROR: version went backwards: $old -> $new" - exit 1 -fi - -echo "$image_path correctly updated: $old -> $new" \ No newline at end of file diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index 255bedc3..d5766e71 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -35,38 +35,12 @@ jobs: printf 'changed: %s\n' "$changed_image_paths" echo "image_paths=$changed_image_paths" >> "$GITHUB_OUTPUT" - # version.txt must be bumped, must go up, and must not already be tagged. - version: - needs: detect - if: needs.detect.outputs.image_paths != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} - steps: - # https://github.com/actions/checkout/releases - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: "Check Version Update" - env: - IMAGE_PATH: ${{ matrix.image_path }} - BASE: ${{ github.event.pull_request.base.sha }} - run: | - .github/scripts/validate_version.sh "$IMAGE_PATH" "$BASE" - - # Build + push a dev image — trusted authors only. + # Build + push a dev image — only works when harbor secret exists # Tag format: dev-, e.g. 1.0.0dev-ase241f dev-image: needs: [detect, version] # The `env` context isn't available in a job-level `if`, so the trusted-bot # list has to be inline. - if: >- - ${{ needs.detect.outputs.image_paths != '[]' && - (contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) || - contains(fromJSON('["dependabot[bot]"]'), github.event.pull_request.user.login)) }} runs-on: ubuntu-latest strategy: @@ -107,9 +81,27 @@ jobs: username: ${{ secrets.STAGING_HARBOR_USERNAME }} password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - env: - IMAGE_PATH: ${{ matrix.image_path }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PUSH: "true" - BUILD_CACHE: "1" - run: bash .github/scripts/build_dev_image.sh "$IMAGE_PATH" "$HEAD_SHA" + - name: Set image tag + id: image + shell: bash + run: | + image_path="${{ matrix.image_path }}" + image_name="$(basename "$image_path")" + version="$(cat "$image_path/version.txt")" + + echo "name=${image_name}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "tag=${image_name}:${version}-dev.${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" + + - name: Build and push + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.tag }} + cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} + cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max + context: "{{ matrix.image_path }}" + file: ${{ matrix.image_path }}/Dockerfile diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 2a84afe7..72cd9b9c 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -92,16 +92,51 @@ jobs: username: ${{ secrets.STAGING_HARBOR_USERNAME }} password: ${{ secrets.STAGING_HARBOR_TOKEN }} - # No-ops if this version is already tagged. - - id: release - env: - IMAGE_PATH: ${{ matrix.image_path }} - PUSH: "true" - BUILD_CACHE: "1" - run: bash .github/scripts/build_release_image.sh "$IMAGE_PATH" + - name: Set image tag + id: image + shell: bash + run: | + image_path="${{ matrix.image_path }}" + image_name="$(basename "$image_path")" + version="$(cat "$image_path/version.txt")" + + echo "name=${image_name}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "tag=${image_name}:${version}" >> "$GITHUB_OUTPUT" + echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" + + # if image already exists with the same version, don't build + - name: Check image tag does not exist + id: check + shell: bash + run: | + image="${{ steps.image.outputs.image }}" + tag="${{ steps.image.outputs.version_tag }}" + echo "skip=false" >> "$GITHUB_OUTPUT" + + if docker manifest inspect "${image}:${tag}" >/dev/null 2>&1; then + echo "::error::Image ${image}:${tag} already exists, not building + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + echo "Image ${image}:${tag} does not exist; continuing" + + - if: steps.check.outputs.skip == 'false' + name: Build and push + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.tag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.name }}:latest + cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} + cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max + context: "{{ matrix.image_path }}" + file: ${{ matrix.image_path }}/Dockerfile # Tag last, so a failed build doesn't leave a tag behind. - - if: steps.release.outputs.skip == 'false' + - if: steps.check.outputs.skip == 'false' env: TAG: ${{ steps.release.outputs.tag }} run: | diff --git a/README.md b/README.md index 7e57fffd..f801582f 100644 --- a/README.md +++ b/README.md @@ -16,26 +16,23 @@ If adding source-code - please a new CI github action for performing tests and l Docker images are built and pushed to https://harbor.stfc.ac.uk by CI jobs -`.github/workflows/create_dev_image.yml` runs when a PR is created and will build and push a dev image following the format: `-dev--` - e.g. `cloud-monitoring-dev-1.0.0-ase34e1` +`.github/workflows/create_dev_image.yml` runs when a PR is created and will build and push a dev image following the format: `-dev.-` - e.g. `cloud-monitoring-dev.1.0.0-ase34e1` `.github/workflows/create_release_image.yml` runs when the PR is merged. A new version - specified in `version.txt` is required to be bumped before merging. Once merged - a new image with that version will be built and pushed to harbor -both scripts can be invoked locally by utilising `.github/scripts/build_dev_image.sh` and `.github/scripts/build_release_iamge.sh` repectively - see script file for details on how to use it. - -In general, they can be invoked like so: +To manually build and push images, you can run: ```sh -# dev image: dev-, sha defaults to HEAD -REGISTRY=ghcr.io IMAGE_NAMESPACE=my-org ./.github/scripts/build-dev-image.sh cloud-monitoring +VERSION=$(cat cloud-monitoring/version.txt) + +# docker build //:$VERSION +docker build harbor.stfc.ac.uk/stfc-cloud/cloud-monitoring:$VERSION --push -# release image: : and :latest -REGISTRY=ghcr.io IMAGE_NAMESPACE=my-org ./.github/scripts/build-release-image.sh cloud-monitoring +# for a dev image +docker build harbor.stfc.ac.uk/stfc-cloud-staging/cloud-monitoring:$VERSION-dev.1 --push ``` ## Linting and Testing Currently linting and testing CI actions are prefixed with `test_and_lint*`. Since each image is built slightly differently, we have separate CI actions for each. These CI jobs do not build images and are run when a PR is opened -## Regular version bumps - -Each month, we bump the minor version of all our docker images and open a pull request - this forces a new docker image to be built with updated dependencies. From 4b3b6a4cc4beca8bb0b56dfe43af8b70e708dc64 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Fri, 2 Oct 2026 14:49:47 +0100 Subject: [PATCH 09/14] move maximise build space step to later in CI Maximise build space is slow running, run only when we're sure to build an image --- .github/workflows/create_dev_image.yaml | 28 +++++++++---------- .github/workflows/create_release_image.yaml | 30 +++++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index d5766e71..f9538856 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -48,20 +48,6 @@ jobs: matrix: image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} steps: - - name: Maximize build space # free up space for building images - uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - build-mount-path: "/var/lib/docker/" - - - name: Restart docker # restart the docker service - run: sudo service docker restart - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -94,6 +80,20 @@ jobs: echo "tag=${image_name}:${version}-dev.${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" + - name: Maximize build space # free up space for building images + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - name: Restart docker # restart the docker service + run: sudo service docker restart + - name: Build and push # https://github.com/docker/build-push-action/releases uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 72cd9b9c..0b1739f3 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -59,20 +59,6 @@ jobs: image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} steps: - - name: Maximize build space # free up space for building images - uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - build-mount-path: "/var/lib/docker/" - - - name: Restart docker # restart the docker service - run: sudo service docker restart - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -121,6 +107,22 @@ jobs: echo "Image ${image}:${tag} does not exist; continuing" + - if: steps.check.outputs.skip == 'false' + name: Maximize build space # free up space for building images + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - if: steps.check.outputs.skip == 'false' + name: Restart docker # restart the docker service + run: sudo service docker restart + - if: steps.check.outputs.skip == 'false' name: Build and push # https://github.com/docker/build-push-action/releases From 9577514f14c21aea1fd8c21f2edb777434c8b496 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Fri, 2 Oct 2026 15:03:32 +0100 Subject: [PATCH 10/14] bump github CI dependency versions add comment link releases --- .github/workflows/create_dev_image.yaml | 13 +++++++++---- .github/workflows/create_release_image.yaml | 13 +++++++++---- .github/workflows/test_and_lint_cloud_chatops.yaml | 2 +- .../workflows/test_and_lint_cloud_monitoring.yaml | 2 +- .../workflows/test_and_lint_rabbit_consumer.yaml | 2 +- .../workflows/test_and_lint_username_service.yaml | 4 ++-- 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index f9538856..39c81522 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -48,20 +48,24 @@ jobs: matrix: image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} steps: + # https://github.com/actions/checkout/releases - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} - - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + # https://github.com/docker/setup-buildx-action/releases + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 - name: Login to DockerHub # increase pull rate limit - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.STAGING_HARBOR_USERNAME }} @@ -81,6 +85,7 @@ jobs: echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" - name: Maximize build space # free up space for building images + # https://github.com/easimon/maximize-build-space/releases uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 with: root-reserve-mb: 512 @@ -96,7 +101,7 @@ jobs: - name: Build and push # https://github.com/docker/build-push-action/releases - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 with: push: true tags: | diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 0b1739f3..201f32ed 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -59,20 +59,24 @@ jobs: image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} steps: + # https://github.com/actions/checkout/releases - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} - - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + # https://github.com/docker/setup-buildx-action/releases + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 - name: Login to DockerHub # increase pull rate limit - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Harbor Staging - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.STAGING_HARBOR_USERNAME }} @@ -109,6 +113,7 @@ jobs: - if: steps.check.outputs.skip == 'false' name: Maximize build space # free up space for building images + # https://github.com/easimon/maximize-build-space/releases uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 with: root-reserve-mb: 512 @@ -126,7 +131,7 @@ jobs: - if: steps.check.outputs.skip == 'false' name: Build and push # https://github.com/docker/build-push-action/releases - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 with: push: true tags: | diff --git a/.github/workflows/test_and_lint_cloud_chatops.yaml b/.github/workflows/test_and_lint_cloud_chatops.yaml index e2afefd5..23adb5fe 100644 --- a/.github/workflows/test_and_lint_cloud_chatops.yaml +++ b/.github/workflows/test_and_lint_cloud_chatops.yaml @@ -57,7 +57,7 @@ jobs: - name: Upload coverage to Codecov # https://github.com/codecov/codecov-action/releases - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 with: token: ${{secrets.CODECOV_TOKEN}} files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_cloud_monitoring.yaml b/.github/workflows/test_and_lint_cloud_monitoring.yaml index a4c6b355..e14b692f 100644 --- a/.github/workflows/test_and_lint_cloud_monitoring.yaml +++ b/.github/workflows/test_and_lint_cloud_monitoring.yaml @@ -51,7 +51,7 @@ jobs: - name: Upload coverage to Codecov # https://github.com/codecov/codecov-action/releases - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 with: token: ${{secrets.CODECOV_TOKEN}} files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_rabbit_consumer.yaml b/.github/workflows/test_and_lint_rabbit_consumer.yaml index b3323dfb..155c5f4e 100644 --- a/.github/workflows/test_and_lint_rabbit_consumer.yaml +++ b/.github/workflows/test_and_lint_rabbit_consumer.yaml @@ -52,7 +52,7 @@ jobs: - name: Upload coverage to codecov # https://github.com/codecov/codecov-action/releases - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 with: files: openstack-rabbit-consumer/coverage.xml fail_ci_if_error: true diff --git a/.github/workflows/test_and_lint_username_service.yaml b/.github/workflows/test_and_lint_username_service.yaml index b03cc7ee..3e6775bc 100644 --- a/.github/workflows/test_and_lint_username_service.yaml +++ b/.github/workflows/test_and_lint_username_service.yaml @@ -34,11 +34,11 @@ jobs: - name: Set up Docker Buildx # https://github.com/docker/setup-buildx-action/releases - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 - name: Run tests in container # https://github.com/docker/build-push-action/releases - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 with: no-cache: true cache-to: type=gha,mode=max From a42f259d5e5e6389e9dfb3c78d601fcba1116a99 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Fri, 2 Oct 2026 15:07:15 +0100 Subject: [PATCH 11/14] bugfix: remove needs: version version step was removed as we no longer check to see if there was a version bump and simply if it was changed --- .github/workflows/create_dev_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index 39c81522..b8dd720c 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -38,7 +38,7 @@ jobs: # Build + push a dev image — only works when harbor secret exists # Tag format: dev-, e.g. 1.0.0dev-ase241f dev-image: - needs: [detect, version] + needs: [detect] # The `env` context isn't available in a job-level `if`, so the trusted-bot # list has to be inline. From 85bb4b0686be5875ca4d2f531b0f2e6120226627 Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 13 Oct 2026 10:44:06 +0100 Subject: [PATCH 12/14] fix ci typo --- .github/workflows/create_dev_image.yaml | 2 +- .github/workflows/create_release_image.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index b8dd720c..a66a584c 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -82,7 +82,7 @@ jobs: echo "name=${image_name}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "tag=${image_name}:${version}-dev.${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" - echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" + echo "cache_ref=${image_name}:buildcache" >> "$GITHUB_OUTPUT" - name: Maximize build space # free up space for building images # https://github.com/easimon/maximize-build-space/releases diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 201f32ed..24ea3d79 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -93,7 +93,7 @@ jobs: echo "name=${image_name}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "tag=${image_name}:${version}" >> "$GITHUB_OUTPUT" - echo "cache_ref="${image}:buildcache" >> "$GITHUB_OUTPUT"" + echo "cache_ref=${image_name}:buildcache" >> "$GITHUB_OUTPUT" # if image already exists with the same version, don't build - name: Check image tag does not exist From c6235e4e4dc5d9345650d83873ae4bc3d98c7f4f Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 13 Oct 2026 11:02:01 +0100 Subject: [PATCH 13/14] set default context properly for docker build --- .github/workflows/create_dev_image.yaml | 3 +-- .github/workflows/create_release_image.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index a66a584c..75f38f11 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -108,5 +108,4 @@ jobs: ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.tag }} cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max - context: "{{ matrix.image_path }}" - file: ${{ matrix.image_path }}/Dockerfile + context: "{{defaultContext}}:${{ matrix.image_path }}" diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index 24ea3d79..b36d8cb5 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -139,8 +139,7 @@ jobs: ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.name }}:latest cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max - context: "{{ matrix.image_path }}" - file: ${{ matrix.image_path }}/Dockerfile + context: "{{defaultContext}}:${{ matrix.image_path }}" # Tag last, so a failed build doesn't leave a tag behind. - if: steps.check.outputs.skip == 'false' From 52307f8483c8c1f1c7fc7e2705f7dba40c356fee Mon Sep 17 00:00:00 2001 From: anish-mudaraddi Date: Tue, 13 Oct 2026 13:20:45 +0100 Subject: [PATCH 14/14] set cache-ref properly --- .github/workflows/create_dev_image.yaml | 2 +- .github/workflows/create_release_image.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml index 75f38f11..f8c44a15 100644 --- a/.github/workflows/create_dev_image.yaml +++ b/.github/workflows/create_dev_image.yaml @@ -82,7 +82,7 @@ jobs: echo "name=${image_name}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "tag=${image_name}:${version}-dev.${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" - echo "cache_ref=${image_name}:buildcache" >> "$GITHUB_OUTPUT" + echo "cache_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${image_name}:buildcache" >> "$GITHUB_OUTPUT" - name: Maximize build space # free up space for building images # https://github.com/easimon/maximize-build-space/releases diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml index b36d8cb5..6be8e554 100644 --- a/.github/workflows/create_release_image.yaml +++ b/.github/workflows/create_release_image.yaml @@ -93,7 +93,7 @@ jobs: echo "name=${image_name}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "tag=${image_name}:${version}" >> "$GITHUB_OUTPUT" - echo "cache_ref=${image_name}:buildcache" >> "$GITHUB_OUTPUT" + echo "cache_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${image_name}:buildcache" >> "$GITHUB_OUTPUT" # if image already exists with the same version, don't build - name: Check image tag does not exist