diff --git a/.github/scripts/bump_patch_version.sh b/.github/scripts/bump_patch_version.sh new file mode 100755 index 00000000..a33ce22b --- /dev/null +++ b/.github/scripts/bump_patch_version.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Bump the patch version of a given version.txt +# Usage: bump_patch_version.sh + +version_file="${1:-}" + +if [[ -z "$version_file" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if [[ ! -f "$version_file" ]]; then + echo "Error: file does not exist: $version_file" >&2 + exit 1 +fi + +version=$(<"$version_file") + + +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: $version_file must contain exactly one version in MAJOR.MINOR.PATCH format" >&2 + exit 1 +fi + +IFS='.' read -r major minor patch <<< "$version" + +printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))" > "$version_file" diff --git a/.github/scripts/detect_changed_images.sh b/.github/scripts/detect_changed_images.sh new file mode 100755 index 00000000..8da7b6bb --- /dev/null +++ b/.github/scripts/detect_changed_images.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +base="$1" +head="$2" + +changed=$(git diff --name-only "$base...$head") + +find . -name Dockerfile -not -path './.git/*' -printf '%h\n' | +while read -r dir; do + dir=${dir#./} + + # Only consider directories containing both Dockerfile and version.txt + # as these directories will be the name of the image that has changed + [[ -f "$dir/Dockerfile" && -f "$dir/version.txt" ]] || continue + + # Check whether anything in the image directory changed + if grep -q "^$dir/" <<< "$changed"; then + # version.txt must also have changed + if ! grep -q "^$dir/version.txt$" <<< "$changed"; then + echo "ERROR: $dir has changed, but version.txt was not changed - bump the version!" >&2 + exit 1 + fi + echo "$dir" + fi +done | +sort -u diff --git a/.github/scripts/utils.sh b/.github/scripts/utils.sh new file mode 100644 index 00000000..eb93ae38 --- /dev/null +++ b/.github/scripts/utils.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Shared helpers. Sourced by the other scripts, not run directly. + +# allows these scripts to be used by github runners +# if run locally will go back to stdout and summaries discarded +: "${GITHUB_OUTPUT:=/dev/stdout}" +: "${GITHUB_STEP_SUMMARY:=/dev/null}" + +# Emit a step output (key=value). +output() { printf '%s=%s\n' "$1" "$2" >> "$GITHUB_OUTPUT"; } + +# Append a line to the job summary. +summary() { printf '%s\n' "$*" >> "$GITHUB_STEP_SUMMARY"; } + +# Logging. All to stderr. +info() { printf '%s\n' "$*" >&2; } +warn() { printf '::warning::%s\n' "$*" >&2; } +fail() { printf '::error::%s\n' "$*" >&2; exit 1; } + +# The image name for a image_path is just its directory name. +get_image_name() { basename "$1"; } + +# Assert that the named variables are set and non-empty. +require() { + local var + for var in "$@"; do + [ -n "${!var:-}" ] || fail "$var is not set" + done +} + +# Read and validate a image_path's version.txt. Prints the version. +read_version() { + local image_path=$1 file version + file="$image_path/version.txt" + [ -f "$file" ] || fail "$file not found" + version=$(tr -d '[:space:]' < "$file") + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || fail "$file is '$version', expected MAJOR.MINOR.PATCH" + printf '%s' "$version" +} + +# Locate a image_path's Dockerfile, at any depth. Prints the path. +find_dockerfile() { + local image_path=$1 dockerfile + dockerfile=$(find "$image_path" -name Dockerfile | head -n1) + [ -n "$dockerfile" ] || fail "no Dockerfile found under $image_path" + printf '%s' "$dockerfile" +} \ No newline at end of file diff --git a/.github/workflows/bin/version_increment.sh b/.github/workflows/bin/version_increment.sh deleted file mode 100755 index 57bbc186..00000000 --- a/.github/workflows/bin/version_increment.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/bash - -set -euo pipefail - -#reads the version.txt file and sets the version to the current version -version=`cat ~0/openstack-rabbit-consumer/version.txt` - -# cuts the $version variable into major, minor and patch numbers, removing the fullstop -major=$(echo $version | cut -f1 -d.) -minor=$(echo $version | cut -f2 -d.) -patch=$(echo $version | cut -f3 -d.) - -#increments the patch by 1 -patch=$((patch + 1)) - -#concatenate the version -newversion="$major.$minor.$patch" - -#overwrites the version.txt file with new new version -printf "$newversion" > ~0/openstack-rabbit-consumer/version.txt diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml deleted file mode 100644 index 5d268a26..00000000 --- a/.github/workflows/build_images.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: docker_images -on: - push: - paths: - - "exporters/**" - - ".github/workflows/codeql.yaml" - - ".github/workflows/build_images.yaml" - pull_request: - paths: - - "exporters/**" - - ".github/workflows/codeql.yaml" - - ".github/workflows/build_images.yaml" - schedule: [ { cron: "14 14 * * TUE" } ] # Every Tuesday at 14:14 -jobs: - setup: - if: github.event_name != 'pull_request' || - !contains('OWNER,MEMBER,COLLABORATOR', - github.event.pull_request.author_association) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} # fix SHA - - name: List all images - id: metadata - run: | - echo "images=$( - find -name Dockerfile | sed -r 's#\./(.+)/Dockerfile#\1#g' | jq -Rsc 'split("\n")[:-1]' - )" >> $GITHUB_OUTPUT - echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - outputs: - images: ${{ steps.metadata.outputs.images }} - sha: ${{ steps.metadata.outputs.sha_short }} - build: - needs: setup - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: ${{ fromJson(needs.setup.outputs.images) }} - steps: - - name: Maximize build space # free up space for building images - uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - build-mount-path: "/var/lib/docker/" - - name: Restart docker # restart the docker service - run: sudo service docker restart - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - name: Login to DockerHub # increase pull rate limit - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to Harbor Staging - if: ${{ github.ref != 'refs/heads/master' }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - name: Login to Harbor - if: ${{ github.ref == 'refs/heads/master' }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - name: Build & push to staging - if: ${{ github.ref != 'refs/heads/master' }} - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:${{ matrix.image }}" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/${{ matrix.image }}:${{ - needs.setup.outputs.sha }}" - - name: Inform of tagged name - if: ${{ github.ref != 'refs/heads/master' }} - run: echo "::notice title=published::harbor.stfc.ac.uk/stfc-cloud-staging/${{ - matrix.image }}:${{ needs.setup.outputs.sha }}" - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat ${{ matrix.image }}/version.txt)" >> $GITHUB_OUTPUT - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - '${{ matrix.image }}/version.txt' - - name: Build & push to prod - if: ${{ github.ref == 'refs/heads/master' }} - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - push: true - context: "{{defaultContext}}:${{ matrix.image }}" - tags: "harbor.stfc.ac.uk/stfc-cloud/${{ matrix.image }}:v${{ - steps.release_tag.outputs.version }}" - finished: - # convenient single job name to apply branch protection to - needs: build - runs-on: ubuntu-latest - steps: [ { run: true } ] diff --git a/.github/workflows/cloud_chatops.yaml b/.github/workflows/cloud_chatops.yaml deleted file mode 100644 index 0c3308d5..00000000 --- a/.github/workflows/cloud_chatops.yaml +++ /dev/null @@ -1,160 +0,0 @@ -name: Cloud Chatops - -permissions: - contents: read - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/cloud_chatops.yaml" - - "cloud-chatops/**" - -jobs: - test_and_lint: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ['ubuntu-22.04','ubuntu-latest'] - python-version: [ "3.12", "3.x" ] - steps: - # https://github.com/actions/checkout/releases - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - # https://github.com/actions/setup-python/releases - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - cd cloud-chatops - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Analyse with pylint - run: | - cd cloud-chatops - pylint . --recursive=true --rcfile=.pylintrc - - - name: Run tests - run: | - cd cloud-chatops - python3 -m pytest tests - - - name: Run tests and collect coverage - run: | - cd cloud-chatops - python3 -m pytest tests --cov-report xml:coverage.xml --cov - - - name: Upload coverage to Codecov - # https://github.com/codecov/codecov-action/releases - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - token: ${{secrets.CODECOV_TOKEN}} - files: cloud-chatops/coverage.xml - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - # https://github.com/docker/setup-buildx-action/releases - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - # https://github.com/docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2/releases - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - # https://github.com/docker/build-push-action/releases - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-chatops" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/cloud-chatops:${{ steps.commit_sha.outputs.sha_short }}" - - check_version_update: - runs-on: ubuntu-latest - steps: - - name: Checkout master - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: 'master' - path: 'main' - - - name: Checkout current working branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - path: 'branch' - - - name: Compare versions - if: ${{ github.ref != 'refs/heads/master' }} - uses: stfc/check-version-action@main - with: - app_version_path: "cloud-chatops/version.txt" - docker_compose_path: "cloud-chatops/docker-compose.yaml" - labels: ${{ toJSON(github.event.pull_request.labels.*.name) }} - - - name: Log App Success - if: ${{ env.app_updated == 'true' }} - run: | - echo "App version has been updated correctly!" - - name: Log Compose Success - if: ${{ env.compose_updated == 'true' }} - run: | - echo "Compose version has been updated correctly!" - - push_prod_image_harbor: - runs-on: ubuntu-latest - needs: [test_and_lint, check_version_update] - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat cloud-chatops/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'cloud-chatops/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-chatops" - tags: "harbor.stfc.ac.uk/stfc-cloud/cloud-chatops:${{ steps.release_tag.outputs.version }}" diff --git a/.github/workflows/cloud_monitoring.yaml b/.github/workflows/cloud_monitoring.yaml deleted file mode 100644 index a92bae33..00000000 --- a/.github/workflows/cloud_monitoring.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: Cloud Monitoring - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/cloud_monitoring.yaml" - - "cloud-monitoring/**" - -jobs: - test_and_lint: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-22.04", "ubuntu-latest"] - python-version: ["3.12", "3.x"] - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - cd cloud-monitoring - python -m pip install --upgrade pip - # Install the package with all optional dependencies for testing - pip install -e ".[test,dev]" - - - name: Check code formatting with Black - run: | - cd cloud-monitoring - black --check --diff cloudmon/ tests/ - - - name: Analyse with pylint - run: | - cd cloud-monitoring - pylint . --recursive=true - - - name: Run tests with Pytest - run: | - cd cloud-monitoring - python -m pytest tests --cov-report xml:coverage.xml --cov - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - token: ${{secrets.CODECOV_TOKEN}} - files: cloud-chatops/coverage.xml - - build_and_push_dev_image: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Checkout the code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-monitoring" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/cloud-monitoring:${{ steps.commit_sha.outputs.sha_short }}" - - build_and_push_prod_image: - runs-on: ubuntu-latest - needs: test_and_lint - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 # Needed for tags - - - name: Check if relevant files changed - id: changes - run: | - # check if any python package files have been changed - CHANGED=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -E '^cloud-monitoring/(cloudmon/.*|tests/.*|pyproject\.toml)$' || true) - if [ -z "$CHANGED" ]; then - echo "changed=false" >> $GITHUB_OUTPUT - else - echo "changed=true" >> $GITHUB_OUTPUT - fi - - - name: Exit if no relevant changes - if: steps.changes.outputs.changed == 'false' - run: echo "No relevant changes, no version bump needed." - - - name: Get latest cloud-monitoring tag - if: steps.changes.outputs.changed == 'true' - id: get_tag - run: | - tag=$(git tag --list 'cloud-monitoring-*' | sort -V | tail -n 1) - if [ -z "$tag" ]; then - tag="cloud-monitoring-0.0.0" - fi - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - - name: Get version from pyproject.toml - id: get_version - run: | - version=$(grep -m1 '^version' cloud-monitoring/pyproject.toml | sed -E 's/version *= *"([^"]+)"/\1/') - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Check if bump is needed - id: check_bump - if: steps.changes.outputs.changed == 'true' - run: | - tag_version="${{ steps.get_tag.outputs.tag#cloud-monitoring- }}" - current_version="${{ steps.get_version.outputs.version }}" - - if [ "$tag_version" = "$current_version" ]; then - echo "same=true" >> "$GITHUB_OUTPUT" - else - echo "same=false" >> "$GITHUB_OUTPUT" - fi - - - name: Bump version (patch) - if: steps.check_bump.outputs.same == 'true' - uses: callowayproject/bump-my-version@e6ecdc3e573698766cd6c2112faeda50bcc2e56a - id: bump - with: - args: patch - github-token: ${{ secrets.GITHUB_TOKEN }} - workdir: cloud-monitoring - - - name: Refresh version after bump - id: final_version - run: | - version=$(grep -m1 '^version' cloud-monitoring/pyproject.toml | sed -E 's/version *= *"([^"]+)"/\1/') - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Create tag for current version - run: | - version="${{ steps.final_version.outputs.version }}" - tag_name="cloud-monitoring-${version}" - - git config user.name "github-actions" - git config user.email "github-actions@github.com" - - # Only tag if it doesn't already exist - if git rev-parse "$tag_name" >/dev/null 2>&1; then - echo "Tag $tag_name already exists. Skipping." - else - git tag "$tag_name" - git push origin "$tag_name" - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:cloud-monitoring" - tags: "harbor.stfc.ac.uk/stfc-cloud/cloud-monitoring:${{ steps.final_version.outputs.version }}" diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index f063f3c0..7e4769ff 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -13,11 +13,11 @@ name: "CodeQL Advanced" on: push: - branches: [ "master" ] + branches: ["master"] pull_request: - branches: [ "master" ] + branches: ["master"] schedule: - - cron: '0 12 * * 1' + - cron: "0 12 * * 1" jobs: analyze: @@ -43,8 +43,8 @@ jobs: fail-fast: false matrix: include: - - language: python - build-mode: none + - language: python + build-mode: none # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -54,39 +54,42 @@ jobs: # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Checkout repository + # https://github.com/actions/checkout/releases + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + # https://github.com/github/codeql-action/releases + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa #v4.36.0 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + # https://github.com/github/codeql-action/releases + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa #v4.36.0 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/create_dev_image.yaml b/.github/workflows/create_dev_image.yaml new file mode 100644 index 00000000..f8c44a15 --- /dev/null +++ b/.github/workflows/create_dev_image.yaml @@ -0,0 +1,111 @@ +name: Create a Dev Image + +on: + pull_request: + branches: + - master + workflow_dispatch: + +env: + REGISTRY: harbor.stfc.ac.uk + IMAGE_NAMESPACE: stfc-cloud-staging + +permissions: + contents: read + packages: write + +jobs: + detect: + runs-on: ubuntu-latest + outputs: + image_paths: ${{ steps.detect.outputs.image_paths }} + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - id: detect + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + # convert image paths into json + changed_image_paths=$(.github/scripts/detect_changed_images.sh "$BASE_SHA" "$HEAD_SHA" | jq -Rsc 'split("\n") | map(select(length > 0))') + printf 'changed: %s\n' "$changed_image_paths" + echo "image_paths=$changed_image_paths" >> "$GITHUB_OUTPUT" + + # Build + push a dev image — only works when harbor secret exists + # Tag format: dev-, e.g. 1.0.0dev-ase241f + dev-image: + needs: [detect] + # The `env` context isn't available in a job-level `if`, so the trusted-bot + # list has to be inline. + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + # https://github.com/docker/setup-buildx-action/releases + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 + + - name: Login to DockerHub # increase pull rate limit + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Harbor + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.STAGING_HARBOR_USERNAME }} + password: ${{ secrets.STAGING_HARBOR_TOKEN }} + + - name: Set image tag + id: image + shell: bash + run: | + image_path="${{ matrix.image_path }}" + image_name="$(basename "$image_path")" + version="$(cat "$image_path/version.txt")" + + echo "name=${image_name}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "tag=${image_name}:${version}-dev.${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + echo "cache_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${image_name}:buildcache" >> "$GITHUB_OUTPUT" + + - name: Maximize build space # free up space for building images + # https://github.com/easimon/maximize-build-space/releases + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - name: Restart docker # restart the docker service + run: sudo service docker restart + + - name: Build and push + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 + with: + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.tag }} + cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} + cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max + context: "{{defaultContext}}:${{ matrix.image_path }}" diff --git a/.github/workflows/create_release_image.yaml b/.github/workflows/create_release_image.yaml new file mode 100644 index 00000000..6be8e554 --- /dev/null +++ b/.github/workflows/create_release_image.yaml @@ -0,0 +1,150 @@ +name: Creata a Release Image + +on: + push: + branches: + - master + +# ───────────────────────────────────────────── +# Keep these in sync with pr.yml +# ───────────────────────────────────────────── +env: + REGISTRY: harbor.stfc.ac.uk + IMAGE_NAMESPACE: stfc-cloud + +permissions: + contents: write # needed to push tags + packages: write # needed to push images + +jobs: + detect: + runs-on: ubuntu-latest + outputs: + image_paths: ${{ steps.detect.outputs.image_paths }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: "Find All Dockerfiles" + id: detect + env: + BEFORE: ${{ github.event.before }} + run: | + + image_paths=$( + find . \ + -name Dockerfile \ + -not -path './.git/*' \ + -printf '%h\n' | + sed 's|^\./||' | + sort -u | + jq -Rc -s 'split("\n") | map(select(length > 0))' + ) + + echo "image_paths=$image_paths" >> "$GITHUB_OUTPUT" + + echo "$image_paths" + + release: + needs: detect + if: >- + ${{ needs.detect.outputs.image_paths != '[]'}} + + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 1 # tag pushes serialise more cleanly + matrix: + image_path: ${{ fromJSON(needs.detect.outputs.image_paths) }} + + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + # https://github.com/docker/setup-buildx-action/releases + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 + + - name: Login to DockerHub # increase pull rate limit + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Harbor Staging + # https://github.com/docker/login-action/releases + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.STAGING_HARBOR_USERNAME }} + password: ${{ secrets.STAGING_HARBOR_TOKEN }} + + - name: Set image tag + id: image + shell: bash + run: | + image_path="${{ matrix.image_path }}" + image_name="$(basename "$image_path")" + version="$(cat "$image_path/version.txt")" + + echo "name=${image_name}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "tag=${image_name}:${version}" >> "$GITHUB_OUTPUT" + echo "cache_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${image_name}:buildcache" >> "$GITHUB_OUTPUT" + + # if image already exists with the same version, don't build + - name: Check image tag does not exist + id: check + shell: bash + run: | + image="${{ steps.image.outputs.image }}" + tag="${{ steps.image.outputs.version_tag }}" + echo "skip=false" >> "$GITHUB_OUTPUT" + + if docker manifest inspect "${image}:${tag}" >/dev/null 2>&1; then + echo "::error::Image ${image}:${tag} already exists, not building + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + echo "Image ${image}:${tag} does not exist; continuing" + + - if: steps.check.outputs.skip == 'false' + name: Maximize build space # free up space for building images + # https://github.com/easimon/maximize-build-space/releases + uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + build-mount-path: "/var/lib/docker/" + + - if: steps.check.outputs.skip == 'false' + name: Restart docker # restart the docker service + run: sudo service docker restart + + - if: steps.check.outputs.skip == 'false' + name: Build and push + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 + with: + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.tag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ steps.image.outputs.name }}:latest + cache-from: type=registry,ref=${{ steps.image.outputs.cache_ref }} + cache-to: type=registry,ref=${{ steps.image.outputs.cache_ref }},mode=max + context: "{{defaultContext}}:${{ matrix.image_path }}" + + # Tag last, so a failed build doesn't leave a tag behind. + - if: steps.check.outputs.skip == 'false' + env: + TAG: ${{ steps.release.outputs.tag }} + run: | + git tag "$TAG" "$GITHUB_SHA" + git push origin "$TAG" diff --git a/.github/workflows/rabbit_consumer.yaml b/.github/workflows/rabbit_consumer.yaml deleted file mode 100644 index 59ba0a29..00000000 --- a/.github/workflows/rabbit_consumer.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: Rabbit Consumer - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/rabbit_consumer.yaml" - - "openstack-rabbit-consumer/**" - -jobs: - test_and_lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Required for requests-kerberos - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev --fix-missing - pip install -r openstack-rabbit-consumer/requirements.txt - pip install -r openstack-rabbit-consumer/requirements-test.txt - - - name: Run tests - # Using Python3 to launch the module sets up the Python path for us - run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . - - - name: Analyse with pylint - run: | - cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') - - - name: Prepare coverage - run: | - cd openstack-rabbit-consumer && python -m coverage xml - - - name: Upload coverage to codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f - with: - files: openstack-rabbit-consumer/coverage.xml - fail_ci_if_error: true - flags: rabbit_consumer - token: ${{secrets.CODECOV_TOKEN}} - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/openstack-rabbit-consumer:${{ steps.commit_sha.outputs.sha_short }}" - - - name: Inform of tagged name - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud-staging/openstack-rabbit-consumer:${{ steps.commit_sha.outputs.sha_short }}" - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat openstack-rabbit-consumer/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'openstack-rabbit-consumer/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer:v${{ steps.release_tag.outputs.version }}" diff --git a/.github/workflows/rabbit_consumer_chart_schedule.yaml b/.github/workflows/rabbit_consumer_chart_schedule.yaml deleted file mode 100644 index 9382eec0..00000000 --- a/.github/workflows/rabbit_consumer_chart_schedule.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: Rabbit Consumer Schedule - -# Incrementing the patch version on a schedule to pull latest containers dependencies. - -on: - schedule: - - cron: "0 10 1 * *" # First day of each month at 10.00am - -jobs: - test_and_lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - sudo apt-get update --fix-missing - python -m pip install --upgrade pip - # Required for requests-kerberos - sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev - pip install -r openstack-rabbit-consumer/requirements.txt - pip install -r openstack-rabbit-consumer/requirements-test.txt - - - name: Run tests - # Using Python3 to launch the module sets up the Python path for us - run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . - - - name: Analyse with pylint - run: | - cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') - - - name: Prepare coverage - run: | - cd openstack-rabbit-consumer && python -m coverage xml - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: test_and_lint - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - #increments ONLY the patch version using the version_incrememnt.sh script and commits the version.txt file to master - - name: Increment Version - run: bash ${{ github.workspace }}/.github/workflows/bin/version_increment.sh - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat openstack-rabbit-consumer/version.txt)" >> $GITHUB_OUTPUT - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:openstack-rabbit-consumer" - tags: "harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer-schedule:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/openstack-rabbit-consumer-schedule:v${{ steps.release_tag.outputs.version }}" - - - name: Commit Changes - uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 - with: - add: ${{ github.workspace }}/openstack-rabbit-consumer/version.txt - default_author: github_actions - committer_name: GitHub Actions - committer_email: actions@github.com - new_branch: rabbitConsumerDependUpdate - - - name: create pull request - run: gh pr create -B master -H rabbitConsumerDependUpdate --title 'Merge rabbitConsumerDependUpdate into master' --body 'Updating the dependencies for Rabbit Consumer Image. Created by Github action' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_and_lint_cloud_chatops.yaml b/.github/workflows/test_and_lint_cloud_chatops.yaml new file mode 100644 index 00000000..23adb5fe --- /dev/null +++ b/.github/workflows/test_and_lint_cloud_chatops.yaml @@ -0,0 +1,63 @@ +name: Cloud ChatOps Lint & Test + +permissions: + contents: read + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/cloud_chatops.yaml" + - "cloud-chatops/**" + +jobs: + test_and_lint: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-22.04", "ubuntu-latest"] + python-version: ["3.12", "3.x"] + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + cd cloud-chatops + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Check code formatting with Black + run: | + cd cloud-chatops/ + black --check --diff chatops/ tests/ + + - name: Analyse with pylint + run: | + cd cloud-chatops + pylint . --recursive=true --rcfile=.pylintrc + + - name: Run tests + run: | + cd cloud-chatops + python3 -m pytest tests + + - name: Run tests and collect coverage + run: | + cd cloud-chatops + python3 -m pytest tests --cov-report xml:coverage.xml --cov + + - name: Upload coverage to Codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_cloud_monitoring.yaml b/.github/workflows/test_and_lint_cloud_monitoring.yaml new file mode 100644 index 00000000..e14b692f --- /dev/null +++ b/.github/workflows/test_and_lint_cloud_monitoring.yaml @@ -0,0 +1,57 @@ +name: Cloud Monitoring Lint & Test + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/cloud_monitoring.yaml" + - "cloud-monitoring/**" + +jobs: + test_and_lint: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-22.04", "ubuntu-latest"] + python-version: ["3.12", "3.x"] + + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + cd cloud-monitoring + python -m pip install --upgrade pip + # Install the package with all optional dependencies for testing + pip install -e ".[test,dev]" + + - name: Check code formatting with Black + run: | + cd cloud-monitoring + black --check --diff cloudmon/ tests/ + + - name: Analyse with pylint + run: | + cd cloud-monitoring + pylint . --recursive=true + + - name: Run tests with Pytest + run: | + cd cloud-monitoring + python -m pytest tests --cov-report xml:coverage.xml --cov + + - name: Upload coverage to Codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: cloud-chatops/coverage.xml diff --git a/.github/workflows/test_and_lint_rabbit_consumer.yaml b/.github/workflows/test_and_lint_rabbit_consumer.yaml new file mode 100644 index 00000000..155c5f4e --- /dev/null +++ b/.github/workflows/test_and_lint_rabbit_consumer.yaml @@ -0,0 +1,60 @@ +name: Rabbit Consumer Lint & Test + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/rabbit_consumer.yaml" + - "openstack-rabbit-consumer/**" + +jobs: + test_and_lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + # https://github.com/actions/setup-python/releases + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Required for requests-kerberos + sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update + sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install libkrb5-dev --fix-missing + pip install -r openstack-rabbit-consumer/requirements.txt + pip install -r openstack-rabbit-consumer/requirements-test.txt + + - name: Check code formatting with Black + run: | + cd openstack-rabbit-consumer + black --check --diff rabbit_consumer/ tests/ + + - name: Run tests + # Using Python3 to launch the module sets up the Python path for us + run: cd openstack-rabbit-consumer && python3 -m coverage run -m pytest . + + - name: Analyse with pylint + run: | + cd openstack-rabbit-consumer && pylint $(git ls-files '*.py') + + - name: Prepare coverage + run: | + cd openstack-rabbit-consumer && python -m coverage xml + + - name: Upload coverage to codecov + # https://github.com/codecov/codecov-action/releases + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.1.1 + with: + files: openstack-rabbit-consumer/coverage.xml + fail_ci_if_error: true + flags: rabbit_consumer + token: ${{secrets.CODECOV_TOKEN}} diff --git a/.github/workflows/test_and_lint_username_service.yaml b/.github/workflows/test_and_lint_username_service.yaml new file mode 100644 index 00000000..3e6775bc --- /dev/null +++ b/.github/workflows/test_and_lint_username_service.yaml @@ -0,0 +1,47 @@ +name: Username Service Test and Lint + +on: + push: + branches: + - master + pull_request: + paths: + - ".github/workflows/username_service.yaml" + - "stfc-username-service/**" + +jobs: + golangci-lint: + name: runner / golangci-lint + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + # https://github.com/actions/checkout/releases + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: golangci-lint + # https://github.com/golangci/golangci-lint-action/releases + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + working-directory: stfc-username-service/ + + run_tests: + runs-on: ubuntu-latest + steps: + # https://github.com/actions/checkout/releases + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Docker Buildx + # https://github.com/docker/setup-buildx-action/releases + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.4.1 + + - name: Run tests in container + # https://github.com/docker/build-push-action/releases + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.4.0 + with: + no-cache: true + cache-to: type=gha,mode=max + target: test-stage + push: false + context: "{{defaultContext}}:stfc-username-service" diff --git a/.github/workflows/username_service.yaml b/.github/workflows/username_service.yaml deleted file mode 100644 index 204acfc9..00000000 --- a/.github/workflows/username_service.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Username Service - -on: - push: - branches: - - master - pull_request: - paths: - - ".github/workflows/username_service.yaml" - - "stfc-username-service/**" - -jobs: - golangci-lint: - name: runner / golangci-lint - runs-on: ubuntu-latest - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - - - name: golangci-lint - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a - with: - working-directory: stfc-username-service/ - - run_tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Run tests in container - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - no-cache: true - cache-to: type=gha,mode=max - target: test-stage - push: false - context: "{{defaultContext}}:stfc-username-service" - - push_dev_image_harbor: - runs-on: ubuntu-latest - needs: [run_tests, golangci-lint] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.STAGING_HARBOR_USERNAME }} - password: ${{ secrets.STAGING_HARBOR_TOKEN }} - - - name: Set commit SHA for later - id: commit_sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Build and push to staging project - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:stfc-username-service" - tags: "harbor.stfc.ac.uk/stfc-cloud-staging/stfc-username-service:${{ steps.commit_sha.outputs.sha_short }}" - - - name: Inform of tagged name - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud-staging/stfc-username-service:${{ steps.commit_sha.outputs.sha_short }}" - - push_release_image_harbor: - runs-on: ubuntu-latest - needs: [run_tests, golangci-lint] - if: github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Login to Harbor - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee - with: - registry: harbor.stfc.ac.uk - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_TOKEN }} - - - name: Get release tag for later - id: release_tag - run: echo "version=$(cat stfc-username-service/version.txt)" >> $GITHUB_OUTPUT - - - name: Check if release file has updated - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d - id: release_updated - with: - filters: | - version: - - 'stfc-username-service/version.txt' - - - name: Build and push on version change - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a - if: steps.release_updated.outputs.version == 'true' - with: - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - context: "{{defaultContext}}:stfc-username-service" - tags: "harbor.stfc.ac.uk/stfc-cloud/stfc-username-service:v${{ steps.release_tag.outputs.version }}" - - - name: Inform of tagged name - if: steps.release_updated.outputs.version == 'true' - run: echo "Image published to harbor.stfc.ac.uk/stfc-cloud/stfc-username-service:v${{ steps.release_tag.outputs.version }}" diff --git a/README.md b/README.md index 879686e5..f801582f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,38 @@ -## Building Images Locally +# Cloud Docker Images -`build_images.sh` will build all docker images locally. +This repo stores various Docker images and build scripts that are used by the STFC Cloud -## CI and Automated Builds +# Adding a new docker image -[![CI](https://img.shields.io/github/actions/workflow/status/stfc/cloud-docker-images/build_images.yaml?logo=docker)](https://github.com/stfc/cloud-docker-images/actions/workflows/build_images.yaml?query=branch%3Amaster) -[![codecov](https://codecov.io/gh/stfc/cloud-docker-images/graph/badge.svg?token=BZEBAE0TQD)](https://codecov.io/gh/stfc/cloud-docker-images) +Create a new unique directory and create a `Dockerfile` and `version.txt`. -Images are built: +The `version.txt` will track the latest version of your image for build tagging and version bumps. -- On Pull Requests -- On merge to `master` -- Weekly +If adding source-code - please a new CI github action for performing tests and linting checks + + +# Image build scripts and CI + +Docker images are built and pushed to https://harbor.stfc.ac.uk by CI jobs + + +`.github/workflows/create_dev_image.yml` runs when a PR is created and will build and push a dev image following the format: `-dev.-` - e.g. `cloud-monitoring-dev.1.0.0-ase34e1` + +`.github/workflows/create_release_image.yml` runs when the PR is merged. A new version - specified in `version.txt` is required to be bumped before merging. Once merged - a new image with that version will be built and pushed to harbor + +To manually build and push images, you can run: + +```sh +VERSION=$(cat cloud-monitoring/version.txt) + +# docker build //:$VERSION +docker build harbor.stfc.ac.uk/stfc-cloud/cloud-monitoring:$VERSION --push + +# for a dev image +docker build harbor.stfc.ac.uk/stfc-cloud-staging/cloud-monitoring:$VERSION-dev.1 --push +``` + +## Linting and Testing + +Currently linting and testing CI actions are prefixed with `test_and_lint*`. Since each image is built slightly differently, we have separate CI actions for each. These CI jobs do not build images and are run when a PR is opened -In addition, the successful builds on `master` will be uploaded to [Harbor](https://harbor.stfc.ac.uk) too. diff --git a/build_images.sh b/build_images.sh deleted file mode 100755 index 3c7bf9dd..00000000 --- a/build_images.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -# Builds all docker images in all subirectories -find -name Dockerfile -execdir docker build . \; -echo "All images built successfully" diff --git a/cloud-monitoring/version.txt b/cloud-monitoring/version.txt new file mode 100644 index 00000000..3eefcb9d --- /dev/null +++ b/cloud-monitoring/version.txt @@ -0,0 +1 @@ +1.0.0 diff --git a/exporters/ipmi_exporter/version.txt b/exporters/ipmi_exporter/version.txt deleted file mode 100644 index e33692ab..00000000 --- a/exporters/ipmi_exporter/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.10.1 \ No newline at end of file diff --git a/exporters/ipmi_exporter/Dockerfile b/ipmi-exporter/Dockerfile similarity index 100% rename from exporters/ipmi_exporter/Dockerfile rename to ipmi-exporter/Dockerfile diff --git a/ipmi-exporter/version.txt b/ipmi-exporter/version.txt new file mode 100644 index 00000000..4dae2985 --- /dev/null +++ b/ipmi-exporter/version.txt @@ -0,0 +1 @@ +1.10.1 diff --git a/openstack-rabbit-consumer/requirements-test.txt b/openstack-rabbit-consumer/requirements-test.txt index c74b26f9..f1b5873f 100644 --- a/openstack-rabbit-consumer/requirements-test.txt +++ b/openstack-rabbit-consumer/requirements-test.txt @@ -1,3 +1,4 @@ coverage pylint -pytest \ No newline at end of file +pytest +black \ No newline at end of file diff --git a/openstack-rabbit-consumer/version.txt b/openstack-rabbit-consumer/version.txt index 0fa4ae48..712bd5a6 100644 --- a/openstack-rabbit-consumer/version.txt +++ b/openstack-rabbit-consumer/version.txt @@ -1 +1 @@ -3.3.0 \ No newline at end of file +3.3.1 \ No newline at end of file diff --git a/stfc-username-service/version.txt b/stfc-username-service/version.txt index 7f207341..6d7de6e6 100644 --- a/stfc-username-service/version.txt +++ b/stfc-username-service/version.txt @@ -1 +1 @@ -1.0.1 \ No newline at end of file +1.0.2